neuron-technologies/neuron
3
0
Fork 0
Code Issues 45 Pull Requests 8 Actions Packages Projects Releases Wiki Activity

feat(engram): BM25-shaped lexical leg + addressability guard on the read path

Browse Source 
engram_search_json ranked its lexical leg by raw distinct-token coverage with
salience as tiebreak: a token in 30,000 nodes counted the same as a token in 1,
and a 1.3 MB record matched nearly every query token by surface area alone.
Score it BM25-shaped instead - Lucene-form IDF and length normalisation over
the corpus mean - with per-token document frequency accumulated in the SAME
corpus pass that finds the hits (no extra scan, no extra round-trip).

Also refuse to return records whose identifier is not printable ASCII. This
corpus carries 1,032 such records (453 by the printable test) from a save-side
corruption; they occupy 125 of 303 returned slots on main. Claims 12, 23 and 27
all key on the node identifier, so such a record is unfetchable by any caller.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Tim Lingo
2026-08-07 15:51:49 -05:00
parent 6f3a048f36
commit 9c39084e60
1 changed files with 107 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vendor/el-runtime/v1.0.0-20260501/el_runtime.c
Vendored
+107 -4
View File
@@ -7387,6 +7387,48 @@ static int engram_node_match_score(const EngramNode* n,
return score;
}
/* Same match test as engram_node_match_score, but returns the SET of matched
* query tokens as a bitmask instead of only their count. ENGRAM_MAX_QTOKENS is
* 32, so one uint32 covers every token the tokenizer can produce. The mask is
* what lets the caller accumulate a per-token document frequency in the SAME
* pass that finds the hits — no second scan of the corpus. */
static uint32_t engram_node_match_mask(const EngramNode* n,
char toks[][ENGRAM_QTOK_LEN], int ntok) {
uint32_t m = 0;
for (int t = 0; t < ntok && t < 32; t++) {
if (istr_contains(n->content, toks[t]) ||
istr_contains(n->label, toks[t]) ||
istr_contains(n->tags, toks[t]))
m |= (uint32_t)1u << t;
}
return m;
}
/* Searchable byte length of a node: the same three fields the match test
* reads. Used as the BM25 document length so a long node does not out-match a
* short one merely by containing more text. */
static double engram_node_len(const EngramNode* n) {
double l = 0.0;
if (n->content) l += (double)strlen(n->content);
if (n->label) l += (double)strlen(n->label);
if (n->tags) l += (double)strlen(n->tags);
return l;
}
/* Addressability guard. Claim 23 stores node records under a key encoding the
* node identifier, claim 12 deduplicates merged results by node identifier,
* and claim 27's competition map is indexed by node identifier — every one of
* those requires the identifier to be a usable string. This corpus contains
* records whose id field is binary garbage (a save-side corruption); they are
* unfetchable by any caller, so returning one wastes a result slot. Printable
* ASCII, non-empty, is the whole test. */
static int eg_node_addressable(const EngramNode* n) {
const unsigned char* p = (const unsigned char*)n->id;
if (!p || !*p) return 0;
for (; *p; p++) if (*p < 0x20 || *p > 0x7e) return 0;
return 1;
}
/* Semantic leg of the read path (engram claim 24). Returns the query/target
* cosine renormalized onto [0,1] over the band [ENGRAM_EMBED_SEED_MIN, 1.0],
* and exactly 0.0 when the pair is not comparable (no query embedding, target
@@ -7406,7 +7448,12 @@ static double eg_sem_term(const EngramNode* n, const float* qv, int32_t qdim) {
* (tiebreak, desc). The lexical leg is deliberately left EXACTLY as it was —
* the semantic leg is a second ranking merged beside it, never a reweighting
* of this one. */
typedef struct { int64_t idx; int score; double salience; } EngramRankEntry;
typedef struct {
int64_t idx; int score; double salience;
uint32_t mask; /* which query tokens matched (BM25 leg) */
double len; /* searchable byte length (BM25 leg) */
double w; /* BM25-shaped weighted score */
} EngramRankEntry;
static int engram_rank_cmp(const void* a, const void* b) {
const EngramRankEntry* ea = (const EngramRankEntry*)a;
const EngramRankEntry* eb = (const EngramRankEntry*)b;
@@ -7416,6 +7463,21 @@ static int engram_rank_cmp(const void* a, const void* b) {
return 0;
}
/* BM25-shaped ordering for the read path's lexical leg: rare-term weight and
* length normalisation instead of a raw distinct-token count. Salience stays
* the tiebreak, exactly as in engram_rank_cmp. */
#define ENGRAM_BM25_K1 1.2
#define ENGRAM_BM25_B 0.75
static int engram_rank_w_cmp(const void* a, const void* b) {
const EngramRankEntry* ea = (const EngramRankEntry*)a;
const EngramRankEntry* eb = (const EngramRankEntry*)b;
if (ea->w < eb->w) return 1; /* desc */
if (ea->w > eb->w) return -1;
if (ea->salience < eb->salience) return 1;
if (ea->salience > eb->salience) return -1;
return 0;
}
/* Semantic rank entry: node index and its renormalized query similarity,
* ordered by similarity desc. This is the claim-24 "embedding search"
* ranking, computed independently of the lexical one. */
@@ -7615,6 +7677,7 @@ static int64_t engram_assoc_leg(EngramStore* g,
if (seen[i] != 1) continue; /* skip unreached and seeds */
if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue;
EngramNode* nd = &g->nodes[i];
if (!eg_node_addressable(nd)) continue; /* unfetchable record */
if (!nd->emb || nd->emb_dim != qdim) continue;
double c = eg_cosine(nd->emb, qv, qdim);
if (c <= 0.0) continue;
@@ -9561,15 +9624,32 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
int64_t semseed[ENGRAM_EMBED_SEED_K];
double semseedc[ENGRAM_EMBED_SEED_K];
int64_t nsemseed = 0;
/* BM25 statistics gathered in this same pass: per-token
* document frequency, and the corpus mean field length. */
int64_t df[ENGRAM_MAX_QTOKENS];
for (int t = 0; t < ntok; t++) df[t] = 0;
double dl_total = 0.0;
int64_t dl_n = 0;
for (int64_t i = 0; i < g->node_count; i++) {
EngramNode* n = &g->nodes[i];
/* Filter transparent layers — same as engram_search. */
if (engram_layer_is_transparent(n->layer_id)) continue;
int sc = engram_node_match_score(n, toks, ntok);
if (sc > 0) {
/* Unaddressable records cannot be fetched by a caller and
* must not consume a result slot (claims 12/23/27). */
if (!eg_node_addressable(n)) continue;
double dl = engram_node_len(n);
dl_total += dl; dl_n++;
uint32_t mask = engram_node_match_mask(n, toks, ntok);
if (mask) {
int sc = 0;
for (int t = 0; t < ntok; t++)
if (mask & ((uint32_t)1u << t)) { sc++; df[t]++; }
hits[nhits].idx = i;
hits[nhits].score = sc;
hits[nhits].salience = n->salience;
hits[nhits].mask = mask;
hits[nhits].len = dl;
hits[nhits].w = 0.0;
nhits++;
}
if (sem && n->emb && n->emb_dim == qdim) {
@@ -9597,7 +9677,30 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
}
}
}
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
/* BM25-shaped lexical score. Binary term frequency (the match
* primitive is a substring test, not a count), Lucene-form IDF,
* and length normalisation over the corpus mean. A token that
* occurs in 30,000 nodes now weighs far less than one that
* occurs in 1, and a 1.3 MB record no longer out-matches a
* 300-byte one by sheer surface area. */
double avgdl = dl_n ? (dl_total / (double)dl_n) : 1.0;
if (avgdl <= 0.0) avgdl = 1.0;
double idf[ENGRAM_MAX_QTOKENS];
for (int t = 0; t < ntok; t++) {
double dfx = (double)df[t];
idf[t] = log(1.0 + ((double)dl_n - dfx + 0.5) / (dfx + 0.5));
}
for (int64_t h = 0; h < nhits; h++) {
double norm = 1.0 - ENGRAM_BM25_B
+ ENGRAM_BM25_B * (hits[h].len / avgdl);
double s = 0.0;
for (int t = 0; t < ntok; t++)
if (hits[h].mask & ((uint32_t)1u << t))
s += idf[t] * (ENGRAM_BM25_K1 + 1.0)
/ (1.0 + ENGRAM_BM25_K1 * norm);
hits[h].w = s;
}
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_w_cmp);
if (sem) qsort(sem, (size_t)nsem, sizeof(EngramSemEntry), engram_sem_cmp);
/* Claim-10 associative leg: expand the top lexical hits along
* structural relations only, order the reached set by query