From 315b2eff00e7cec6be499d47763cddd69b24238b Mon Sep 17 00:00:00 2001 From: Neuron Date: Fri, 7 Aug 2026 14:44:42 -0500 Subject: [PATCH] feat(engram): fuse cosine similarity into the recall read path (claim 24) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit engram_search_json — the function /api/neuron/recall actually reaches — ranked only by distinct-token match count, so the embedding field on every node record was inert. Add the semantic leg as a UNION beside the lexical one, not a replacement for it: fused = (distinct_tokens_matched / query_tokens) + 0.90 * sem sem = clamp01((cos(q,n) - 0.60) / (1 - 0.60)) ; 0 when not comparable Holding the semantic weight strictly below 1.0 means a node matching every query token can never be displaced by semantics alone — the regression guard that PR #135 lacked when it swapped the read path to spreading activation and took phrase recall from 85.7% to 28.6%. No query embedding (embedder down, circuit breaker open) => sem == 0 for all nodes => fused == sc/ntok, a monotone map of the old integer score, so the ordering degrades to the historical behaviour exactly. Restores engram claim 24: 'maintain a vector similarity index over the semantic embedding vectors of all stored node records, and ... respond to embedding search queries by returning the node records whose embedding vectors have the highest cosine similarity to a query vector, independently of the spreading activation traversal.' Co-Authored-By: Claude Opus 5 (1M context) --- .../el-runtime/v1.0.0-20260501/el_runtime.c | 74 +++++++++++++++++-- 1 file changed, 67 insertions(+), 7 deletions(-) diff --git a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c index 765aa6e..85b4595 100644 --- a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c +++ b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c @@ -6099,6 +6099,21 @@ static void engram_bll_parse_access(EngramNode* nn, const char* s) { * propagation loop in engram_activate. 0.25 damps semantically unrelated * branches ~4x without severing them. Unembedded targets are ungated. */ #define ENGRAM_QGATE_FLOOR 0.25 +/* ENGRAM_SEM_WEIGHT: weight of the semantic (cosine) leg in the fused read-path + * score, per engram claim 24 — "maintain a vector similarity index over the + * semantic embedding vectors of all stored node records, and ... respond to + * embedding search queries by returning the node records whose embedding + * vectors have the highest cosine similarity to a query vector, independently + * of the spreading activation traversal." + * The lexical leg contributes distinct-token coverage in [0,1]; the semantic + * leg contributes at most ENGRAM_SEM_WEIGHT. Holding this strictly BELOW 1.0 + * is the safety property: a node matching every query token can never be + * displaced by semantics alone, so exact/phrase retrieval cannot regress the + * way it did when the read path was swapped wholesale to spreading activation + * (PR #135: phrase 85.7% -> 28.6%). Semantics reorders WITHIN and BELOW the + * full-lexical-match band, and admits high-cosine nodes the lexical pass + * missed entirely. */ +#define ENGRAM_SEM_WEIGHT 0.90 #define ENGRAM_EMBED_MAX_CHARS 2000 #define ENGRAM_EMBED_TIMEOUT_MS 4000L #define ENGRAM_EMBED_BREAKER_LIMIT 3 @@ -7381,13 +7396,39 @@ static int engram_node_match_score(const EngramNode* n, return score; } -/* Rank entry: distinct-token match count (primary, desc) then salience - * (tiebreak, desc). */ -typedef struct { int64_t idx; int score; double salience; } EngramRankEntry; +/* Semantic leg of the read path (engram claim 24). Returns the query/target + * cosine renormalized onto [0,1] over the band [ENGRAM_EMBED_SEED_MIN, 1.0], + * and exactly 0.0 when the pair is not comparable (no query embedding, target + * unembedded, dim mismatch) or falls at/below the seed floor. Claim 32's + * clamp-at-zero is subsumed: nothing below the floor can contribute. + * A 0.0 return makes the fused score collapse to the lexical score, which is + * why a dead embedder degrades to the historical behaviour exactly. */ +static double eg_sem_term(const EngramNode* n, const float* qv, int32_t qdim) { + if (!qv || qdim <= 0 || !n->emb || n->emb_dim != qdim) return 0.0; + double c = eg_cosine(n->emb, qv, qdim); + if (c <= ENGRAM_EMBED_SEED_MIN) return 0.0; + double t = (c - ENGRAM_EMBED_SEED_MIN) / (1.0 - ENGRAM_EMBED_SEED_MIN); + return t > 1.0 ? 1.0 : t; +} + +/* Fused rank score: distinct-token coverage in [0,1] plus at most + * ENGRAM_SEM_WEIGHT of semantic similarity. With no query embedding this is + * sc/ntok, a strictly monotone map of the old integer score, so the ordering + * is bit-identical to the pre-semantic ranking. */ +static double eg_fused_score(int sc, int ntok, double sem) { + double f = (double)sc / (double)(ntok > 0 ? ntok : 1); + return f + ENGRAM_SEM_WEIGHT * sem; +} + +/* Rank entry: fused lexical+semantic score (primary, desc) then salience + * (tiebreak, desc). `score` retains the raw distinct-token count for callers + * that want the lexical signal on its own. */ +typedef struct { int64_t idx; int score; double fused; double salience; } EngramRankEntry; static int engram_rank_cmp(const void* a, const void* b) { const EngramRankEntry* ea = (const EngramRankEntry*)a; const EngramRankEntry* eb = (const EngramRankEntry*)b; - if (ea->score != eb->score) return eb->score - ea->score; /* desc */ + if (ea->fused < eb->fused) return 1; /* desc */ + if (ea->fused > eb->fused) return -1; if (ea->salience < eb->salience) return 1; if (ea->salience > eb->salience) return -1; return 0; @@ -7405,6 +7446,10 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { if (ntok == 0) return lst; EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry)); if (!hits) return lst; + /* Claim-24 semantic leg: one query embedding, fetched once per search. + * NULL (embedder down / circuit breaker open) => pure lexical, as before. */ + int32_t qdim = 0; + float* qv = eg_embed_fetch(q, &qdim); int64_t nhits = 0; for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; @@ -7414,20 +7459,26 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { * + engram_compile_layered_json — that's the legitimate path. */ if (engram_layer_is_transparent(n->layer_id)) continue; int sc = engram_node_match_score(n, toks, ntok); - if (sc > 0) { + double sem = eg_sem_term(n, qv, qdim); + /* Union, not replacement: a node enters the candidate set on EITHER + * leg. sc == 0 && sem > 0 is the embedding-search half of claim 24 — + * nodes the lexical pass cannot see at all. */ + if (sc > 0 || sem > 0.0) { hits[nhits].idx = i; hits[nhits].score = sc; + hits[nhits].fused = eg_fused_score(sc, ntok, sem); hits[nhits].salience = n->salience; nhits++; } } - /* Rank by distinct tokens matched (desc) then salience (desc), then cap. */ + /* Rank by fused score (desc) then salience (desc), then cap. */ qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); int64_t end = nhits < lim ? nhits : lim; for (int64_t k = 0; k < end; k++) { lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx])); } free(hits); + free(qv); return lst; } @@ -9259,15 +9310,23 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { if (ntok > 0) { EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry)); if (hits) { + /* Claim-24 semantic leg. This is the function /api/neuron/recall + * actually reaches (routes.el -> neuron-api.el handle_api_recall), + * so the semantic half of the retrieval surface has to land HERE + * to be observable to the MCP wrapper and the app. */ + int32_t qdim = 0; + float* qv = eg_embed_fetch(q, &qdim); int64_t nhits = 0; for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; /* Filter transparent layers — same as engram_search. */ if (engram_layer_is_transparent(n->layer_id)) continue; int sc = engram_node_match_score(n, toks, ntok); - if (sc > 0) { + double sem = eg_sem_term(n, qv, qdim); + if (sc > 0 || sem > 0.0) { hits[nhits].idx = i; hits[nhits].score = sc; + hits[nhits].fused = eg_fused_score(sc, ntok, sem); hits[nhits].salience = n->salience; nhits++; } @@ -9280,6 +9339,7 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { first = 0; } free(hits); + free(qv); } } }