diff --git a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c index 765aa6e..85b4595 100644 --- a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c +++ b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c @@ -6099,6 +6099,21 @@ static void engram_bll_parse_access(EngramNode* nn, const char* s) { * propagation loop in engram_activate. 0.25 damps semantically unrelated * branches ~4x without severing them. Unembedded targets are ungated. */ #define ENGRAM_QGATE_FLOOR 0.25 +/* ENGRAM_SEM_WEIGHT: weight of the semantic (cosine) leg in the fused read-path + * score, per engram claim 24 — "maintain a vector similarity index over the + * semantic embedding vectors of all stored node records, and ... respond to + * embedding search queries by returning the node records whose embedding + * vectors have the highest cosine similarity to a query vector, independently + * of the spreading activation traversal." + * The lexical leg contributes distinct-token coverage in [0,1]; the semantic + * leg contributes at most ENGRAM_SEM_WEIGHT. Holding this strictly BELOW 1.0 + * is the safety property: a node matching every query token can never be + * displaced by semantics alone, so exact/phrase retrieval cannot regress the + * way it did when the read path was swapped wholesale to spreading activation + * (PR #135: phrase 85.7% -> 28.6%). Semantics reorders WITHIN and BELOW the + * full-lexical-match band, and admits high-cosine nodes the lexical pass + * missed entirely. */ +#define ENGRAM_SEM_WEIGHT 0.90 #define ENGRAM_EMBED_MAX_CHARS 2000 #define ENGRAM_EMBED_TIMEOUT_MS 4000L #define ENGRAM_EMBED_BREAKER_LIMIT 3 @@ -7381,13 +7396,39 @@ static int engram_node_match_score(const EngramNode* n, return score; } -/* Rank entry: distinct-token match count (primary, desc) then salience - * (tiebreak, desc). */ -typedef struct { int64_t idx; int score; double salience; } EngramRankEntry; +/* Semantic leg of the read path (engram claim 24). Returns the query/target + * cosine renormalized onto [0,1] over the band [ENGRAM_EMBED_SEED_MIN, 1.0], + * and exactly 0.0 when the pair is not comparable (no query embedding, target + * unembedded, dim mismatch) or falls at/below the seed floor. Claim 32's + * clamp-at-zero is subsumed: nothing below the floor can contribute. + * A 0.0 return makes the fused score collapse to the lexical score, which is + * why a dead embedder degrades to the historical behaviour exactly. */ +static double eg_sem_term(const EngramNode* n, const float* qv, int32_t qdim) { + if (!qv || qdim <= 0 || !n->emb || n->emb_dim != qdim) return 0.0; + double c = eg_cosine(n->emb, qv, qdim); + if (c <= ENGRAM_EMBED_SEED_MIN) return 0.0; + double t = (c - ENGRAM_EMBED_SEED_MIN) / (1.0 - ENGRAM_EMBED_SEED_MIN); + return t > 1.0 ? 1.0 : t; +} + +/* Fused rank score: distinct-token coverage in [0,1] plus at most + * ENGRAM_SEM_WEIGHT of semantic similarity. With no query embedding this is + * sc/ntok, a strictly monotone map of the old integer score, so the ordering + * is bit-identical to the pre-semantic ranking. */ +static double eg_fused_score(int sc, int ntok, double sem) { + double f = (double)sc / (double)(ntok > 0 ? ntok : 1); + return f + ENGRAM_SEM_WEIGHT * sem; +} + +/* Rank entry: fused lexical+semantic score (primary, desc) then salience + * (tiebreak, desc). `score` retains the raw distinct-token count for callers + * that want the lexical signal on its own. */ +typedef struct { int64_t idx; int score; double fused; double salience; } EngramRankEntry; static int engram_rank_cmp(const void* a, const void* b) { const EngramRankEntry* ea = (const EngramRankEntry*)a; const EngramRankEntry* eb = (const EngramRankEntry*)b; - if (ea->score != eb->score) return eb->score - ea->score; /* desc */ + if (ea->fused < eb->fused) return 1; /* desc */ + if (ea->fused > eb->fused) return -1; if (ea->salience < eb->salience) return 1; if (ea->salience > eb->salience) return -1; return 0; @@ -7405,6 +7446,10 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { if (ntok == 0) return lst; EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry)); if (!hits) return lst; + /* Claim-24 semantic leg: one query embedding, fetched once per search. + * NULL (embedder down / circuit breaker open) => pure lexical, as before. */ + int32_t qdim = 0; + float* qv = eg_embed_fetch(q, &qdim); int64_t nhits = 0; for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; @@ -7414,20 +7459,26 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { * + engram_compile_layered_json — that's the legitimate path. */ if (engram_layer_is_transparent(n->layer_id)) continue; int sc = engram_node_match_score(n, toks, ntok); - if (sc > 0) { + double sem = eg_sem_term(n, qv, qdim); + /* Union, not replacement: a node enters the candidate set on EITHER + * leg. sc == 0 && sem > 0 is the embedding-search half of claim 24 — + * nodes the lexical pass cannot see at all. */ + if (sc > 0 || sem > 0.0) { hits[nhits].idx = i; hits[nhits].score = sc; + hits[nhits].fused = eg_fused_score(sc, ntok, sem); hits[nhits].salience = n->salience; nhits++; } } - /* Rank by distinct tokens matched (desc) then salience (desc), then cap. */ + /* Rank by fused score (desc) then salience (desc), then cap. */ qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); int64_t end = nhits < lim ? nhits : lim; for (int64_t k = 0; k < end; k++) { lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx])); } free(hits); + free(qv); return lst; } @@ -9259,15 +9310,23 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { if (ntok > 0) { EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry)); if (hits) { + /* Claim-24 semantic leg. This is the function /api/neuron/recall + * actually reaches (routes.el -> neuron-api.el handle_api_recall), + * so the semantic half of the retrieval surface has to land HERE + * to be observable to the MCP wrapper and the app. */ + int32_t qdim = 0; + float* qv = eg_embed_fetch(q, &qdim); int64_t nhits = 0; for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; /* Filter transparent layers — same as engram_search. */ if (engram_layer_is_transparent(n->layer_id)) continue; int sc = engram_node_match_score(n, toks, ntok); - if (sc > 0) { + double sem = eg_sem_term(n, qv, qdim); + if (sc > 0 || sem > 0.0) { hits[nhits].idx = i; hits[nhits].score = sc; + hits[nhits].fused = eg_fused_score(sc, ntok, sem); hits[nhits].salience = n->salience; nhits++; } @@ -9280,6 +9339,7 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { first = 0; } free(hits); + free(qv); } } }