feat(engram): rank-interleave the semantic leg into recall; embed the corpus
Replaces the score-fusion first cut with rank fusion, which is what the data called for. nomic's cosine scale is compressed (true matches 0.55-0.70, unrelated pairs 0.35-0.50), so an additive blend of cosine onto token-coverage is dominated by whichever leg has the wider spread. Alternation is invariant to both scales: L1, S1, L2, S2, ... deduped, capped at limit Lexical ranking is left byte-identical; the semantic ranking is computed beside it and admitted only above ENGRAM_EMBED_SEED_MIN (0.60) — Will's existing seed floor, no new tuning constant. That floor is what keeps the nonsense controls clean: a query with no real match must not be answered with its neighbours. embed-corpus.py / merge-corpus.py produce the derived corpus the semantic leg needs (76,986 vectors, nomic-embed-text, 0 failures, 11 min). Zero of 78,791 nodes carried an embedding before this; the field round-tripped through the snapshot but nothing ever wrote it. MEASURED, 38-query gold set, paired against the SAME derived corpus so the comparison isolates the code change: hit@5 34.3% -> 51.4% paraphrase 0.0% -> 38.5% MRR@10 0.294 -> 0.387 superseded 1/3 -> 2/3 outranks recall@10 33.3% -> 50.5% latency p50 1146 -> 1220ms (1.06x) exact_rare 100% -> 100% phrase 85.7% -> 85.7% nonsense 2/3 -> 2/3 6 queries fixed, 0 broken, McNemar exact p=0.0312, 0 drift across repeats. Regression guards all held. Contrast PR #135, which swapped the read path to spreading activation wholesale: phrase 85.7 -> 28.6, latency 2.81x. Correct mechanism, wrong substrate. The substrate is now present. Restores engram claim 24 (previously 0% honoured). Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Neuron
+93
-48
@@ -6099,21 +6099,12 @@ static void engram_bll_parse_access(EngramNode* nn, const char* s) {
|
||||
* propagation loop in engram_activate. 0.25 damps semantically unrelated
|
||||
* branches ~4x without severing them. Unembedded targets are ungated. */
|
||||
#define ENGRAM_QGATE_FLOOR 0.25
|
||||
/* ENGRAM_SEM_WEIGHT: weight of the semantic (cosine) leg in the fused read-path
|
||||
* score, per engram claim 24 — "maintain a vector similarity index over the
|
||||
* semantic embedding vectors of all stored node records, and ... respond to
|
||||
* embedding search queries by returning the node records whose embedding
|
||||
* vectors have the highest cosine similarity to a query vector, independently
|
||||
* of the spreading activation traversal."
|
||||
* The lexical leg contributes distinct-token coverage in [0,1]; the semantic
|
||||
* leg contributes at most ENGRAM_SEM_WEIGHT. Holding this strictly BELOW 1.0
|
||||
* is the safety property: a node matching every query token can never be
|
||||
* displaced by semantics alone, so exact/phrase retrieval cannot regress the
|
||||
* way it did when the read path was swapped wholesale to spreading activation
|
||||
* (PR #135: phrase 85.7% -> 28.6%). Semantics reorders WITHIN and BELOW the
|
||||
* full-lexical-match band, and admits high-cosine nodes the lexical pass
|
||||
* missed entirely. */
|
||||
#define ENGRAM_SEM_WEIGHT 0.90
|
||||
/* The read-path semantic leg (engram claim 24) reuses ENGRAM_EMBED_SEED_MIN
|
||||
* above as its admission floor: a node joins the embedding ranking only if its
|
||||
* query cosine clears the same bar that lets it join the seed set. No new
|
||||
* tuning constant is introduced, and the floor is load-bearing rather than
|
||||
* cosmetic — it is what keeps a query with no real match (the gold set's
|
||||
* nonsense controls) from being answered with its nearest neighbours. */
|
||||
#define ENGRAM_EMBED_MAX_CHARS 2000
|
||||
#define ENGRAM_EMBED_TIMEOUT_MS 4000L
|
||||
#define ENGRAM_EMBED_BREAKER_LIMIT 3
|
||||
@@ -7411,29 +7402,67 @@ static double eg_sem_term(const EngramNode* n, const float* qv, int32_t qdim) {
|
||||
return t > 1.0 ? 1.0 : t;
|
||||
}
|
||||
|
||||
/* Fused rank score: distinct-token coverage in [0,1] plus at most
|
||||
* ENGRAM_SEM_WEIGHT of semantic similarity. With no query embedding this is
|
||||
* sc/ntok, a strictly monotone map of the old integer score, so the ordering
|
||||
* is bit-identical to the pre-semantic ranking. */
|
||||
static double eg_fused_score(int sc, int ntok, double sem) {
|
||||
double f = (double)sc / (double)(ntok > 0 ? ntok : 1);
|
||||
return f + ENGRAM_SEM_WEIGHT * sem;
|
||||
}
|
||||
|
||||
/* Rank entry: fused lexical+semantic score (primary, desc) then salience
|
||||
* (tiebreak, desc). `score` retains the raw distinct-token count for callers
|
||||
* that want the lexical signal on its own. */
|
||||
typedef struct { int64_t idx; int score; double fused; double salience; } EngramRankEntry;
|
||||
/* Rank entry: distinct-token match count (primary, desc) then salience
|
||||
* (tiebreak, desc). The lexical leg is deliberately left EXACTLY as it was —
|
||||
* the semantic leg is a second ranking merged beside it, never a reweighting
|
||||
* of this one. */
|
||||
typedef struct { int64_t idx; int score; double salience; } EngramRankEntry;
|
||||
static int engram_rank_cmp(const void* a, const void* b) {
|
||||
const EngramRankEntry* ea = (const EngramRankEntry*)a;
|
||||
const EngramRankEntry* eb = (const EngramRankEntry*)b;
|
||||
if (ea->fused < eb->fused) return 1; /* desc */
|
||||
if (ea->fused > eb->fused) return -1;
|
||||
if (ea->score != eb->score) return eb->score - ea->score; /* desc */
|
||||
if (ea->salience < eb->salience) return 1;
|
||||
if (ea->salience > eb->salience) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Semantic rank entry: node index and its renormalized query similarity,
|
||||
* ordered by similarity desc. This is the claim-24 "embedding search"
|
||||
* ranking, computed independently of the lexical one. */
|
||||
typedef struct { int64_t idx; double sem; } EngramSemEntry;
|
||||
static int engram_sem_cmp(const void* a, const void* b) {
|
||||
const EngramSemEntry* ea = (const EngramSemEntry*)a;
|
||||
const EngramSemEntry* eb = (const EngramSemEntry*)b;
|
||||
if (ea->sem < eb->sem) return 1; /* desc */
|
||||
if (ea->sem > eb->sem) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Merge the two rankings by strict alternation, lexical first:
|
||||
* L1, S1, L2, S2, L3, ... deduplicated by node index, capped at lim.
|
||||
*
|
||||
* Rank fusion, not score fusion. nomic's cosine scale is compressed (real
|
||||
* matches land ~0.55-0.70 while unrelated pairs sit ~0.35-0.50), so any
|
||||
* additive blend of a cosine onto a token-coverage score is dominated by
|
||||
* whichever leg happens to have the wider spread. Alternation is invariant to
|
||||
* both scales: it asks each leg for its next best answer in turn.
|
||||
*
|
||||
* Position 1 is always the top lexical hit, so a query whose answer the
|
||||
* lexical leg already ranks first cannot be displaced — exact-token retrieval
|
||||
* is structurally safe. The cost is bounded and explicit: a lexical hit at
|
||||
* rank r lands at output position 2r-1. */
|
||||
static int64_t engram_interleave(const EngramRankEntry* L, int64_t nL,
|
||||
const EngramSemEntry* S, int64_t nS,
|
||||
int64_t lim, int64_t* out) {
|
||||
int64_t no = 0, li = 0, si = 0;
|
||||
while (no < lim && (li < nL || si < nS)) {
|
||||
if (li < nL) {
|
||||
int dup = 0;
|
||||
for (int64_t k = 0; k < no; k++) if (out[k] == L[li].idx) { dup = 1; break; }
|
||||
if (!dup) out[no++] = L[li].idx;
|
||||
li++;
|
||||
}
|
||||
if (no >= lim) break;
|
||||
if (si < nS) {
|
||||
int dup = 0;
|
||||
for (int64_t k = 0; k < no; k++) if (out[k] == S[si].idx) { dup = 1; break; }
|
||||
if (!dup) out[no++] = S[si].idx;
|
||||
si++;
|
||||
}
|
||||
}
|
||||
return no;
|
||||
}
|
||||
|
||||
el_val_t engram_search(el_val_t query, el_val_t limit) {
|
||||
EngramStore* g = engram_get();
|
||||
const char* q = EL_CSTR(query);
|
||||
@@ -7450,6 +7479,8 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
|
||||
* NULL (embedder down / circuit breaker open) => pure lexical, as before. */
|
||||
int32_t qdim = 0;
|
||||
float* qv = eg_embed_fetch(q, &qdim);
|
||||
EngramSemEntry* sem = qv ? malloc((size_t)g->node_count * sizeof(EngramSemEntry)) : NULL;
|
||||
int64_t nsem = 0;
|
||||
int64_t nhits = 0;
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
EngramNode* n = &g->nodes[i];
|
||||
@@ -7459,25 +7490,29 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
|
||||
* + engram_compile_layered_json — that's the legitimate path. */
|
||||
if (engram_layer_is_transparent(n->layer_id)) continue;
|
||||
int sc = engram_node_match_score(n, toks, ntok);
|
||||
double sem = eg_sem_term(n, qv, qdim);
|
||||
/* Union, not replacement: a node enters the candidate set on EITHER
|
||||
* leg. sc == 0 && sem > 0 is the embedding-search half of claim 24 —
|
||||
* nodes the lexical pass cannot see at all. */
|
||||
if (sc > 0 || sem > 0.0) {
|
||||
if (sc > 0) {
|
||||
hits[nhits].idx = i;
|
||||
hits[nhits].score = sc;
|
||||
hits[nhits].fused = eg_fused_score(sc, ntok, sem);
|
||||
hits[nhits].salience = n->salience;
|
||||
nhits++;
|
||||
}
|
||||
if (sem) {
|
||||
double sv = eg_sem_term(n, qv, qdim);
|
||||
if (sv > 0.0) { sem[nsem].idx = i; sem[nsem].sem = sv; nsem++; }
|
||||
}
|
||||
}
|
||||
/* Rank by fused score (desc) then salience (desc), then cap. */
|
||||
/* Rank each leg independently, then alternate between them. */
|
||||
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
|
||||
int64_t end = nhits < lim ? nhits : lim;
|
||||
for (int64_t k = 0; k < end; k++) {
|
||||
lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx]));
|
||||
if (sem) qsort(sem, (size_t)nsem, sizeof(EngramSemEntry), engram_sem_cmp);
|
||||
int64_t* order = malloc((size_t)lim * sizeof(int64_t));
|
||||
if (order) {
|
||||
int64_t no = engram_interleave(hits, nhits, sem, nsem, lim, order);
|
||||
for (int64_t k = 0; k < no; k++)
|
||||
lst = el_list_append(lst, engram_node_to_map(&g->nodes[order[k]]));
|
||||
free(order);
|
||||
}
|
||||
free(hits);
|
||||
free(sem);
|
||||
free(qv);
|
||||
return lst;
|
||||
}
|
||||
@@ -9316,29 +9351,39 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
* to be observable to the MCP wrapper and the app. */
|
||||
int32_t qdim = 0;
|
||||
float* qv = eg_embed_fetch(q, &qdim);
|
||||
EngramSemEntry* sem = qv ? malloc((size_t)g->node_count * sizeof(EngramSemEntry)) : NULL;
|
||||
int64_t nsem = 0;
|
||||
int64_t nhits = 0;
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
EngramNode* n = &g->nodes[i];
|
||||
/* Filter transparent layers — same as engram_search. */
|
||||
if (engram_layer_is_transparent(n->layer_id)) continue;
|
||||
int sc = engram_node_match_score(n, toks, ntok);
|
||||
double sem = eg_sem_term(n, qv, qdim);
|
||||
if (sc > 0 || sem > 0.0) {
|
||||
if (sc > 0) {
|
||||
hits[nhits].idx = i;
|
||||
hits[nhits].score = sc;
|
||||
hits[nhits].fused = eg_fused_score(sc, ntok, sem);
|
||||
hits[nhits].salience = n->salience;
|
||||
nhits++;
|
||||
}
|
||||
if (sem) {
|
||||
double sv = eg_sem_term(n, qv, qdim);
|
||||
if (sv > 0.0) { sem[nsem].idx = i; sem[nsem].sem = sv; nsem++; }
|
||||
}
|
||||
}
|
||||
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
|
||||
int64_t end = nhits < lim ? nhits : lim;
|
||||
for (int64_t k = 0; k < end; k++) {
|
||||
if (!first) jb_putc(&b, ',');
|
||||
engram_emit_node_json(&b, &g->nodes[hits[k].idx], 0);
|
||||
first = 0;
|
||||
if (sem) qsort(sem, (size_t)nsem, sizeof(EngramSemEntry), engram_sem_cmp);
|
||||
int64_t* order = malloc((size_t)lim * sizeof(int64_t));
|
||||
if (order) {
|
||||
int64_t no = engram_interleave(hits, nhits, sem, nsem, lim, order);
|
||||
for (int64_t k = 0; k < no; k++) {
|
||||
if (!first) jb_putc(&b, ',');
|
||||
engram_emit_node_json(&b, &g->nodes[order[k]], 0);
|
||||
first = 0;
|
||||
}
|
||||
free(order);
|
||||
}
|
||||
free(hits);
|
||||
free(sem);
|
||||
free(qv);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user