From e3dabe3e082cb95d143a2084995dbc3ae40d4d28 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Tue, 14 Jul 2026 18:39:07 -0500 Subject: [PATCH] fix(engram): tokenized + ranked lexical search, not whole-query Ctrl-F MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit engram search/activate/goal-bias matched the ENTIRE raw query string as a single case-insensitive substring (istr_contains(field, q)). Multi-word queries like "windows msi signing" only matched a node containing that exact contiguous run, so real multi-word queries returned ZERO on a graph saturated with the answer. This is Ctrl-F, not search — and search is the core of the engram being useful. Fix: split the query on whitespace into distinct tokens; a node matches if it contains ANY token in content/label/tags. Rank by distinct tokens matched (desc) then salience (desc). istr_contains is kept unchanged as the per-token primitive. Single-token queries are a strict special case (score 0 or 1) so the many single-word callers do not regress. Sites changed (all in el_runtime.c): - new helpers engram_tokenize_query / engram_node_match_score / engram_rank_cmp - engram_search (internal el_val_t path) - engram_search_json (HTTP /api/search path) - engram_activate seed loop (HTTP /api/activate path; seed activation scaled by token coverage so full-query matches seed more strongly) - engram_goal_bias overlap bonus upgraded to graded token coverage Proof (6591-node snapshot copy, rebuilt binary on :8799, POST JSON path): windows msi signing 0 -> 20 Will Anderson 0 -> 20 windows msi 0 -> 20 tokenized search fix 0 -> 20 Single-word parity preserved (VBD/volatility/elc capped at limit; unkey = all matching nodes). Top hits are relevant (e.g. "Will Anderson" surfaces the Project Design and VBD whitepapers). Note: GET ?q=a%20b still returns 0 because query_param (server.el) does not URL-decode — a separate EL-layer bug; the soul's POST-JSON path is fixed here. --- lang/el-compiler/runtime/el_runtime.c | 164 +++++++++++++++++++++----- 1 file changed, 137 insertions(+), 27 deletions(-) diff --git a/lang/el-compiler/runtime/el_runtime.c b/lang/el-compiler/runtime/el_runtime.c index 68e6dca..1d4d9ac 100644 --- a/lang/el-compiler/runtime/el_runtime.c +++ b/lang/el-compiler/runtime/el_runtime.c @@ -6826,6 +6826,75 @@ static int istr_contains(const char* hay, const char* needle) { return 0; } +/* ── Tokenized query matching ─────────────────────────────────────────── + * The engram query surface (search / activate / goal-bias) historically + * matched the ENTIRE raw query string as a single case-insensitive + * substring via istr_contains(field, q). That is Ctrl-F, not search: + * a multi-word query like "windows msi signing" only matched a node whose + * text contained that exact contiguous run, so real multi-word queries + * returned zero. istr_contains stays as the per-TOKEN primitive; these + * helpers split the query on whitespace and match ANY token, then rank by + * how many DISTINCT tokens a node covers. Single-token queries are a strict + * special case (score is 0 or 1) so single-word callers never regress. */ +#define ENGRAM_MAX_QTOKENS 32 +#define ENGRAM_QTOK_LEN 256 + +/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct + * (case-insensitive) tokens. Returns the token count. Over-long tokens are + * truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */ +static int engram_tokenize_query(const char* q, + char toks[][ENGRAM_QTOK_LEN], int maxtok) { + int n = 0; + if (!q) return 0; + const char* p = q; + while (*p && n < maxtok) { + while (*p && isspace((unsigned char)*p)) p++; + if (!*p) break; + char buf[ENGRAM_QTOK_LEN]; + size_t tl = 0; + while (*p && !isspace((unsigned char)*p)) { + if (tl < sizeof(buf) - 1) buf[tl++] = *p; + p++; + } + buf[tl] = '\0'; + if (tl == 0) continue; + int dup = 0; + for (int s = 0; s < n; s++) { + if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; } + } + if (dup) continue; + memcpy(toks[n], buf, tl + 1); + n++; + } + return n; +} + +/* Count how many of the ntok distinct query tokens appear (case-insensitive) + * in the node's content, label, or tags. 0 == no match. */ +static int engram_node_match_score(const EngramNode* n, + char toks[][ENGRAM_QTOK_LEN], int ntok) { + int score = 0; + for (int t = 0; t < ntok; t++) { + if (istr_contains(n->content, toks[t]) || + istr_contains(n->label, toks[t]) || + istr_contains(n->tags, toks[t])) + score++; + } + return score; +} + +/* Rank entry: distinct-token match count (primary, desc) then salience + * (tiebreak, desc). */ +typedef struct { int64_t idx; int score; double salience; } EngramRankEntry; +static int engram_rank_cmp(const void* a, const void* b) { + const EngramRankEntry* ea = (const EngramRankEntry*)a; + const EngramRankEntry* eb = (const EngramRankEntry*)b; + if (ea->score != eb->score) return eb->score - ea->score; /* desc */ + if (ea->salience < eb->salience) return 1; + if (ea->salience > eb->salience) return -1; + return 0; +} + el_val_t engram_search(el_val_t query, el_val_t limit) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); @@ -6833,21 +6902,34 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { if (lim <= 0) lim = 100; el_val_t lst = el_list_empty(); if (!q || !*q) return lst; - int64_t found = 0; - for (int64_t i = 0; i < g->node_count && found < lim; i++) { + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS); + if (ntok == 0) return lst; + EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry)); + if (!hits) return lst; + int64_t nhits = 0; + for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; /* Filter transparent layers: nodes whose layer is `transparent=1` * shape output but are invisible to introspection ("what do you * know about yourself"). They still surface via engram_activate * + engram_compile_layered_json — that's the legitimate path. */ if (engram_layer_is_transparent(n->layer_id)) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { - lst = el_list_append(lst, engram_node_to_map(n)); - found++; + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0) { + hits[nhits].idx = i; + hits[nhits].score = sc; + hits[nhits].salience = n->salience; + nhits++; } } + /* Rank by distinct tokens matched (desc) then salience (desc), then cap. */ + qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); + int64_t end = nhits < lim ? nhits : lim; + for (int64_t k = 0; k < end; k++) { + lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx])); + } + free(hits); return lst; } @@ -7124,10 +7206,14 @@ static double engram_temporal_proximity_bonus(int64_t node_created, static double engram_goal_bias(const EngramNode* n, const char* query) { if (!query || !*query) return 1.0; double bias = 1.0; - /* Direct lexical overlap: node content/label/tags share text with query. */ - if (istr_contains(n->content, query) || istr_contains(n->label, query) || - istr_contains(n->tags, query)) { - bias += 0.5; + /* Direct lexical overlap, graded by token coverage: a node covering all + * query tokens gets the full +0.5; partial coverage gets a proportional + * share. Single-token queries → full +0.5 on match, identical to before. */ + { + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(query, toks, ENGRAM_MAX_QTOKENS); + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0 && ntok > 0) bias += 0.5 * ((double)sc / (double)ntok); } /* Node-type resonance with query intent. */ int technical_query = istr_contains(query, "code") || @@ -7193,14 +7279,21 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { if (!seeds) { free(best_bg); free(best_hops); free(reached); return out; } + /* Tokenize once: a node seeds if it matches ANY query token, and its seed + * activation is scaled by token coverage (fraction of distinct query + * tokens it contains) so a node matching all words seeds more strongly + * than one matching a single word. Single-word queries → coverage 1.0, + * identical to the prior whole-query behavior. */ + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS); for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0) { double tdecay = engram_temporal_decay(n, now_ms); double dampen = engram_activation_dampen(n); - double act = n->salience * tdecay * dampen; + double cover = ntok > 0 ? (double)sc / (double)ntok : 1.0; + double act = n->salience * tdecay * dampen * cover; seeds[seed_count].idx = i; seeds[seed_count].act = act; seeds[seed_count].created_at = n->created_at; @@ -7769,19 +7862,36 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { JsonBuf b; jb_init(&b); jb_putc(&b, '['); int first = 1; - int64_t found = 0; if (q && *q) { - for (int64_t i = 0; i < g->node_count && found < lim; i++) { - EngramNode* n = &g->nodes[i]; - /* Filter transparent layers — same as engram_search. */ - if (engram_layer_is_transparent(n->layer_id)) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { - if (!first) jb_putc(&b, ','); - engram_emit_node_json(&b, n); - first = 0; - found++; + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS); + if (ntok > 0) { + EngramRankEntry* hits = + malloc((size_t)g->node_count * sizeof(EngramRankEntry)); + if (hits) { + int64_t nhits = 0; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + /* Filter transparent layers — same as engram_search. */ + if (engram_layer_is_transparent(n->layer_id)) continue; + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0) { + hits[nhits].idx = i; + hits[nhits].score = sc; + hits[nhits].salience = n->salience; + nhits++; + } + } + /* Rank by distinct tokens matched (desc) then salience (desc). */ + qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), + engram_rank_cmp); + int64_t end = nhits < lim ? nhits : lim; + for (int64_t k = 0; k < end; k++) { + if (!first) jb_putc(&b, ','); + engram_emit_node_json(&b, &g->nodes[hits[k].idx]); + first = 0; + } + free(hits); } } }