fix(engram): tokenized + ranked lexical search, not whole-query Ctrl-F

engram search/activate/goal-bias matched the ENTIRE raw query string as a single case-insensitive substring (istr_contains(field, q)). Multi-word queries like "windows msi signing" only matched a node containing that exact contiguous run, so real multi-word queries returned ZERO on a graph saturated with the answer. This is Ctrl-F, not search — and search is the core of the engram being useful. Fix: split the query on whitespace into distinct tokens; a node matches if it contains ANY token in content/label/tags. Rank by distinct tokens matched (desc) then salience (desc). istr_contains is kept unchanged as the per-token primitive. Single-token queries are a strict special case (score 0 or 1) so the many single-word callers do not regress. Sites changed (all in el_runtime.c): - new helpers engram_tokenize_query / engram_node_match_score / engram_rank_cmp - engram_search (internal el_val_t path) - engram_search_json (HTTP /api/search path) - engram_activate seed loop (HTTP /api/activate path; seed activation scaled by token coverage so full-query matches seed more strongly) - engram_goal_bias overlap bonus upgraded to graded token coverage Proof (6591-node snapshot copy, rebuilt binary on :8799, POST JSON path): windows msi signing 0 -> 20 Will Anderson 0 -> 20 windows msi 0 -> 20 tokenized search fix 0 -> 20 Single-word parity preserved (VBD/volatility/elc capped at limit; unkey = all matching nodes). Top hits are relevant (e.g. "Will Anderson" surfaces the Project Design and VBD whitepapers). Note: GET ?q=a%20b still returns 0 because query_param (server.el) does not URL-decode — a separate EL-layer bug; the soul's POST-JSON path is fixed here.
2026-07-14 18:39:07 -05:00
parent 0a0a2bcb44
commit e3dabe3e08
1 changed files with 137 additions and 27 deletions
@@ -6826,6 +6826,75 @@ static int istr_contains(const char* hay, const char* needle) {
    return 0;
 }

+/* ── Tokenized query matching ───────────────────────────────────────────
+ * The engram query surface (search / activate / goal-bias) historically
+ * matched the ENTIRE raw query string as a single case-insensitive
+ * substring via istr_contains(field, q). That is Ctrl-F, not search:
+ * a multi-word query like "windows msi signing" only matched a node whose
+ * text contained that exact contiguous run, so real multi-word queries
+ * returned zero. istr_contains stays as the per-TOKEN primitive; these
+ * helpers split the query on whitespace and match ANY token, then rank by
+ * how many DISTINCT tokens a node covers. Single-token queries are a strict
+ * special case (score is 0 or 1) so single-word callers never regress. */
+#define ENGRAM_MAX_QTOKENS 32
+#define ENGRAM_QTOK_LEN    256
+
+/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct
+ * (case-insensitive) tokens. Returns the token count. Over-long tokens are
+ * truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */
+static int engram_tokenize_query(const char* q,
+                                 char toks[][ENGRAM_QTOK_LEN], int maxtok) {
+    int n = 0;
+    if (!q) return 0;
+    const char* p = q;
+    while (*p && n < maxtok) {
+        while (*p && isspace((unsigned char)*p)) p++;
+        if (!*p) break;
+        char buf[ENGRAM_QTOK_LEN];
+        size_t tl = 0;
+        while (*p && !isspace((unsigned char)*p)) {
+            if (tl < sizeof(buf) - 1) buf[tl++] = *p;
+            p++;
+        }
+        buf[tl] = '\0';
+        if (tl == 0) continue;
+        int dup = 0;
+        for (int s = 0; s < n; s++) {
+            if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; }
+        }
+        if (dup) continue;
+        memcpy(toks[n], buf, tl + 1);
+        n++;
+    }
+    return n;
+}
+
+/* Count how many of the ntok distinct query tokens appear (case-insensitive)
+ * in the node's content, label, or tags. 0 == no match. */
+static int engram_node_match_score(const EngramNode* n,
+                                   char toks[][ENGRAM_QTOK_LEN], int ntok) {
+    int score = 0;
+    for (int t = 0; t < ntok; t++) {
+        if (istr_contains(n->content, toks[t]) ||
+            istr_contains(n->label,   toks[t]) ||
+            istr_contains(n->tags,    toks[t]))
+            score++;
+    }
+    return score;
+}
+
+/* Rank entry: distinct-token match count (primary, desc) then salience
+ * (tiebreak, desc). */
+typedef struct { int64_t idx; int score; double salience; } EngramRankEntry;
+static int engram_rank_cmp(const void* a, const void* b) {
+    const EngramRankEntry* ea = (const EngramRankEntry*)a;
+    const EngramRankEntry* eb = (const EngramRankEntry*)b;
+    if (ea->score != eb->score) return eb->score - ea->score;      /* desc */
+    if (ea->salience < eb->salience) return 1;
+    if (ea->salience > eb->salience) return -1;
+    return 0;
+}
+
 el_val_t engram_search(el_val_t query, el_val_t limit) {
    EngramStore* g = engram_get();
    const char* q = EL_CSTR(query);
@@ -6833,21 +6902,34 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
    if (lim <= 0) lim = 100;
    el_val_t lst = el_list_empty();
    if (!q || !*q) return lst;
-    int64_t found = 0;
-    for (int64_t i = 0; i < g->node_count && found < lim; i++) {
+    char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
+    int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
+    if (ntok == 0) return lst;
+    EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry));
+    if (!hits) return lst;
+    int64_t nhits = 0;
+    for (int64_t i = 0; i < g->node_count; i++) {
        EngramNode* n = &g->nodes[i];
        /* Filter transparent layers: nodes whose layer is `transparent=1`
         * shape output but are invisible to introspection ("what do you
         * know about yourself"). They still surface via engram_activate
         * + engram_compile_layered_json — that's the legitimate path. */
        if (engram_layer_is_transparent(n->layer_id)) continue;
-        if (istr_contains(n->content, q) ||
-            istr_contains(n->label,   q) ||
-            istr_contains(n->tags,    q)) {
-            lst = el_list_append(lst, engram_node_to_map(n));
-            found++;
+        int sc = engram_node_match_score(n, toks, ntok);
+        if (sc > 0) {
+            hits[nhits].idx      = i;
+            hits[nhits].score    = sc;
+            hits[nhits].salience = n->salience;
+            nhits++;
        }
    }
+    /* Rank by distinct tokens matched (desc) then salience (desc), then cap. */
+    qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
+    int64_t end = nhits < lim ? nhits : lim;
+    for (int64_t k = 0; k < end; k++) {
+        lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx]));
+    }
+    free(hits);
    return lst;
 }

@@ -7124,10 +7206,14 @@ static double engram_temporal_proximity_bonus(int64_t node_created,
 static double engram_goal_bias(const EngramNode* n, const char* query) {
    if (!query || !*query) return 1.0;
    double bias = 1.0;
-    /* Direct lexical overlap: node content/label/tags share text with query. */
-    if (istr_contains(n->content, query) || istr_contains(n->label, query) ||
-        istr_contains(n->tags, query)) {
-        bias += 0.5;
+    /* Direct lexical overlap, graded by token coverage: a node covering all
+     * query tokens gets the full +0.5; partial coverage gets a proportional
+     * share. Single-token queries → full +0.5 on match, identical to before. */
+    {
+        char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
+        int ntok = engram_tokenize_query(query, toks, ENGRAM_MAX_QTOKENS);
+        int sc   = engram_node_match_score(n, toks, ntok);
+        if (sc > 0 && ntok > 0) bias += 0.5 * ((double)sc / (double)ntok);
    }
    /* Node-type resonance with query intent. */
    int technical_query = istr_contains(query, "code")         ||
@@ -7193,14 +7279,21 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
    if (!seeds) {
        free(best_bg); free(best_hops); free(reached); return out;
    }
+    /* Tokenize once: a node seeds if it matches ANY query token, and its seed
+     * activation is scaled by token coverage (fraction of distinct query
+     * tokens it contains) so a node matching all words seeds more strongly
+     * than one matching a single word. Single-word queries → coverage 1.0,
+     * identical to the prior whole-query behavior. */
+    char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
+    int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
    for (int64_t i = 0; i < g->node_count; i++) {
        EngramNode* n = &g->nodes[i];
-        if (istr_contains(n->content, q) ||
-            istr_contains(n->label,   q) ||
-            istr_contains(n->tags,    q)) {
+        int sc = engram_node_match_score(n, toks, ntok);
+        if (sc > 0) {
            double tdecay = engram_temporal_decay(n, now_ms);
            double dampen = engram_activation_dampen(n);
-            double act    = n->salience * tdecay * dampen;
+            double cover  = ntok > 0 ? (double)sc / (double)ntok : 1.0;
+            double act    = n->salience * tdecay * dampen * cover;
            seeds[seed_count].idx        = i;
            seeds[seed_count].act        = act;
            seeds[seed_count].created_at = n->created_at;
@@ -7769,19 +7862,36 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
    JsonBuf b; jb_init(&b);
    jb_putc(&b, '[');
    int first = 1;
-    int64_t found = 0;
    if (q && *q) {
-        for (int64_t i = 0; i < g->node_count && found < lim; i++) {
-            EngramNode* n = &g->nodes[i];
-            /* Filter transparent layers — same as engram_search. */
-            if (engram_layer_is_transparent(n->layer_id)) continue;
-            if (istr_contains(n->content, q) ||
-                istr_contains(n->label,   q) ||
-                istr_contains(n->tags,    q)) {
-                if (!first) jb_putc(&b, ',');
-                engram_emit_node_json(&b, n);
-                first = 0;
-                found++;
+        char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
+        int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
+        if (ntok > 0) {
+            EngramRankEntry* hits =
+                malloc((size_t)g->node_count * sizeof(EngramRankEntry));
+            if (hits) {
+                int64_t nhits = 0;
+                for (int64_t i = 0; i < g->node_count; i++) {
+                    EngramNode* n = &g->nodes[i];
+                    /* Filter transparent layers — same as engram_search. */
+                    if (engram_layer_is_transparent(n->layer_id)) continue;
+                    int sc = engram_node_match_score(n, toks, ntok);
+                    if (sc > 0) {
+                        hits[nhits].idx      = i;
+                        hits[nhits].score    = sc;
+                        hits[nhits].salience = n->salience;
+                        nhits++;
+                    }
+                }
+                /* Rank by distinct tokens matched (desc) then salience (desc). */
+                qsort(hits, (size_t)nhits, sizeof(EngramRankEntry),
+                      engram_rank_cmp);
+                int64_t end = nhits < lim ? nhits : lim;
+                for (int64_t k = 0; k < end; k++) {
+                    if (!first) jb_putc(&b, ',');
+                    engram_emit_node_json(&b, &g->nodes[hits[k].idx]);
+                    first = 0;
+                }
+                free(hits);
            }
        }
    }