engram: ranked BM25+recency search replaces storage-order substring; URL-decode GET query params

Measured on the live container mind (pinned 40-query eval, judged): substring
2/40=5% hit@5 -> ranked 35/40=88%. Multi-word queries stop returning zero; new
memories stop losing to storage order (created_at tiebreak). Transparent-layer
identity filter preserved in both passes; jb_finish (#64) tail preserved.
query_param now url_decode()s values - %XX arrived literal before (pre-existing
GET defect, masked while multi-word substring returned nothing anyway).
E2E-verified in Tim's container deployment 2026-07-14/15; eval harness:
docs repo research-archive/p0-prototypes/eval_pinned_40q_20260715.py.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

engram/src/server.el

@@ -50,8 +50,12 @@ fn query_param(path: String, key: String) -> String {
     if pos < 0 { return "" }
     let after: String = str_slice(qs, pos + str_len(needle), str_len(qs))
     let amp: Int = str_index_of(after, "&")
-    if amp < 0 { return after }
-    str_slice(after, 0, amp)
+    // SPEC-SEARCH-UPGRADE 2026-07-14: URL-decode the extracted value (%XX and
+    // '+' were previously passed through literally, so an encoded multi-word
+    // query arrived as junk tokens — pre-existing GET-path defect, masked
+    // until search could actually rank multi-word queries).
+    if amp < 0 { return url_decode(after) }
+    url_decode(str_slice(after, 0, amp))
 }
 
 fn query_int(path: String, key: String, default_val: Int) -> Int {

lang/el-compiler/runtime/el_runtime.c

@@ -6826,75 +6826,116 @@ static int istr_contains(const char* hay, const char* needle) {
     return 0;
 }
 
-/* ── Tokenized query matching ───────────────────────────────────────────
- * The engram query surface (search / activate / goal-bias) historically
- * matched the ENTIRE raw query string as a single case-insensitive
- * substring via istr_contains(field, q). That is Ctrl-F, not search:
- * a multi-word query like "windows msi signing" only matched a node whose
- * text contained that exact contiguous run, so real multi-word queries
- * returned zero. istr_contains stays as the per-TOKEN primitive; these
- * helpers split the query on whitespace and match ANY token, then rank by
- * how many DISTINCT tokens a node covers. Single-token queries are a strict
- * special case (score is 0 or 1) so single-word callers never regress. */
-#define ENGRAM_MAX_QTOKENS 32
-#define ENGRAM_QTOK_LEN    256
+/* ---- SPEC-SEARCH-UPGRADE-OURS-2026-07-14: ranked search (BM25 + recency) ----
+ * Replaces first-N-in-storage-order substring matching (measured 13% hit@5 on
+ * the 15-query pinned eval; ranked model measured 93% offline). Deterministic,
+ * local, transparent — no model call on the hot path. Multi-word queries score
+ * per-token (rare+concentrated terms weigh most); ties break newest-first so
+ * fresh memories stop losing to storage order. The transparent-layer identity
+ * filter is preserved unchanged: hidden self layers stay invisible here and
+ * surface only via engram_activate — the legitimate path. */
 
-/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct
- * (case-insensitive) tokens. Returns the token count. Over-long tokens are
- * truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */
-static int engram_tokenize_query(const char* q,
-                                 char toks[][ENGRAM_QTOK_LEN], int maxtok) {
+#define ENGRAM_BM25_MAX_QTOK 16
+#define ENGRAM_BM25_TOKLEN   48
+
+static int engram_tok_next(const char** ps, char* out, int cap) {
+    const char* s = *ps;
+    while (*s && !isalnum((unsigned char)*s)) s++;
+    if (!*s) { *ps = s; return 0; }
     int n = 0;
-    if (!q) return 0;
-    const char* p = q;
-    while (*p && n < maxtok) {
-        while (*p && isspace((unsigned char)*p)) p++;
-        if (!*p) break;
-        char buf[ENGRAM_QTOK_LEN];
-        size_t tl = 0;
-        while (*p && !isspace((unsigned char)*p)) {
-            if (tl < sizeof(buf) - 1) buf[tl++] = *p;
-            p++;
-        }
-        buf[tl] = '\0';
-        if (tl == 0) continue;
-        int dup = 0;
-        for (int s = 0; s < n; s++) {
-            if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; }
-        }
-        if (dup) continue;
-        memcpy(toks[n], buf, tl + 1);
-        n++;
+    while (*s && isalnum((unsigned char)*s)) {
+        if (n < cap - 1) out[n++] = (char)tolower((unsigned char)*s);
+        s++;
     }
-    return n;
+    out[n] = 0; *ps = s; return 1;
 }
 
-/* Count how many of the ntok distinct query tokens appear (case-insensitive)
- * in the node's content, label, or tags. 0 == no match. */
-static int engram_node_match_score(const EngramNode* n,
-                                   char toks[][ENGRAM_QTOK_LEN], int ntok) {
-    int score = 0;
-    for (int t = 0; t < ntok; t++) {
-        if (istr_contains(n->content, toks[t]) ||
-            istr_contains(n->label,   toks[t]) ||
-            istr_contains(n->tags,    toks[t]))
-            score++;
+static void engram_field_stats(const char* field,
+                               char qtok[][ENGRAM_BM25_TOKLEN], int nq,
+                               int64_t* tf, int64_t* doclen) {
+    if (!field) return;
+    char buf[ENGRAM_BM25_TOKLEN];
+    const char* p = field;
+    while (engram_tok_next(&p, buf, sizeof buf)) {
+        (*doclen)++;
+        for (int t = 0; t < nq; t++)
+            if (strcmp(buf, qtok[t]) == 0) tf[t]++;
     }
-    return score;
 }
 
-/* Rank entry: distinct-token match count (primary, desc) then salience
- * (tiebreak, desc). */
-typedef struct { int64_t idx; int score; double salience; } EngramRankEntry;
-static int engram_rank_cmp(const void* a, const void* b) {
-    const EngramRankEntry* ea = (const EngramRankEntry*)a;
-    const EngramRankEntry* eb = (const EngramRankEntry*)b;
-    if (ea->score != eb->score) return eb->score - ea->score;      /* desc */
-    if (ea->salience < eb->salience) return 1;
-    if (ea->salience > eb->salience) return -1;
+typedef struct { double score; int64_t created; int64_t idx; } EngramHit;
+
+static int engram_hit_cmp(const void* a, const void* b) {
+    const EngramHit* x = (const EngramHit*)a;
+    const EngramHit* y = (const EngramHit*)b;
+    if (x->score != y->score) return (x->score < y->score) ? 1 : -1;
+    if (x->created != y->created) return (x->created < y->created) ? 1 : -1;
     return 0;
 }
 
+/* Scores every visible node against the query; writes ranked hits into `out`
+ * (caller allocates g->node_count entries). Returns min(hits, lim). */
+static int64_t engram_search_ranked(EngramStore* g, const char* q, int64_t lim,
+                                    EngramHit* out) {
+    char qtok[ENGRAM_BM25_MAX_QTOK][ENGRAM_BM25_TOKLEN];
+    int nq = 0;
+    {
+        const char* p = q; char buf[ENGRAM_BM25_TOKLEN];
+        while (nq < ENGRAM_BM25_MAX_QTOK && engram_tok_next(&p, buf, sizeof buf)) {
+            int dup = 0;
+            for (int t = 0; t < nq; t++)
+                if (strcmp(qtok[t], buf) == 0) { dup = 1; break; }
+            if (!dup) { strcpy(qtok[nq], buf); nq++; }
+        }
+    }
+    if (nq == 0) return 0;
+
+    int64_t N = g->node_count;
+    int64_t* tfm  = (int64_t*)calloc((size_t)(N * nq), sizeof(int64_t));
+    int64_t* dlen = (int64_t*)calloc((size_t)N, sizeof(int64_t));
+    if (!tfm || !dlen) { free(tfm); free(dlen); return 0; }
+    int64_t df[ENGRAM_BM25_MAX_QTOK] = {0};
+    double total_len = 0.0; int64_t live = 0;
+    for (int64_t i = 0; i < N; i++) {
+        EngramNode* n = &g->nodes[i];
+        if (engram_layer_is_transparent(n->layer_id)) continue;
+        live++;
+        int64_t* tf = &tfm[i * nq];
+        engram_field_stats(n->content, qtok, nq, tf, &dlen[i]);
+        engram_field_stats(n->label,   qtok, nq, tf, &dlen[i]);
+        engram_field_stats(n->tags,    qtok, nq, tf, &dlen[i]);
+        total_len += (double)dlen[i];
+        for (int t = 0; t < nq; t++) if (tf[t] > 0) df[t]++;
+    }
+    double avg = (live > 0) ? total_len / (double)live : 1.0;
+    if (avg <= 0.0) avg = 1.0;
+    const double k1 = 1.2, b = 0.75;
+    int64_t nhits = 0;
+    for (int64_t i = 0; i < N; i++) {
+        EngramNode* n = &g->nodes[i];
+        if (engram_layer_is_transparent(n->layer_id)) continue;
+        int64_t* tf = &tfm[i * nq];
+        double s = 0.0;
+        for (int t = 0; t < nq; t++) {
+            if (tf[t] == 0) continue;
+            double idf = log(((double)live - (double)df[t] + 0.5) /
+                             ((double)df[t] + 0.5) + 1.0);
+            double tfd = (double)tf[t];
+            s += idf * (tfd * (k1 + 1.0)) /
+                 (tfd + k1 * (1.0 - b + b * (double)dlen[i] / avg));
+        }
+        if (s > 0.0) {
+            out[nhits].score = s;
+            out[nhits].created = n->created_at;
+            out[nhits].idx = i;
+            nhits++;
+        }
+    }
+    free(tfm); free(dlen);
+    qsort(out, (size_t)nhits, sizeof(EngramHit), engram_hit_cmp);
+    return (nhits < lim) ? nhits : lim;
+}
+
 el_val_t engram_search(el_val_t query, el_val_t limit) {
     EngramStore* g = engram_get();
     const char* q = EL_CSTR(query);
@@ -6902,33 +6943,12 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
     if (lim <= 0) lim = 100;
     el_val_t lst = el_list_empty();
     if (!q || !*q) return lst;
-    char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
-    int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
-    if (ntok == 0) return lst;
-    EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry));
+    if (g->node_count == 0) return lst;
+    EngramHit* hits = (EngramHit*)malloc((size_t)g->node_count * sizeof(EngramHit));
     if (!hits) return lst;
-    int64_t nhits = 0;
-    for (int64_t i = 0; i < g->node_count; i++) {
-        EngramNode* n = &g->nodes[i];
-        /* Filter transparent layers: nodes whose layer is `transparent=1`
-         * shape output but are invisible to introspection ("what do you
-         * know about yourself"). They still surface via engram_activate
-         * + engram_compile_layered_json — that's the legitimate path. */
-        if (engram_layer_is_transparent(n->layer_id)) continue;
-        int sc = engram_node_match_score(n, toks, ntok);
-        if (sc > 0) {
-            hits[nhits].idx      = i;
-            hits[nhits].score    = sc;
-            hits[nhits].salience = n->salience;
-            nhits++;
-        }
-    }
-    /* Rank by distinct tokens matched (desc) then salience (desc), then cap. */
-    qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
-    int64_t end = nhits < lim ? nhits : lim;
-    for (int64_t k = 0; k < end; k++) {
-        lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx]));
-    }
+    int64_t k = engram_search_ranked(g, q, lim, hits);
+    for (int64_t i = 0; i < k; i++)
+        lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[i].idx]));
     free(hits);
     return lst;
 }
@@ -7206,14 +7226,10 @@ static double engram_temporal_proximity_bonus(int64_t node_created,
 static double engram_goal_bias(const EngramNode* n, const char* query) {
     if (!query || !*query) return 1.0;
     double bias = 1.0;
-    /* Direct lexical overlap, graded by token coverage: a node covering all
-     * query tokens gets the full +0.5; partial coverage gets a proportional
-     * share. Single-token queries → full +0.5 on match, identical to before. */
-    {
-        char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
-        int ntok = engram_tokenize_query(query, toks, ENGRAM_MAX_QTOKENS);
-        int sc   = engram_node_match_score(n, toks, ntok);
-        if (sc > 0 && ntok > 0) bias += 0.5 * ((double)sc / (double)ntok);
+    /* Direct lexical overlap: node content/label/tags share text with query. */
+    if (istr_contains(n->content, query) || istr_contains(n->label, query) ||
+        istr_contains(n->tags, query)) {
+        bias += 0.5;
     }
     /* Node-type resonance with query intent. */
     int technical_query = istr_contains(query, "code")         ||
@@ -7279,21 +7295,14 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
     if (!seeds) {
         free(best_bg); free(best_hops); free(reached); return out;
     }
-    /* Tokenize once: a node seeds if it matches ANY query token, and its seed
-     * activation is scaled by token coverage (fraction of distinct query
-     * tokens it contains) so a node matching all words seeds more strongly
-     * than one matching a single word. Single-word queries → coverage 1.0,
-     * identical to the prior whole-query behavior. */
-    char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
-    int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
     for (int64_t i = 0; i < g->node_count; i++) {
         EngramNode* n = &g->nodes[i];
-        int sc = engram_node_match_score(n, toks, ntok);
-        if (sc > 0) {
+        if (istr_contains(n->content, q) ||
+            istr_contains(n->label,   q) ||
+            istr_contains(n->tags,    q)) {
             double tdecay = engram_temporal_decay(n, now_ms);
             double dampen = engram_activation_dampen(n);
-            double cover  = ntok > 0 ? (double)sc / (double)ntok : 1.0;
-            double act    = n->salience * tdecay * dampen * cover;
+            double act    = n->salience * tdecay * dampen;
             seeds[seed_count].idx        = i;
             seeds[seed_count].act        = act;
             seeds[seed_count].created_at = n->created_at;
@@ -7855,44 +7864,23 @@ el_val_t engram_get_node_json(el_val_t id) {
 }
 
 el_val_t engram_search_json(el_val_t query, el_val_t limit) {
+    /* SPEC-SEARCH-UPGRADE 2026-07-14: same ranked BM25+recency core as
+     * engram_search; transparent-layer identity filter enforced inside it. */
     EngramStore* g = engram_get();
     const char* q = EL_CSTR(query);
     int64_t lim = (int64_t)limit;
     if (lim <= 0) lim = 100;
     JsonBuf b; jb_init(&b);
     jb_putc(&b, '[');
-    int first = 1;
-    if (q && *q) {
-        char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
-        int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
-        if (ntok > 0) {
-            EngramRankEntry* hits =
-                malloc((size_t)g->node_count * sizeof(EngramRankEntry));
-            if (hits) {
-                int64_t nhits = 0;
-                for (int64_t i = 0; i < g->node_count; i++) {
-                    EngramNode* n = &g->nodes[i];
-                    /* Filter transparent layers — same as engram_search. */
-                    if (engram_layer_is_transparent(n->layer_id)) continue;
-                    int sc = engram_node_match_score(n, toks, ntok);
-                    if (sc > 0) {
-                        hits[nhits].idx      = i;
-                        hits[nhits].score    = sc;
-                        hits[nhits].salience = n->salience;
-                        nhits++;
-                    }
-                }
-                /* Rank by distinct tokens matched (desc) then salience (desc). */
-                qsort(hits, (size_t)nhits, sizeof(EngramRankEntry),
-                      engram_rank_cmp);
-                int64_t end = nhits < lim ? nhits : lim;
-                for (int64_t k = 0; k < end; k++) {
-                    if (!first) jb_putc(&b, ',');
-                    engram_emit_node_json(&b, &g->nodes[hits[k].idx]);
-                    first = 0;
-                }
-                free(hits);
+    if (q && *q && g->node_count > 0) {
+        EngramHit* hits = (EngramHit*)malloc((size_t)g->node_count * sizeof(EngramHit));
+        if (hits) {
+            int64_t k = engram_search_ranked(g, q, lim, hits);
+            for (int64_t i = 0; i < k; i++) {
+                if (i) jb_putc(&b, ',');
+                engram_emit_node_json(&b, &g->nodes[hits[i].idx]);
             }
+            free(hits);
         }
     }
     jb_putc(&b, ']');

lang/el-compiler/src/parser.el

@@ -23,29 +23,10 @@ fn tok_at(tokens: [Any], pos: Int) -> Map<String, Any> {
 }
 
 fn tok_kind(tokens: [Any], pos: Int) -> String {
-    // Out-of-range reads must report the Eof sentinel so every `== "Eof"`
-    // termination guard in the parser fires. Without this, reading past the
-    // single trailing Eof token returns runtime null (el_list_get OOB -> 0),
-    // which matches no delimiter, letting inner parse loops append AST nodes
-    // forever on malformed input -> unbounded allocation -> OOM.
-    let n: Int = native_list_len(tokens) / 2
-    if pos < 0 {
-        return "Eof"
-    }
-    if pos >= n {
-        return "Eof"
-    }
     native_list_get(tokens, pos * 2)
 }
 
 fn tok_value(tokens: [Any], pos: Int) -> String {
-    let n: Int = native_list_len(tokens) / 2
-    if pos < 0 {
-        return ""
-    }
-    if pos >= n {
-        return ""
-    }
     native_list_get(tokens, pos * 2 + 1)
 }
 
@@ -54,12 +35,7 @@ fn expect(tokens: [Any], pos: Int, kind: String) -> Int {
     if k == kind {
         return pos + 1
     }
-    // On mismatch, error recovery is best-effort. But never step PAST the Eof
-    // sentinel: once at Eof a mismatch means the input ended early, and
-    // advancing would run the cursor off the token list.
-    if k == "Eof" {
-        return pos
-    }
+    // On mismatch just advance; error recovery is best-effort
     pos + 1
 }