engram/src/server.el

@@ -50,8 +50,12 @@ fn query_param(path: String, key: String) -> String {
     if pos < 0 { return "" }
     let after: String = str_slice(qs, pos + str_len(needle), str_len(qs))
     let amp: Int = str_index_of(after, "&")
-    if amp < 0 { return after }
+    // SPEC-SEARCH-UPGRADE 2026-07-14: URL-decode the extracted value (%XX and
-    str_slice(after, 0, amp)
+    // '+' were previously passed through literally, so an encoded multi-word
+    // query arrived as junk tokens — pre-existing GET-path defect, masked
+    // until search could actually rank multi-word queries).
+    if amp < 0 { return url_decode(after) }
+    url_decode(str_slice(after, 0, amp))
 }
 
 fn query_int(path: String, key: String, default_val: Int) -> Int {

lang/el-compiler/runtime/el_runtime.c

@@ -6826,6 +6826,116 @@ static int istr_contains(const char* hay, const char* needle) {
     return 0;
 }
 
+/* ---- SPEC-SEARCH-UPGRADE-OURS-2026-07-14: ranked search (BM25 + recency) ----
+ * Replaces first-N-in-storage-order substring matching (measured 13% hit@5 on
+ * the 15-query pinned eval; ranked model measured 93% offline). Deterministic,
+ * local, transparent — no model call on the hot path. Multi-word queries score
+ * per-token (rare+concentrated terms weigh most); ties break newest-first so
+ * fresh memories stop losing to storage order. The transparent-layer identity
+ * filter is preserved unchanged: hidden self layers stay invisible here and
+ * surface only via engram_activate — the legitimate path. */
+
+#define ENGRAM_BM25_MAX_QTOK 16
+#define ENGRAM_BM25_TOKLEN   48
+
+static int engram_tok_next(const char** ps, char* out, int cap) {
+    const char* s = *ps;
+    while (*s && !isalnum((unsigned char)*s)) s++;
+    if (!*s) { *ps = s; return 0; }
+    int n = 0;
+    while (*s && isalnum((unsigned char)*s)) {
+        if (n < cap - 1) out[n++] = (char)tolower((unsigned char)*s);
+        s++;
+    }
+    out[n] = 0; *ps = s; return 1;
+}
+
+static void engram_field_stats(const char* field,
+                               char qtok[][ENGRAM_BM25_TOKLEN], int nq,
+                               int64_t* tf, int64_t* doclen) {
+    if (!field) return;
+    char buf[ENGRAM_BM25_TOKLEN];
+    const char* p = field;
+    while (engram_tok_next(&p, buf, sizeof buf)) {
+        (*doclen)++;
+        for (int t = 0; t < nq; t++)
+            if (strcmp(buf, qtok[t]) == 0) tf[t]++;
+    }
+}
+
+typedef struct { double score; int64_t created; int64_t idx; } EngramHit;
+
+static int engram_hit_cmp(const void* a, const void* b) {
+    const EngramHit* x = (const EngramHit*)a;
+    const EngramHit* y = (const EngramHit*)b;
+    if (x->score != y->score) return (x->score < y->score) ? 1 : -1;
+    if (x->created != y->created) return (x->created < y->created) ? 1 : -1;
+    return 0;
+}
+
+/* Scores every visible node against the query; writes ranked hits into `out`
+ * (caller allocates g->node_count entries). Returns min(hits, lim). */
+static int64_t engram_search_ranked(EngramStore* g, const char* q, int64_t lim,
+                                    EngramHit* out) {
+    char qtok[ENGRAM_BM25_MAX_QTOK][ENGRAM_BM25_TOKLEN];
+    int nq = 0;
+    {
+        const char* p = q; char buf[ENGRAM_BM25_TOKLEN];
+        while (nq < ENGRAM_BM25_MAX_QTOK && engram_tok_next(&p, buf, sizeof buf)) {
+            int dup = 0;
+            for (int t = 0; t < nq; t++)
+                if (strcmp(qtok[t], buf) == 0) { dup = 1; break; }
+            if (!dup) { strcpy(qtok[nq], buf); nq++; }
+        }
+    }
+    if (nq == 0) return 0;
+
+    int64_t N = g->node_count;
+    int64_t* tfm  = (int64_t*)calloc((size_t)(N * nq), sizeof(int64_t));
+    int64_t* dlen = (int64_t*)calloc((size_t)N, sizeof(int64_t));
+    if (!tfm || !dlen) { free(tfm); free(dlen); return 0; }
+    int64_t df[ENGRAM_BM25_MAX_QTOK] = {0};
+    double total_len = 0.0; int64_t live = 0;
+    for (int64_t i = 0; i < N; i++) {
+        EngramNode* n = &g->nodes[i];
+        if (engram_layer_is_transparent(n->layer_id)) continue;
+        live++;
+        int64_t* tf = &tfm[i * nq];
+        engram_field_stats(n->content, qtok, nq, tf, &dlen[i]);
+        engram_field_stats(n->label,   qtok, nq, tf, &dlen[i]);
+        engram_field_stats(n->tags,    qtok, nq, tf, &dlen[i]);
+        total_len += (double)dlen[i];
+        for (int t = 0; t < nq; t++) if (tf[t] > 0) df[t]++;
+    }
+    double avg = (live > 0) ? total_len / (double)live : 1.0;
+    if (avg <= 0.0) avg = 1.0;
+    const double k1 = 1.2, b = 0.75;
+    int64_t nhits = 0;
+    for (int64_t i = 0; i < N; i++) {
+        EngramNode* n = &g->nodes[i];
+        if (engram_layer_is_transparent(n->layer_id)) continue;
+        int64_t* tf = &tfm[i * nq];
+        double s = 0.0;
+        for (int t = 0; t < nq; t++) {
+            if (tf[t] == 0) continue;
+            double idf = log(((double)live - (double)df[t] + 0.5) /
+                             ((double)df[t] + 0.5) + 1.0);
+            double tfd = (double)tf[t];
+            s += idf * (tfd * (k1 + 1.0)) /
+                 (tfd + k1 * (1.0 - b + b * (double)dlen[i] / avg));
+        }
+        if (s > 0.0) {
+            out[nhits].score = s;
+            out[nhits].created = n->created_at;
+            out[nhits].idx = i;
+            nhits++;
+        }
+    }
+    free(tfm); free(dlen);
+    qsort(out, (size_t)nhits, sizeof(EngramHit), engram_hit_cmp);
+    return (nhits < lim) ? nhits : lim;
+}
+
 el_val_t engram_search(el_val_t query, el_val_t limit) {
     EngramStore* g = engram_get();
     const char* q = EL_CSTR(query);
@@ -6833,21 +6943,13 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
     if (lim <= 0) lim = 100;
     el_val_t lst = el_list_empty();
     if (!q || !*q) return lst;
-    int64_t found = 0;
+    if (g->node_count == 0) return lst;
-    for (int64_t i = 0; i < g->node_count && found < lim; i++) {
+    EngramHit* hits = (EngramHit*)malloc((size_t)g->node_count * sizeof(EngramHit));
-        EngramNode* n = &g->nodes[i];
+    if (!hits) return lst;
-        /* Filter transparent layers: nodes whose layer is `transparent=1`
+    int64_t k = engram_search_ranked(g, q, lim, hits);
-         * shape output but are invisible to introspection ("what do you
+    for (int64_t i = 0; i < k; i++)
-         * know about yourself"). They still surface via engram_activate
+        lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[i].idx]));
-         * + engram_compile_layered_json — that's the legitimate path. */
+    free(hits);
-        if (engram_layer_is_transparent(n->layer_id)) continue;
-        if (istr_contains(n->content, q) ||
-            istr_contains(n->label,   q) ||
-            istr_contains(n->tags,    q)) {
-            lst = el_list_append(lst, engram_node_to_map(n));
-            found++;
-        }
-    }
     return lst;
 }
 
@@ -7762,27 +7864,23 @@ el_val_t engram_get_node_json(el_val_t id) {
 }
 
 el_val_t engram_search_json(el_val_t query, el_val_t limit) {
+    /* SPEC-SEARCH-UPGRADE 2026-07-14: same ranked BM25+recency core as
+     * engram_search; transparent-layer identity filter enforced inside it. */
     EngramStore* g = engram_get();
     const char* q = EL_CSTR(query);
     int64_t lim = (int64_t)limit;
     if (lim <= 0) lim = 100;
     JsonBuf b; jb_init(&b);
     jb_putc(&b, '[');
-    int first = 1;
+    if (q && *q && g->node_count > 0) {
-    int64_t found = 0;
+        EngramHit* hits = (EngramHit*)malloc((size_t)g->node_count * sizeof(EngramHit));
-    if (q && *q) {
+        if (hits) {
-        for (int64_t i = 0; i < g->node_count && found < lim; i++) {
+            int64_t k = engram_search_ranked(g, q, lim, hits);
-            EngramNode* n = &g->nodes[i];
+            for (int64_t i = 0; i < k; i++) {
-            /* Filter transparent layers — same as engram_search. */
+                if (i) jb_putc(&b, ',');
-            if (engram_layer_is_transparent(n->layer_id)) continue;
+                engram_emit_node_json(&b, &g->nodes[hits[i].idx]);
-            if (istr_contains(n->content, q) ||
-                istr_contains(n->label,   q) ||
-                istr_contains(n->tags,    q)) {
-                if (!first) jb_putc(&b, ',');
-                engram_emit_node_json(&b, n);
-                first = 0;
-                found++;
             }
+            free(hits);
         }
     }
     jb_putc(&b, ']');

lang/el-compiler/src/codegen.el

@@ -3626,6 +3626,24 @@ fn codegen_streaming(tokens: [Any], sigs: [Map<String, Any>], source: String) ->
     let pos: Int = 0
     let el_main_body: [Map<String, Any>] = native_list_empty()
     let toplevel_exec_stmts: [Map<String, Any>] = native_list_empty()
+    // CGI IDENTITY CAPTURE (2026-08-09). A cgi block is a top-level DECLARATION, so
+    // the classifier below correctly excludes it from toplevel_exec_stmts and calls
+    // el_release on it. The identity emission further down then searched
+    // toplevel_exec_stmts for it — a list that structurally can never contain it —
+    // found nothing, and emitted nothing, silently. Measured: that search sees only
+    // [Let, Expr] for a program whose first statement is a cgi block.
+    // Fix: copy the values out BEFORE the release (strings, so no dangling reference)
+    // and emit from these. No search, so the failure mode is removed rather than moved.
+    let cgi_have: Bool = false
+    let cgi_name_v: String = ""
+    let cgi_did_v: String = ""
+    let cgi_prin_v: String = ""
+    let cgi_net_v: String = ""
+    let cgi_eng_v: String = ""
+    let cgi_has_did: Bool = false
+    let cgi_has_prin: Bool = false
+    let cgi_has_net: Bool = false
+    let cgi_has_eng: Bool = false
     let has_toplevel_exec: Bool = false
 
     let stream_running: Bool = true
@@ -3736,6 +3754,20 @@ fn codegen_streaming(tokens: [Any], sigs: [Map<String, Any>], source: String) ->
                         if is_top_level_decl(stmt) {
                             // Import, TypeDef, EnumDef, CgiBlock, ServiceBlock, ExternFn
                             // These are no-ops in codegen (forward decls already emitted)
+                            // — except a CgiBlock, whose declared identity must survive
+                            // this release to be emitted as a compiled constant.
+                            if str_eq(sk, "CgiBlock") {
+                                let cgi_have = true
+                                let cgi_name_v = stmt["name"]
+                                let cgi_did_v = stmt["dharma_id"]
+                                let cgi_prin_v = stmt["principal"]
+                                let cgi_net_v = stmt["network"]
+                                let cgi_eng_v = stmt["engram"]
+                                let cgi_has_did = stmt["has_dharma_id"]
+                                let cgi_has_prin = stmt["has_principal"]
+                                let cgi_has_net = stmt["has_network"]
+                                let cgi_has_eng = stmt["has_engram"]
+                            }
                             el_release(stmt)
                         } else {
                             if str_eq(sk, "Let") {
@@ -3814,33 +3846,17 @@ fn codegen_streaming(tokens: [Any], sigs: [Map<String, Any>], source: String) ->
         let sig2 = native_list_get(sigs, si2)
         let sk3: String = sig2["kind"]
         if str_eq(sk3, "cgi_block") {
-            // We need the full cgi_block data — it was parsed by scan_fn_sigs
+            // Emit from the values captured before the declaration was released.
-            // but scan only stored the name. For cgi_init we need dharma_id etc.
+            // The previous implementation searched toplevel_exec_stmts, which by
-            // Since cgi blocks are rare and small, they end up in toplevel_exec_stmts.
+            // construction never contains a declaration — so it emitted nothing and
-            // Find the CgiBlock in toplevel_exec_stmts.
+            // said nothing. See the capture block near toplevel_exec_stmts init.
-            let tes_n: Int = native_list_len(toplevel_exec_stmts)
+            if cgi_have {
-            let tes_i: Int = 0
+                let arg_name2: String = "EL_STR(" + c_str_lit(cgi_name_v) + ")"
-            while tes_i < tes_n {
+                let arg_did2: String = cgi_arg(cgi_did_v, cgi_has_did)
-                let tes = native_list_get(toplevel_exec_stmts, tes_i)
+                let arg_prin2: String = cgi_arg(cgi_prin_v, cgi_has_prin)
-                let tes_k: String = tes["stmt"]
+                let arg_net2: String = cgi_arg(cgi_net_v, cgi_has_net)
-                if str_eq(tes_k, "CgiBlock") {
+                let arg_eng2: String = cgi_arg(cgi_eng_v, cgi_has_eng)
-                    let cname2: String = tes["name"]
+                emit_line("  el_cgi_init(" + arg_name2 + ", " + arg_did2 + ", " + arg_prin2 + ", " + arg_net2 + ", " + arg_eng2 + ");")
-                    let cdid2: String = tes["dharma_id"]
-                    let cprin2: String = tes["principal"]
-                    let cnet2: String = tes["network"]
-                    let ceng2: String = tes["engram"]
-                    let has_did2: Bool = tes["has_dharma_id"]
-                    let has_prin2: Bool = tes["has_principal"]
-                    let has_net2: Bool = tes["has_network"]
-                    let has_eng2: Bool = tes["has_engram"]
-                    let arg_name2: String = "EL_STR(" + c_str_lit(cname2) + ")"
-                    let arg_did2: String = cgi_arg(cdid2, has_did2)
-                    let arg_prin2: String = cgi_arg(cprin2, has_prin2)
-                    let arg_net2: String = cgi_arg(cnet2, has_net2)
-                    let arg_eng2: String = cgi_arg(ceng2, has_eng2)
-                    emit_line("  el_cgi_init(" + arg_name2 + ", " + arg_did2 + ", " + arg_prin2 + ", " + arg_net2 + ", " + arg_eng2 + ");")
-                }
-                let tes_i = tes_i + 1
             }
         }
         let si2 = si2 + 1