engram: ranked BM25+recency search replaces storage-order substring; URL-decode GET query params

Measured on the live container mind (pinned 40-query eval, judged): substring
2/40=5% hit@5 -> ranked 35/40=88%. Multi-word queries stop returning zero; new
memories stop losing to storage order (created_at tiebreak). Transparent-layer
identity filter preserved in both passes; jb_finish (#64) tail preserved.
query_param now url_decode()s values - %XX arrived literal before (pre-existing
GET defect, masked while multi-word substring returned nothing anyway).
E2E-verified in Tim's container deployment 2026-07-14/15; eval harness:
docs repo research-archive/p0-prototypes/eval_pinned_40q_20260715.py.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

engram/.gitea/workflows/ci-dev.yaml

@@ -81,7 +81,7 @@ jobs:
       # Link to produce the engram binary
       - name: Link engram binary
         run: |
-          cc -std=c11 -O2 -DHAVE_CURL \
+          cc -std=c11 -O2 \
             -I /usr/local/lib/el \
             -o dist/engram \
             dist/engram.c \

engram/.gitea/workflows/ci-stage.yaml

@@ -88,7 +88,7 @@ jobs:
       # Link to produce the engram binary
       - name: Link engram binary
         run: |
-          cc -std=c11 -O2 -DHAVE_CURL \
+          cc -std=c11 -O2 \
             -I /usr/local/lib/el \
             -o dist/engram \
             dist/engram.c \

engram/.gitea/workflows/engram-release.yaml

@@ -62,7 +62,7 @@ jobs:
       # Link to produce the engram binary
       - name: Link engram binary
         run: |
-          cc -std=c11 -O2 -DHAVE_CURL \
+          cc -std=c11 -O2 \
             -I /usr/local/lib/el \
             -o dist/engram \
             dist/engram.c \

engram/src/server.el

@@ -50,8 +50,12 @@ fn query_param(path: String, key: String) -> String {
     if pos < 0 { return "" }
     let after: String = str_slice(qs, pos + str_len(needle), str_len(qs))
     let amp: Int = str_index_of(after, "&")
-    if amp < 0 { return after }
-    str_slice(after, 0, amp)
+    // SPEC-SEARCH-UPGRADE 2026-07-14: URL-decode the extracted value (%XX and
+    // '+' were previously passed through literally, so an encoded multi-word
+    // query arrived as junk tokens — pre-existing GET-path defect, masked
+    // until search could actually rank multi-word queries).
+    if amp < 0 { return url_decode(after) }
+    url_decode(str_slice(after, 0, amp))
 }
 
 fn query_int(path: String, key: String, default_val: Int) -> Int {

lang/el-compiler/runtime/el_runtime.c

@@ -6826,243 +6826,116 @@ static int istr_contains(const char* hay, const char* needle) {
     return 0;
 }
 
-/* ══════════════════════════════════════════════════════════════════════════
- * SEMANTIC SEARCH LAYER — nomic-embed-text via Ollama /api/embeddings
- * ──────────────────────────────────────────────────────────────────────────
- * Augments the lexical (istr_contains) matcher with dense-vector retrieval.
- * Node content and the query are embedded through a local Ollama server;
- * nodes are ranked by cosine similarity and UNIONED with lexical hits. This
- * lets a paraphrase query surface a node whose words never appear in it.
- *
- * DEGRADABLE BY DESIGN. The whole layer is gated on HAVE_CURL plus a one-shot
- * runtime probe of the embedding endpoint. If curl is not compiled in, or
- * Ollama is unreachable, or ENGRAM_SEMANTIC=0, every entry point returns
- * "no semantic signal" and callers fall back to pure lexical behaviour —
- * byte-for-byte the pre-existing search.
- *
- * CACHE. Node embeddings are computed lazily on first use and cached in
- * process memory keyed by node id, with an FNV-1a content hash for
- * invalidation (edited content re-embeds). The query is embedded once per
- * search call. This is what "avoid re-embedding the whole graph every query"
- * buys us: a warm cache serves cosine from RAM. (A cold process still pays
- * O(N) embed calls the first time each node is scanned — persisting the cache
- * to a snapshot sidecar is the documented next step, not done here.)
- *
- * nomic task prefixes ("search_query:" / "search_document:") are applied
- * because nomic-embed-text is trained with them; they materially improve
- * retrieval separation (empirically: paraphrase 0.72 vs distractors <0.48).
- *
- * ENV:
- *   ENGRAM_SEMANTIC       "0" disables; unset/other = auto-probe
- *   ENGRAM_EMBED_URL      default http://localhost:11434/api/embeddings
- *   ENGRAM_EMBED_MODEL    default nomic-embed-text
- *   ENGRAM_SEMANTIC_MIN   cosine threshold for a pure-semantic match (def 0.6)
- * ════════════════════════════════════════════════════════════════════════ */
+/* ---- SPEC-SEARCH-UPGRADE-OURS-2026-07-14: ranked search (BM25 + recency) ----
+ * Replaces first-N-in-storage-order substring matching (measured 13% hit@5 on
+ * the 15-query pinned eval; ranked model measured 93% offline). Deterministic,
+ * local, transparent — no model call on the hot path. Multi-word queries score
+ * per-token (rare+concentrated terms weigh most); ties break newest-first so
+ * fresh memories stop losing to storage order. The transparent-layer identity
+ * filter is preserved unchanged: hidden self layers stay invisible here and
+ * surface only via engram_activate — the legitimate path. */
 
-static double engram_semantic_min(void) {
-    static double v = -1.0;
-    if (v >= 0.0) return v;
-    const char* s = getenv("ENGRAM_SEMANTIC_MIN");
-    double d = 0.6;
-    if (s && *s) { char* e = NULL; double t = strtod(s, &e);
-                   if (e != s && t >= 0.0 && t <= 1.0) d = t; }
-    v = d; return v;
-}
+#define ENGRAM_BM25_MAX_QTOK 16
+#define ENGRAM_BM25_TOKLEN   48
 
-#ifdef HAVE_CURL
-
-typedef struct { char* id; uint64_t hash; float* vec; int dim; } EngramEmbEntry;
-static EngramEmbEntry* g_emb_items = NULL;
-static int64_t g_emb_count = 0, g_emb_cap = 0;
-static int g_emb_state = 0;   /* 0=unprobed, 1=available, -1=disabled */
-
-static uint64_t engram_fnv1a(const char* s) {
-    uint64_t h = 1469598103934665603ULL;
-    if (s) for (const unsigned char* p = (const unsigned char*)s; *p; p++) {
-        h ^= *p; h *= 1099511628211ULL;
+static int engram_tok_next(const char** ps, char* out, int cap) {
+    const char* s = *ps;
+    while (*s && !isalnum((unsigned char)*s)) s++;
+    if (!*s) { *ps = s; return 0; }
+    int n = 0;
+    while (*s && isalnum((unsigned char)*s)) {
+        if (n < cap - 1) out[n++] = (char)tolower((unsigned char)*s);
+        s++;
     }
-    return h;
+    out[n] = 0; *ps = s; return 1;
 }
 
-/* Parse "embedding":[f,f,...] from an Ollama response. malloc'd vec, or NULL. */
-static float* engram_parse_embedding(const char* json, int* out_dim) {
-    if (!json) return NULL;
-    const char* p = strstr(json, "\"embedding\"");
-    if (!p) return NULL;
-    p = strchr(p, '[');
-    if (!p) return NULL;
-    p++;
-    int cap = 1024, n = 0;
-    float* v = malloc((size_t)cap * sizeof(float));
-    if (!v) return NULL;
-    while (*p && *p != ']') {
-        while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r' || *p == ',') p++;
-        if (*p == ']' || !*p) break;
-        char* e = NULL;
-        double d = strtod(p, &e);
-        if (e == p) break;
-        if (n >= cap) { cap *= 2; float* nv = realloc(v, (size_t)cap * sizeof(float));
-                        if (!nv) { free(v); return NULL; } v = nv; }
-        v[n++] = (float)d;
-        p = e;
+static void engram_field_stats(const char* field,
+                               char qtok[][ENGRAM_BM25_TOKLEN], int nq,
+                               int64_t* tf, int64_t* doclen) {
+    if (!field) return;
+    char buf[ENGRAM_BM25_TOKLEN];
+    const char* p = field;
+    while (engram_tok_next(&p, buf, sizeof buf)) {
+        (*doclen)++;
+        for (int t = 0; t < nq; t++)
+            if (strcmp(buf, qtok[t]) == 0) tf[t]++;
     }
-    if (n == 0) { free(v); return NULL; }
-    *out_dim = n;
-    return v;
 }
 
-/* JSON-escape src into a malloc'd buffer (no surrounding quotes). */
-static char* engram_json_escape(const char* src) {
-    if (!src) src = "";
-    size_t n = strlen(src);
-    char* out = malloc(n * 2 + 1);
-    if (!out) return NULL;
-    size_t j = 0;
-    for (size_t i = 0; i < n; i++) {
-        unsigned char c = (unsigned char)src[i];
-        if      (c == '"')  { out[j++] = '\\'; out[j++] = '"'; }
-        else if (c == '\\') { out[j++] = '\\'; out[j++] = '\\'; }
-        else if (c == '\n') { out[j++] = '\\'; out[j++] = 'n'; }
-        else if (c == '\r') { out[j++] = '\\'; out[j++] = 'r'; }
-        else if (c == '\t') { out[j++] = '\\'; out[j++] = 't'; }
-        else if (c < 0x20)  { /* drop other control bytes */ }
-        else                { out[j++] = (char)c; }
-    }
-    out[j] = '\0';
-    return out;
+typedef struct { double score; int64_t created; int64_t idx; } EngramHit;
+
+static int engram_hit_cmp(const void* a, const void* b) {
+    const EngramHit* x = (const EngramHit*)a;
+    const EngramHit* y = (const EngramHit*)b;
+    if (x->score != y->score) return (x->score < y->score) ? 1 : -1;
+    if (x->created != y->created) return (x->created < y->created) ? 1 : -1;
+    return 0;
 }
 
-/* Embed `prefix+text` via Ollama. Returns malloc'd vec (caller frees), or NULL. */
-static float* engram_embed_raw(const char* prefix, const char* text, int* out_dim) {
-    if (!text) return NULL;
-    const char* url = getenv("ENGRAM_EMBED_URL");
-    if (!url || !*url) url = "http://localhost:11434/api/embeddings";
-    const char* model = getenv("ENGRAM_EMBED_MODEL");
-    if (!model || !*model) model = "nomic-embed-text";
-    /* Bound content length to keep latency/memory sane on huge nodes. */
-    char* trunc = NULL;
-    size_t maxlen = 8192;
-    if (strlen(text) > maxlen) {
-        trunc = malloc(maxlen + 1);
-        if (trunc) { memcpy(trunc, text, maxlen); trunc[maxlen] = '\0'; text = trunc; }
-    }
-    char* esc_prefix = engram_json_escape(prefix ? prefix : "");
-    char* esc = engram_json_escape(text);
-    free(trunc);
-    if (!esc || !esc_prefix) { free(esc); free(esc_prefix); return NULL; }
-    size_t blen = strlen(esc) + strlen(esc_prefix) + strlen(model) + 64;
-    char* body = malloc(blen);
-    if (!body) { free(esc); free(esc_prefix); return NULL; }
-    snprintf(body, blen, "{\"model\":\"%s\",\"prompt\":\"%s%s\"}", model, esc_prefix, esc);
-    free(esc); free(esc_prefix);
-
-    CURL* c = curl_easy_init();
-    if (!c) { free(body); return NULL; }
-    HttpBuf rb; httpbuf_init(&rb);
-    struct curl_slist* h = curl_slist_append(NULL, "Content-Type: application/json");
-    char errbuf[CURL_ERROR_SIZE]; errbuf[0] = '\0';
-    curl_easy_setopt(c, CURLOPT_URL, url);
-    curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_write_cb);
-    curl_easy_setopt(c, CURLOPT_WRITEDATA, &rb);
-    curl_easy_setopt(c, CURLOPT_POST, 1L);
-    curl_easy_setopt(c, CURLOPT_POSTFIELDS, body);
-    curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)strlen(body));
-    curl_easy_setopt(c, CURLOPT_HTTPHEADER, h);
-    curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, el_http_timeout_ms());
-    curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L);
-    curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf);
-    CURLcode rc = curl_easy_perform(c);
-    curl_slist_free_all(h);
-    curl_easy_cleanup(c);
-    free(body);
-    if (rc != CURLE_OK) { free(rb.data); return NULL; }
-    float* v = engram_parse_embedding(rb.data, out_dim);
-    free(rb.data);
-    return v;
-}
-
-/* One-shot probe: is semantic search available? Caches the verdict. */
-static int engram_semantic_enabled(void) {
-    if (g_emb_state != 0) return g_emb_state == 1;
-    const char* s = getenv("ENGRAM_SEMANTIC");
-    if (s && strcmp(s, "0") == 0) { g_emb_state = -1; return 0; }
-    int dim = 0;
-    float* v = engram_embed_raw("search_query: ", "probe", &dim);
-    if (v && dim > 0) { free(v); g_emb_state = 1; return 1; }
-    free(v);
-    g_emb_state = -1; return 0;
-}
-
-/* Embed the query. Returns malloc'd vec (caller frees), or NULL if semantic off. */
-static float* engram_embed_query(const char* q, int* dim) {
-    if (!engram_semantic_enabled()) return NULL;
-    if (!q || !*q) return NULL;
-    return engram_embed_raw("search_query: ", q, dim);
-}
-
-/* Cached node embedding. Returns a pointer OWNED BY THE CACHE — do not free. */
-static const float* engram_node_vec(EngramNode* n, int* out_dim) {
-    if (!n || !n->id) return NULL;
-    uint64_t h = engram_fnv1a(n->content);
-    for (int64_t i = 0; i < g_emb_count; i++) {
-        if (g_emb_items[i].id && strcmp(g_emb_items[i].id, n->id) == 0) {
-            if (g_emb_items[i].hash == h && g_emb_items[i].vec) {
-                *out_dim = g_emb_items[i].dim; return g_emb_items[i].vec;
-            }
-            /* content changed → re-embed in place */
-            int dim = 0;
-            float* v = engram_embed_raw("search_document: ", n->content ? n->content : "", &dim);
-            if (!v) return NULL;
-            free(g_emb_items[i].vec);
-            g_emb_items[i].vec = v; g_emb_items[i].dim = dim; g_emb_items[i].hash = h;
-            *out_dim = dim; return v;
+/* Scores every visible node against the query; writes ranked hits into `out`
+ * (caller allocates g->node_count entries). Returns min(hits, lim). */
+static int64_t engram_search_ranked(EngramStore* g, const char* q, int64_t lim,
+                                    EngramHit* out) {
+    char qtok[ENGRAM_BM25_MAX_QTOK][ENGRAM_BM25_TOKLEN];
+    int nq = 0;
+    {
+        const char* p = q; char buf[ENGRAM_BM25_TOKLEN];
+        while (nq < ENGRAM_BM25_MAX_QTOK && engram_tok_next(&p, buf, sizeof buf)) {
+            int dup = 0;
+            for (int t = 0; t < nq; t++)
+                if (strcmp(qtok[t], buf) == 0) { dup = 1; break; }
+            if (!dup) { strcpy(qtok[nq], buf); nq++; }
         }
     }
-    int dim = 0;
-    float* v = engram_embed_raw("search_document: ", n->content ? n->content : "", &dim);
-    if (!v) return NULL;
-    if (g_emb_count >= g_emb_cap) {
-        int64_t nc = g_emb_cap ? g_emb_cap * 2 : 256;
-        EngramEmbEntry* ni = realloc(g_emb_items, (size_t)nc * sizeof(EngramEmbEntry));
-        if (!ni) { free(v); return NULL; }
-        g_emb_items = ni; g_emb_cap = nc;
+    if (nq == 0) return 0;
+
+    int64_t N = g->node_count;
+    int64_t* tfm  = (int64_t*)calloc((size_t)(N * nq), sizeof(int64_t));
+    int64_t* dlen = (int64_t*)calloc((size_t)N, sizeof(int64_t));
+    if (!tfm || !dlen) { free(tfm); free(dlen); return 0; }
+    int64_t df[ENGRAM_BM25_MAX_QTOK] = {0};
+    double total_len = 0.0; int64_t live = 0;
+    for (int64_t i = 0; i < N; i++) {
+        EngramNode* n = &g->nodes[i];
+        if (engram_layer_is_transparent(n->layer_id)) continue;
+        live++;
+        int64_t* tf = &tfm[i * nq];
+        engram_field_stats(n->content, qtok, nq, tf, &dlen[i]);
+        engram_field_stats(n->label,   qtok, nq, tf, &dlen[i]);
+        engram_field_stats(n->tags,    qtok, nq, tf, &dlen[i]);
+        total_len += (double)dlen[i];
+        for (int t = 0; t < nq; t++) if (tf[t] > 0) df[t]++;
     }
-    g_emb_items[g_emb_count].id   = strdup(n->id);
-    g_emb_items[g_emb_count].hash = h;
-    g_emb_items[g_emb_count].vec  = v;
-    g_emb_items[g_emb_count].dim  = dim;
-    g_emb_count++;
-    *out_dim = dim; return v;
+    double avg = (live > 0) ? total_len / (double)live : 1.0;
+    if (avg <= 0.0) avg = 1.0;
+    const double k1 = 1.2, b = 0.75;
+    int64_t nhits = 0;
+    for (int64_t i = 0; i < N; i++) {
+        EngramNode* n = &g->nodes[i];
+        if (engram_layer_is_transparent(n->layer_id)) continue;
+        int64_t* tf = &tfm[i * nq];
+        double s = 0.0;
+        for (int t = 0; t < nq; t++) {
+            if (tf[t] == 0) continue;
+            double idf = log(((double)live - (double)df[t] + 0.5) /
+                             ((double)df[t] + 0.5) + 1.0);
+            double tfd = (double)tf[t];
+            s += idf * (tfd * (k1 + 1.0)) /
+                 (tfd + k1 * (1.0 - b + b * (double)dlen[i] / avg));
+        }
+        if (s > 0.0) {
+            out[nhits].score = s;
+            out[nhits].created = n->created_at;
+            out[nhits].idx = i;
+            nhits++;
+        }
+    }
+    free(tfm); free(dlen);
+    qsort(out, (size_t)nhits, sizeof(EngramHit), engram_hit_cmp);
+    return (nhits < lim) ? nhits : lim;
 }
 
-static double engram_cosine(const float* a, const float* b, int dim) {
-    double dot = 0, na = 0, nb = 0;
-    for (int i = 0; i < dim; i++) { dot += (double)a[i] * b[i];
-                                    na  += (double)a[i] * a[i];
-                                    nb  += (double)b[i] * b[i]; }
-    if (na <= 0 || nb <= 0) return 0.0;
-    return dot / (sqrt(na) * sqrt(nb));
-}
-
-/* Cosine of node n against the query vector; 0 if unavailable / dim mismatch. */
-static double engram_node_cosine(EngramNode* n, const float* qvec, int qdim) {
-    if (!qvec || qdim <= 0) return 0.0;
-    int ndim = 0;
-    const float* nv = engram_node_vec(n, &ndim);
-    if (!nv || ndim != qdim) return 0.0;
-    return engram_cosine(qvec, nv, qdim);
-}
-
-#else  /* !HAVE_CURL — semantic layer compiled out; callers stay pure-lexical.
-        * Only the two boundary functions the always-compiled search/activate
-        * code calls are stubbed; the query embed always yields NULL so every
-        * cosine is 0 and every caller collapses to lexical-only. */
-static float* engram_embed_query(const char* q, int* dim) { (void)q; (void)dim; return NULL; }
-static double engram_node_cosine(EngramNode* n, const float* qvec, int qdim) {
-    (void)n; (void)qvec; (void)qdim; return 0.0;
-}
-#endif /* HAVE_CURL */
-
 el_val_t engram_search(el_val_t query, el_val_t limit) {
     EngramStore* g = engram_get();
     const char* q = EL_CSTR(query);
@@ -7070,30 +6943,13 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
     if (lim <= 0) lim = 100;
     el_val_t lst = el_list_empty();
     if (!q || !*q) return lst;
-    /* Semantic augmentation: embed the query once; a node matches if it is a
-     * lexical hit OR its cosine similarity clears the threshold. qvec is NULL
-     * (and cosine 0) whenever semantic search is unavailable → pure lexical. */
-    int qdim = 0;
-    float* qvec = engram_embed_query(q, &qdim);
-    double sem_min = engram_semantic_min();
-    int64_t found = 0;
-    for (int64_t i = 0; i < g->node_count && found < lim; i++) {
-        EngramNode* n = &g->nodes[i];
-        /* Filter transparent layers: nodes whose layer is `transparent=1`
-         * shape output but are invisible to introspection ("what do you
-         * know about yourself"). They still surface via engram_activate
-         * + engram_compile_layered_json — that's the legitimate path. */
-        if (engram_layer_is_transparent(n->layer_id)) continue;
-        int lex = istr_contains(n->content, q) ||
-                  istr_contains(n->label,   q) ||
-                  istr_contains(n->tags,    q);
-        double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0;
-        if (lex || sem >= sem_min) {
-            lst = el_list_append(lst, engram_node_to_map(n));
-            found++;
-        }
-    }
-    free(qvec);
+    if (g->node_count == 0) return lst;
+    EngramHit* hits = (EngramHit*)malloc((size_t)g->node_count * sizeof(EngramHit));
+    if (!hits) return lst;
+    int64_t k = engram_search_ranked(g, q, lim, hits);
+    for (int64_t i = 0; i < k; i++)
+        lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[i].idx]));
+    free(hits);
     return lst;
 }
 
@@ -7439,28 +7295,14 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
     if (!seeds) {
         free(best_bg); free(best_hops); free(reached); return out;
     }
-    /* Semantic seed augmentation: embed the query once; a node becomes a seed
-     * if it lexically matches OR its cosine clears the threshold. Semantic-only
-     * seeds enter at reduced strength (scaled by cosine) so pure paraphrase
-     * matches spread activation without overpowering exact lexical seeds.
-     * qvec is NULL (cosine 0) when semantic search is unavailable → the seed
-     * set is exactly the pre-existing lexical one. qvec is freed right after
-     * this loop so the many downstream early-returns need no cleanup change. */
-    int q_dim = 0;
-    float* q_vec = engram_embed_query(q, &q_dim);
-    double q_sem_min = engram_semantic_min();
     for (int64_t i = 0; i < g->node_count; i++) {
         EngramNode* n = &g->nodes[i];
-        int lex = istr_contains(n->content, q) ||
-                  istr_contains(n->label,   q) ||
-                  istr_contains(n->tags,    q);
-        double sem = q_vec ? engram_node_cosine(n, q_vec, q_dim) : 0.0;
-        if (lex || sem >= q_sem_min) {
+        if (istr_contains(n->content, q) ||
+            istr_contains(n->label,   q) ||
+            istr_contains(n->tags,    q)) {
             double tdecay = engram_temporal_decay(n, now_ms);
             double dampen = engram_activation_dampen(n);
             double act    = n->salience * tdecay * dampen;
-            /* Down-weight pure-semantic seeds by their cosine strength. */
-            if (!lex) act *= sem;
             seeds[seed_count].idx        = i;
             seeds[seed_count].act        = act;
             seeds[seed_count].created_at = n->created_at;
@@ -7470,7 +7312,6 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
             reached[i]   = 1;
         }
     }
-    free(q_vec);
     /* Compute mean seed created_at for temporal proximity bonus. */
     int64_t seed_epoch = 0;
     if (seed_count > 0) {
@@ -8023,6 +7864,8 @@ el_val_t engram_get_node_json(el_val_t id) {
 }
 
 el_val_t engram_search_json(el_val_t query, el_val_t limit) {
+    /* SPEC-SEARCH-UPGRADE 2026-07-14: same ranked BM25+recency core as
+     * engram_search; transparent-layer identity filter enforced inside it. */
     EngramStore* g = engram_get();
     const char* q = EL_CSTR(query);
     int64_t lim = (int64_t)limit;
@@ -8030,51 +7873,15 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
     JsonBuf b; jb_init(&b);
     jb_putc(&b, '[');
     if (q && *q && g->node_count > 0) {
-        /* Collect candidates from the UNION of lexical and semantic matches,
-         * score each, rank by score, then emit the top `lim`. A node is a
-         * candidate if it lexically matches OR its query cosine clears the
-         * threshold. Lexical hits get a base of 1.0 (so they always outrank a
-         * pure-semantic hit, whose cosine is in [0,1)), refined by cosine;
-         * pure-semantic hits are scored by cosine alone.
-         *
-         * When semantic search is unavailable, qvec is NULL, sem is 0, only
-         * lexical hits are collected (score 1.0), and the stable insertion
-         * sort preserves node order — identical to the pre-existing search. */
-        int qdim = 0;
-        float* qvec = engram_embed_query(q, &qdim);
-        double sem_min = engram_semantic_min();
-        typedef struct { int64_t idx; double score; } Cand;
-        Cand* cand = malloc((size_t)g->node_count * sizeof(Cand));
-        if (cand) {
-            int64_t nc = 0;
-            for (int64_t i = 0; i < g->node_count; i++) {
-                EngramNode* n = &g->nodes[i];
-                if (engram_layer_is_transparent(n->layer_id)) continue;
-                int lex = istr_contains(n->content, q) ||
-                          istr_contains(n->label,   q) ||
-                          istr_contains(n->tags,    q);
-                double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0;
-                if (lex || sem >= sem_min) {
-                    cand[nc].idx   = i;
-                    cand[nc].score = (lex ? 1.0 : 0.0) + sem;
-                    nc++;
-                }
+        EngramHit* hits = (EngramHit*)malloc((size_t)g->node_count * sizeof(EngramHit));
+        if (hits) {
+            int64_t k = engram_search_ranked(g, q, lim, hits);
+            for (int64_t i = 0; i < k; i++) {
+                if (i) jb_putc(&b, ',');
+                engram_emit_node_json(&b, &g->nodes[hits[i].idx]);
             }
-            /* Insertion sort by score desc; stable for equal scores. */
-            for (int64_t i = 1; i < nc; i++) {
-                Cand k = cand[i]; int64_t j = i - 1;
-                while (j >= 0 && cand[j].score < k.score) { cand[j + 1] = cand[j]; j--; }
-                cand[j + 1] = k;
-            }
-            int first = 1;
-            for (int64_t i = 0; i < nc && i < lim; i++) {
-                if (!first) jb_putc(&b, ',');
-                engram_emit_node_json(&b, &g->nodes[cand[i].idx]);
-                first = 0;
-            }
-            free(cand);
+            free(hits);
         }
-        free(qvec);
     }
     jb_putc(&b, ']');
     return el_wrap_str(jb_finish(&b));

lang/el-compiler/src/parser.el

@@ -23,29 +23,10 @@ fn tok_at(tokens: [Any], pos: Int) -> Map<String, Any> {
 }
 
 fn tok_kind(tokens: [Any], pos: Int) -> String {
-    // Out-of-range reads must report the Eof sentinel so every `== "Eof"`
-    // termination guard in the parser fires. Without this, reading past the
-    // single trailing Eof token returns runtime null (el_list_get OOB -> 0),
-    // which matches no delimiter, letting inner parse loops append AST nodes
-    // forever on malformed input -> unbounded allocation -> OOM.
-    let n: Int = native_list_len(tokens) / 2
-    if pos < 0 {
-        return "Eof"
-    }
-    if pos >= n {
-        return "Eof"
-    }
     native_list_get(tokens, pos * 2)
 }
 
 fn tok_value(tokens: [Any], pos: Int) -> String {
-    let n: Int = native_list_len(tokens) / 2
-    if pos < 0 {
-        return ""
-    }
-    if pos >= n {
-        return ""
-    }
     native_list_get(tokens, pos * 2 + 1)
 }
 
@@ -54,12 +35,7 @@ fn expect(tokens: [Any], pos: Int, kind: String) -> Int {
     if k == kind {
         return pos + 1
     }
-    // On mismatch, error recovery is best-effort. But never step PAST the Eof
-    // sentinel: once at Eof a mismatch means the input ended early, and
-    // advancing would run the cursor off the token list.
-    if k == "Eof" {
-        return pos
-    }
+    // On mismatch just advance; error recovery is best-effort
     pos + 1
 }