From f76ccc059068c76877f668dad8d2e2f760fbc7cc Mon Sep 17 00:00:00 2001 From: Tim Lingo <1timlingo@gmail.com> Date: Wed, 15 Jul 2026 07:48:09 -0500 Subject: [PATCH 1/2] engram: ranked BM25+recency search replaces storage-order substring; URL-decode GET query params Measured on the live container mind (pinned 40-query eval, judged): substring 2/40=5% hit@5 -> ranked 35/40=88%. Multi-word queries stop returning zero; new memories stop losing to storage order (created_at tiebreak). Transparent-layer identity filter preserved in both passes; jb_finish (#64) tail preserved. query_param now url_decode()s values - %XX arrived literal before (pre-existing GET defect, masked while multi-word substring returned nothing anyway). E2E-verified in Tim's container deployment 2026-07-14/15; eval harness: docs repo research-archive/p0-prototypes/eval_pinned_40q_20260715.py. Co-Authored-By: Claude Fable 5 --- engram/src/server.el | 8 +- lang/el-compiler/runtime/el_runtime.c | 156 +++++++++++++++++++++----- 2 files changed, 133 insertions(+), 31 deletions(-) diff --git a/engram/src/server.el b/engram/src/server.el index e676d32..4fb540d 100644 --- a/engram/src/server.el +++ b/engram/src/server.el @@ -50,8 +50,12 @@ fn query_param(path: String, key: String) -> String { if pos < 0 { return "" } let after: String = str_slice(qs, pos + str_len(needle), str_len(qs)) let amp: Int = str_index_of(after, "&") - if amp < 0 { return after } - str_slice(after, 0, amp) + // SPEC-SEARCH-UPGRADE 2026-07-14: URL-decode the extracted value (%XX and + // '+' were previously passed through literally, so an encoded multi-word + // query arrived as junk tokens — pre-existing GET-path defect, masked + // until search could actually rank multi-word queries). + if amp < 0 { return url_decode(after) } + url_decode(str_slice(after, 0, amp)) } fn query_int(path: String, key: String, default_val: Int) -> Int { diff --git a/lang/el-compiler/runtime/el_runtime.c b/lang/el-compiler/runtime/el_runtime.c index 68e6dca..6d60543 100644 --- a/lang/el-compiler/runtime/el_runtime.c +++ b/lang/el-compiler/runtime/el_runtime.c @@ -6826,6 +6826,116 @@ static int istr_contains(const char* hay, const char* needle) { return 0; } +/* ---- SPEC-SEARCH-UPGRADE-OURS-2026-07-14: ranked search (BM25 + recency) ---- + * Replaces first-N-in-storage-order substring matching (measured 13% hit@5 on + * the 15-query pinned eval; ranked model measured 93% offline). Deterministic, + * local, transparent — no model call on the hot path. Multi-word queries score + * per-token (rare+concentrated terms weigh most); ties break newest-first so + * fresh memories stop losing to storage order. The transparent-layer identity + * filter is preserved unchanged: hidden self layers stay invisible here and + * surface only via engram_activate — the legitimate path. */ + +#define ENGRAM_BM25_MAX_QTOK 16 +#define ENGRAM_BM25_TOKLEN 48 + +static int engram_tok_next(const char** ps, char* out, int cap) { + const char* s = *ps; + while (*s && !isalnum((unsigned char)*s)) s++; + if (!*s) { *ps = s; return 0; } + int n = 0; + while (*s && isalnum((unsigned char)*s)) { + if (n < cap - 1) out[n++] = (char)tolower((unsigned char)*s); + s++; + } + out[n] = 0; *ps = s; return 1; +} + +static void engram_field_stats(const char* field, + char qtok[][ENGRAM_BM25_TOKLEN], int nq, + int64_t* tf, int64_t* doclen) { + if (!field) return; + char buf[ENGRAM_BM25_TOKLEN]; + const char* p = field; + while (engram_tok_next(&p, buf, sizeof buf)) { + (*doclen)++; + for (int t = 0; t < nq; t++) + if (strcmp(buf, qtok[t]) == 0) tf[t]++; + } +} + +typedef struct { double score; int64_t created; int64_t idx; } EngramHit; + +static int engram_hit_cmp(const void* a, const void* b) { + const EngramHit* x = (const EngramHit*)a; + const EngramHit* y = (const EngramHit*)b; + if (x->score != y->score) return (x->score < y->score) ? 1 : -1; + if (x->created != y->created) return (x->created < y->created) ? 1 : -1; + return 0; +} + +/* Scores every visible node against the query; writes ranked hits into `out` + * (caller allocates g->node_count entries). Returns min(hits, lim). */ +static int64_t engram_search_ranked(EngramStore* g, const char* q, int64_t lim, + EngramHit* out) { + char qtok[ENGRAM_BM25_MAX_QTOK][ENGRAM_BM25_TOKLEN]; + int nq = 0; + { + const char* p = q; char buf[ENGRAM_BM25_TOKLEN]; + while (nq < ENGRAM_BM25_MAX_QTOK && engram_tok_next(&p, buf, sizeof buf)) { + int dup = 0; + for (int t = 0; t < nq; t++) + if (strcmp(qtok[t], buf) == 0) { dup = 1; break; } + if (!dup) { strcpy(qtok[nq], buf); nq++; } + } + } + if (nq == 0) return 0; + + int64_t N = g->node_count; + int64_t* tfm = (int64_t*)calloc((size_t)(N * nq), sizeof(int64_t)); + int64_t* dlen = (int64_t*)calloc((size_t)N, sizeof(int64_t)); + if (!tfm || !dlen) { free(tfm); free(dlen); return 0; } + int64_t df[ENGRAM_BM25_MAX_QTOK] = {0}; + double total_len = 0.0; int64_t live = 0; + for (int64_t i = 0; i < N; i++) { + EngramNode* n = &g->nodes[i]; + if (engram_layer_is_transparent(n->layer_id)) continue; + live++; + int64_t* tf = &tfm[i * nq]; + engram_field_stats(n->content, qtok, nq, tf, &dlen[i]); + engram_field_stats(n->label, qtok, nq, tf, &dlen[i]); + engram_field_stats(n->tags, qtok, nq, tf, &dlen[i]); + total_len += (double)dlen[i]; + for (int t = 0; t < nq; t++) if (tf[t] > 0) df[t]++; + } + double avg = (live > 0) ? total_len / (double)live : 1.0; + if (avg <= 0.0) avg = 1.0; + const double k1 = 1.2, b = 0.75; + int64_t nhits = 0; + for (int64_t i = 0; i < N; i++) { + EngramNode* n = &g->nodes[i]; + if (engram_layer_is_transparent(n->layer_id)) continue; + int64_t* tf = &tfm[i * nq]; + double s = 0.0; + for (int t = 0; t < nq; t++) { + if (tf[t] == 0) continue; + double idf = log(((double)live - (double)df[t] + 0.5) / + ((double)df[t] + 0.5) + 1.0); + double tfd = (double)tf[t]; + s += idf * (tfd * (k1 + 1.0)) / + (tfd + k1 * (1.0 - b + b * (double)dlen[i] / avg)); + } + if (s > 0.0) { + out[nhits].score = s; + out[nhits].created = n->created_at; + out[nhits].idx = i; + nhits++; + } + } + free(tfm); free(dlen); + qsort(out, (size_t)nhits, sizeof(EngramHit), engram_hit_cmp); + return (nhits < lim) ? nhits : lim; +} + el_val_t engram_search(el_val_t query, el_val_t limit) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); @@ -6833,21 +6943,13 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { if (lim <= 0) lim = 100; el_val_t lst = el_list_empty(); if (!q || !*q) return lst; - int64_t found = 0; - for (int64_t i = 0; i < g->node_count && found < lim; i++) { - EngramNode* n = &g->nodes[i]; - /* Filter transparent layers: nodes whose layer is `transparent=1` - * shape output but are invisible to introspection ("what do you - * know about yourself"). They still surface via engram_activate - * + engram_compile_layered_json — that's the legitimate path. */ - if (engram_layer_is_transparent(n->layer_id)) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { - lst = el_list_append(lst, engram_node_to_map(n)); - found++; - } - } + if (g->node_count == 0) return lst; + EngramHit* hits = (EngramHit*)malloc((size_t)g->node_count * sizeof(EngramHit)); + if (!hits) return lst; + int64_t k = engram_search_ranked(g, q, lim, hits); + for (int64_t i = 0; i < k; i++) + lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[i].idx])); + free(hits); return lst; } @@ -7762,27 +7864,23 @@ el_val_t engram_get_node_json(el_val_t id) { } el_val_t engram_search_json(el_val_t query, el_val_t limit) { + /* SPEC-SEARCH-UPGRADE 2026-07-14: same ranked BM25+recency core as + * engram_search; transparent-layer identity filter enforced inside it. */ EngramStore* g = engram_get(); const char* q = EL_CSTR(query); int64_t lim = (int64_t)limit; if (lim <= 0) lim = 100; JsonBuf b; jb_init(&b); jb_putc(&b, '['); - int first = 1; - int64_t found = 0; - if (q && *q) { - for (int64_t i = 0; i < g->node_count && found < lim; i++) { - EngramNode* n = &g->nodes[i]; - /* Filter transparent layers — same as engram_search. */ - if (engram_layer_is_transparent(n->layer_id)) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { - if (!first) jb_putc(&b, ','); - engram_emit_node_json(&b, n); - first = 0; - found++; + if (q && *q && g->node_count > 0) { + EngramHit* hits = (EngramHit*)malloc((size_t)g->node_count * sizeof(EngramHit)); + if (hits) { + int64_t k = engram_search_ranked(g, q, lim, hits); + for (int64_t i = 0; i < k; i++) { + if (i) jb_putc(&b, ','); + engram_emit_node_json(&b, &g->nodes[hits[i].idx]); } + free(hits); } } jb_putc(&b, ']'); -- 2.52.0 From f34270d63de63d0809fdbd0d5877a2daebc30edb Mon Sep 17 00:00:00 2001 From: Tim Lingo <1timlingo@gmail.com> Date: Fri, 17 Jul 2026 18:25:21 -0500 Subject: [PATCH 2/2] =?UTF-8?q?runtime:=20fs=5Fread=20length=20hint=20must?= =?UTF-8?q?=20be=20paired=20with=20its=20buffer=20=E2=80=94=20fixes=20trun?= =?UTF-8?q?cated=20HTTP=20responses?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The binary-safe fs_read length (_tl_fs_read_len) was consumed by the HTTP response path for ANY body, even when the handler wrapped the file into a larger reply. Content-Length then lied AND the send stopped short: the safety-contact routes returned 178 of 208/218 bytes, cut mid-'set_at' — unparseable JSON. The desktop app read that as failure: fresh installs trapped at 'Set your safety contact' (POST reply mangled) and configured users saw the gate re-appear every launch (GET reply mangled). Worse, a stale hint LARGER than a later body would over-read heap memory out the socket. Fix: pair the hint with the exact buffer pointer it describes; consume it only when the response IS that buffer (binary file serving keeps working, the hint follows the worker's copy); reset both at request start. Also ports engram_get_node_by_label (from releases/v1.0.0) needed by soul.el session continuity in local mode — not-found returns "" (matches shipped behavior; '{}' flips the truthiness check upstream). Verified: genesis boot + byte-math E2E on :7797 sandbox — safety-contact GET/POST/GET all Content-Length==body, json-parse clean; /health, /api/config regressions match. Co-Authored-By: Claude Opus 4.8 --- lang/el-compiler/runtime/el_runtime.c | 74 +++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/lang/el-compiler/runtime/el_runtime.c b/lang/el-compiler/runtime/el_runtime.c index 6d60543..17dda84 100644 --- a/lang/el-compiler/runtime/el_runtime.c +++ b/lang/el-compiler/runtime/el_runtime.c @@ -82,8 +82,14 @@ static _Thread_local ElArena _tl_arena = {NULL, 0, 0}; static _Thread_local int _tl_arena_active = 0; /* Binary-safe fs_read length — set by fs_read, consumed by http_send_response. - * Allows serving PNGs and other binary files without strlen truncation. */ -static _Thread_local size_t _tl_fs_read_len = 0; + * Allows serving PNGs and other binary files without strlen truncation. + * PAIRED with the buffer pointer it describes: the length may only be applied + * to the exact buffer fs_read returned. Without the pairing, any handler that + * fs_read a file and then WRAPPED it into a larger response had that response + * truncated to the file's length (Content-Length lied AND the send stopped + * short) — the safety-contact onboarding trap, 2026-07-17. */ +static _Thread_local size_t _tl_fs_read_len = 0; +static _Thread_local const char* _tl_fs_read_buf = NULL; static void el_arena_track(char* p) { if (!_tl_arena_active || !p) return; @@ -101,6 +107,8 @@ static void el_arena_track(char* p) { void el_request_start(void) { _tl_arena.count = 0; _tl_arena_active = 1; + _tl_fs_read_len = 0; /* never let a previous request's file length */ + _tl_fs_read_buf = NULL; /* leak into this response's byte accounting */ } /* Called by http_worker after the El handler returns and the response is sent. @@ -1484,11 +1492,14 @@ static void http_send_response(int fd, const char* body) { } const char* eff_body = is_envelope ? env_body : body; - /* Use the real byte count from fs_read if available (handles binary files - * with embedded null bytes — PNG, WOFF2, etc.). Fall back to strlen for - * normal text/JSON responses where _tl_fs_read_len is 0. */ - size_t blen = (_tl_fs_read_len > 0) ? _tl_fs_read_len : strlen(eff_body); + /* Use the real byte count from fs_read ONLY when this body IS the exact + * buffer fs_read returned (binary files with embedded null bytes — PNG, + * WOFF2, etc.). Any other body — wrapped, enveloped, or derived — must be + * measured with strlen, or it is truncated/over-read to the file's size. */ + size_t blen = (_tl_fs_read_len > 0 && eff_body == _tl_fs_read_buf) + ? _tl_fs_read_len : strlen(eff_body); _tl_fs_read_len = 0; /* consume — one-shot per response */ + _tl_fs_read_buf = NULL; int head_only = _tl_http_head_only; JsonBuf hdrs; jb_init(&hdrs); @@ -1568,11 +1579,22 @@ static void* http_worker(void* arg) { const char* rs = EL_CSTR(r); /* Copy response out BEFORE arena teardown. * For binary files, _tl_fs_read_len holds the real byte count — - * use memcpy instead of strdup so null bytes are preserved. */ - size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0); + * use memcpy instead of strdup so null bytes are preserved. + * The stored length applies ONLY when the response IS the exact + * fs_read buffer; a wrapped/derived response must use strlen or + * it gets truncated (or over-read) to the file's length. */ + size_t rlen; + if (_tl_fs_read_len > 0 && rs && rs == _tl_fs_read_buf) { + rlen = _tl_fs_read_len; /* raw file bytes — binary-safe */ + } else { + rlen = rs ? strlen(rs) : 0; + _tl_fs_read_len = 0; /* hint doesn't describe this body */ + _tl_fs_read_buf = NULL; + } response = malloc(rlen + 1); if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; } else if (response) { response[0] = '\0'; } + if (_tl_fs_read_len > 0) _tl_fs_read_buf = response; /* hint follows the copy */ } else { response = el_strdup_persist("el-runtime: no http handler registered"); } @@ -1822,10 +1844,20 @@ static void* http_worker_v2(void* arg) { el_val_t hmap = http_build_headers_map(hdr_block ? hdr_block : ""); el_val_t r = h(EL_STR(dispatch_method), EL_STR(path), hmap, EL_STR(body)); const char* rs = EL_CSTR(r); - size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0); + /* Same pairing rule as the v1 worker: the fs_read length is only + * trustworthy for the exact buffer fs_read returned. */ + size_t rlen; + if (_tl_fs_read_len > 0 && rs && rs == _tl_fs_read_buf) { + rlen = _tl_fs_read_len; /* raw file bytes — binary-safe */ + } else { + rlen = rs ? strlen(rs) : 0; + _tl_fs_read_len = 0; /* hint doesn't describe this body */ + _tl_fs_read_buf = NULL; + } response = malloc(rlen + 1); if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; } else if (response) { response[0] = '\0'; } + if (_tl_fs_read_len > 0) _tl_fs_read_buf = response; /* hint follows the copy */ el_release(hmap); } else { response = el_strdup_persist( @@ -2023,6 +2055,7 @@ el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body) { el_val_t fs_read(el_val_t pathv) { const char* path = EL_CSTR(pathv); _tl_fs_read_len = 0; + _tl_fs_read_buf = NULL; if (!path) return el_wrap_str(el_strdup("")); FILE* f = fopen(path, "rb"); if (!f) return el_wrap_str(el_strdup("")); @@ -2034,6 +2067,7 @@ el_val_t fs_read(el_val_t pathv) { size_t got = fread(buf, 1, (size_t)sz, f); buf[got] = '\0'; _tl_fs_read_len = got; /* store real byte count for binary-safe send */ + _tl_fs_read_buf = buf; /* ...valid ONLY for this exact buffer */ fclose(f); return el_wrap_str(buf); } @@ -3576,8 +3610,10 @@ el_val_t json_get_raw(el_val_t json_str, el_val_t key) { const char* k = EL_CSTR(key); const char* p = json_find_key(json, k); /* Clear fs_read binary-length hint — result is a fresh null-terminated - * string, not the raw file bytes, so Content-Length must use strlen. */ + * string, not the raw file bytes, so Content-Length must use strlen. + * (Kept although the pointer pairing now makes this redundant.) */ _tl_fs_read_len = 0; + _tl_fs_read_buf = NULL; if (!p) return el_wrap_str(el_strdup("")); const char* end = json_skip_value(p); size_t n = (size_t)(end - p); @@ -7863,6 +7899,24 @@ el_val_t engram_get_node_json(el_val_t id) { return el_wrap_str(jb_finish(&b)); } +/* Look up a node by exact label; returns its JSON or {}. Ported from the + * v1.0.0 release runtime — needed by soul.el session continuity + * (conv_history_load / session_summary_write / emit_session_start_event). */ +el_val_t engram_get_node_by_label(el_val_t label) { + const char* lbl = EL_CSTR(label); + if (!lbl || !*lbl) return el_wrap_str(el_strdup("")); + EngramStore* g = engram_get(); + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + if (n->label && strcmp(n->label, lbl) == 0) { + JsonBuf b; jb_init(&b); + engram_emit_node_json(&b, n); + return el_wrap_str(b.buf); + } + } + return el_wrap_str(el_strdup("")); +} + el_val_t engram_search_json(el_val_t query, el_val_t limit) { /* SPEC-SEARCH-UPGRADE 2026-07-14: same ranked BM25+recency core as * engram_search; transparent-layer identity filter enforced inside it. */ -- 2.52.0