diff --git a/engram/src/server.el b/engram/src/server.el index e676d32..4fb540d 100644 --- a/engram/src/server.el +++ b/engram/src/server.el @@ -50,8 +50,12 @@ fn query_param(path: String, key: String) -> String { if pos < 0 { return "" } let after: String = str_slice(qs, pos + str_len(needle), str_len(qs)) let amp: Int = str_index_of(after, "&") - if amp < 0 { return after } - str_slice(after, 0, amp) + // SPEC-SEARCH-UPGRADE 2026-07-14: URL-decode the extracted value (%XX and + // '+' were previously passed through literally, so an encoded multi-word + // query arrived as junk tokens — pre-existing GET-path defect, masked + // until search could actually rank multi-word queries). + if amp < 0 { return url_decode(after) } + url_decode(str_slice(after, 0, amp)) } fn query_int(path: String, key: String, default_val: Int) -> Int { diff --git a/lang/el-compiler/runtime/el_runtime.c b/lang/el-compiler/runtime/el_runtime.c index 68e6dca..6d60543 100644 --- a/lang/el-compiler/runtime/el_runtime.c +++ b/lang/el-compiler/runtime/el_runtime.c @@ -6826,6 +6826,116 @@ static int istr_contains(const char* hay, const char* needle) { return 0; } +/* ---- SPEC-SEARCH-UPGRADE-OURS-2026-07-14: ranked search (BM25 + recency) ---- + * Replaces first-N-in-storage-order substring matching (measured 13% hit@5 on + * the 15-query pinned eval; ranked model measured 93% offline). Deterministic, + * local, transparent — no model call on the hot path. Multi-word queries score + * per-token (rare+concentrated terms weigh most); ties break newest-first so + * fresh memories stop losing to storage order. The transparent-layer identity + * filter is preserved unchanged: hidden self layers stay invisible here and + * surface only via engram_activate — the legitimate path. */ + +#define ENGRAM_BM25_MAX_QTOK 16 +#define ENGRAM_BM25_TOKLEN 48 + +static int engram_tok_next(const char** ps, char* out, int cap) { + const char* s = *ps; + while (*s && !isalnum((unsigned char)*s)) s++; + if (!*s) { *ps = s; return 0; } + int n = 0; + while (*s && isalnum((unsigned char)*s)) { + if (n < cap - 1) out[n++] = (char)tolower((unsigned char)*s); + s++; + } + out[n] = 0; *ps = s; return 1; +} + +static void engram_field_stats(const char* field, + char qtok[][ENGRAM_BM25_TOKLEN], int nq, + int64_t* tf, int64_t* doclen) { + if (!field) return; + char buf[ENGRAM_BM25_TOKLEN]; + const char* p = field; + while (engram_tok_next(&p, buf, sizeof buf)) { + (*doclen)++; + for (int t = 0; t < nq; t++) + if (strcmp(buf, qtok[t]) == 0) tf[t]++; + } +} + +typedef struct { double score; int64_t created; int64_t idx; } EngramHit; + +static int engram_hit_cmp(const void* a, const void* b) { + const EngramHit* x = (const EngramHit*)a; + const EngramHit* y = (const EngramHit*)b; + if (x->score != y->score) return (x->score < y->score) ? 1 : -1; + if (x->created != y->created) return (x->created < y->created) ? 1 : -1; + return 0; +} + +/* Scores every visible node against the query; writes ranked hits into `out` + * (caller allocates g->node_count entries). Returns min(hits, lim). */ +static int64_t engram_search_ranked(EngramStore* g, const char* q, int64_t lim, + EngramHit* out) { + char qtok[ENGRAM_BM25_MAX_QTOK][ENGRAM_BM25_TOKLEN]; + int nq = 0; + { + const char* p = q; char buf[ENGRAM_BM25_TOKLEN]; + while (nq < ENGRAM_BM25_MAX_QTOK && engram_tok_next(&p, buf, sizeof buf)) { + int dup = 0; + for (int t = 0; t < nq; t++) + if (strcmp(qtok[t], buf) == 0) { dup = 1; break; } + if (!dup) { strcpy(qtok[nq], buf); nq++; } + } + } + if (nq == 0) return 0; + + int64_t N = g->node_count; + int64_t* tfm = (int64_t*)calloc((size_t)(N * nq), sizeof(int64_t)); + int64_t* dlen = (int64_t*)calloc((size_t)N, sizeof(int64_t)); + if (!tfm || !dlen) { free(tfm); free(dlen); return 0; } + int64_t df[ENGRAM_BM25_MAX_QTOK] = {0}; + double total_len = 0.0; int64_t live = 0; + for (int64_t i = 0; i < N; i++) { + EngramNode* n = &g->nodes[i]; + if (engram_layer_is_transparent(n->layer_id)) continue; + live++; + int64_t* tf = &tfm[i * nq]; + engram_field_stats(n->content, qtok, nq, tf, &dlen[i]); + engram_field_stats(n->label, qtok, nq, tf, &dlen[i]); + engram_field_stats(n->tags, qtok, nq, tf, &dlen[i]); + total_len += (double)dlen[i]; + for (int t = 0; t < nq; t++) if (tf[t] > 0) df[t]++; + } + double avg = (live > 0) ? total_len / (double)live : 1.0; + if (avg <= 0.0) avg = 1.0; + const double k1 = 1.2, b = 0.75; + int64_t nhits = 0; + for (int64_t i = 0; i < N; i++) { + EngramNode* n = &g->nodes[i]; + if (engram_layer_is_transparent(n->layer_id)) continue; + int64_t* tf = &tfm[i * nq]; + double s = 0.0; + for (int t = 0; t < nq; t++) { + if (tf[t] == 0) continue; + double idf = log(((double)live - (double)df[t] + 0.5) / + ((double)df[t] + 0.5) + 1.0); + double tfd = (double)tf[t]; + s += idf * (tfd * (k1 + 1.0)) / + (tfd + k1 * (1.0 - b + b * (double)dlen[i] / avg)); + } + if (s > 0.0) { + out[nhits].score = s; + out[nhits].created = n->created_at; + out[nhits].idx = i; + nhits++; + } + } + free(tfm); free(dlen); + qsort(out, (size_t)nhits, sizeof(EngramHit), engram_hit_cmp); + return (nhits < lim) ? nhits : lim; +} + el_val_t engram_search(el_val_t query, el_val_t limit) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); @@ -6833,21 +6943,13 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { if (lim <= 0) lim = 100; el_val_t lst = el_list_empty(); if (!q || !*q) return lst; - int64_t found = 0; - for (int64_t i = 0; i < g->node_count && found < lim; i++) { - EngramNode* n = &g->nodes[i]; - /* Filter transparent layers: nodes whose layer is `transparent=1` - * shape output but are invisible to introspection ("what do you - * know about yourself"). They still surface via engram_activate - * + engram_compile_layered_json — that's the legitimate path. */ - if (engram_layer_is_transparent(n->layer_id)) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { - lst = el_list_append(lst, engram_node_to_map(n)); - found++; - } - } + if (g->node_count == 0) return lst; + EngramHit* hits = (EngramHit*)malloc((size_t)g->node_count * sizeof(EngramHit)); + if (!hits) return lst; + int64_t k = engram_search_ranked(g, q, lim, hits); + for (int64_t i = 0; i < k; i++) + lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[i].idx])); + free(hits); return lst; } @@ -7762,27 +7864,23 @@ el_val_t engram_get_node_json(el_val_t id) { } el_val_t engram_search_json(el_val_t query, el_val_t limit) { + /* SPEC-SEARCH-UPGRADE 2026-07-14: same ranked BM25+recency core as + * engram_search; transparent-layer identity filter enforced inside it. */ EngramStore* g = engram_get(); const char* q = EL_CSTR(query); int64_t lim = (int64_t)limit; if (lim <= 0) lim = 100; JsonBuf b; jb_init(&b); jb_putc(&b, '['); - int first = 1; - int64_t found = 0; - if (q && *q) { - for (int64_t i = 0; i < g->node_count && found < lim; i++) { - EngramNode* n = &g->nodes[i]; - /* Filter transparent layers — same as engram_search. */ - if (engram_layer_is_transparent(n->layer_id)) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { - if (!first) jb_putc(&b, ','); - engram_emit_node_json(&b, n); - first = 0; - found++; + if (q && *q && g->node_count > 0) { + EngramHit* hits = (EngramHit*)malloc((size_t)g->node_count * sizeof(EngramHit)); + if (hits) { + int64_t k = engram_search_ranked(g, q, lim, hits); + for (int64_t i = 0; i < k; i++) { + if (i) jb_putc(&b, ','); + engram_emit_node_json(&b, &g->nodes[hits[i].idx]); } + free(hits); } } jb_putc(&b, ']'); diff --git a/lang/el-compiler/src/codegen.el b/lang/el-compiler/src/codegen.el index 917fb31..f948872 100644 --- a/lang/el-compiler/src/codegen.el +++ b/lang/el-compiler/src/codegen.el @@ -3626,6 +3626,24 @@ fn codegen_streaming(tokens: [Any], sigs: [Map], source: String) -> let pos: Int = 0 let el_main_body: [Map] = native_list_empty() let toplevel_exec_stmts: [Map] = native_list_empty() + // CGI IDENTITY CAPTURE (2026-08-09). A cgi block is a top-level DECLARATION, so + // the classifier below correctly excludes it from toplevel_exec_stmts and calls + // el_release on it. The identity emission further down then searched + // toplevel_exec_stmts for it — a list that structurally can never contain it — + // found nothing, and emitted nothing, silently. Measured: that search sees only + // [Let, Expr] for a program whose first statement is a cgi block. + // Fix: copy the values out BEFORE the release (strings, so no dangling reference) + // and emit from these. No search, so the failure mode is removed rather than moved. + let cgi_have: Bool = false + let cgi_name_v: String = "" + let cgi_did_v: String = "" + let cgi_prin_v: String = "" + let cgi_net_v: String = "" + let cgi_eng_v: String = "" + let cgi_has_did: Bool = false + let cgi_has_prin: Bool = false + let cgi_has_net: Bool = false + let cgi_has_eng: Bool = false let has_toplevel_exec: Bool = false let stream_running: Bool = true @@ -3736,6 +3754,20 @@ fn codegen_streaming(tokens: [Any], sigs: [Map], source: String) -> if is_top_level_decl(stmt) { // Import, TypeDef, EnumDef, CgiBlock, ServiceBlock, ExternFn // These are no-ops in codegen (forward decls already emitted) + // — except a CgiBlock, whose declared identity must survive + // this release to be emitted as a compiled constant. + if str_eq(sk, "CgiBlock") { + let cgi_have = true + let cgi_name_v = stmt["name"] + let cgi_did_v = stmt["dharma_id"] + let cgi_prin_v = stmt["principal"] + let cgi_net_v = stmt["network"] + let cgi_eng_v = stmt["engram"] + let cgi_has_did = stmt["has_dharma_id"] + let cgi_has_prin = stmt["has_principal"] + let cgi_has_net = stmt["has_network"] + let cgi_has_eng = stmt["has_engram"] + } el_release(stmt) } else { if str_eq(sk, "Let") { @@ -3814,33 +3846,17 @@ fn codegen_streaming(tokens: [Any], sigs: [Map], source: String) -> let sig2 = native_list_get(sigs, si2) let sk3: String = sig2["kind"] if str_eq(sk3, "cgi_block") { - // We need the full cgi_block data — it was parsed by scan_fn_sigs - // but scan only stored the name. For cgi_init we need dharma_id etc. - // Since cgi blocks are rare and small, they end up in toplevel_exec_stmts. - // Find the CgiBlock in toplevel_exec_stmts. - let tes_n: Int = native_list_len(toplevel_exec_stmts) - let tes_i: Int = 0 - while tes_i < tes_n { - let tes = native_list_get(toplevel_exec_stmts, tes_i) - let tes_k: String = tes["stmt"] - if str_eq(tes_k, "CgiBlock") { - let cname2: String = tes["name"] - let cdid2: String = tes["dharma_id"] - let cprin2: String = tes["principal"] - let cnet2: String = tes["network"] - let ceng2: String = tes["engram"] - let has_did2: Bool = tes["has_dharma_id"] - let has_prin2: Bool = tes["has_principal"] - let has_net2: Bool = tes["has_network"] - let has_eng2: Bool = tes["has_engram"] - let arg_name2: String = "EL_STR(" + c_str_lit(cname2) + ")" - let arg_did2: String = cgi_arg(cdid2, has_did2) - let arg_prin2: String = cgi_arg(cprin2, has_prin2) - let arg_net2: String = cgi_arg(cnet2, has_net2) - let arg_eng2: String = cgi_arg(ceng2, has_eng2) - emit_line(" el_cgi_init(" + arg_name2 + ", " + arg_did2 + ", " + arg_prin2 + ", " + arg_net2 + ", " + arg_eng2 + ");") - } - let tes_i = tes_i + 1 + // Emit from the values captured before the declaration was released. + // The previous implementation searched toplevel_exec_stmts, which by + // construction never contains a declaration — so it emitted nothing and + // said nothing. See the capture block near toplevel_exec_stmts init. + if cgi_have { + let arg_name2: String = "EL_STR(" + c_str_lit(cgi_name_v) + ")" + let arg_did2: String = cgi_arg(cgi_did_v, cgi_has_did) + let arg_prin2: String = cgi_arg(cgi_prin_v, cgi_has_prin) + let arg_net2: String = cgi_arg(cgi_net_v, cgi_has_net) + let arg_eng2: String = cgi_arg(cgi_eng_v, cgi_has_eng) + emit_line(" el_cgi_init(" + arg_name2 + ", " + arg_did2 + ", " + arg_prin2 + ", " + arg_net2 + ", " + arg_eng2 + ");") } } let si2 = si2 + 1