Merge remote-tracking branch 'origin/feat/engram-semantic-search' into hotfix/stage-elc-engram-integration

# Conflicts:
#	lang/el-compiler/runtime/el_runtime.c

lang/el-compiler/runtime/el_platform_win.h

@@ -75,6 +75,7 @@ static inline void* el_win_dlsym(void* handle, const char* name) {
 #include <direct.h>            /* _mkdir */
 #define mkdir(path, mode) _mkdir(path)     /* POSIX mkdir(path,mode) → _mkdir(path) */
 #define timegm _mkgmtime                   /* UTC tm → time_t */
+#define fsync(fd) _commit(fd)              /* no fsync() on Windows; _commit() (<io.h>) is the equiv */
 
 /* setenv/unsetenv: not in the Windows CRT; map to _putenv_s / SetEnvironmentVariable. */
 static inline int setenv(const char* name, const char* value, int overwrite) {

lang/el-compiler/runtime/el_runtime.c

@@ -1963,8 +1963,9 @@ void http_serve_async(el_val_t port, el_val_t handler) {
     int sock = socket(AF_INET6, SOCK_STREAM, 0);
     if (sock < 0) { perror("socket"); return; }
     int yes = 1; int no = 0;
-    setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));
-    setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &no, sizeof(no));
+    /* Win32/mingw setsockopt takes optval as (const char*); the cast is portable on POSIX too. */
+    setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (const char*)&yes, sizeof(yes));
+    setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (const char*)&no, sizeof(no));
     struct sockaddr_in6 addr;
     memset(&addr, 0, sizeof(addr));
     addr.sin6_family = AF_INET6;
@@ -6826,6 +6827,75 @@ static int istr_contains(const char* hay, const char* needle) {
     return 0;
 }
 
+/* ── Tokenized query matching ───────────────────────────────────────────
+ * The engram query surface (search / activate / goal-bias) historically
+ * matched the ENTIRE raw query string as a single case-insensitive
+ * substring via istr_contains(field, q). That is Ctrl-F, not search:
+ * a multi-word query like "windows msi signing" only matched a node whose
+ * text contained that exact contiguous run, so real multi-word queries
+ * returned zero. istr_contains stays as the per-TOKEN primitive; these
+ * helpers split the query on whitespace and match ANY token, then rank by
+ * how many DISTINCT tokens a node covers. Single-token queries are a strict
+ * special case (score is 0 or 1) so single-word callers never regress. */
+#define ENGRAM_MAX_QTOKENS 32
+#define ENGRAM_QTOK_LEN    256
+
+/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct
+ * (case-insensitive) tokens. Returns the token count. Over-long tokens are
+ * truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */
+static int engram_tokenize_query(const char* q,
+                                 char toks[][ENGRAM_QTOK_LEN], int maxtok) {
+    int n = 0;
+    if (!q) return 0;
+    const char* p = q;
+    while (*p && n < maxtok) {
+        while (*p && isspace((unsigned char)*p)) p++;
+        if (!*p) break;
+        char buf[ENGRAM_QTOK_LEN];
+        size_t tl = 0;
+        while (*p && !isspace((unsigned char)*p)) {
+            if (tl < sizeof(buf) - 1) buf[tl++] = *p;
+            p++;
+        }
+        buf[tl] = '\0';
+        if (tl == 0) continue;
+        int dup = 0;
+        for (int s = 0; s < n; s++) {
+            if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; }
+        }
+        if (dup) continue;
+        memcpy(toks[n], buf, tl + 1);
+        n++;
+    }
+    return n;
+}
+
+/* Count how many of the ntok distinct query tokens appear (case-insensitive)
+ * in the node's content, label, or tags. 0 == no match. */
+static int engram_node_match_score(const EngramNode* n,
+                                   char toks[][ENGRAM_QTOK_LEN], int ntok) {
+    int score = 0;
+    for (int t = 0; t < ntok; t++) {
+        if (istr_contains(n->content, toks[t]) ||
+            istr_contains(n->label,   toks[t]) ||
+            istr_contains(n->tags,    toks[t]))
+            score++;
+    }
+    return score;
+}
+
+/* Rank entry: distinct-token match count (primary, desc) then salience
+ * (tiebreak, desc). */
+typedef struct { int64_t idx; int score; double salience; } EngramRankEntry;
+static int engram_rank_cmp(const void* a, const void* b) {
+    const EngramRankEntry* ea = (const EngramRankEntry*)a;
+    const EngramRankEntry* eb = (const EngramRankEntry*)b;
+    if (ea->score != eb->score) return eb->score - ea->score;      /* desc */
+    if (ea->salience < eb->salience) return 1;
+    if (ea->salience > eb->salience) return -1;
+    return 0;
+}
+
 /* ══════════════════════════════════════════════════════════════════════════
  * SEMANTIC SEARCH LAYER — nomic-embed-text via Ollama /api/embeddings
  * ──────────────────────────────────────────────────────────────────────────
@@ -7070,29 +7140,44 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
     if (lim <= 0) lim = 100;
     el_val_t lst = el_list_empty();
     if (!q || !*q) return lst;
-    /* Semantic augmentation: embed the query once; a node matches if it is a
-     * lexical hit OR its cosine similarity clears the threshold. qvec is NULL
-     * (and cosine 0) whenever semantic search is unavailable → pure lexical. */
+    char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
+    int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
+    if (ntok == 0) return lst;
+    /* Semantic augmentation: embed the query once; a node is a hit if it covers
+     * >=1 query token (tokenized-lexical, #66) OR its cosine clears the
+     * threshold (#67). qvec is NULL (cosine 0) when semantic is unavailable →
+     * pure tokenized-lexical, byte-identical to the lexical-only behaviour. */
     int qdim = 0;
     float* qvec = engram_embed_query(q, &qdim);
     double sem_min = engram_semantic_min();
-    int64_t found = 0;
-    for (int64_t i = 0; i < g->node_count && found < lim; i++) {
+    EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry));
+    if (!hits) { free(qvec); return lst; }
+    int64_t nhits = 0;
+    for (int64_t i = 0; i < g->node_count; i++) {
         EngramNode* n = &g->nodes[i];
         /* Filter transparent layers: nodes whose layer is `transparent=1`
          * shape output but are invisible to introspection ("what do you
          * know about yourself"). They still surface via engram_activate
          * + engram_compile_layered_json — that's the legitimate path. */
         if (engram_layer_is_transparent(n->layer_id)) continue;
-        int lex = istr_contains(n->content, q) ||
-                  istr_contains(n->label,   q) ||
-                  istr_contains(n->tags,    q);
+        int sc = engram_node_match_score(n, toks, ntok);
         double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0;
-        if (lex || sem >= sem_min) {
-            lst = el_list_append(lst, engram_node_to_map(n));
-            found++;
+        if (sc > 0 || sem >= sem_min) {
+            hits[nhits].idx      = i;
+            hits[nhits].score    = sc;
+            hits[nhits].salience = n->salience;
+            nhits++;
         }
     }
+    /* Rank by distinct tokens matched (desc) then salience (desc), then cap.
+     * Pure-semantic hits (token score 0) sort after every lexical hit — a
+     * lexical ∪ semantic union with lexical precedence. */
+    qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
+    int64_t end = nhits < lim ? nhits : lim;
+    for (int64_t k = 0; k < end; k++) {
+        lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx]));
+    }
+    free(hits);
     free(qvec);
     return lst;
 }
@@ -7370,10 +7455,14 @@ static double engram_temporal_proximity_bonus(int64_t node_created,
 static double engram_goal_bias(const EngramNode* n, const char* query) {
     if (!query || !*query) return 1.0;
     double bias = 1.0;
-    /* Direct lexical overlap: node content/label/tags share text with query. */
-    if (istr_contains(n->content, query) || istr_contains(n->label, query) ||
-        istr_contains(n->tags, query)) {
-        bias += 0.5;
+    /* Direct lexical overlap, graded by token coverage: a node covering all
+     * query tokens gets the full +0.5; partial coverage gets a proportional
+     * share. Single-token queries → full +0.5 on match, identical to before. */
+    {
+        char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
+        int ntok = engram_tokenize_query(query, toks, ENGRAM_MAX_QTOKENS);
+        int sc   = engram_node_match_score(n, toks, ntok);
+        if (sc > 0 && ntok > 0) bias += 0.5 * ((double)sc / (double)ntok);
     }
     /* Node-type resonance with query intent. */
     int technical_query = istr_contains(query, "code")         ||
@@ -7439,28 +7528,31 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
     if (!seeds) {
         free(best_bg); free(best_hops); free(reached); return out;
     }
-    /* Semantic seed augmentation: embed the query once; a node becomes a seed
-     * if it lexically matches OR its cosine clears the threshold. Semantic-only
-     * seeds enter at reduced strength (scaled by cosine) so pure paraphrase
-     * matches spread activation without overpowering exact lexical seeds.
-     * qvec is NULL (cosine 0) when semantic search is unavailable → the seed
-     * set is exactly the pre-existing lexical one. qvec is freed right after
-     * this loop so the many downstream early-returns need no cleanup change. */
+    /* Tokenized + semantic seeding: a node seeds if it covers >=1 query token
+     * (tokenized-lexical, #66) OR its cosine clears the threshold (#67). A
+     * lexical seed's activation is scaled by token coverage (fraction of
+     * distinct query tokens covered) so a node matching all words seeds more
+     * strongly than one matching a single word; single-word queries → coverage
+     * 1.0. A pure-semantic seed (no token match) is instead down-weighted by
+     * its cosine so paraphrase matches spread without overpowering exact seeds.
+     * q_vec is NULL (cosine 0) when semantic is unavailable → the seed set is
+     * exactly the tokenized-lexical one. q_vec is freed right after this loop
+     * so the many downstream early-returns need no cleanup change. */
+    char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
+    int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
     int q_dim = 0;
     float* q_vec = engram_embed_query(q, &q_dim);
     double q_sem_min = engram_semantic_min();
     for (int64_t i = 0; i < g->node_count; i++) {
         EngramNode* n = &g->nodes[i];
-        int lex = istr_contains(n->content, q) ||
-                  istr_contains(n->label,   q) ||
-                  istr_contains(n->tags,    q);
+        int sc = engram_node_match_score(n, toks, ntok);
         double sem = q_vec ? engram_node_cosine(n, q_vec, q_dim) : 0.0;
-        if (lex || sem >= q_sem_min) {
+        if (sc > 0 || sem >= q_sem_min) {
             double tdecay = engram_temporal_decay(n, now_ms);
             double dampen = engram_activation_dampen(n);
             double act    = n->salience * tdecay * dampen;
-            /* Down-weight pure-semantic seeds by their cosine strength. */
-            if (!lex) act *= sem;
+            if (sc > 0) act *= (ntok > 0 ? (double)sc / (double)ntok : 1.0);
+            else        act *= sem;  /* pure-semantic seed: down-weight by cosine */
             seeds[seed_count].idx        = i;
             seeds[seed_count].act        = act;
             seeds[seed_count].created_at = n->created_at;
@@ -8022,6 +8114,35 @@ el_val_t engram_get_node_json(el_val_t id) {
     return el_wrap_str(jb_finish(&b));
 }
 
+/* engram_get_node_by_label — find the first node whose label field exactly
+ * matches the given string. Returns the node as a JSON object string, or "{}"
+ * if no match is found.
+ *
+ * Used by chat.el to retrieve well-known nodes (e.g. "conv:history",
+ * "session:summary") by their stable label rather than by ID, which is immune
+ * to vector index drift across restarts.
+ *
+ * Exact match (strcmp, not istr_contains) because labels like "conv:history"
+ * must not collide with nodes whose content happens to contain that substring.
+ *
+ * Backported verbatim (idiom-adapted to jb_finish) from release runtime
+ * v1.0.0-20260501 to unblock the soul regen link: chat.el references this
+ * native but the current runtime lacked its definition. */
+el_val_t engram_get_node_by_label(el_val_t label) {
+    const char* lbl = EL_CSTR(label);
+    if (!lbl || !*lbl) return el_wrap_str(el_strdup("{}"));
+    EngramStore* g = engram_get();
+    for (int64_t i = 0; i < g->node_count; i++) {
+        EngramNode* n = &g->nodes[i];
+        if (n->label && strcmp(n->label, lbl) == 0) {
+            JsonBuf b; jb_init(&b);
+            engram_emit_node_json(&b, n);
+            return el_wrap_str(jb_finish(&b));
+        }
+    }
+    return el_wrap_str(el_strdup("{}"));
+}
+
 el_val_t engram_search_json(el_val_t query, el_val_t limit) {
     EngramStore* g = engram_get();
     const char* q = EL_CSTR(query);
@@ -8030,16 +8151,16 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
     JsonBuf b; jb_init(&b);
     jb_putc(&b, '[');
     if (q && *q && g->node_count > 0) {
-        /* Collect candidates from the UNION of lexical and semantic matches,
-         * score each, rank by score, then emit the top `lim`. A node is a
-         * candidate if it lexically matches OR its query cosine clears the
-         * threshold. Lexical hits get a base of 1.0 (so they always outrank a
-         * pure-semantic hit, whose cosine is in [0,1)), refined by cosine;
-         * pure-semantic hits are scored by cosine alone.
-         *
-         * When semantic search is unavailable, qvec is NULL, sem is 0, only
-         * lexical hits are collected (score 1.0), and the stable insertion
-         * sort preserves node order — identical to the pre-existing search. */
+        /* Collect candidates from the UNION of tokenized-lexical and semantic
+         * matches, score each, rank by score, emit the top `lim`. A node is a
+         * candidate if it covers >=1 query token (tokenized-lexical, #66) OR its
+         * query cosine clears the threshold (#67). Lexical score is the distinct
+         * token count (>=1), so any lexical hit outranks a pure-semantic hit
+         * (cosine < 1); pure-semantic hits are scored by cosine alone. When
+         * semantic is unavailable qvec is NULL, sem is 0, only tokenized-lexical
+         * hits are collected, and the stable insertion sort preserves order. */
+        char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
+        int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
         int qdim = 0;
         float* qvec = engram_embed_query(q, &qdim);
         double sem_min = engram_semantic_min();
@@ -8050,13 +8171,11 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
             for (int64_t i = 0; i < g->node_count; i++) {
                 EngramNode* n = &g->nodes[i];
                 if (engram_layer_is_transparent(n->layer_id)) continue;
-                int lex = istr_contains(n->content, q) ||
-                          istr_contains(n->label,   q) ||
-                          istr_contains(n->tags,    q);
+                int sc = engram_node_match_score(n, toks, ntok);
                 double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0;
-                if (lex || sem >= sem_min) {
+                if (sc > 0 || sem >= sem_min) {
                     cand[nc].idx   = i;
-                    cand[nc].score = (lex ? 1.0 : 0.0) + sem;
+                    cand[nc].score = (double)sc + sem;
                     nc++;
                 }
             }

lang/el-compiler/runtime/el_runtime.h

@@ -632,6 +632,7 @@ el_val_t  engram_load(el_val_t path);
  * can pass results straight through without round-tripping ElList/ElMap
  * through json_stringify. */
 el_val_t  engram_get_node_json(el_val_t id);
+el_val_t  engram_get_node_by_label(el_val_t label);
 el_val_t  engram_search_json(el_val_t query, el_val_t limit);
 el_val_t  engram_scan_nodes_json(el_val_t limit, el_val_t offset);
 el_val_t  engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);

lang/el-compiler/runtime/el_seed.c

@@ -1072,6 +1072,7 @@ el_val_t __engram_save(el_val_t path)                       { return engram_save
 el_val_t __engram_load(el_val_t path)                       { return engram_load(path); }
 
 el_val_t __engram_get_node_json(el_val_t id)               { return engram_get_node_json(id); }
+el_val_t __engram_get_node_by_label(el_val_t label)        { return engram_get_node_by_label(label); }
 
 el_val_t __engram_search_json(el_val_t query, el_val_t limit) {
     return engram_search_json(query, limit);

lang/el-compiler/runtime/el_seed.h

@@ -226,6 +226,7 @@ el_val_t  __engram_activate(el_val_t query, el_val_t depth);
 el_val_t  __engram_save(el_val_t path);
 el_val_t  __engram_load(el_val_t path);
 el_val_t  __engram_get_node_json(el_val_t id);
+el_val_t  __engram_get_node_by_label(el_val_t label);
 el_val_t  __engram_search_json(el_val_t query, el_val_t limit);
 el_val_t  __engram_scan_nodes_json(el_val_t limit, el_val_t offset);
 el_val_t  __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);

lang/el-compiler/src/codegen.el

@@ -2670,6 +2670,7 @@ fn builtin_arity(name: String) -> Int {
     if str_eq(name, "engram_save") { return 1 }
     if str_eq(name, "engram_load") { return 1 }
     if str_eq(name, "engram_get_node_json") { return 1 }
+    if str_eq(name, "engram_get_node_by_label") { return 1 }
     if str_eq(name, "engram_search_json") { return 2 }
     if str_eq(name, "engram_scan_nodes_json") { return 2 }
     if str_eq(name, "engram_neighbors_json") { return 3 }