Merge remote-tracking branch 'origin/feat/engram-semantic-search' into hotfix/stage-elc-engram-integration

# Conflicts:
#	lang/el-compiler/runtime/el_runtime.c

engram/.gitea/workflows/ci-dev.yaml

@@ -81,7 +81,7 @@ jobs:
       # Link to produce the engram binary
       - name: Link engram binary
         run: |
-          cc -std=c11 -O2 \
+          cc -std=c11 -O2 -DHAVE_CURL \
             -I /usr/local/lib/el \
             -o dist/engram \
             dist/engram.c \

engram/.gitea/workflows/ci-stage.yaml

@@ -88,7 +88,7 @@ jobs:
       # Link to produce the engram binary
       - name: Link engram binary
         run: |
-          cc -std=c11 -O2 \
+          cc -std=c11 -O2 -DHAVE_CURL \
             -I /usr/local/lib/el \
             -o dist/engram \
             dist/engram.c \

engram/.gitea/workflows/engram-release.yaml

@@ -62,7 +62,7 @@ jobs:
       # Link to produce the engram binary
       - name: Link engram binary
         run: |
-          cc -std=c11 -O2 \
+          cc -std=c11 -O2 -DHAVE_CURL \
             -I /usr/local/lib/el \
             -o dist/engram \
             dist/engram.c \

lang/el-compiler/runtime/el_runtime.c

@@ -6896,6 +6896,243 @@ static int engram_rank_cmp(const void* a, const void* b) {
     return 0;
 }
 
+/* ══════════════════════════════════════════════════════════════════════════
+ * SEMANTIC SEARCH LAYER — nomic-embed-text via Ollama /api/embeddings
+ * ──────────────────────────────────────────────────────────────────────────
+ * Augments the lexical (istr_contains) matcher with dense-vector retrieval.
+ * Node content and the query are embedded through a local Ollama server;
+ * nodes are ranked by cosine similarity and UNIONED with lexical hits. This
+ * lets a paraphrase query surface a node whose words never appear in it.
+ *
+ * DEGRADABLE BY DESIGN. The whole layer is gated on HAVE_CURL plus a one-shot
+ * runtime probe of the embedding endpoint. If curl is not compiled in, or
+ * Ollama is unreachable, or ENGRAM_SEMANTIC=0, every entry point returns
+ * "no semantic signal" and callers fall back to pure lexical behaviour —
+ * byte-for-byte the pre-existing search.
+ *
+ * CACHE. Node embeddings are computed lazily on first use and cached in
+ * process memory keyed by node id, with an FNV-1a content hash for
+ * invalidation (edited content re-embeds). The query is embedded once per
+ * search call. This is what "avoid re-embedding the whole graph every query"
+ * buys us: a warm cache serves cosine from RAM. (A cold process still pays
+ * O(N) embed calls the first time each node is scanned — persisting the cache
+ * to a snapshot sidecar is the documented next step, not done here.)
+ *
+ * nomic task prefixes ("search_query:" / "search_document:") are applied
+ * because nomic-embed-text is trained with them; they materially improve
+ * retrieval separation (empirically: paraphrase 0.72 vs distractors <0.48).
+ *
+ * ENV:
+ *   ENGRAM_SEMANTIC       "0" disables; unset/other = auto-probe
+ *   ENGRAM_EMBED_URL      default http://localhost:11434/api/embeddings
+ *   ENGRAM_EMBED_MODEL    default nomic-embed-text
+ *   ENGRAM_SEMANTIC_MIN   cosine threshold for a pure-semantic match (def 0.6)
+ * ════════════════════════════════════════════════════════════════════════ */
+
+static double engram_semantic_min(void) {
+    static double v = -1.0;
+    if (v >= 0.0) return v;
+    const char* s = getenv("ENGRAM_SEMANTIC_MIN");
+    double d = 0.6;
+    if (s && *s) { char* e = NULL; double t = strtod(s, &e);
+                   if (e != s && t >= 0.0 && t <= 1.0) d = t; }
+    v = d; return v;
+}
+
+#ifdef HAVE_CURL
+
+typedef struct { char* id; uint64_t hash; float* vec; int dim; } EngramEmbEntry;
+static EngramEmbEntry* g_emb_items = NULL;
+static int64_t g_emb_count = 0, g_emb_cap = 0;
+static int g_emb_state = 0;   /* 0=unprobed, 1=available, -1=disabled */
+
+static uint64_t engram_fnv1a(const char* s) {
+    uint64_t h = 1469598103934665603ULL;
+    if (s) for (const unsigned char* p = (const unsigned char*)s; *p; p++) {
+        h ^= *p; h *= 1099511628211ULL;
+    }
+    return h;
+}
+
+/* Parse "embedding":[f,f,...] from an Ollama response. malloc'd vec, or NULL. */
+static float* engram_parse_embedding(const char* json, int* out_dim) {
+    if (!json) return NULL;
+    const char* p = strstr(json, "\"embedding\"");
+    if (!p) return NULL;
+    p = strchr(p, '[');
+    if (!p) return NULL;
+    p++;
+    int cap = 1024, n = 0;
+    float* v = malloc((size_t)cap * sizeof(float));
+    if (!v) return NULL;
+    while (*p && *p != ']') {
+        while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r' || *p == ',') p++;
+        if (*p == ']' || !*p) break;
+        char* e = NULL;
+        double d = strtod(p, &e);
+        if (e == p) break;
+        if (n >= cap) { cap *= 2; float* nv = realloc(v, (size_t)cap * sizeof(float));
+                        if (!nv) { free(v); return NULL; } v = nv; }
+        v[n++] = (float)d;
+        p = e;
+    }
+    if (n == 0) { free(v); return NULL; }
+    *out_dim = n;
+    return v;
+}
+
+/* JSON-escape src into a malloc'd buffer (no surrounding quotes). */
+static char* engram_json_escape(const char* src) {
+    if (!src) src = "";
+    size_t n = strlen(src);
+    char* out = malloc(n * 2 + 1);
+    if (!out) return NULL;
+    size_t j = 0;
+    for (size_t i = 0; i < n; i++) {
+        unsigned char c = (unsigned char)src[i];
+        if      (c == '"')  { out[j++] = '\\'; out[j++] = '"'; }
+        else if (c == '\\') { out[j++] = '\\'; out[j++] = '\\'; }
+        else if (c == '\n') { out[j++] = '\\'; out[j++] = 'n'; }
+        else if (c == '\r') { out[j++] = '\\'; out[j++] = 'r'; }
+        else if (c == '\t') { out[j++] = '\\'; out[j++] = 't'; }
+        else if (c < 0x20)  { /* drop other control bytes */ }
+        else                { out[j++] = (char)c; }
+    }
+    out[j] = '\0';
+    return out;
+}
+
+/* Embed `prefix+text` via Ollama. Returns malloc'd vec (caller frees), or NULL. */
+static float* engram_embed_raw(const char* prefix, const char* text, int* out_dim) {
+    if (!text) return NULL;
+    const char* url = getenv("ENGRAM_EMBED_URL");
+    if (!url || !*url) url = "http://localhost:11434/api/embeddings";
+    const char* model = getenv("ENGRAM_EMBED_MODEL");
+    if (!model || !*model) model = "nomic-embed-text";
+    /* Bound content length to keep latency/memory sane on huge nodes. */
+    char* trunc = NULL;
+    size_t maxlen = 8192;
+    if (strlen(text) > maxlen) {
+        trunc = malloc(maxlen + 1);
+        if (trunc) { memcpy(trunc, text, maxlen); trunc[maxlen] = '\0'; text = trunc; }
+    }
+    char* esc_prefix = engram_json_escape(prefix ? prefix : "");
+    char* esc = engram_json_escape(text);
+    free(trunc);
+    if (!esc || !esc_prefix) { free(esc); free(esc_prefix); return NULL; }
+    size_t blen = strlen(esc) + strlen(esc_prefix) + strlen(model) + 64;
+    char* body = malloc(blen);
+    if (!body) { free(esc); free(esc_prefix); return NULL; }
+    snprintf(body, blen, "{\"model\":\"%s\",\"prompt\":\"%s%s\"}", model, esc_prefix, esc);
+    free(esc); free(esc_prefix);
+
+    CURL* c = curl_easy_init();
+    if (!c) { free(body); return NULL; }
+    HttpBuf rb; httpbuf_init(&rb);
+    struct curl_slist* h = curl_slist_append(NULL, "Content-Type: application/json");
+    char errbuf[CURL_ERROR_SIZE]; errbuf[0] = '\0';
+    curl_easy_setopt(c, CURLOPT_URL, url);
+    curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_write_cb);
+    curl_easy_setopt(c, CURLOPT_WRITEDATA, &rb);
+    curl_easy_setopt(c, CURLOPT_POST, 1L);
+    curl_easy_setopt(c, CURLOPT_POSTFIELDS, body);
+    curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)strlen(body));
+    curl_easy_setopt(c, CURLOPT_HTTPHEADER, h);
+    curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, el_http_timeout_ms());
+    curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L);
+    curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf);
+    CURLcode rc = curl_easy_perform(c);
+    curl_slist_free_all(h);
+    curl_easy_cleanup(c);
+    free(body);
+    if (rc != CURLE_OK) { free(rb.data); return NULL; }
+    float* v = engram_parse_embedding(rb.data, out_dim);
+    free(rb.data);
+    return v;
+}
+
+/* One-shot probe: is semantic search available? Caches the verdict. */
+static int engram_semantic_enabled(void) {
+    if (g_emb_state != 0) return g_emb_state == 1;
+    const char* s = getenv("ENGRAM_SEMANTIC");
+    if (s && strcmp(s, "0") == 0) { g_emb_state = -1; return 0; }
+    int dim = 0;
+    float* v = engram_embed_raw("search_query: ", "probe", &dim);
+    if (v && dim > 0) { free(v); g_emb_state = 1; return 1; }
+    free(v);
+    g_emb_state = -1; return 0;
+}
+
+/* Embed the query. Returns malloc'd vec (caller frees), or NULL if semantic off. */
+static float* engram_embed_query(const char* q, int* dim) {
+    if (!engram_semantic_enabled()) return NULL;
+    if (!q || !*q) return NULL;
+    return engram_embed_raw("search_query: ", q, dim);
+}
+
+/* Cached node embedding. Returns a pointer OWNED BY THE CACHE — do not free. */
+static const float* engram_node_vec(EngramNode* n, int* out_dim) {
+    if (!n || !n->id) return NULL;
+    uint64_t h = engram_fnv1a(n->content);
+    for (int64_t i = 0; i < g_emb_count; i++) {
+        if (g_emb_items[i].id && strcmp(g_emb_items[i].id, n->id) == 0) {
+            if (g_emb_items[i].hash == h && g_emb_items[i].vec) {
+                *out_dim = g_emb_items[i].dim; return g_emb_items[i].vec;
+            }
+            /* content changed → re-embed in place */
+            int dim = 0;
+            float* v = engram_embed_raw("search_document: ", n->content ? n->content : "", &dim);
+            if (!v) return NULL;
+            free(g_emb_items[i].vec);
+            g_emb_items[i].vec = v; g_emb_items[i].dim = dim; g_emb_items[i].hash = h;
+            *out_dim = dim; return v;
+        }
+    }
+    int dim = 0;
+    float* v = engram_embed_raw("search_document: ", n->content ? n->content : "", &dim);
+    if (!v) return NULL;
+    if (g_emb_count >= g_emb_cap) {
+        int64_t nc = g_emb_cap ? g_emb_cap * 2 : 256;
+        EngramEmbEntry* ni = realloc(g_emb_items, (size_t)nc * sizeof(EngramEmbEntry));
+        if (!ni) { free(v); return NULL; }
+        g_emb_items = ni; g_emb_cap = nc;
+    }
+    g_emb_items[g_emb_count].id   = strdup(n->id);
+    g_emb_items[g_emb_count].hash = h;
+    g_emb_items[g_emb_count].vec  = v;
+    g_emb_items[g_emb_count].dim  = dim;
+    g_emb_count++;
+    *out_dim = dim; return v;
+}
+
+static double engram_cosine(const float* a, const float* b, int dim) {
+    double dot = 0, na = 0, nb = 0;
+    for (int i = 0; i < dim; i++) { dot += (double)a[i] * b[i];
+                                    na  += (double)a[i] * a[i];
+                                    nb  += (double)b[i] * b[i]; }
+    if (na <= 0 || nb <= 0) return 0.0;
+    return dot / (sqrt(na) * sqrt(nb));
+}
+
+/* Cosine of node n against the query vector; 0 if unavailable / dim mismatch. */
+static double engram_node_cosine(EngramNode* n, const float* qvec, int qdim) {
+    if (!qvec || qdim <= 0) return 0.0;
+    int ndim = 0;
+    const float* nv = engram_node_vec(n, &ndim);
+    if (!nv || ndim != qdim) return 0.0;
+    return engram_cosine(qvec, nv, qdim);
+}
+
+#else  /* !HAVE_CURL — semantic layer compiled out; callers stay pure-lexical.
+        * Only the two boundary functions the always-compiled search/activate
+        * code calls are stubbed; the query embed always yields NULL so every
+        * cosine is 0 and every caller collapses to lexical-only. */
+static float* engram_embed_query(const char* q, int* dim) { (void)q; (void)dim; return NULL; }
+static double engram_node_cosine(EngramNode* n, const float* qvec, int qdim) {
+    (void)n; (void)qvec; (void)qdim; return 0.0;
+}
+#endif /* HAVE_CURL */
+
 el_val_t engram_search(el_val_t query, el_val_t limit) {
     EngramStore* g = engram_get();
     const char* q = EL_CSTR(query);
@@ -6906,8 +7143,15 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
     char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
     int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
     if (ntok == 0) return lst;
+    /* Semantic augmentation: embed the query once; a node is a hit if it covers
+     * >=1 query token (tokenized-lexical, #66) OR its cosine clears the
+     * threshold (#67). qvec is NULL (cosine 0) when semantic is unavailable →
+     * pure tokenized-lexical, byte-identical to the lexical-only behaviour. */
+    int qdim = 0;
+    float* qvec = engram_embed_query(q, &qdim);
+    double sem_min = engram_semantic_min();
     EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry));
-    if (!hits) return lst;
+    if (!hits) { free(qvec); return lst; }
     int64_t nhits = 0;
     for (int64_t i = 0; i < g->node_count; i++) {
         EngramNode* n = &g->nodes[i];
@@ -6917,20 +7161,24 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
          * + engram_compile_layered_json — that's the legitimate path. */
         if (engram_layer_is_transparent(n->layer_id)) continue;
         int sc = engram_node_match_score(n, toks, ntok);
-        if (sc > 0) {
+        double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0;
+        if (sc > 0 || sem >= sem_min) {
             hits[nhits].idx      = i;
             hits[nhits].score    = sc;
             hits[nhits].salience = n->salience;
             nhits++;
         }
     }
-    /* Rank by distinct tokens matched (desc) then salience (desc), then cap. */
+    /* Rank by distinct tokens matched (desc) then salience (desc), then cap.
+     * Pure-semantic hits (token score 0) sort after every lexical hit — a
+     * lexical ∪ semantic union with lexical precedence. */
     qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
     int64_t end = nhits < lim ? nhits : lim;
     for (int64_t k = 0; k < end; k++) {
         lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx]));
     }
     free(hits);
+    free(qvec);
     return lst;
 }
 
@@ -7280,21 +7528,31 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
     if (!seeds) {
         free(best_bg); free(best_hops); free(reached); return out;
     }
-    /* Tokenize once: a node seeds if it matches ANY query token, and its seed
-     * activation is scaled by token coverage (fraction of distinct query
-     * tokens it contains) so a node matching all words seeds more strongly
-     * than one matching a single word. Single-word queries → coverage 1.0,
-     * identical to the prior whole-query behavior. */
+    /* Tokenized + semantic seeding: a node seeds if it covers >=1 query token
+     * (tokenized-lexical, #66) OR its cosine clears the threshold (#67). A
+     * lexical seed's activation is scaled by token coverage (fraction of
+     * distinct query tokens covered) so a node matching all words seeds more
+     * strongly than one matching a single word; single-word queries → coverage
+     * 1.0. A pure-semantic seed (no token match) is instead down-weighted by
+     * its cosine so paraphrase matches spread without overpowering exact seeds.
+     * q_vec is NULL (cosine 0) when semantic is unavailable → the seed set is
+     * exactly the tokenized-lexical one. q_vec is freed right after this loop
+     * so the many downstream early-returns need no cleanup change. */
     char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
     int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
+    int q_dim = 0;
+    float* q_vec = engram_embed_query(q, &q_dim);
+    double q_sem_min = engram_semantic_min();
     for (int64_t i = 0; i < g->node_count; i++) {
         EngramNode* n = &g->nodes[i];
         int sc = engram_node_match_score(n, toks, ntok);
-        if (sc > 0) {
+        double sem = q_vec ? engram_node_cosine(n, q_vec, q_dim) : 0.0;
+        if (sc > 0 || sem >= q_sem_min) {
             double tdecay = engram_temporal_decay(n, now_ms);
             double dampen = engram_activation_dampen(n);
-            double cover  = ntok > 0 ? (double)sc / (double)ntok : 1.0;
-            double act    = n->salience * tdecay * dampen * cover;
+            double act    = n->salience * tdecay * dampen;
+            if (sc > 0) act *= (ntok > 0 ? (double)sc / (double)ntok : 1.0);
+            else        act *= sem;  /* pure-semantic seed: down-weight by cosine */
             seeds[seed_count].idx        = i;
             seeds[seed_count].act        = act;
             seeds[seed_count].created_at = n->created_at;
@@ -7304,6 +7562,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
             reached[i]   = 1;
         }
     }
+    free(q_vec);
     /* Compute mean seed created_at for temporal proximity bonus. */
     int64_t seed_epoch = 0;
     if (seed_count > 0) {
@@ -7891,39 +8150,50 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
     if (lim <= 0) lim = 100;
     JsonBuf b; jb_init(&b);
     jb_putc(&b, '[');
-    int first = 1;
-    if (q && *q) {
+    if (q && *q && g->node_count > 0) {
+        /* Collect candidates from the UNION of tokenized-lexical and semantic
+         * matches, score each, rank by score, emit the top `lim`. A node is a
+         * candidate if it covers >=1 query token (tokenized-lexical, #66) OR its
+         * query cosine clears the threshold (#67). Lexical score is the distinct
+         * token count (>=1), so any lexical hit outranks a pure-semantic hit
+         * (cosine < 1); pure-semantic hits are scored by cosine alone. When
+         * semantic is unavailable qvec is NULL, sem is 0, only tokenized-lexical
+         * hits are collected, and the stable insertion sort preserves order. */
         char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
         int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
-        if (ntok > 0) {
-            EngramRankEntry* hits =
-                malloc((size_t)g->node_count * sizeof(EngramRankEntry));
-            if (hits) {
-                int64_t nhits = 0;
-                for (int64_t i = 0; i < g->node_count; i++) {
-                    EngramNode* n = &g->nodes[i];
-                    /* Filter transparent layers — same as engram_search. */
-                    if (engram_layer_is_transparent(n->layer_id)) continue;
-                    int sc = engram_node_match_score(n, toks, ntok);
-                    if (sc > 0) {
-                        hits[nhits].idx      = i;
-                        hits[nhits].score    = sc;
-                        hits[nhits].salience = n->salience;
-                        nhits++;
-                    }
+        int qdim = 0;
+        float* qvec = engram_embed_query(q, &qdim);
+        double sem_min = engram_semantic_min();
+        typedef struct { int64_t idx; double score; } Cand;
+        Cand* cand = malloc((size_t)g->node_count * sizeof(Cand));
+        if (cand) {
+            int64_t nc = 0;
+            for (int64_t i = 0; i < g->node_count; i++) {
+                EngramNode* n = &g->nodes[i];
+                if (engram_layer_is_transparent(n->layer_id)) continue;
+                int sc = engram_node_match_score(n, toks, ntok);
+                double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0;
+                if (sc > 0 || sem >= sem_min) {
+                    cand[nc].idx   = i;
+                    cand[nc].score = (double)sc + sem;
+                    nc++;
                 }
-                /* Rank by distinct tokens matched (desc) then salience (desc). */
-                qsort(hits, (size_t)nhits, sizeof(EngramRankEntry),
-                      engram_rank_cmp);
-                int64_t end = nhits < lim ? nhits : lim;
-                for (int64_t k = 0; k < end; k++) {
-                    if (!first) jb_putc(&b, ',');
-                    engram_emit_node_json(&b, &g->nodes[hits[k].idx]);
-                    first = 0;
-                }
-                free(hits);
             }
+            /* Insertion sort by score desc; stable for equal scores. */
+            for (int64_t i = 1; i < nc; i++) {
+                Cand k = cand[i]; int64_t j = i - 1;
+                while (j >= 0 && cand[j].score < k.score) { cand[j + 1] = cand[j]; j--; }
+                cand[j + 1] = k;
+            }
+            int first = 1;
+            for (int64_t i = 0; i < nc && i < lim; i++) {
+                if (!first) jb_putc(&b, ',');
+                engram_emit_node_json(&b, &g->nodes[cand[i].idx]);
+                first = 0;
+            }
+            free(cand);
         }
+        free(qvec);
     }
     jb_putc(&b, ']');
     return el_wrap_str(jb_finish(&b));