diff --git a/engram/.gitea/workflows/ci-dev.yaml b/engram/.gitea/workflows/ci-dev.yaml index 7463aa8..a432869 100644 --- a/engram/.gitea/workflows/ci-dev.yaml +++ b/engram/.gitea/workflows/ci-dev.yaml @@ -81,7 +81,7 @@ jobs: # Link to produce the engram binary - name: Link engram binary run: | - cc -std=c11 -O2 \ + cc -std=c11 -O2 -DHAVE_CURL \ -I /usr/local/lib/el \ -o dist/engram \ dist/engram.c \ diff --git a/engram/.gitea/workflows/ci-stage.yaml b/engram/.gitea/workflows/ci-stage.yaml index 69ed162..a134117 100644 --- a/engram/.gitea/workflows/ci-stage.yaml +++ b/engram/.gitea/workflows/ci-stage.yaml @@ -88,7 +88,7 @@ jobs: # Link to produce the engram binary - name: Link engram binary run: | - cc -std=c11 -O2 \ + cc -std=c11 -O2 -DHAVE_CURL \ -I /usr/local/lib/el \ -o dist/engram \ dist/engram.c \ diff --git a/engram/.gitea/workflows/engram-release.yaml b/engram/.gitea/workflows/engram-release.yaml index bdf3258..aa02e9f 100644 --- a/engram/.gitea/workflows/engram-release.yaml +++ b/engram/.gitea/workflows/engram-release.yaml @@ -62,7 +62,7 @@ jobs: # Link to produce the engram binary - name: Link engram binary run: | - cc -std=c11 -O2 \ + cc -std=c11 -O2 -DHAVE_CURL \ -I /usr/local/lib/el \ -o dist/engram \ dist/engram.c \ diff --git a/lang/el-compiler/runtime/el_runtime.c b/lang/el-compiler/runtime/el_runtime.c index 68e6dca..4e41611 100644 --- a/lang/el-compiler/runtime/el_runtime.c +++ b/lang/el-compiler/runtime/el_runtime.c @@ -6826,6 +6826,243 @@ static int istr_contains(const char* hay, const char* needle) { return 0; } +/* ══════════════════════════════════════════════════════════════════════════ + * SEMANTIC SEARCH LAYER — nomic-embed-text via Ollama /api/embeddings + * ────────────────────────────────────────────────────────────────────────── + * Augments the lexical (istr_contains) matcher with dense-vector retrieval. + * Node content and the query are embedded through a local Ollama server; + * nodes are ranked by cosine similarity and UNIONED with lexical hits. This + * lets a paraphrase query surface a node whose words never appear in it. + * + * DEGRADABLE BY DESIGN. The whole layer is gated on HAVE_CURL plus a one-shot + * runtime probe of the embedding endpoint. If curl is not compiled in, or + * Ollama is unreachable, or ENGRAM_SEMANTIC=0, every entry point returns + * "no semantic signal" and callers fall back to pure lexical behaviour — + * byte-for-byte the pre-existing search. + * + * CACHE. Node embeddings are computed lazily on first use and cached in + * process memory keyed by node id, with an FNV-1a content hash for + * invalidation (edited content re-embeds). The query is embedded once per + * search call. This is what "avoid re-embedding the whole graph every query" + * buys us: a warm cache serves cosine from RAM. (A cold process still pays + * O(N) embed calls the first time each node is scanned — persisting the cache + * to a snapshot sidecar is the documented next step, not done here.) + * + * nomic task prefixes ("search_query:" / "search_document:") are applied + * because nomic-embed-text is trained with them; they materially improve + * retrieval separation (empirically: paraphrase 0.72 vs distractors <0.48). + * + * ENV: + * ENGRAM_SEMANTIC "0" disables; unset/other = auto-probe + * ENGRAM_EMBED_URL default http://localhost:11434/api/embeddings + * ENGRAM_EMBED_MODEL default nomic-embed-text + * ENGRAM_SEMANTIC_MIN cosine threshold for a pure-semantic match (def 0.6) + * ════════════════════════════════════════════════════════════════════════ */ + +static double engram_semantic_min(void) { + static double v = -1.0; + if (v >= 0.0) return v; + const char* s = getenv("ENGRAM_SEMANTIC_MIN"); + double d = 0.6; + if (s && *s) { char* e = NULL; double t = strtod(s, &e); + if (e != s && t >= 0.0 && t <= 1.0) d = t; } + v = d; return v; +} + +#ifdef HAVE_CURL + +typedef struct { char* id; uint64_t hash; float* vec; int dim; } EngramEmbEntry; +static EngramEmbEntry* g_emb_items = NULL; +static int64_t g_emb_count = 0, g_emb_cap = 0; +static int g_emb_state = 0; /* 0=unprobed, 1=available, -1=disabled */ + +static uint64_t engram_fnv1a(const char* s) { + uint64_t h = 1469598103934665603ULL; + if (s) for (const unsigned char* p = (const unsigned char*)s; *p; p++) { + h ^= *p; h *= 1099511628211ULL; + } + return h; +} + +/* Parse "embedding":[f,f,...] from an Ollama response. malloc'd vec, or NULL. */ +static float* engram_parse_embedding(const char* json, int* out_dim) { + if (!json) return NULL; + const char* p = strstr(json, "\"embedding\""); + if (!p) return NULL; + p = strchr(p, '['); + if (!p) return NULL; + p++; + int cap = 1024, n = 0; + float* v = malloc((size_t)cap * sizeof(float)); + if (!v) return NULL; + while (*p && *p != ']') { + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r' || *p == ',') p++; + if (*p == ']' || !*p) break; + char* e = NULL; + double d = strtod(p, &e); + if (e == p) break; + if (n >= cap) { cap *= 2; float* nv = realloc(v, (size_t)cap * sizeof(float)); + if (!nv) { free(v); return NULL; } v = nv; } + v[n++] = (float)d; + p = e; + } + if (n == 0) { free(v); return NULL; } + *out_dim = n; + return v; +} + +/* JSON-escape src into a malloc'd buffer (no surrounding quotes). */ +static char* engram_json_escape(const char* src) { + if (!src) src = ""; + size_t n = strlen(src); + char* out = malloc(n * 2 + 1); + if (!out) return NULL; + size_t j = 0; + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)src[i]; + if (c == '"') { out[j++] = '\\'; out[j++] = '"'; } + else if (c == '\\') { out[j++] = '\\'; out[j++] = '\\'; } + else if (c == '\n') { out[j++] = '\\'; out[j++] = 'n'; } + else if (c == '\r') { out[j++] = '\\'; out[j++] = 'r'; } + else if (c == '\t') { out[j++] = '\\'; out[j++] = 't'; } + else if (c < 0x20) { /* drop other control bytes */ } + else { out[j++] = (char)c; } + } + out[j] = '\0'; + return out; +} + +/* Embed `prefix+text` via Ollama. Returns malloc'd vec (caller frees), or NULL. */ +static float* engram_embed_raw(const char* prefix, const char* text, int* out_dim) { + if (!text) return NULL; + const char* url = getenv("ENGRAM_EMBED_URL"); + if (!url || !*url) url = "http://localhost:11434/api/embeddings"; + const char* model = getenv("ENGRAM_EMBED_MODEL"); + if (!model || !*model) model = "nomic-embed-text"; + /* Bound content length to keep latency/memory sane on huge nodes. */ + char* trunc = NULL; + size_t maxlen = 8192; + if (strlen(text) > maxlen) { + trunc = malloc(maxlen + 1); + if (trunc) { memcpy(trunc, text, maxlen); trunc[maxlen] = '\0'; text = trunc; } + } + char* esc_prefix = engram_json_escape(prefix ? prefix : ""); + char* esc = engram_json_escape(text); + free(trunc); + if (!esc || !esc_prefix) { free(esc); free(esc_prefix); return NULL; } + size_t blen = strlen(esc) + strlen(esc_prefix) + strlen(model) + 64; + char* body = malloc(blen); + if (!body) { free(esc); free(esc_prefix); return NULL; } + snprintf(body, blen, "{\"model\":\"%s\",\"prompt\":\"%s%s\"}", model, esc_prefix, esc); + free(esc); free(esc_prefix); + + CURL* c = curl_easy_init(); + if (!c) { free(body); return NULL; } + HttpBuf rb; httpbuf_init(&rb); + struct curl_slist* h = curl_slist_append(NULL, "Content-Type: application/json"); + char errbuf[CURL_ERROR_SIZE]; errbuf[0] = '\0'; + curl_easy_setopt(c, CURLOPT_URL, url); + curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_write_cb); + curl_easy_setopt(c, CURLOPT_WRITEDATA, &rb); + curl_easy_setopt(c, CURLOPT_POST, 1L); + curl_easy_setopt(c, CURLOPT_POSTFIELDS, body); + curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)strlen(body)); + curl_easy_setopt(c, CURLOPT_HTTPHEADER, h); + curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, el_http_timeout_ms()); + curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf); + CURLcode rc = curl_easy_perform(c); + curl_slist_free_all(h); + curl_easy_cleanup(c); + free(body); + if (rc != CURLE_OK) { free(rb.data); return NULL; } + float* v = engram_parse_embedding(rb.data, out_dim); + free(rb.data); + return v; +} + +/* One-shot probe: is semantic search available? Caches the verdict. */ +static int engram_semantic_enabled(void) { + if (g_emb_state != 0) return g_emb_state == 1; + const char* s = getenv("ENGRAM_SEMANTIC"); + if (s && strcmp(s, "0") == 0) { g_emb_state = -1; return 0; } + int dim = 0; + float* v = engram_embed_raw("search_query: ", "probe", &dim); + if (v && dim > 0) { free(v); g_emb_state = 1; return 1; } + free(v); + g_emb_state = -1; return 0; +} + +/* Embed the query. Returns malloc'd vec (caller frees), or NULL if semantic off. */ +static float* engram_embed_query(const char* q, int* dim) { + if (!engram_semantic_enabled()) return NULL; + if (!q || !*q) return NULL; + return engram_embed_raw("search_query: ", q, dim); +} + +/* Cached node embedding. Returns a pointer OWNED BY THE CACHE — do not free. */ +static const float* engram_node_vec(EngramNode* n, int* out_dim) { + if (!n || !n->id) return NULL; + uint64_t h = engram_fnv1a(n->content); + for (int64_t i = 0; i < g_emb_count; i++) { + if (g_emb_items[i].id && strcmp(g_emb_items[i].id, n->id) == 0) { + if (g_emb_items[i].hash == h && g_emb_items[i].vec) { + *out_dim = g_emb_items[i].dim; return g_emb_items[i].vec; + } + /* content changed → re-embed in place */ + int dim = 0; + float* v = engram_embed_raw("search_document: ", n->content ? n->content : "", &dim); + if (!v) return NULL; + free(g_emb_items[i].vec); + g_emb_items[i].vec = v; g_emb_items[i].dim = dim; g_emb_items[i].hash = h; + *out_dim = dim; return v; + } + } + int dim = 0; + float* v = engram_embed_raw("search_document: ", n->content ? n->content : "", &dim); + if (!v) return NULL; + if (g_emb_count >= g_emb_cap) { + int64_t nc = g_emb_cap ? g_emb_cap * 2 : 256; + EngramEmbEntry* ni = realloc(g_emb_items, (size_t)nc * sizeof(EngramEmbEntry)); + if (!ni) { free(v); return NULL; } + g_emb_items = ni; g_emb_cap = nc; + } + g_emb_items[g_emb_count].id = strdup(n->id); + g_emb_items[g_emb_count].hash = h; + g_emb_items[g_emb_count].vec = v; + g_emb_items[g_emb_count].dim = dim; + g_emb_count++; + *out_dim = dim; return v; +} + +static double engram_cosine(const float* a, const float* b, int dim) { + double dot = 0, na = 0, nb = 0; + for (int i = 0; i < dim; i++) { dot += (double)a[i] * b[i]; + na += (double)a[i] * a[i]; + nb += (double)b[i] * b[i]; } + if (na <= 0 || nb <= 0) return 0.0; + return dot / (sqrt(na) * sqrt(nb)); +} + +/* Cosine of node n against the query vector; 0 if unavailable / dim mismatch. */ +static double engram_node_cosine(EngramNode* n, const float* qvec, int qdim) { + if (!qvec || qdim <= 0) return 0.0; + int ndim = 0; + const float* nv = engram_node_vec(n, &ndim); + if (!nv || ndim != qdim) return 0.0; + return engram_cosine(qvec, nv, qdim); +} + +#else /* !HAVE_CURL — semantic layer compiled out; callers stay pure-lexical. + * Only the two boundary functions the always-compiled search/activate + * code calls are stubbed; the query embed always yields NULL so every + * cosine is 0 and every caller collapses to lexical-only. */ +static float* engram_embed_query(const char* q, int* dim) { (void)q; (void)dim; return NULL; } +static double engram_node_cosine(EngramNode* n, const float* qvec, int qdim) { + (void)n; (void)qvec; (void)qdim; return 0.0; +} +#endif /* HAVE_CURL */ + el_val_t engram_search(el_val_t query, el_val_t limit) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); @@ -6833,6 +7070,12 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { if (lim <= 0) lim = 100; el_val_t lst = el_list_empty(); if (!q || !*q) return lst; + /* Semantic augmentation: embed the query once; a node matches if it is a + * lexical hit OR its cosine similarity clears the threshold. qvec is NULL + * (and cosine 0) whenever semantic search is unavailable → pure lexical. */ + int qdim = 0; + float* qvec = engram_embed_query(q, &qdim); + double sem_min = engram_semantic_min(); int64_t found = 0; for (int64_t i = 0; i < g->node_count && found < lim; i++) { EngramNode* n = &g->nodes[i]; @@ -6841,13 +7084,16 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { * know about yourself"). They still surface via engram_activate * + engram_compile_layered_json — that's the legitimate path. */ if (engram_layer_is_transparent(n->layer_id)) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { + int lex = istr_contains(n->content, q) || + istr_contains(n->label, q) || + istr_contains(n->tags, q); + double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0; + if (lex || sem >= sem_min) { lst = el_list_append(lst, engram_node_to_map(n)); found++; } } + free(qvec); return lst; } @@ -7193,14 +7439,28 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { if (!seeds) { free(best_bg); free(best_hops); free(reached); return out; } + /* Semantic seed augmentation: embed the query once; a node becomes a seed + * if it lexically matches OR its cosine clears the threshold. Semantic-only + * seeds enter at reduced strength (scaled by cosine) so pure paraphrase + * matches spread activation without overpowering exact lexical seeds. + * qvec is NULL (cosine 0) when semantic search is unavailable → the seed + * set is exactly the pre-existing lexical one. qvec is freed right after + * this loop so the many downstream early-returns need no cleanup change. */ + int q_dim = 0; + float* q_vec = engram_embed_query(q, &q_dim); + double q_sem_min = engram_semantic_min(); for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { + int lex = istr_contains(n->content, q) || + istr_contains(n->label, q) || + istr_contains(n->tags, q); + double sem = q_vec ? engram_node_cosine(n, q_vec, q_dim) : 0.0; + if (lex || sem >= q_sem_min) { double tdecay = engram_temporal_decay(n, now_ms); double dampen = engram_activation_dampen(n); double act = n->salience * tdecay * dampen; + /* Down-weight pure-semantic seeds by their cosine strength. */ + if (!lex) act *= sem; seeds[seed_count].idx = i; seeds[seed_count].act = act; seeds[seed_count].created_at = n->created_at; @@ -7210,6 +7470,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { reached[i] = 1; } } + free(q_vec); /* Compute mean seed created_at for temporal proximity bonus. */ int64_t seed_epoch = 0; if (seed_count > 0) { @@ -7768,22 +8029,52 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { if (lim <= 0) lim = 100; JsonBuf b; jb_init(&b); jb_putc(&b, '['); - int first = 1; - int64_t found = 0; - if (q && *q) { - for (int64_t i = 0; i < g->node_count && found < lim; i++) { - EngramNode* n = &g->nodes[i]; - /* Filter transparent layers — same as engram_search. */ - if (engram_layer_is_transparent(n->layer_id)) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { - if (!first) jb_putc(&b, ','); - engram_emit_node_json(&b, n); - first = 0; - found++; + if (q && *q && g->node_count > 0) { + /* Collect candidates from the UNION of lexical and semantic matches, + * score each, rank by score, then emit the top `lim`. A node is a + * candidate if it lexically matches OR its query cosine clears the + * threshold. Lexical hits get a base of 1.0 (so they always outrank a + * pure-semantic hit, whose cosine is in [0,1)), refined by cosine; + * pure-semantic hits are scored by cosine alone. + * + * When semantic search is unavailable, qvec is NULL, sem is 0, only + * lexical hits are collected (score 1.0), and the stable insertion + * sort preserves node order — identical to the pre-existing search. */ + int qdim = 0; + float* qvec = engram_embed_query(q, &qdim); + double sem_min = engram_semantic_min(); + typedef struct { int64_t idx; double score; } Cand; + Cand* cand = malloc((size_t)g->node_count * sizeof(Cand)); + if (cand) { + int64_t nc = 0; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + if (engram_layer_is_transparent(n->layer_id)) continue; + int lex = istr_contains(n->content, q) || + istr_contains(n->label, q) || + istr_contains(n->tags, q); + double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0; + if (lex || sem >= sem_min) { + cand[nc].idx = i; + cand[nc].score = (lex ? 1.0 : 0.0) + sem; + nc++; + } } + /* Insertion sort by score desc; stable for equal scores. */ + for (int64_t i = 1; i < nc; i++) { + Cand k = cand[i]; int64_t j = i - 1; + while (j >= 0 && cand[j].score < k.score) { cand[j + 1] = cand[j]; j--; } + cand[j + 1] = k; + } + int first = 1; + for (int64_t i = 0; i < nc && i < lim; i++) { + if (!first) jb_putc(&b, ','); + engram_emit_node_json(&b, &g->nodes[cand[i].idx]); + first = 0; + } + free(cand); } + free(qvec); } jb_putc(&b, ']'); return el_wrap_str(jb_finish(&b));