|
|
|
@@ -6826,243 +6826,75 @@ static int istr_contains(const char* hay, const char* needle) {
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* ══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
* SEMANTIC SEARCH LAYER — nomic-embed-text via Ollama /api/embeddings
|
|
|
|
|
* ──────────────────────────────────────────────────────────────────────────
|
|
|
|
|
* Augments the lexical (istr_contains) matcher with dense-vector retrieval.
|
|
|
|
|
* Node content and the query are embedded through a local Ollama server;
|
|
|
|
|
* nodes are ranked by cosine similarity and UNIONED with lexical hits. This
|
|
|
|
|
* lets a paraphrase query surface a node whose words never appear in it.
|
|
|
|
|
*
|
|
|
|
|
* DEGRADABLE BY DESIGN. The whole layer is gated on HAVE_CURL plus a one-shot
|
|
|
|
|
* runtime probe of the embedding endpoint. If curl is not compiled in, or
|
|
|
|
|
* Ollama is unreachable, or ENGRAM_SEMANTIC=0, every entry point returns
|
|
|
|
|
* "no semantic signal" and callers fall back to pure lexical behaviour —
|
|
|
|
|
* byte-for-byte the pre-existing search.
|
|
|
|
|
*
|
|
|
|
|
* CACHE. Node embeddings are computed lazily on first use and cached in
|
|
|
|
|
* process memory keyed by node id, with an FNV-1a content hash for
|
|
|
|
|
* invalidation (edited content re-embeds). The query is embedded once per
|
|
|
|
|
* search call. This is what "avoid re-embedding the whole graph every query"
|
|
|
|
|
* buys us: a warm cache serves cosine from RAM. (A cold process still pays
|
|
|
|
|
* O(N) embed calls the first time each node is scanned — persisting the cache
|
|
|
|
|
* to a snapshot sidecar is the documented next step, not done here.)
|
|
|
|
|
*
|
|
|
|
|
* nomic task prefixes ("search_query:" / "search_document:") are applied
|
|
|
|
|
* because nomic-embed-text is trained with them; they materially improve
|
|
|
|
|
* retrieval separation (empirically: paraphrase 0.72 vs distractors <0.48).
|
|
|
|
|
*
|
|
|
|
|
* ENV:
|
|
|
|
|
* ENGRAM_SEMANTIC "0" disables; unset/other = auto-probe
|
|
|
|
|
* ENGRAM_EMBED_URL default http://localhost:11434/api/embeddings
|
|
|
|
|
* ENGRAM_EMBED_MODEL default nomic-embed-text
|
|
|
|
|
* ENGRAM_SEMANTIC_MIN cosine threshold for a pure-semantic match (def 0.6)
|
|
|
|
|
* ════════════════════════════════════════════════════════════════════════ */
|
|
|
|
|
/* ── Tokenized query matching ───────────────────────────────────────────
|
|
|
|
|
* The engram query surface (search / activate / goal-bias) historically
|
|
|
|
|
* matched the ENTIRE raw query string as a single case-insensitive
|
|
|
|
|
* substring via istr_contains(field, q). That is Ctrl-F, not search:
|
|
|
|
|
* a multi-word query like "windows msi signing" only matched a node whose
|
|
|
|
|
* text contained that exact contiguous run, so real multi-word queries
|
|
|
|
|
* returned zero. istr_contains stays as the per-TOKEN primitive; these
|
|
|
|
|
* helpers split the query on whitespace and match ANY token, then rank by
|
|
|
|
|
* how many DISTINCT tokens a node covers. Single-token queries are a strict
|
|
|
|
|
* special case (score is 0 or 1) so single-word callers never regress. */
|
|
|
|
|
#define ENGRAM_MAX_QTOKENS 32
|
|
|
|
|
#define ENGRAM_QTOK_LEN 256
|
|
|
|
|
|
|
|
|
|
static double engram_semantic_min(void) {
|
|
|
|
|
static double v = -1.0;
|
|
|
|
|
if (v >= 0.0) return v;
|
|
|
|
|
const char* s = getenv("ENGRAM_SEMANTIC_MIN");
|
|
|
|
|
double d = 0.6;
|
|
|
|
|
if (s && *s) { char* e = NULL; double t = strtod(s, &e);
|
|
|
|
|
if (e != s && t >= 0.0 && t <= 1.0) d = t; }
|
|
|
|
|
v = d; return v;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#ifdef HAVE_CURL
|
|
|
|
|
|
|
|
|
|
typedef struct { char* id; uint64_t hash; float* vec; int dim; } EngramEmbEntry;
|
|
|
|
|
static EngramEmbEntry* g_emb_items = NULL;
|
|
|
|
|
static int64_t g_emb_count = 0, g_emb_cap = 0;
|
|
|
|
|
static int g_emb_state = 0; /* 0=unprobed, 1=available, -1=disabled */
|
|
|
|
|
|
|
|
|
|
static uint64_t engram_fnv1a(const char* s) {
|
|
|
|
|
uint64_t h = 1469598103934665603ULL;
|
|
|
|
|
if (s) for (const unsigned char* p = (const unsigned char*)s; *p; p++) {
|
|
|
|
|
h ^= *p; h *= 1099511628211ULL;
|
|
|
|
|
}
|
|
|
|
|
return h;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Parse "embedding":[f,f,...] from an Ollama response. malloc'd vec, or NULL. */
|
|
|
|
|
static float* engram_parse_embedding(const char* json, int* out_dim) {
|
|
|
|
|
if (!json) return NULL;
|
|
|
|
|
const char* p = strstr(json, "\"embedding\"");
|
|
|
|
|
if (!p) return NULL;
|
|
|
|
|
p = strchr(p, '[');
|
|
|
|
|
if (!p) return NULL;
|
|
|
|
|
p++;
|
|
|
|
|
int cap = 1024, n = 0;
|
|
|
|
|
float* v = malloc((size_t)cap * sizeof(float));
|
|
|
|
|
if (!v) return NULL;
|
|
|
|
|
while (*p && *p != ']') {
|
|
|
|
|
while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r' || *p == ',') p++;
|
|
|
|
|
if (*p == ']' || !*p) break;
|
|
|
|
|
char* e = NULL;
|
|
|
|
|
double d = strtod(p, &e);
|
|
|
|
|
if (e == p) break;
|
|
|
|
|
if (n >= cap) { cap *= 2; float* nv = realloc(v, (size_t)cap * sizeof(float));
|
|
|
|
|
if (!nv) { free(v); return NULL; } v = nv; }
|
|
|
|
|
v[n++] = (float)d;
|
|
|
|
|
p = e;
|
|
|
|
|
}
|
|
|
|
|
if (n == 0) { free(v); return NULL; }
|
|
|
|
|
*out_dim = n;
|
|
|
|
|
return v;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* JSON-escape src into a malloc'd buffer (no surrounding quotes). */
|
|
|
|
|
static char* engram_json_escape(const char* src) {
|
|
|
|
|
if (!src) src = "";
|
|
|
|
|
size_t n = strlen(src);
|
|
|
|
|
char* out = malloc(n * 2 + 1);
|
|
|
|
|
if (!out) return NULL;
|
|
|
|
|
size_t j = 0;
|
|
|
|
|
for (size_t i = 0; i < n; i++) {
|
|
|
|
|
unsigned char c = (unsigned char)src[i];
|
|
|
|
|
if (c == '"') { out[j++] = '\\'; out[j++] = '"'; }
|
|
|
|
|
else if (c == '\\') { out[j++] = '\\'; out[j++] = '\\'; }
|
|
|
|
|
else if (c == '\n') { out[j++] = '\\'; out[j++] = 'n'; }
|
|
|
|
|
else if (c == '\r') { out[j++] = '\\'; out[j++] = 'r'; }
|
|
|
|
|
else if (c == '\t') { out[j++] = '\\'; out[j++] = 't'; }
|
|
|
|
|
else if (c < 0x20) { /* drop other control bytes */ }
|
|
|
|
|
else { out[j++] = (char)c; }
|
|
|
|
|
}
|
|
|
|
|
out[j] = '\0';
|
|
|
|
|
return out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Embed `prefix+text` via Ollama. Returns malloc'd vec (caller frees), or NULL. */
|
|
|
|
|
static float* engram_embed_raw(const char* prefix, const char* text, int* out_dim) {
|
|
|
|
|
if (!text) return NULL;
|
|
|
|
|
const char* url = getenv("ENGRAM_EMBED_URL");
|
|
|
|
|
if (!url || !*url) url = "http://localhost:11434/api/embeddings";
|
|
|
|
|
const char* model = getenv("ENGRAM_EMBED_MODEL");
|
|
|
|
|
if (!model || !*model) model = "nomic-embed-text";
|
|
|
|
|
/* Bound content length to keep latency/memory sane on huge nodes. */
|
|
|
|
|
char* trunc = NULL;
|
|
|
|
|
size_t maxlen = 8192;
|
|
|
|
|
if (strlen(text) > maxlen) {
|
|
|
|
|
trunc = malloc(maxlen + 1);
|
|
|
|
|
if (trunc) { memcpy(trunc, text, maxlen); trunc[maxlen] = '\0'; text = trunc; }
|
|
|
|
|
}
|
|
|
|
|
char* esc_prefix = engram_json_escape(prefix ? prefix : "");
|
|
|
|
|
char* esc = engram_json_escape(text);
|
|
|
|
|
free(trunc);
|
|
|
|
|
if (!esc || !esc_prefix) { free(esc); free(esc_prefix); return NULL; }
|
|
|
|
|
size_t blen = strlen(esc) + strlen(esc_prefix) + strlen(model) + 64;
|
|
|
|
|
char* body = malloc(blen);
|
|
|
|
|
if (!body) { free(esc); free(esc_prefix); return NULL; }
|
|
|
|
|
snprintf(body, blen, "{\"model\":\"%s\",\"prompt\":\"%s%s\"}", model, esc_prefix, esc);
|
|
|
|
|
free(esc); free(esc_prefix);
|
|
|
|
|
|
|
|
|
|
CURL* c = curl_easy_init();
|
|
|
|
|
if (!c) { free(body); return NULL; }
|
|
|
|
|
HttpBuf rb; httpbuf_init(&rb);
|
|
|
|
|
struct curl_slist* h = curl_slist_append(NULL, "Content-Type: application/json");
|
|
|
|
|
char errbuf[CURL_ERROR_SIZE]; errbuf[0] = '\0';
|
|
|
|
|
curl_easy_setopt(c, CURLOPT_URL, url);
|
|
|
|
|
curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_write_cb);
|
|
|
|
|
curl_easy_setopt(c, CURLOPT_WRITEDATA, &rb);
|
|
|
|
|
curl_easy_setopt(c, CURLOPT_POST, 1L);
|
|
|
|
|
curl_easy_setopt(c, CURLOPT_POSTFIELDS, body);
|
|
|
|
|
curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)strlen(body));
|
|
|
|
|
curl_easy_setopt(c, CURLOPT_HTTPHEADER, h);
|
|
|
|
|
curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, el_http_timeout_ms());
|
|
|
|
|
curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L);
|
|
|
|
|
curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf);
|
|
|
|
|
CURLcode rc = curl_easy_perform(c);
|
|
|
|
|
curl_slist_free_all(h);
|
|
|
|
|
curl_easy_cleanup(c);
|
|
|
|
|
free(body);
|
|
|
|
|
if (rc != CURLE_OK) { free(rb.data); return NULL; }
|
|
|
|
|
float* v = engram_parse_embedding(rb.data, out_dim);
|
|
|
|
|
free(rb.data);
|
|
|
|
|
return v;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* One-shot probe: is semantic search available? Caches the verdict. */
|
|
|
|
|
static int engram_semantic_enabled(void) {
|
|
|
|
|
if (g_emb_state != 0) return g_emb_state == 1;
|
|
|
|
|
const char* s = getenv("ENGRAM_SEMANTIC");
|
|
|
|
|
if (s && strcmp(s, "0") == 0) { g_emb_state = -1; return 0; }
|
|
|
|
|
int dim = 0;
|
|
|
|
|
float* v = engram_embed_raw("search_query: ", "probe", &dim);
|
|
|
|
|
if (v && dim > 0) { free(v); g_emb_state = 1; return 1; }
|
|
|
|
|
free(v);
|
|
|
|
|
g_emb_state = -1; return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Embed the query. Returns malloc'd vec (caller frees), or NULL if semantic off. */
|
|
|
|
|
static float* engram_embed_query(const char* q, int* dim) {
|
|
|
|
|
if (!engram_semantic_enabled()) return NULL;
|
|
|
|
|
if (!q || !*q) return NULL;
|
|
|
|
|
return engram_embed_raw("search_query: ", q, dim);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Cached node embedding. Returns a pointer OWNED BY THE CACHE — do not free. */
|
|
|
|
|
static const float* engram_node_vec(EngramNode* n, int* out_dim) {
|
|
|
|
|
if (!n || !n->id) return NULL;
|
|
|
|
|
uint64_t h = engram_fnv1a(n->content);
|
|
|
|
|
for (int64_t i = 0; i < g_emb_count; i++) {
|
|
|
|
|
if (g_emb_items[i].id && strcmp(g_emb_items[i].id, n->id) == 0) {
|
|
|
|
|
if (g_emb_items[i].hash == h && g_emb_items[i].vec) {
|
|
|
|
|
*out_dim = g_emb_items[i].dim; return g_emb_items[i].vec;
|
|
|
|
|
}
|
|
|
|
|
/* content changed → re-embed in place */
|
|
|
|
|
int dim = 0;
|
|
|
|
|
float* v = engram_embed_raw("search_document: ", n->content ? n->content : "", &dim);
|
|
|
|
|
if (!v) return NULL;
|
|
|
|
|
free(g_emb_items[i].vec);
|
|
|
|
|
g_emb_items[i].vec = v; g_emb_items[i].dim = dim; g_emb_items[i].hash = h;
|
|
|
|
|
*out_dim = dim; return v;
|
|
|
|
|
/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct
|
|
|
|
|
* (case-insensitive) tokens. Returns the token count. Over-long tokens are
|
|
|
|
|
* truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */
|
|
|
|
|
static int engram_tokenize_query(const char* q,
|
|
|
|
|
char toks[][ENGRAM_QTOK_LEN], int maxtok) {
|
|
|
|
|
int n = 0;
|
|
|
|
|
if (!q) return 0;
|
|
|
|
|
const char* p = q;
|
|
|
|
|
while (*p && n < maxtok) {
|
|
|
|
|
while (*p && isspace((unsigned char)*p)) p++;
|
|
|
|
|
if (!*p) break;
|
|
|
|
|
char buf[ENGRAM_QTOK_LEN];
|
|
|
|
|
size_t tl = 0;
|
|
|
|
|
while (*p && !isspace((unsigned char)*p)) {
|
|
|
|
|
if (tl < sizeof(buf) - 1) buf[tl++] = *p;
|
|
|
|
|
p++;
|
|
|
|
|
}
|
|
|
|
|
buf[tl] = '\0';
|
|
|
|
|
if (tl == 0) continue;
|
|
|
|
|
int dup = 0;
|
|
|
|
|
for (int s = 0; s < n; s++) {
|
|
|
|
|
if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; }
|
|
|
|
|
}
|
|
|
|
|
if (dup) continue;
|
|
|
|
|
memcpy(toks[n], buf, tl + 1);
|
|
|
|
|
n++;
|
|
|
|
|
}
|
|
|
|
|
int dim = 0;
|
|
|
|
|
float* v = engram_embed_raw("search_document: ", n->content ? n->content : "", &dim);
|
|
|
|
|
if (!v) return NULL;
|
|
|
|
|
if (g_emb_count >= g_emb_cap) {
|
|
|
|
|
int64_t nc = g_emb_cap ? g_emb_cap * 2 : 256;
|
|
|
|
|
EngramEmbEntry* ni = realloc(g_emb_items, (size_t)nc * sizeof(EngramEmbEntry));
|
|
|
|
|
if (!ni) { free(v); return NULL; }
|
|
|
|
|
g_emb_items = ni; g_emb_cap = nc;
|
|
|
|
|
return n;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Count how many of the ntok distinct query tokens appear (case-insensitive)
|
|
|
|
|
* in the node's content, label, or tags. 0 == no match. */
|
|
|
|
|
static int engram_node_match_score(const EngramNode* n,
|
|
|
|
|
char toks[][ENGRAM_QTOK_LEN], int ntok) {
|
|
|
|
|
int score = 0;
|
|
|
|
|
for (int t = 0; t < ntok; t++) {
|
|
|
|
|
if (istr_contains(n->content, toks[t]) ||
|
|
|
|
|
istr_contains(n->label, toks[t]) ||
|
|
|
|
|
istr_contains(n->tags, toks[t]))
|
|
|
|
|
score++;
|
|
|
|
|
}
|
|
|
|
|
g_emb_items[g_emb_count].id = strdup(n->id);
|
|
|
|
|
g_emb_items[g_emb_count].hash = h;
|
|
|
|
|
g_emb_items[g_emb_count].vec = v;
|
|
|
|
|
g_emb_items[g_emb_count].dim = dim;
|
|
|
|
|
g_emb_count++;
|
|
|
|
|
*out_dim = dim; return v;
|
|
|
|
|
return score;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static double engram_cosine(const float* a, const float* b, int dim) {
|
|
|
|
|
double dot = 0, na = 0, nb = 0;
|
|
|
|
|
for (int i = 0; i < dim; i++) { dot += (double)a[i] * b[i];
|
|
|
|
|
na += (double)a[i] * a[i];
|
|
|
|
|
nb += (double)b[i] * b[i]; }
|
|
|
|
|
if (na <= 0 || nb <= 0) return 0.0;
|
|
|
|
|
return dot / (sqrt(na) * sqrt(nb));
|
|
|
|
|
/* Rank entry: distinct-token match count (primary, desc) then salience
|
|
|
|
|
* (tiebreak, desc). */
|
|
|
|
|
typedef struct { int64_t idx; int score; double salience; } EngramRankEntry;
|
|
|
|
|
static int engram_rank_cmp(const void* a, const void* b) {
|
|
|
|
|
const EngramRankEntry* ea = (const EngramRankEntry*)a;
|
|
|
|
|
const EngramRankEntry* eb = (const EngramRankEntry*)b;
|
|
|
|
|
if (ea->score != eb->score) return eb->score - ea->score; /* desc */
|
|
|
|
|
if (ea->salience < eb->salience) return 1;
|
|
|
|
|
if (ea->salience > eb->salience) return -1;
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Cosine of node n against the query vector; 0 if unavailable / dim mismatch. */
|
|
|
|
|
static double engram_node_cosine(EngramNode* n, const float* qvec, int qdim) {
|
|
|
|
|
if (!qvec || qdim <= 0) return 0.0;
|
|
|
|
|
int ndim = 0;
|
|
|
|
|
const float* nv = engram_node_vec(n, &ndim);
|
|
|
|
|
if (!nv || ndim != qdim) return 0.0;
|
|
|
|
|
return engram_cosine(qvec, nv, qdim);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#else /* !HAVE_CURL — semantic layer compiled out; callers stay pure-lexical.
|
|
|
|
|
* Only the two boundary functions the always-compiled search/activate
|
|
|
|
|
* code calls are stubbed; the query embed always yields NULL so every
|
|
|
|
|
* cosine is 0 and every caller collapses to lexical-only. */
|
|
|
|
|
static float* engram_embed_query(const char* q, int* dim) { (void)q; (void)dim; return NULL; }
|
|
|
|
|
static double engram_node_cosine(EngramNode* n, const float* qvec, int qdim) {
|
|
|
|
|
(void)n; (void)qvec; (void)qdim; return 0.0;
|
|
|
|
|
}
|
|
|
|
|
#endif /* HAVE_CURL */
|
|
|
|
|
|
|
|
|
|
el_val_t engram_search(el_val_t query, el_val_t limit) {
|
|
|
|
|
EngramStore* g = engram_get();
|
|
|
|
|
const char* q = EL_CSTR(query);
|
|
|
|
@@ -7070,30 +6902,34 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
|
|
|
|
|
if (lim <= 0) lim = 100;
|
|
|
|
|
el_val_t lst = el_list_empty();
|
|
|
|
|
if (!q || !*q) return lst;
|
|
|
|
|
/* Semantic augmentation: embed the query once; a node matches if it is a
|
|
|
|
|
* lexical hit OR its cosine similarity clears the threshold. qvec is NULL
|
|
|
|
|
* (and cosine 0) whenever semantic search is unavailable → pure lexical. */
|
|
|
|
|
int qdim = 0;
|
|
|
|
|
float* qvec = engram_embed_query(q, &qdim);
|
|
|
|
|
double sem_min = engram_semantic_min();
|
|
|
|
|
int64_t found = 0;
|
|
|
|
|
for (int64_t i = 0; i < g->node_count && found < lim; i++) {
|
|
|
|
|
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
|
|
|
|
int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
|
|
|
|
|
if (ntok == 0) return lst;
|
|
|
|
|
EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry));
|
|
|
|
|
if (!hits) return lst;
|
|
|
|
|
int64_t nhits = 0;
|
|
|
|
|
for (int64_t i = 0; i < g->node_count; i++) {
|
|
|
|
|
EngramNode* n = &g->nodes[i];
|
|
|
|
|
/* Filter transparent layers: nodes whose layer is `transparent=1`
|
|
|
|
|
* shape output but are invisible to introspection ("what do you
|
|
|
|
|
* know about yourself"). They still surface via engram_activate
|
|
|
|
|
* + engram_compile_layered_json — that's the legitimate path. */
|
|
|
|
|
if (engram_layer_is_transparent(n->layer_id)) continue;
|
|
|
|
|
int lex = istr_contains(n->content, q) ||
|
|
|
|
|
istr_contains(n->label, q) ||
|
|
|
|
|
istr_contains(n->tags, q);
|
|
|
|
|
double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0;
|
|
|
|
|
if (lex || sem >= sem_min) {
|
|
|
|
|
lst = el_list_append(lst, engram_node_to_map(n));
|
|
|
|
|
found++;
|
|
|
|
|
int sc = engram_node_match_score(n, toks, ntok);
|
|
|
|
|
if (sc > 0) {
|
|
|
|
|
hits[nhits].idx = i;
|
|
|
|
|
hits[nhits].score = sc;
|
|
|
|
|
hits[nhits].salience = n->salience;
|
|
|
|
|
nhits++;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
free(qvec);
|
|
|
|
|
/* Rank by distinct tokens matched (desc) then salience (desc), then cap. */
|
|
|
|
|
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
|
|
|
|
|
int64_t end = nhits < lim ? nhits : lim;
|
|
|
|
|
for (int64_t k = 0; k < end; k++) {
|
|
|
|
|
lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx]));
|
|
|
|
|
}
|
|
|
|
|
free(hits);
|
|
|
|
|
return lst;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -7370,10 +7206,14 @@ static double engram_temporal_proximity_bonus(int64_t node_created,
|
|
|
|
|
static double engram_goal_bias(const EngramNode* n, const char* query) {
|
|
|
|
|
if (!query || !*query) return 1.0;
|
|
|
|
|
double bias = 1.0;
|
|
|
|
|
/* Direct lexical overlap: node content/label/tags share text with query. */
|
|
|
|
|
if (istr_contains(n->content, query) || istr_contains(n->label, query) ||
|
|
|
|
|
istr_contains(n->tags, query)) {
|
|
|
|
|
bias += 0.5;
|
|
|
|
|
/* Direct lexical overlap, graded by token coverage: a node covering all
|
|
|
|
|
* query tokens gets the full +0.5; partial coverage gets a proportional
|
|
|
|
|
* share. Single-token queries → full +0.5 on match, identical to before. */
|
|
|
|
|
{
|
|
|
|
|
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
|
|
|
|
int ntok = engram_tokenize_query(query, toks, ENGRAM_MAX_QTOKENS);
|
|
|
|
|
int sc = engram_node_match_score(n, toks, ntok);
|
|
|
|
|
if (sc > 0 && ntok > 0) bias += 0.5 * ((double)sc / (double)ntok);
|
|
|
|
|
}
|
|
|
|
|
/* Node-type resonance with query intent. */
|
|
|
|
|
int technical_query = istr_contains(query, "code") ||
|
|
|
|
@@ -7439,28 +7279,21 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
|
|
|
|
|
if (!seeds) {
|
|
|
|
|
free(best_bg); free(best_hops); free(reached); return out;
|
|
|
|
|
}
|
|
|
|
|
/* Semantic seed augmentation: embed the query once; a node becomes a seed
|
|
|
|
|
* if it lexically matches OR its cosine clears the threshold. Semantic-only
|
|
|
|
|
* seeds enter at reduced strength (scaled by cosine) so pure paraphrase
|
|
|
|
|
* matches spread activation without overpowering exact lexical seeds.
|
|
|
|
|
* qvec is NULL (cosine 0) when semantic search is unavailable → the seed
|
|
|
|
|
* set is exactly the pre-existing lexical one. qvec is freed right after
|
|
|
|
|
* this loop so the many downstream early-returns need no cleanup change. */
|
|
|
|
|
int q_dim = 0;
|
|
|
|
|
float* q_vec = engram_embed_query(q, &q_dim);
|
|
|
|
|
double q_sem_min = engram_semantic_min();
|
|
|
|
|
/* Tokenize once: a node seeds if it matches ANY query token, and its seed
|
|
|
|
|
* activation is scaled by token coverage (fraction of distinct query
|
|
|
|
|
* tokens it contains) so a node matching all words seeds more strongly
|
|
|
|
|
* than one matching a single word. Single-word queries → coverage 1.0,
|
|
|
|
|
* identical to the prior whole-query behavior. */
|
|
|
|
|
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
|
|
|
|
int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
|
|
|
|
|
for (int64_t i = 0; i < g->node_count; i++) {
|
|
|
|
|
EngramNode* n = &g->nodes[i];
|
|
|
|
|
int lex = istr_contains(n->content, q) ||
|
|
|
|
|
istr_contains(n->label, q) ||
|
|
|
|
|
istr_contains(n->tags, q);
|
|
|
|
|
double sem = q_vec ? engram_node_cosine(n, q_vec, q_dim) : 0.0;
|
|
|
|
|
if (lex || sem >= q_sem_min) {
|
|
|
|
|
int sc = engram_node_match_score(n, toks, ntok);
|
|
|
|
|
if (sc > 0) {
|
|
|
|
|
double tdecay = engram_temporal_decay(n, now_ms);
|
|
|
|
|
double dampen = engram_activation_dampen(n);
|
|
|
|
|
double act = n->salience * tdecay * dampen;
|
|
|
|
|
/* Down-weight pure-semantic seeds by their cosine strength. */
|
|
|
|
|
if (!lex) act *= sem;
|
|
|
|
|
double cover = ntok > 0 ? (double)sc / (double)ntok : 1.0;
|
|
|
|
|
double act = n->salience * tdecay * dampen * cover;
|
|
|
|
|
seeds[seed_count].idx = i;
|
|
|
|
|
seeds[seed_count].act = act;
|
|
|
|
|
seeds[seed_count].created_at = n->created_at;
|
|
|
|
@@ -7470,7 +7303,6 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
|
|
|
|
|
reached[i] = 1;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
free(q_vec);
|
|
|
|
|
/* Compute mean seed created_at for temporal proximity bonus. */
|
|
|
|
|
int64_t seed_epoch = 0;
|
|
|
|
|
if (seed_count > 0) {
|
|
|
|
@@ -8029,52 +7861,39 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
|
|
|
|
if (lim <= 0) lim = 100;
|
|
|
|
|
JsonBuf b; jb_init(&b);
|
|
|
|
|
jb_putc(&b, '[');
|
|
|
|
|
if (q && *q && g->node_count > 0) {
|
|
|
|
|
/* Collect candidates from the UNION of lexical and semantic matches,
|
|
|
|
|
* score each, rank by score, then emit the top `lim`. A node is a
|
|
|
|
|
* candidate if it lexically matches OR its query cosine clears the
|
|
|
|
|
* threshold. Lexical hits get a base of 1.0 (so they always outrank a
|
|
|
|
|
* pure-semantic hit, whose cosine is in [0,1)), refined by cosine;
|
|
|
|
|
* pure-semantic hits are scored by cosine alone.
|
|
|
|
|
*
|
|
|
|
|
* When semantic search is unavailable, qvec is NULL, sem is 0, only
|
|
|
|
|
* lexical hits are collected (score 1.0), and the stable insertion
|
|
|
|
|
* sort preserves node order — identical to the pre-existing search. */
|
|
|
|
|
int qdim = 0;
|
|
|
|
|
float* qvec = engram_embed_query(q, &qdim);
|
|
|
|
|
double sem_min = engram_semantic_min();
|
|
|
|
|
typedef struct { int64_t idx; double score; } Cand;
|
|
|
|
|
Cand* cand = malloc((size_t)g->node_count * sizeof(Cand));
|
|
|
|
|
if (cand) {
|
|
|
|
|
int64_t nc = 0;
|
|
|
|
|
for (int64_t i = 0; i < g->node_count; i++) {
|
|
|
|
|
EngramNode* n = &g->nodes[i];
|
|
|
|
|
if (engram_layer_is_transparent(n->layer_id)) continue;
|
|
|
|
|
int lex = istr_contains(n->content, q) ||
|
|
|
|
|
istr_contains(n->label, q) ||
|
|
|
|
|
istr_contains(n->tags, q);
|
|
|
|
|
double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0;
|
|
|
|
|
if (lex || sem >= sem_min) {
|
|
|
|
|
cand[nc].idx = i;
|
|
|
|
|
cand[nc].score = (lex ? 1.0 : 0.0) + sem;
|
|
|
|
|
nc++;
|
|
|
|
|
int first = 1;
|
|
|
|
|
if (q && *q) {
|
|
|
|
|
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
|
|
|
|
|
int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
|
|
|
|
|
if (ntok > 0) {
|
|
|
|
|
EngramRankEntry* hits =
|
|
|
|
|
malloc((size_t)g->node_count * sizeof(EngramRankEntry));
|
|
|
|
|
if (hits) {
|
|
|
|
|
int64_t nhits = 0;
|
|
|
|
|
for (int64_t i = 0; i < g->node_count; i++) {
|
|
|
|
|
EngramNode* n = &g->nodes[i];
|
|
|
|
|
/* Filter transparent layers — same as engram_search. */
|
|
|
|
|
if (engram_layer_is_transparent(n->layer_id)) continue;
|
|
|
|
|
int sc = engram_node_match_score(n, toks, ntok);
|
|
|
|
|
if (sc > 0) {
|
|
|
|
|
hits[nhits].idx = i;
|
|
|
|
|
hits[nhits].score = sc;
|
|
|
|
|
hits[nhits].salience = n->salience;
|
|
|
|
|
nhits++;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
/* Rank by distinct tokens matched (desc) then salience (desc). */
|
|
|
|
|
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry),
|
|
|
|
|
engram_rank_cmp);
|
|
|
|
|
int64_t end = nhits < lim ? nhits : lim;
|
|
|
|
|
for (int64_t k = 0; k < end; k++) {
|
|
|
|
|
if (!first) jb_putc(&b, ',');
|
|
|
|
|
engram_emit_node_json(&b, &g->nodes[hits[k].idx]);
|
|
|
|
|
first = 0;
|
|
|
|
|
}
|
|
|
|
|
free(hits);
|
|
|
|
|
}
|
|
|
|
|
/* Insertion sort by score desc; stable for equal scores. */
|
|
|
|
|
for (int64_t i = 1; i < nc; i++) {
|
|
|
|
|
Cand k = cand[i]; int64_t j = i - 1;
|
|
|
|
|
while (j >= 0 && cand[j].score < k.score) { cand[j + 1] = cand[j]; j--; }
|
|
|
|
|
cand[j + 1] = k;
|
|
|
|
|
}
|
|
|
|
|
int first = 1;
|
|
|
|
|
for (int64_t i = 0; i < nc && i < lim; i++) {
|
|
|
|
|
if (!first) jb_putc(&b, ',');
|
|
|
|
|
engram_emit_node_json(&b, &g->nodes[cand[i].idx]);
|
|
|
|
|
first = 0;
|
|
|
|
|
}
|
|
|
|
|
free(cand);
|
|
|
|
|
}
|
|
|
|
|
free(qvec);
|
|
|
|
|
}
|
|
|
|
|
jb_putc(&b, ']');
|
|
|
|
|
return el_wrap_str(jb_finish(&b));
|
|
|
|
|
will.anderson