neuron-technologies/el
2
0
Fork 0
Code Issues 1 Pull Requests 6 Actions Packages Projects Releases 5 Wiki Activity

add BM25+ text ranking in EL, remove Ollama query-embedding dependency

Browse Source 
- Add list_set, math_exp, and float_add/sub/mul/div/gt/lt/eq/gte/lte builtins to
  el_runtime.c + el_runtime.h (float arithmetic builtins needed because EL operators
  +*/ operate on raw el_val_t bits, not IEEE 754 doubles)
- Remove engram_embed_query() and its forward declaration from el_runtime.c
- Remove Ollama cosine-similarity blend from activation scoring (reverts 9af2482):
  drops query_emb/query_edim variables, bias *= (1 + 0.3 * sim) block, and all
  free(query_emb) calls from the activation loop
- Implement BM25+ scoring in server.el (k1=1.2, b=0.75, delta=1.0):
  bm25_tokenize, bm25_count_term, bm25_score_doc, bm25_search_json
  V1 uses n_t=1 approximation (constant IDF per corpus size); acceptable as a
  first pass without an inverted index
- Wire /api/bm25/search POST/GET route in server.el dispatcher
- Zero Ollama calls in the activation/search path; embeddings on nodes are
  untouched (still written at node-creation time)
This commit is contained in:
Will Anderson
2026-05-14 11:52:43 -05:00
parent 1a8a16002e
commit 6121b33d25
5 changed files with 446 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lang/releases/v1.0.0-20260501/el_runtime.c
+25 -76
View File
@@ -4851,6 +4851,21 @@ el_val_t str_to_float(el_val_t s) {
el_val_t math_sqrt(el_val_t f) { return el_from_float(sqrt(el_to_float(f))); }
el_val_t math_log(el_val_t f) { return el_from_float(log(el_to_float(f))); }
el_val_t math_ln(el_val_t f) { return el_from_float(log(el_to_float(f))); }
el_val_t math_exp(el_val_t f) { return el_from_float(exp(el_to_float(f))); }
/* ── Float arithmetic builtins ───────────────────────────────────────────────
* EL operators (+, *, /) operate on raw el_val_t bits, which is wrong for
* IEEE 754 floats. These builtins do correct float arithmetic and can be
* called from EL source as float_add(a, b), float_mul(a, b), etc. */
el_val_t float_add(el_val_t a, el_val_t b) { return el_from_float(el_to_float(a) + el_to_float(b)); }
el_val_t float_sub(el_val_t a, el_val_t b) { return el_from_float(el_to_float(a) - el_to_float(b)); }
el_val_t float_mul(el_val_t a, el_val_t b) { return el_from_float(el_to_float(a) * el_to_float(b)); }
el_val_t float_div(el_val_t a, el_val_t b) { double db = el_to_float(b); return el_from_float(db != 0.0 ? el_to_float(a) / db : 0.0); }
el_val_t float_gt(el_val_t a, el_val_t b) { return el_to_float(a) > el_to_float(b) ? 1 : 0; }
el_val_t float_lt(el_val_t a, el_val_t b) { return el_to_float(a) < el_to_float(b) ? 1 : 0; }
el_val_t float_eq(el_val_t a, el_val_t b) { return el_to_float(a) == el_to_float(b) ? 1 : 0; }
el_val_t float_gte(el_val_t a, el_val_t b) { return el_to_float(a) >= el_to_float(b) ? 1 : 0; }
el_val_t float_lte(el_val_t a, el_val_t b) { return el_to_float(a) <= el_to_float(b) ? 1 : 0; }
el_val_t math_sin(el_val_t f) { return el_from_float(sin(el_to_float(f))); }
el_val_t math_cos(el_val_t f) { return el_from_float(cos(el_to_float(f))); }
el_val_t math_pi(void) { return el_from_float(3.141592653589793238462643383279502884); }
@@ -5358,6 +5373,16 @@ el_val_t list_push(el_val_t list, el_val_t elem) {
return el_list_append(list, elem);
}
/* list_set(list, idx, value) — mutate list in-place at idx, return the list.
* Out-of-bounds idx is a no-op (returns list unchanged). */
el_val_t list_set(el_val_t listv, el_val_t index, el_val_t value) {
ElList* lst = (ElList*)(uintptr_t)listv;
if (!lst) return listv;
if (index < 0 || index >= lst->length) return listv;
lst->elems[index] = value;
return listv;
}
el_val_t list_push_front(el_val_t listv, el_val_t elem) {
ElList* lst = (ElList*)(uintptr_t)listv;
if (!lst) {
@@ -5959,7 +5984,6 @@ static int engram_keys_init(void);
static int engram_write_binary(const char* path);
static int engram_load_binary(const char* path);
static void engram_embed_node(EngramNode* n);
static uint32_t engram_embed_query(const char* text, float** vec_out);
static float engram_cosine_sim(const float* a, const float* b, uint32_t dim);
static void engram_checkpoint(void);
static void engram_emit_ise_internal(const char* content, const char* label);
@@ -6871,17 +6895,9 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
double inh = best_bg[src] * e->weight;
if (inh > inhibition[tgt]) inhibition[tgt] = inh;
}
/* Embed the query string once for semantic similarity in Layer 2.
* Uses a 5s timeout so a slow/absent Ollama never blocks activation.
* query_emb is NULL and query_edim is 0 if embedding fails — all
* downstream cosine-sim paths guard on this and degrade to bias=1.0. */
float* query_emb = NULL;
uint32_t query_edim = engram_embed_query(q, &query_emb);
/* Step B: compute working_memory_weight per candidate node. */
double* wm_weights = calloc((size_t)g->node_count, sizeof(double));
if (!wm_weights) {
free(query_emb);
free(best_bg); free(best_hops); free(reached); free(seeds);
free(fr); free(inhibition); return out;
}
@@ -6892,19 +6908,6 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
double type_threshold = engram_type_threshold(n->node_type, n->tier);
/* Goal bias weights the node's relevance to current intent. */
double bias = engram_goal_bias(n, q);
/* Cosine similarity boost: if both query and node have embeddings,
* blend semantic similarity into the bias with weight α=0.3.
* sim ∈ [-1, 1]; clamp to [0, 1] before blending.
* bias_final = bias * (1 + 0.3 * max(0, sim))
* This boosts semantically close nodes even when lexical overlap is low. */
if (query_emb && query_edim > 0 &&
n->embedding && n->embedding_dim == query_edim) {
float sim = engram_cosine_sim(query_emb, n->embedding, query_edim);
if (sim > 0.0f) {
bias *= (1.0 + 0.3 * (double)sim);
if (bias > 2.0) bias = 2.0;
}
}
/* Raw working memory score. */
double raw_wm = best_bg[i] * bias * n->confidence;
/* Apply inhibitory suppression. Full inhibition → scale by factor. */
@@ -7054,7 +7057,6 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
Result* results = malloc((size_t)g->node_count * sizeof(Result));
int64_t rcount = 0;
if (!results) {
free(query_emb);
free(best_bg); free(best_hops); free(reached); free(seeds);
free(fr); free(inhibition); free(wm_weights); return out;
}
@@ -7099,7 +7101,6 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
(el_val_t)(results[i].wm > 0.0 ? 1 : 0));
out = el_list_append(out, entry);
}
free(query_emb);
free(best_bg); free(best_hops); free(reached);
free(seeds); free(fr); free(inhibition); free(wm_weights); free(results);
return out;
@@ -7345,58 +7346,6 @@ static void engram_embed_node(EngramNode* n) {
/* ── Engram: cosine similarity ───────────────────────────────────────────── */
/* Embed an arbitrary text string into a float vector via Ollama.
* Returns the dimension (0 on failure). Caller must free *vec_out. */
static uint32_t engram_embed_query(const char* text, float** vec_out) {
*vec_out = NULL;
if (!text || !*text) return 0;
size_t clen = strlen(text);
if (clen > 2048) clen = 2048;
char* body = malloc(clen * 6 + 128);
if (!body) return 0;
char* bp = body;
bp += sprintf(bp, "{\"model\":\"nomic-embed-text\",\"prompt\":\"");
const char* cp = text;
size_t written = 0;
while (*cp && written < clen) {
if (*cp == '"') { *bp++ = '\\'; *bp++ = '"'; }
else if (*cp == '\\') { *bp++ = '\\'; *bp++ = '\\'; }
else if (*cp == '\n') { *bp++ = '\\'; *bp++ = 'n'; }
else if (*cp == '\r') { *bp++ = '\\'; *bp++ = 'r'; }
else if (*cp == '\t') { *bp++ = '\\'; *bp++ = 't'; }
else { *bp++ = *cp; }
cp++; written++;
}
sprintf(bp, "\"}");
CURL* curl = curl_easy_init();
if (!curl) { free(body); return 0; }
char* resp = NULL;
struct curl_slist* hdrs = NULL;
hdrs = curl_slist_append(hdrs, "Content-Type: application/json");
curl_easy_setopt(curl, CURLOPT_URL, "http://localhost:11434/api/embeddings");
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, hdrs);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, engram_embed_write_cb);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &resp);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5L); /* short timeout — don't block activation */
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
CURLcode rc = curl_easy_perform(curl);
curl_slist_free_all(hdrs);
curl_easy_cleanup(curl);
free(body);
if (rc != CURLE_OK || !resp) { free(resp); return 0; }
const char* ep = strstr(resp, "\"embedding\"");
if (!ep) { free(resp); return 0; }
ep += strlen("\"embedding\"");
while (*ep && *ep != '[') ep++;
float* vec = NULL;
uint32_t dim = engram_parse_float_array(ep, &vec);
free(resp);
if (dim == 0) return 0;
*vec_out = vec;
return dim;
}
static float engram_cosine_sim(const float* a, const float* b, uint32_t dim) {
if (!a || !b || dim == 0) return 0.0f;
double dot = 0.0, na = 0.0, nb = 0.0;
lang/releases/v1.0.0-20260501/el_runtime.h
+13
View File
@@ -430,10 +430,22 @@ el_val_t str_to_float(el_val_t s);
el_val_t math_sqrt(el_val_t f);
el_val_t math_log(el_val_t f);
el_val_t math_ln(el_val_t f);
el_val_t math_exp(el_val_t f);
el_val_t math_sin(el_val_t f);
el_val_t math_cos(el_val_t f);
el_val_t math_pi(void);
/* ── Float arithmetic builtins (correct IEEE 754 via bit-cast round-trip) ─── */
el_val_t float_add(el_val_t a, el_val_t b);
el_val_t float_sub(el_val_t a, el_val_t b);
el_val_t float_mul(el_val_t a, el_val_t b);
el_val_t float_div(el_val_t a, el_val_t b);
el_val_t float_gt(el_val_t a, el_val_t b);
el_val_t float_lt(el_val_t a, el_val_t b);
el_val_t float_eq(el_val_t a, el_val_t b);
el_val_t float_gte(el_val_t a, el_val_t b);
el_val_t float_lte(el_val_t a, el_val_t b);
/* ── String additions ────────────────────────────────────────────────────── */
el_val_t str_index_of(el_val_t s, el_val_t sub);
@@ -493,6 +505,7 @@ el_val_t str_join(el_val_t list, el_val_t sep); /* alias of list_joi
el_val_t list_push(el_val_t list, el_val_t elem);
el_val_t list_push_front(el_val_t list, el_val_t elem);
el_val_t list_set(el_val_t list, el_val_t index, el_val_t value);
el_val_t list_join(el_val_t list, el_val_t sep);
el_val_t list_range(el_val_t start, el_val_t end);