neuron-technologies/el
3
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases 5 Wiki Activity

add BM25+ text ranking in EL, remove Ollama query-embedding dependency

Browse Source 
- Add list_set, math_exp, and float_add/sub/mul/div/gt/lt/eq/gte/lte builtins to
  el_runtime.c + el_runtime.h (float arithmetic builtins needed because EL operators
  +*/ operate on raw el_val_t bits, not IEEE 754 doubles)
- Remove engram_embed_query() and its forward declaration from el_runtime.c
- Remove Ollama cosine-similarity blend from activation scoring (reverts 9af2482):
  drops query_emb/query_edim variables, bias *= (1 + 0.3 * sim) block, and all
  free(query_emb) calls from the activation loop
- Implement BM25+ scoring in server.el (k1=1.2, b=0.75, delta=1.0):
  bm25_tokenize, bm25_count_term, bm25_score_doc, bm25_search_json
  V1 uses n_t=1 approximation (constant IDF per corpus size); acceptable as a
  first pass without an inverted index
- Wire /api/bm25/search POST/GET route in server.el dispatcher
- Zero Ollama calls in the activation/search path; embeddings on nodes are
  untouched (still written at node-creation time)
This commit is contained in:
Will Anderson
2026-05-14 11:52:43 -05:00
parent 1a8a16002e
commit 6121b33d25
5 changed files with 446 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engram/dist/engram
Vendored
BIN
View File
Binary file not shown.
engram/dist/engram.c
Vendored
+193 -4
View File
@@ -2,6 +2,10 @@
#include <stdlib.h>
#include "el_runtime.h"
el_val_t bm25_tokenize(el_val_t text);
el_val_t bm25_count_term(el_val_t term, el_val_t doc_tokens);
el_val_t bm25_score_doc(el_val_t doc_content, el_val_t query_tokens, el_val_t corpus_size, el_val_t avg_doc_len);
el_val_t bm25_search_json(el_val_t query, el_val_t limit);
el_val_t parse_port(el_val_t bind);
el_val_t ok_json(void);
el_val_t err_json(el_val_t msg);
@@ -41,6 +45,7 @@ el_val_t route_neuron_state_events(el_val_t method, el_val_t path, el_val_t body
el_val_t route_neuron_processes(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_events_next(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_events_ack(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_bm25_search(el_val_t method, el_val_t path, el_val_t body);
el_val_t check_auth_ok(el_val_t method, el_val_t body);
el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body);
@@ -50,6 +55,166 @@ el_val_t data_dir;
el_val_t db_path;
el_val_t loaded;
el_val_t bm25_tokenize(el_val_t text) {
el_val_t t = str_to_lower(text);
t = str_replace(t, EL_STR("."), EL_STR(" "));
t = str_replace(t, EL_STR(","), EL_STR(" "));
t = str_replace(t, EL_STR("!"), EL_STR(" "));
t = str_replace(t, EL_STR("?"), EL_STR(" "));
t = str_replace(t, EL_STR("\""), EL_STR(" "));
t = str_replace(t, EL_STR(":"), EL_STR(" "));
t = str_replace(t, EL_STR(";"), EL_STR(" "));
t = str_replace(t, EL_STR("("), EL_STR(" "));
t = str_replace(t, EL_STR(")"), EL_STR(" "));
t = str_replace(t, EL_STR("["), EL_STR(" "));
t = str_replace(t, EL_STR("]"), EL_STR(" "));
t = str_replace(t, EL_STR("{"), EL_STR(" "));
t = str_replace(t, EL_STR("}"), EL_STR(" "));
t = str_replace(t, EL_STR("/"), EL_STR(" "));
t = str_replace(t, EL_STR("\\"), EL_STR(" "));
t = str_replace(t, EL_STR("'"), EL_STR(" "));
t = str_replace(t, EL_STR("-"), EL_STR(" "));
t = str_replace(t, EL_STR("_"), EL_STR(" "));
return str_trim(t);
return 0;
}
el_val_t bm25_count_term(el_val_t term, el_val_t doc_tokens) {
el_val_t padded_term = el_str_concat(el_str_concat(EL_STR(" "), term), EL_STR(" "));
el_val_t padded_doc = el_str_concat(el_str_concat(EL_STR(" "), doc_tokens), EL_STR(" "));
return str_count(padded_doc, padded_term);
return 0;
}
el_val_t bm25_score_doc(el_val_t doc_content, el_val_t query_tokens, el_val_t corpus_size, el_val_t avg_doc_len) {
el_val_t k1 = el_from_float(1.2);
el_val_t b = el_from_float(0.75);
el_val_t delta = el_from_float(1.0);
el_val_t doc_tokens = bm25_tokenize(doc_content);
el_val_t doc_wc = str_count_words(doc_tokens);
if (doc_wc == 0) {
return EL_STR("0.0");
}
el_val_t doc_len = int_to_float(doc_wc);
el_val_t avg_len = str_to_float(avg_doc_len);
el_val_t N = int_to_float(corpus_size);
el_val_t idf_arg = float_add(float_div(float_add(N, el_from_float(1.2)), el_from_float(1.5)), el_from_float(1.0));
el_val_t idf = math_log(idf_arg);
el_val_t terms = str_split(query_tokens, EL_STR(" "));
el_val_t n_terms = len(terms);
el_val_t score = el_from_float(0.0);
el_val_t i = 0;
while (i < n_terms) {
el_val_t term = get(terms, i);
el_val_t tlen = str_len(term);
if (tlen >= 2) {
el_val_t tf_count = bm25_count_term(term, doc_tokens);
if (tf_count > 0) {
el_val_t tf_raw = int_to_float(tf_count);
el_val_t norm_factor = float_add(float_sub(el_from_float(1.0), b), float_div(float_mul(b, doc_len), avg_len));
el_val_t numerator = float_mul(tf_raw, float_add(k1, el_from_float(1.0)));
el_val_t denominator = float_add(tf_raw, float_mul(k1, norm_factor));
el_val_t tf_comp = float_add(delta, float_div(numerator, denominator));
score = float_add(score, float_mul(idf, tf_comp));
}
}
i = (i + 1);
}
return float_to_str(score);
return 0;
}
el_val_t bm25_search_json(el_val_t query, el_val_t limit) {
el_val_t scan_limit = (limit * 10);
if (scan_limit > 500) {
scan_limit = 500;
}
el_val_t nodes_json = engram_scan_nodes_json(scan_limit, 0);
el_val_t n = json_array_len(nodes_json);
if (n == 0) {
return EL_STR("[]");
}
el_val_t total_words = 0;
el_val_t i = 0;
while (i < n) {
el_val_t node = json_array_get(nodes_json, i);
el_val_t content = json_get_string(node, EL_STR("content"));
el_val_t tokens = bm25_tokenize(content);
el_val_t wc = str_count_words(tokens);
total_words = (total_words + wc);
i = (i + 1);
}
el_val_t avg_doc_len_f = float_div(int_to_float(total_words), int_to_float(n));
el_val_t avg_doc_len = ({ el_val_t _if_result_1 = 0; if (float_gt(avg_doc_len_f, el_from_float(0.0))) { _if_result_1 = (float_to_str(avg_doc_len_f)); } else { _if_result_1 = (EL_STR("1.0")); } _if_result_1; });
el_val_t query_tokens = bm25_tokenize(query);
if (str_eq(str_trim(query_tokens), EL_STR(""))) {
return EL_STR("[]");
}
el_val_t result_nodes = 0;
el_val_t result_scores = 0;
el_val_t result_count = 0;
el_val_t j = 0;
while (j < n) {
el_val_t node = json_array_get(nodes_json, j);
el_val_t content = json_get_string(node, EL_STR("content"));
el_val_t sc_str = bm25_score_doc(content, query_tokens, n, avg_doc_len);
if (!str_eq(sc_str, EL_STR("0.0"))) {
if (!str_eq(sc_str, EL_STR(""))) {
result_nodes = list_push(result_nodes, node);
result_scores = list_push(result_scores, sc_str);
result_count = (result_count + 1);
}
}
j = (j + 1);
}
if (result_count == 0) {
return EL_STR("[]");
}
el_val_t out_limit = ({ el_val_t _if_result_2 = 0; if ((result_count < limit)) { _if_result_2 = (result_count); } else { _if_result_2 = (limit); } _if_result_2; });
el_val_t k = 0;
while (k < out_limit) {
el_val_t max_idx = k;
el_val_t max_sc_str = get(result_scores, k);
el_val_t max_sc_f = str_to_float(max_sc_str);
el_val_t p = (k + 1);
while (p < result_count) {
el_val_t sc2_str = get(result_scores, p);
el_val_t sc2_f = str_to_float(sc2_str);
if (float_gt(sc2_f, max_sc_f)) {
max_sc_f = sc2_f;
max_sc_str = sc2_str;
max_idx = p;
}
p = (p + 1);
}
if (max_idx != k) {
el_val_t tmp_node = get(result_nodes, k);
el_val_t tmp_sc = get(result_scores, k);
result_nodes = list_set(result_nodes, k, get(result_nodes, max_idx));
result_scores = list_set(result_scores, k, get(result_scores, max_idx));
result_nodes = list_set(result_nodes, max_idx, tmp_node);
result_scores = list_set(result_scores, max_idx, tmp_sc);
}
k = (k + 1);
}
el_val_t out = EL_STR("[");
el_val_t r = 0;
while (r < out_limit) {
el_val_t node = get(result_nodes, r);
el_val_t sc_str = get(result_scores, r);
el_val_t node_len = str_len(node);
el_val_t node_body = str_slice(node, 0, (node_len - 1));
el_val_t entry = el_str_concat(el_str_concat(el_str_concat(node_body, EL_STR(",\"bm25_score\":")), sc_str), EL_STR("}"));
if (r > 0) {
out = el_str_concat(out, EL_STR(","));
}
out = el_str_concat(out, entry);
r = (r + 1);
}
return el_str_concat(out, EL_STR("]"));
return 0;
}
el_val_t parse_port(el_val_t bind) {
el_val_t colon = str_index_of(bind, EL_STR(":"));
if (colon < 0) {
@@ -371,7 +536,7 @@ el_val_t route_neuron_session_begin(el_val_t method, el_val_t path, el_val_t bod
el_val_t route_neuron_ctx(el_val_t method, el_val_t path, el_val_t body) {
el_val_t results = engram_activate_json(EL_STR("architecture decision memory"), 2);
el_val_t n = json_array_len(results);
el_val_t limit = ({ el_val_t _if_result_1 = 0; if ((n > 10)) { _if_result_1 = (10); } else { _if_result_1 = (n); } _if_result_1; });
el_val_t limit = ({ el_val_t _if_result_3 = 0; if ((n > 10)) { _if_result_3 = (10); } else { _if_result_3 = (n); } _if_result_3; });
el_val_t ctx = EL_STR("Recent working memory:\n");
el_val_t i = 0;
el_val_t ctx_body = EL_STR("");
@@ -380,7 +545,7 @@ el_val_t route_neuron_ctx(el_val_t method, el_val_t path, el_val_t body) {
el_val_t label = json_get_string(elem, EL_STR("label"));
el_val_t content = json_get_string(elem, EL_STR("content"));
el_val_t clen = str_len(content);
el_val_t snippet = ({ el_val_t _if_result_2 = 0; if ((clen > 200)) { _if_result_2 = (str_slice(content, 0, 200)); } else { _if_result_2 = (content); } _if_result_2; });
el_val_t snippet = ({ el_val_t _if_result_4 = 0; if ((clen > 200)) { _if_result_4 = (str_slice(content, 0, 200)); } else { _if_result_4 = (content); } _if_result_4; });
ctx_body = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(ctx_body, EL_STR("- [")), label), EL_STR("]: ")), snippet), EL_STR("\n"));
i = (i + 1);
}
@@ -518,7 +683,7 @@ el_val_t route_neuron_recall(el_val_t method, el_val_t path, el_val_t body) {
if (limit == 0) {
limit = 20;
}
el_val_t q = ({ el_val_t _if_result_3 = 0; if (str_eq(query, EL_STR(""))) { _if_result_3 = (chain); } else { _if_result_3 = (query); } _if_result_3; });
el_val_t q = ({ el_val_t _if_result_5 = 0; if (str_eq(query, EL_STR(""))) { _if_result_5 = (chain); } else { _if_result_5 = (query); } _if_result_5; });
if (str_eq(q, EL_STR(""))) {
return engram_scan_nodes_json(limit, 0);
}
@@ -589,7 +754,7 @@ el_val_t route_neuron_config(el_val_t method, el_val_t path, el_val_t body) {
el_val_t route_neuron_state_events(el_val_t method, el_val_t path, el_val_t body) {
if (str_eq(method, EL_STR("GET"))) {
el_val_t limit_str = query_param(path, EL_STR("limit"));
el_val_t limit = ({ el_val_t _if_result_4 = 0; if (str_eq(limit_str, EL_STR(""))) { _if_result_4 = (50); } else { _if_result_4 = (str_to_int(limit_str)); } _if_result_4; });
el_val_t limit = ({ el_val_t _if_result_6 = 0; if (str_eq(limit_str, EL_STR(""))) { _if_result_6 = (50); } else { _if_result_6 = (str_to_int(limit_str)); } _if_result_6; });
return engram_scan_nodes_by_type_json(EL_STR("InternalStateEvent"), limit, 0);
}
el_val_t content = json_get_string(body, EL_STR("content"));
@@ -616,6 +781,27 @@ el_val_t route_events_ack(el_val_t method, el_val_t path, el_val_t body) {
return 0;
}
el_val_t route_bm25_search(el_val_t method, el_val_t path, el_val_t body) {
el_val_t q = EL_STR("");
if (str_eq(method, EL_STR("GET"))) {
q = query_param(path, EL_STR("q"));
} else {
q = json_get_string(body, EL_STR("query"));
}
if (str_eq(q, EL_STR(""))) {
return EL_STR("{\"error\":\"query is required\"}");
}
el_val_t limit = query_int(path, EL_STR("limit"), 20);
if (limit == 0) {
limit = json_get_int(body, EL_STR("limit"));
}
if (limit == 0) {
limit = 20;
}
return bm25_search_json(q, limit);
return 0;
}
el_val_t check_auth_ok(el_val_t method, el_val_t body) {
el_val_t key = env(EL_STR("ENGRAM_API_KEY"));
if (str_eq(key, EL_STR(""))) {
@@ -732,6 +918,9 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
if (str_eq(method, EL_STR("GET")) && str_starts_with(clean, EL_STR("/api/search"))) {
return route_search(method, path, body);
}
if (str_eq(clean, EL_STR("/api/bm25/search"))) {
return route_bm25_search(method, path, body);
}
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/strengthen")) || str_eq(clean, EL_STR("/strengthen")))) {
return route_strengthen(method, path, body);
}