add BM25+ text ranking in EL, remove Ollama query-embedding dependency
- Add list_set, math_exp, and float_add/sub/mul/div/gt/lt/eq/gte/lte builtins to el_runtime.c + el_runtime.h (float arithmetic builtins needed because EL operators +*/ operate on raw el_val_t bits, not IEEE 754 doubles) - Remove engram_embed_query() and its forward declaration from el_runtime.c - Remove Ollama cosine-similarity blend from activation scoring (reverts 9af2482): drops query_emb/query_edim variables, bias *= (1 + 0.3 * sim) block, and all free(query_emb) calls from the activation loop - Implement BM25+ scoring in server.el (k1=1.2, b=0.75, delta=1.0): bm25_tokenize, bm25_count_term, bm25_score_doc, bm25_search_json V1 uses n_t=1 approximation (constant IDF per corpus size); acceptable as a first pass without an inverted index - Wire /api/bm25/search POST/GET route in server.el dispatcher - Zero Ollama calls in the activation/search path; embeddings on nodes are untouched (still written at node-creation time)
This commit is contained in:
@@ -23,6 +23,202 @@
|
||||
// ENGRAM_API_KEY — bearer auth (optional)
|
||||
// ENGRAM_DATA_DIR — snapshot location (default ~/.neuron/engram)
|
||||
|
||||
// ── BM25+ text ranking ────────────────────────────────────────────────────────
|
||||
//
|
||||
// Implements BM25+ (Lv & Zhai 2011) for in-process keyword search over the
|
||||
// engram node store. No external dependencies — pure EL, zero Ollama calls.
|
||||
//
|
||||
// Parameters: k1=1.2, b=0.75, delta=1.0
|
||||
//
|
||||
// V1 simplification: n(t) (number of docs containing term t) is approximated
|
||||
// as 1 for all terms. This collapses IDF to a constant per corpus size:
|
||||
// IDF = ln((N - 1 + 0.5) / (1 + 0.5) + 1) = ln((N + 0.5) / 1.5 + 1)
|
||||
// Scoring effectively becomes TF-length-normalised BM25+ (delta term present).
|
||||
// Acceptable for V1; a real inverted index can replace this later.
|
||||
|
||||
fn bm25_tokenize(text: String) -> String {
|
||||
// Lowercase and strip punctuation (replace with spaces), then trim.
|
||||
let t: String = str_to_lower(text)
|
||||
let t = str_replace(t, ".", " ")
|
||||
let t = str_replace(t, ",", " ")
|
||||
let t = str_replace(t, "!", " ")
|
||||
let t = str_replace(t, "?", " ")
|
||||
let t = str_replace(t, "\"", " ")
|
||||
let t = str_replace(t, ":", " ")
|
||||
let t = str_replace(t, ";", " ")
|
||||
let t = str_replace(t, "(", " ")
|
||||
let t = str_replace(t, ")", " ")
|
||||
let t = str_replace(t, "[", " ")
|
||||
let t = str_replace(t, "]", " ")
|
||||
let t = str_replace(t, "{", " ")
|
||||
let t = str_replace(t, "}", " ")
|
||||
let t = str_replace(t, "/", " ")
|
||||
let t = str_replace(t, "\\", " ")
|
||||
let t = str_replace(t, "'", " ")
|
||||
let t = str_replace(t, "-", " ")
|
||||
let t = str_replace(t, "_", " ")
|
||||
str_trim(t)
|
||||
}
|
||||
|
||||
fn bm25_count_term(term: String, doc_tokens: String) -> Int {
|
||||
// Pad with spaces to avoid prefix/suffix partial matches.
|
||||
let padded_term: String = " " + term + " "
|
||||
let padded_doc: String = " " + doc_tokens + " "
|
||||
str_count(padded_doc, padded_term)
|
||||
}
|
||||
|
||||
fn bm25_score_doc(doc_content: String, query_tokens: String, corpus_size: Int, avg_doc_len: String) -> String {
|
||||
// BM25+ parameters (stored as strings = float-encoded el_val_t from el_from_float)
|
||||
// We use float_add/float_mul/float_div builtins to avoid EL operator issues.
|
||||
// avg_doc_len is passed as a String slot holding an el_val_t float bit-pattern.
|
||||
// (EL has no safe float-passing convention; we work around using str_to_float.)
|
||||
//
|
||||
// V1: n_t=1 for all terms. IDF = ln((N+0.5)/1.5 + 1) = constant per corpus.
|
||||
// This collapses BM25+ to TF-length-normalised scoring — acceptable for V1.
|
||||
let k1: Float = 1.2
|
||||
let b: Float = 0.75
|
||||
let delta: Float = 1.0
|
||||
|
||||
let doc_tokens: String = bm25_tokenize(doc_content)
|
||||
let doc_wc: Int = str_count_words(doc_tokens)
|
||||
if doc_wc == 0 { return "0.0" }
|
||||
|
||||
let doc_len: Float = int_to_float(doc_wc)
|
||||
let avg_len: Float = str_to_float(avg_doc_len)
|
||||
|
||||
// IDF constant
|
||||
let N: Float = int_to_float(corpus_size)
|
||||
// (N + 0.5) / 1.5 + 1.0
|
||||
let idf_arg: Float = float_add(float_div(float_add(N, 1.2), 1.5), 1.0)
|
||||
let idf: Float = math_log(idf_arg)
|
||||
|
||||
// Sum TF component over query terms
|
||||
let terms: List = str_split(query_tokens, " ")
|
||||
let n_terms: Int = len(terms)
|
||||
let score: Float = 0.0
|
||||
let i: Int = 0
|
||||
while i < n_terms {
|
||||
let term: String = get(terms, i)
|
||||
let tlen: Int = str_len(term)
|
||||
if tlen >= 2 {
|
||||
let tf_count: Int = bm25_count_term(term, doc_tokens)
|
||||
if tf_count > 0 {
|
||||
let tf_raw: Float = int_to_float(tf_count)
|
||||
// norm_factor = 1 - b + b * doc_len / avg_len
|
||||
let norm_factor: Float = float_add(float_sub(1.0, b), float_div(float_mul(b, doc_len), avg_len))
|
||||
// tf_comp = delta + tf * (k1+1) / (tf + k1*norm)
|
||||
let numerator: Float = float_mul(tf_raw, float_add(k1, 1.0))
|
||||
let denominator: Float = float_add(tf_raw, float_mul(k1, norm_factor))
|
||||
let tf_comp: Float = float_add(delta, float_div(numerator, denominator))
|
||||
let score = float_add(score, float_mul(idf, tf_comp))
|
||||
}
|
||||
}
|
||||
let i = i + 1
|
||||
}
|
||||
// Return score as a string so it survives EL's lack of float-in-list support
|
||||
float_to_str(score)
|
||||
}
|
||||
|
||||
fn bm25_search_json(query: String, limit: Int) -> String {
|
||||
// 1. Determine scan size (fetch 10x or up to 500 nodes)
|
||||
let scan_limit: Int = limit * 10
|
||||
if scan_limit > 500 { let scan_limit = 500 }
|
||||
|
||||
// 2. Fetch node sample
|
||||
let nodes_json: String = engram_scan_nodes_json(scan_limit, 0)
|
||||
let n: Int = json_array_len(nodes_json)
|
||||
if n == 0 { return "[]" }
|
||||
|
||||
// 3. Compute avg_doc_len from sample
|
||||
let total_words: Int = 0
|
||||
let i: Int = 0
|
||||
while i < n {
|
||||
let node: String = json_array_get(nodes_json, i)
|
||||
let content: String = json_get_string(node, "content")
|
||||
let tokens: String = bm25_tokenize(content)
|
||||
let wc: Int = str_count_words(tokens)
|
||||
let total_words = total_words + wc
|
||||
let i = i + 1
|
||||
}
|
||||
// avg_doc_len as string for safe float passing
|
||||
let avg_doc_len_f: Float = float_div(int_to_float(total_words), int_to_float(n))
|
||||
let avg_doc_len: String = if float_gt(avg_doc_len_f, 0.0) { float_to_str(avg_doc_len_f) } else { "1.0" }
|
||||
|
||||
// 4. Tokenize query
|
||||
let query_tokens: String = bm25_tokenize(query)
|
||||
if str_eq(str_trim(query_tokens), "") { return "[]" }
|
||||
|
||||
// 5. Score each node; collect results as parallel JSON and score lists.
|
||||
// Scores are stored as strings (float_to_str) to avoid float-in-list issues.
|
||||
let result_nodes: List = 0
|
||||
let result_scores: List = 0
|
||||
let result_count: Int = 0
|
||||
let j: Int = 0
|
||||
while j < n {
|
||||
let node: String = json_array_get(nodes_json, j)
|
||||
let content: String = json_get_string(node, "content")
|
||||
let sc_str: String = bm25_score_doc(content, query_tokens, n, avg_doc_len)
|
||||
// Only include nodes with score > 0 (str check: not "0.0" and not empty)
|
||||
if !str_eq(sc_str, "0.0") {
|
||||
if !str_eq(sc_str, "") {
|
||||
let result_nodes = list_push(result_nodes, node)
|
||||
let result_scores = list_push(result_scores, sc_str)
|
||||
let result_count = result_count + 1
|
||||
}
|
||||
}
|
||||
let j = j + 1
|
||||
}
|
||||
|
||||
if result_count == 0 { return "[]" }
|
||||
|
||||
// 6. Selection-sort descending by score, take top `limit`
|
||||
let out_limit: Int = if result_count < limit { result_count } else { limit }
|
||||
let k: Int = 0
|
||||
while k < out_limit {
|
||||
// Find max score index in [k, result_count)
|
||||
let max_idx: Int = k
|
||||
let max_sc_str: String = get(result_scores, k)
|
||||
let max_sc_f: Float = str_to_float(max_sc_str)
|
||||
let p: Int = k + 1
|
||||
while p < result_count {
|
||||
let sc2_str: String = get(result_scores, p)
|
||||
let sc2_f: Float = str_to_float(sc2_str)
|
||||
if float_gt(sc2_f, max_sc_f) {
|
||||
let max_sc_f = sc2_f
|
||||
let max_sc_str = sc2_str
|
||||
let max_idx = p
|
||||
}
|
||||
let p = p + 1
|
||||
}
|
||||
// Swap k <-> max_idx
|
||||
if max_idx != k {
|
||||
let tmp_node: String = get(result_nodes, k)
|
||||
let tmp_sc: String = get(result_scores, k)
|
||||
let result_nodes = list_set(result_nodes, k, get(result_nodes, max_idx))
|
||||
let result_scores = list_set(result_scores, k, get(result_scores, max_idx))
|
||||
let result_nodes = list_set(result_nodes, max_idx, tmp_node)
|
||||
let result_scores = list_set(result_scores, max_idx, tmp_sc)
|
||||
}
|
||||
let k = k + 1
|
||||
}
|
||||
|
||||
// 7. Build JSON array of top `out_limit` nodes with bm25_score field
|
||||
let out: String = "["
|
||||
let r: Int = 0
|
||||
while r < out_limit {
|
||||
let node: String = get(result_nodes, r)
|
||||
let sc_str: String = get(result_scores, r)
|
||||
// Inject bm25_score: trim the closing } and append field
|
||||
let node_len: Int = str_len(node)
|
||||
let node_body: String = str_slice(node, 0, node_len - 1)
|
||||
let entry: String = node_body + ",\"bm25_score\":" + sc_str + "}"
|
||||
if r > 0 { let out = out + "," }
|
||||
let out = out + entry
|
||||
let r = r + 1
|
||||
}
|
||||
out + "]"
|
||||
}
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
fn parse_port(bind: String) -> Int {
|
||||
@@ -521,6 +717,20 @@ fn route_events_ack(method: String, path: String, body: String) -> String {
|
||||
"{\"ok\":true}"
|
||||
}
|
||||
|
||||
fn route_bm25_search(method: String, path: String, body: String) -> String {
|
||||
let q: String = ""
|
||||
if str_eq(method, "GET") {
|
||||
let q = query_param(path, "q")
|
||||
} else {
|
||||
let q = json_get_string(body, "query")
|
||||
}
|
||||
if str_eq(q, "") { return "{\"error\":\"query is required\"}" }
|
||||
let limit: Int = query_int(path, "limit", 20)
|
||||
if limit == 0 { let limit = json_get_int(body, "limit") }
|
||||
if limit == 0 { let limit = 20 }
|
||||
bm25_search_json(q, limit)
|
||||
}
|
||||
|
||||
// ── Auth ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn check_auth_ok(method: String, body: String) -> Bool {
|
||||
@@ -653,6 +863,11 @@ fn handle_request(method: String, path: String, body: String) -> String {
|
||||
return route_search(method, path, body)
|
||||
}
|
||||
|
||||
// BM25+ text ranking
|
||||
if str_eq(clean, "/api/bm25/search") {
|
||||
return route_bm25_search(method, path, body)
|
||||
}
|
||||
|
||||
// Strengthen
|
||||
if str_eq(method, "POST") && (str_eq(clean, "/api/strengthen") || str_eq(clean, "/strengthen")) {
|
||||
return route_strengthen(method, path, body)
|
||||
|
||||
Reference in New Issue
Block a user