diff --git a/engram/dist/engram b/engram/dist/engram index 23e3e1a..9910b3c 100755 Binary files a/engram/dist/engram and b/engram/dist/engram differ diff --git a/engram/dist/engram.c b/engram/dist/engram.c index f58b7f3..a1bbd6d 100644 --- a/engram/dist/engram.c +++ b/engram/dist/engram.c @@ -2,6 +2,10 @@ #include #include "el_runtime.h" +el_val_t bm25_tokenize(el_val_t text); +el_val_t bm25_count_term(el_val_t term, el_val_t doc_tokens); +el_val_t bm25_score_doc(el_val_t doc_content, el_val_t query_tokens, el_val_t corpus_size, el_val_t avg_doc_len); +el_val_t bm25_search_json(el_val_t query, el_val_t limit); el_val_t parse_port(el_val_t bind); el_val_t ok_json(void); el_val_t err_json(el_val_t msg); @@ -41,6 +45,7 @@ el_val_t route_neuron_state_events(el_val_t method, el_val_t path, el_val_t body el_val_t route_neuron_processes(el_val_t method, el_val_t path, el_val_t body); el_val_t route_events_next(el_val_t method, el_val_t path, el_val_t body); el_val_t route_events_ack(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_bm25_search(el_val_t method, el_val_t path, el_val_t body); el_val_t check_auth_ok(el_val_t method, el_val_t body); el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body); @@ -50,6 +55,166 @@ el_val_t data_dir; el_val_t db_path; el_val_t loaded; +el_val_t bm25_tokenize(el_val_t text) { + el_val_t t = str_to_lower(text); + t = str_replace(t, EL_STR("."), EL_STR(" ")); + t = str_replace(t, EL_STR(","), EL_STR(" ")); + t = str_replace(t, EL_STR("!"), EL_STR(" ")); + t = str_replace(t, EL_STR("?"), EL_STR(" ")); + t = str_replace(t, EL_STR("\""), EL_STR(" ")); + t = str_replace(t, EL_STR(":"), EL_STR(" ")); + t = str_replace(t, EL_STR(";"), EL_STR(" ")); + t = str_replace(t, EL_STR("("), EL_STR(" ")); + t = str_replace(t, EL_STR(")"), EL_STR(" ")); + t = str_replace(t, EL_STR("["), EL_STR(" ")); + t = str_replace(t, EL_STR("]"), EL_STR(" ")); + t = str_replace(t, EL_STR("{"), EL_STR(" ")); + t = str_replace(t, EL_STR("}"), EL_STR(" ")); + t = str_replace(t, EL_STR("/"), EL_STR(" ")); + t = str_replace(t, EL_STR("\\"), EL_STR(" ")); + t = str_replace(t, EL_STR("'"), EL_STR(" ")); + t = str_replace(t, EL_STR("-"), EL_STR(" ")); + t = str_replace(t, EL_STR("_"), EL_STR(" ")); + return str_trim(t); + return 0; +} + +el_val_t bm25_count_term(el_val_t term, el_val_t doc_tokens) { + el_val_t padded_term = el_str_concat(el_str_concat(EL_STR(" "), term), EL_STR(" ")); + el_val_t padded_doc = el_str_concat(el_str_concat(EL_STR(" "), doc_tokens), EL_STR(" ")); + return str_count(padded_doc, padded_term); + return 0; +} + +el_val_t bm25_score_doc(el_val_t doc_content, el_val_t query_tokens, el_val_t corpus_size, el_val_t avg_doc_len) { + el_val_t k1 = el_from_float(1.2); + el_val_t b = el_from_float(0.75); + el_val_t delta = el_from_float(1.0); + el_val_t doc_tokens = bm25_tokenize(doc_content); + el_val_t doc_wc = str_count_words(doc_tokens); + if (doc_wc == 0) { + return EL_STR("0.0"); + } + el_val_t doc_len = int_to_float(doc_wc); + el_val_t avg_len = str_to_float(avg_doc_len); + el_val_t N = int_to_float(corpus_size); + el_val_t idf_arg = float_add(float_div(float_add(N, el_from_float(1.2)), el_from_float(1.5)), el_from_float(1.0)); + el_val_t idf = math_log(idf_arg); + el_val_t terms = str_split(query_tokens, EL_STR(" ")); + el_val_t n_terms = len(terms); + el_val_t score = el_from_float(0.0); + el_val_t i = 0; + while (i < n_terms) { + el_val_t term = get(terms, i); + el_val_t tlen = str_len(term); + if (tlen >= 2) { + el_val_t tf_count = bm25_count_term(term, doc_tokens); + if (tf_count > 0) { + el_val_t tf_raw = int_to_float(tf_count); + el_val_t norm_factor = float_add(float_sub(el_from_float(1.0), b), float_div(float_mul(b, doc_len), avg_len)); + el_val_t numerator = float_mul(tf_raw, float_add(k1, el_from_float(1.0))); + el_val_t denominator = float_add(tf_raw, float_mul(k1, norm_factor)); + el_val_t tf_comp = float_add(delta, float_div(numerator, denominator)); + score = float_add(score, float_mul(idf, tf_comp)); + } + } + i = (i + 1); + } + return float_to_str(score); + return 0; +} + +el_val_t bm25_search_json(el_val_t query, el_val_t limit) { + el_val_t scan_limit = (limit * 10); + if (scan_limit > 500) { + scan_limit = 500; + } + el_val_t nodes_json = engram_scan_nodes_json(scan_limit, 0); + el_val_t n = json_array_len(nodes_json); + if (n == 0) { + return EL_STR("[]"); + } + el_val_t total_words = 0; + el_val_t i = 0; + while (i < n) { + el_val_t node = json_array_get(nodes_json, i); + el_val_t content = json_get_string(node, EL_STR("content")); + el_val_t tokens = bm25_tokenize(content); + el_val_t wc = str_count_words(tokens); + total_words = (total_words + wc); + i = (i + 1); + } + el_val_t avg_doc_len_f = float_div(int_to_float(total_words), int_to_float(n)); + el_val_t avg_doc_len = ({ el_val_t _if_result_1 = 0; if (float_gt(avg_doc_len_f, el_from_float(0.0))) { _if_result_1 = (float_to_str(avg_doc_len_f)); } else { _if_result_1 = (EL_STR("1.0")); } _if_result_1; }); + el_val_t query_tokens = bm25_tokenize(query); + if (str_eq(str_trim(query_tokens), EL_STR(""))) { + return EL_STR("[]"); + } + el_val_t result_nodes = 0; + el_val_t result_scores = 0; + el_val_t result_count = 0; + el_val_t j = 0; + while (j < n) { + el_val_t node = json_array_get(nodes_json, j); + el_val_t content = json_get_string(node, EL_STR("content")); + el_val_t sc_str = bm25_score_doc(content, query_tokens, n, avg_doc_len); + if (!str_eq(sc_str, EL_STR("0.0"))) { + if (!str_eq(sc_str, EL_STR(""))) { + result_nodes = list_push(result_nodes, node); + result_scores = list_push(result_scores, sc_str); + result_count = (result_count + 1); + } + } + j = (j + 1); + } + if (result_count == 0) { + return EL_STR("[]"); + } + el_val_t out_limit = ({ el_val_t _if_result_2 = 0; if ((result_count < limit)) { _if_result_2 = (result_count); } else { _if_result_2 = (limit); } _if_result_2; }); + el_val_t k = 0; + while (k < out_limit) { + el_val_t max_idx = k; + el_val_t max_sc_str = get(result_scores, k); + el_val_t max_sc_f = str_to_float(max_sc_str); + el_val_t p = (k + 1); + while (p < result_count) { + el_val_t sc2_str = get(result_scores, p); + el_val_t sc2_f = str_to_float(sc2_str); + if (float_gt(sc2_f, max_sc_f)) { + max_sc_f = sc2_f; + max_sc_str = sc2_str; + max_idx = p; + } + p = (p + 1); + } + if (max_idx != k) { + el_val_t tmp_node = get(result_nodes, k); + el_val_t tmp_sc = get(result_scores, k); + result_nodes = list_set(result_nodes, k, get(result_nodes, max_idx)); + result_scores = list_set(result_scores, k, get(result_scores, max_idx)); + result_nodes = list_set(result_nodes, max_idx, tmp_node); + result_scores = list_set(result_scores, max_idx, tmp_sc); + } + k = (k + 1); + } + el_val_t out = EL_STR("["); + el_val_t r = 0; + while (r < out_limit) { + el_val_t node = get(result_nodes, r); + el_val_t sc_str = get(result_scores, r); + el_val_t node_len = str_len(node); + el_val_t node_body = str_slice(node, 0, (node_len - 1)); + el_val_t entry = el_str_concat(el_str_concat(el_str_concat(node_body, EL_STR(",\"bm25_score\":")), sc_str), EL_STR("}")); + if (r > 0) { + out = el_str_concat(out, EL_STR(",")); + } + out = el_str_concat(out, entry); + r = (r + 1); + } + return el_str_concat(out, EL_STR("]")); + return 0; +} + el_val_t parse_port(el_val_t bind) { el_val_t colon = str_index_of(bind, EL_STR(":")); if (colon < 0) { @@ -371,7 +536,7 @@ el_val_t route_neuron_session_begin(el_val_t method, el_val_t path, el_val_t bod el_val_t route_neuron_ctx(el_val_t method, el_val_t path, el_val_t body) { el_val_t results = engram_activate_json(EL_STR("architecture decision memory"), 2); el_val_t n = json_array_len(results); - el_val_t limit = ({ el_val_t _if_result_1 = 0; if ((n > 10)) { _if_result_1 = (10); } else { _if_result_1 = (n); } _if_result_1; }); + el_val_t limit = ({ el_val_t _if_result_3 = 0; if ((n > 10)) { _if_result_3 = (10); } else { _if_result_3 = (n); } _if_result_3; }); el_val_t ctx = EL_STR("Recent working memory:\n"); el_val_t i = 0; el_val_t ctx_body = EL_STR(""); @@ -380,7 +545,7 @@ el_val_t route_neuron_ctx(el_val_t method, el_val_t path, el_val_t body) { el_val_t label = json_get_string(elem, EL_STR("label")); el_val_t content = json_get_string(elem, EL_STR("content")); el_val_t clen = str_len(content); - el_val_t snippet = ({ el_val_t _if_result_2 = 0; if ((clen > 200)) { _if_result_2 = (str_slice(content, 0, 200)); } else { _if_result_2 = (content); } _if_result_2; }); + el_val_t snippet = ({ el_val_t _if_result_4 = 0; if ((clen > 200)) { _if_result_4 = (str_slice(content, 0, 200)); } else { _if_result_4 = (content); } _if_result_4; }); ctx_body = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(ctx_body, EL_STR("- [")), label), EL_STR("]: ")), snippet), EL_STR("\n")); i = (i + 1); } @@ -518,7 +683,7 @@ el_val_t route_neuron_recall(el_val_t method, el_val_t path, el_val_t body) { if (limit == 0) { limit = 20; } - el_val_t q = ({ el_val_t _if_result_3 = 0; if (str_eq(query, EL_STR(""))) { _if_result_3 = (chain); } else { _if_result_3 = (query); } _if_result_3; }); + el_val_t q = ({ el_val_t _if_result_5 = 0; if (str_eq(query, EL_STR(""))) { _if_result_5 = (chain); } else { _if_result_5 = (query); } _if_result_5; }); if (str_eq(q, EL_STR(""))) { return engram_scan_nodes_json(limit, 0); } @@ -589,7 +754,7 @@ el_val_t route_neuron_config(el_val_t method, el_val_t path, el_val_t body) { el_val_t route_neuron_state_events(el_val_t method, el_val_t path, el_val_t body) { if (str_eq(method, EL_STR("GET"))) { el_val_t limit_str = query_param(path, EL_STR("limit")); - el_val_t limit = ({ el_val_t _if_result_4 = 0; if (str_eq(limit_str, EL_STR(""))) { _if_result_4 = (50); } else { _if_result_4 = (str_to_int(limit_str)); } _if_result_4; }); + el_val_t limit = ({ el_val_t _if_result_6 = 0; if (str_eq(limit_str, EL_STR(""))) { _if_result_6 = (50); } else { _if_result_6 = (str_to_int(limit_str)); } _if_result_6; }); return engram_scan_nodes_by_type_json(EL_STR("InternalStateEvent"), limit, 0); } el_val_t content = json_get_string(body, EL_STR("content")); @@ -616,6 +781,27 @@ el_val_t route_events_ack(el_val_t method, el_val_t path, el_val_t body) { return 0; } +el_val_t route_bm25_search(el_val_t method, el_val_t path, el_val_t body) { + el_val_t q = EL_STR(""); + if (str_eq(method, EL_STR("GET"))) { + q = query_param(path, EL_STR("q")); + } else { + q = json_get_string(body, EL_STR("query")); + } + if (str_eq(q, EL_STR(""))) { + return EL_STR("{\"error\":\"query is required\"}"); + } + el_val_t limit = query_int(path, EL_STR("limit"), 20); + if (limit == 0) { + limit = json_get_int(body, EL_STR("limit")); + } + if (limit == 0) { + limit = 20; + } + return bm25_search_json(q, limit); + return 0; +} + el_val_t check_auth_ok(el_val_t method, el_val_t body) { el_val_t key = env(EL_STR("ENGRAM_API_KEY")); if (str_eq(key, EL_STR(""))) { @@ -732,6 +918,9 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) { if (str_eq(method, EL_STR("GET")) && str_starts_with(clean, EL_STR("/api/search"))) { return route_search(method, path, body); } + if (str_eq(clean, EL_STR("/api/bm25/search"))) { + return route_bm25_search(method, path, body); + } if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/strengthen")) || str_eq(clean, EL_STR("/strengthen")))) { return route_strengthen(method, path, body); } diff --git a/engram/src/server.el b/engram/src/server.el index bea33e0..2f5f5b7 100644 --- a/engram/src/server.el +++ b/engram/src/server.el @@ -23,6 +23,202 @@ // ENGRAM_API_KEY — bearer auth (optional) // ENGRAM_DATA_DIR — snapshot location (default ~/.neuron/engram) +// ── BM25+ text ranking ──────────────────────────────────────────────────────── +// +// Implements BM25+ (Lv & Zhai 2011) for in-process keyword search over the +// engram node store. No external dependencies — pure EL, zero Ollama calls. +// +// Parameters: k1=1.2, b=0.75, delta=1.0 +// +// V1 simplification: n(t) (number of docs containing term t) is approximated +// as 1 for all terms. This collapses IDF to a constant per corpus size: +// IDF = ln((N - 1 + 0.5) / (1 + 0.5) + 1) = ln((N + 0.5) / 1.5 + 1) +// Scoring effectively becomes TF-length-normalised BM25+ (delta term present). +// Acceptable for V1; a real inverted index can replace this later. + +fn bm25_tokenize(text: String) -> String { + // Lowercase and strip punctuation (replace with spaces), then trim. + let t: String = str_to_lower(text) + let t = str_replace(t, ".", " ") + let t = str_replace(t, ",", " ") + let t = str_replace(t, "!", " ") + let t = str_replace(t, "?", " ") + let t = str_replace(t, "\"", " ") + let t = str_replace(t, ":", " ") + let t = str_replace(t, ";", " ") + let t = str_replace(t, "(", " ") + let t = str_replace(t, ")", " ") + let t = str_replace(t, "[", " ") + let t = str_replace(t, "]", " ") + let t = str_replace(t, "{", " ") + let t = str_replace(t, "}", " ") + let t = str_replace(t, "/", " ") + let t = str_replace(t, "\\", " ") + let t = str_replace(t, "'", " ") + let t = str_replace(t, "-", " ") + let t = str_replace(t, "_", " ") + str_trim(t) +} + +fn bm25_count_term(term: String, doc_tokens: String) -> Int { + // Pad with spaces to avoid prefix/suffix partial matches. + let padded_term: String = " " + term + " " + let padded_doc: String = " " + doc_tokens + " " + str_count(padded_doc, padded_term) +} + +fn bm25_score_doc(doc_content: String, query_tokens: String, corpus_size: Int, avg_doc_len: String) -> String { + // BM25+ parameters (stored as strings = float-encoded el_val_t from el_from_float) + // We use float_add/float_mul/float_div builtins to avoid EL operator issues. + // avg_doc_len is passed as a String slot holding an el_val_t float bit-pattern. + // (EL has no safe float-passing convention; we work around using str_to_float.) + // + // V1: n_t=1 for all terms. IDF = ln((N+0.5)/1.5 + 1) = constant per corpus. + // This collapses BM25+ to TF-length-normalised scoring — acceptable for V1. + let k1: Float = 1.2 + let b: Float = 0.75 + let delta: Float = 1.0 + + let doc_tokens: String = bm25_tokenize(doc_content) + let doc_wc: Int = str_count_words(doc_tokens) + if doc_wc == 0 { return "0.0" } + + let doc_len: Float = int_to_float(doc_wc) + let avg_len: Float = str_to_float(avg_doc_len) + + // IDF constant + let N: Float = int_to_float(corpus_size) + // (N + 0.5) / 1.5 + 1.0 + let idf_arg: Float = float_add(float_div(float_add(N, 1.2), 1.5), 1.0) + let idf: Float = math_log(idf_arg) + + // Sum TF component over query terms + let terms: List = str_split(query_tokens, " ") + let n_terms: Int = len(terms) + let score: Float = 0.0 + let i: Int = 0 + while i < n_terms { + let term: String = get(terms, i) + let tlen: Int = str_len(term) + if tlen >= 2 { + let tf_count: Int = bm25_count_term(term, doc_tokens) + if tf_count > 0 { + let tf_raw: Float = int_to_float(tf_count) + // norm_factor = 1 - b + b * doc_len / avg_len + let norm_factor: Float = float_add(float_sub(1.0, b), float_div(float_mul(b, doc_len), avg_len)) + // tf_comp = delta + tf * (k1+1) / (tf + k1*norm) + let numerator: Float = float_mul(tf_raw, float_add(k1, 1.0)) + let denominator: Float = float_add(tf_raw, float_mul(k1, norm_factor)) + let tf_comp: Float = float_add(delta, float_div(numerator, denominator)) + let score = float_add(score, float_mul(idf, tf_comp)) + } + } + let i = i + 1 + } + // Return score as a string so it survives EL's lack of float-in-list support + float_to_str(score) +} + +fn bm25_search_json(query: String, limit: Int) -> String { + // 1. Determine scan size (fetch 10x or up to 500 nodes) + let scan_limit: Int = limit * 10 + if scan_limit > 500 { let scan_limit = 500 } + + // 2. Fetch node sample + let nodes_json: String = engram_scan_nodes_json(scan_limit, 0) + let n: Int = json_array_len(nodes_json) + if n == 0 { return "[]" } + + // 3. Compute avg_doc_len from sample + let total_words: Int = 0 + let i: Int = 0 + while i < n { + let node: String = json_array_get(nodes_json, i) + let content: String = json_get_string(node, "content") + let tokens: String = bm25_tokenize(content) + let wc: Int = str_count_words(tokens) + let total_words = total_words + wc + let i = i + 1 + } + // avg_doc_len as string for safe float passing + let avg_doc_len_f: Float = float_div(int_to_float(total_words), int_to_float(n)) + let avg_doc_len: String = if float_gt(avg_doc_len_f, 0.0) { float_to_str(avg_doc_len_f) } else { "1.0" } + + // 4. Tokenize query + let query_tokens: String = bm25_tokenize(query) + if str_eq(str_trim(query_tokens), "") { return "[]" } + + // 5. Score each node; collect results as parallel JSON and score lists. + // Scores are stored as strings (float_to_str) to avoid float-in-list issues. + let result_nodes: List = 0 + let result_scores: List = 0 + let result_count: Int = 0 + let j: Int = 0 + while j < n { + let node: String = json_array_get(nodes_json, j) + let content: String = json_get_string(node, "content") + let sc_str: String = bm25_score_doc(content, query_tokens, n, avg_doc_len) + // Only include nodes with score > 0 (str check: not "0.0" and not empty) + if !str_eq(sc_str, "0.0") { + if !str_eq(sc_str, "") { + let result_nodes = list_push(result_nodes, node) + let result_scores = list_push(result_scores, sc_str) + let result_count = result_count + 1 + } + } + let j = j + 1 + } + + if result_count == 0 { return "[]" } + + // 6. Selection-sort descending by score, take top `limit` + let out_limit: Int = if result_count < limit { result_count } else { limit } + let k: Int = 0 + while k < out_limit { + // Find max score index in [k, result_count) + let max_idx: Int = k + let max_sc_str: String = get(result_scores, k) + let max_sc_f: Float = str_to_float(max_sc_str) + let p: Int = k + 1 + while p < result_count { + let sc2_str: String = get(result_scores, p) + let sc2_f: Float = str_to_float(sc2_str) + if float_gt(sc2_f, max_sc_f) { + let max_sc_f = sc2_f + let max_sc_str = sc2_str + let max_idx = p + } + let p = p + 1 + } + // Swap k <-> max_idx + if max_idx != k { + let tmp_node: String = get(result_nodes, k) + let tmp_sc: String = get(result_scores, k) + let result_nodes = list_set(result_nodes, k, get(result_nodes, max_idx)) + let result_scores = list_set(result_scores, k, get(result_scores, max_idx)) + let result_nodes = list_set(result_nodes, max_idx, tmp_node) + let result_scores = list_set(result_scores, max_idx, tmp_sc) + } + let k = k + 1 + } + + // 7. Build JSON array of top `out_limit` nodes with bm25_score field + let out: String = "[" + let r: Int = 0 + while r < out_limit { + let node: String = get(result_nodes, r) + let sc_str: String = get(result_scores, r) + // Inject bm25_score: trim the closing } and append field + let node_len: Int = str_len(node) + let node_body: String = str_slice(node, 0, node_len - 1) + let entry: String = node_body + ",\"bm25_score\":" + sc_str + "}" + if r > 0 { let out = out + "," } + let out = out + entry + let r = r + 1 + } + out + "]" +} + // ── Helpers ─────────────────────────────────────────────────────────────────── fn parse_port(bind: String) -> Int { @@ -521,6 +717,20 @@ fn route_events_ack(method: String, path: String, body: String) -> String { "{\"ok\":true}" } +fn route_bm25_search(method: String, path: String, body: String) -> String { + let q: String = "" + if str_eq(method, "GET") { + let q = query_param(path, "q") + } else { + let q = json_get_string(body, "query") + } + if str_eq(q, "") { return "{\"error\":\"query is required\"}" } + let limit: Int = query_int(path, "limit", 20) + if limit == 0 { let limit = json_get_int(body, "limit") } + if limit == 0 { let limit = 20 } + bm25_search_json(q, limit) +} + // ── Auth ────────────────────────────────────────────────────────────────────── fn check_auth_ok(method: String, body: String) -> Bool { @@ -653,6 +863,11 @@ fn handle_request(method: String, path: String, body: String) -> String { return route_search(method, path, body) } + // BM25+ text ranking + if str_eq(clean, "/api/bm25/search") { + return route_bm25_search(method, path, body) + } + // Strengthen if str_eq(method, "POST") && (str_eq(clean, "/api/strengthen") || str_eq(clean, "/strengthen")) { return route_strengthen(method, path, body) diff --git a/lang/releases/v1.0.0-20260501/el_runtime.c b/lang/releases/v1.0.0-20260501/el_runtime.c index d75661d..8bed1ad 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.c +++ b/lang/releases/v1.0.0-20260501/el_runtime.c @@ -4851,6 +4851,21 @@ el_val_t str_to_float(el_val_t s) { el_val_t math_sqrt(el_val_t f) { return el_from_float(sqrt(el_to_float(f))); } el_val_t math_log(el_val_t f) { return el_from_float(log(el_to_float(f))); } el_val_t math_ln(el_val_t f) { return el_from_float(log(el_to_float(f))); } +el_val_t math_exp(el_val_t f) { return el_from_float(exp(el_to_float(f))); } + +/* ── Float arithmetic builtins ─────────────────────────────────────────────── + * EL operators (+, *, /) operate on raw el_val_t bits, which is wrong for + * IEEE 754 floats. These builtins do correct float arithmetic and can be + * called from EL source as float_add(a, b), float_mul(a, b), etc. */ +el_val_t float_add(el_val_t a, el_val_t b) { return el_from_float(el_to_float(a) + el_to_float(b)); } +el_val_t float_sub(el_val_t a, el_val_t b) { return el_from_float(el_to_float(a) - el_to_float(b)); } +el_val_t float_mul(el_val_t a, el_val_t b) { return el_from_float(el_to_float(a) * el_to_float(b)); } +el_val_t float_div(el_val_t a, el_val_t b) { double db = el_to_float(b); return el_from_float(db != 0.0 ? el_to_float(a) / db : 0.0); } +el_val_t float_gt(el_val_t a, el_val_t b) { return el_to_float(a) > el_to_float(b) ? 1 : 0; } +el_val_t float_lt(el_val_t a, el_val_t b) { return el_to_float(a) < el_to_float(b) ? 1 : 0; } +el_val_t float_eq(el_val_t a, el_val_t b) { return el_to_float(a) == el_to_float(b) ? 1 : 0; } +el_val_t float_gte(el_val_t a, el_val_t b) { return el_to_float(a) >= el_to_float(b) ? 1 : 0; } +el_val_t float_lte(el_val_t a, el_val_t b) { return el_to_float(a) <= el_to_float(b) ? 1 : 0; } el_val_t math_sin(el_val_t f) { return el_from_float(sin(el_to_float(f))); } el_val_t math_cos(el_val_t f) { return el_from_float(cos(el_to_float(f))); } el_val_t math_pi(void) { return el_from_float(3.141592653589793238462643383279502884); } @@ -5358,6 +5373,16 @@ el_val_t list_push(el_val_t list, el_val_t elem) { return el_list_append(list, elem); } +/* list_set(list, idx, value) — mutate list in-place at idx, return the list. + * Out-of-bounds idx is a no-op (returns list unchanged). */ +el_val_t list_set(el_val_t listv, el_val_t index, el_val_t value) { + ElList* lst = (ElList*)(uintptr_t)listv; + if (!lst) return listv; + if (index < 0 || index >= lst->length) return listv; + lst->elems[index] = value; + return listv; +} + el_val_t list_push_front(el_val_t listv, el_val_t elem) { ElList* lst = (ElList*)(uintptr_t)listv; if (!lst) { @@ -5959,7 +5984,6 @@ static int engram_keys_init(void); static int engram_write_binary(const char* path); static int engram_load_binary(const char* path); static void engram_embed_node(EngramNode* n); -static uint32_t engram_embed_query(const char* text, float** vec_out); static float engram_cosine_sim(const float* a, const float* b, uint32_t dim); static void engram_checkpoint(void); static void engram_emit_ise_internal(const char* content, const char* label); @@ -6871,17 +6895,9 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { double inh = best_bg[src] * e->weight; if (inh > inhibition[tgt]) inhibition[tgt] = inh; } - /* Embed the query string once for semantic similarity in Layer 2. - * Uses a 5s timeout so a slow/absent Ollama never blocks activation. - * query_emb is NULL and query_edim is 0 if embedding fails — all - * downstream cosine-sim paths guard on this and degrade to bias=1.0. */ - float* query_emb = NULL; - uint32_t query_edim = engram_embed_query(q, &query_emb); - /* Step B: compute working_memory_weight per candidate node. */ double* wm_weights = calloc((size_t)g->node_count, sizeof(double)); if (!wm_weights) { - free(query_emb); free(best_bg); free(best_hops); free(reached); free(seeds); free(fr); free(inhibition); return out; } @@ -6892,19 +6908,6 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { double type_threshold = engram_type_threshold(n->node_type, n->tier); /* Goal bias weights the node's relevance to current intent. */ double bias = engram_goal_bias(n, q); - /* Cosine similarity boost: if both query and node have embeddings, - * blend semantic similarity into the bias with weight α=0.3. - * sim ∈ [-1, 1]; clamp to [0, 1] before blending. - * bias_final = bias * (1 + 0.3 * max(0, sim)) - * This boosts semantically close nodes even when lexical overlap is low. */ - if (query_emb && query_edim > 0 && - n->embedding && n->embedding_dim == query_edim) { - float sim = engram_cosine_sim(query_emb, n->embedding, query_edim); - if (sim > 0.0f) { - bias *= (1.0 + 0.3 * (double)sim); - if (bias > 2.0) bias = 2.0; - } - } /* Raw working memory score. */ double raw_wm = best_bg[i] * bias * n->confidence; /* Apply inhibitory suppression. Full inhibition → scale by factor. */ @@ -7054,7 +7057,6 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { Result* results = malloc((size_t)g->node_count * sizeof(Result)); int64_t rcount = 0; if (!results) { - free(query_emb); free(best_bg); free(best_hops); free(reached); free(seeds); free(fr); free(inhibition); free(wm_weights); return out; } @@ -7099,7 +7101,6 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { (el_val_t)(results[i].wm > 0.0 ? 1 : 0)); out = el_list_append(out, entry); } - free(query_emb); free(best_bg); free(best_hops); free(reached); free(seeds); free(fr); free(inhibition); free(wm_weights); free(results); return out; @@ -7345,58 +7346,6 @@ static void engram_embed_node(EngramNode* n) { /* ── Engram: cosine similarity ───────────────────────────────────────────── */ -/* Embed an arbitrary text string into a float vector via Ollama. - * Returns the dimension (0 on failure). Caller must free *vec_out. */ -static uint32_t engram_embed_query(const char* text, float** vec_out) { - *vec_out = NULL; - if (!text || !*text) return 0; - size_t clen = strlen(text); - if (clen > 2048) clen = 2048; - char* body = malloc(clen * 6 + 128); - if (!body) return 0; - char* bp = body; - bp += sprintf(bp, "{\"model\":\"nomic-embed-text\",\"prompt\":\""); - const char* cp = text; - size_t written = 0; - while (*cp && written < clen) { - if (*cp == '"') { *bp++ = '\\'; *bp++ = '"'; } - else if (*cp == '\\') { *bp++ = '\\'; *bp++ = '\\'; } - else if (*cp == '\n') { *bp++ = '\\'; *bp++ = 'n'; } - else if (*cp == '\r') { *bp++ = '\\'; *bp++ = 'r'; } - else if (*cp == '\t') { *bp++ = '\\'; *bp++ = 't'; } - else { *bp++ = *cp; } - cp++; written++; - } - sprintf(bp, "\"}"); - CURL* curl = curl_easy_init(); - if (!curl) { free(body); return 0; } - char* resp = NULL; - struct curl_slist* hdrs = NULL; - hdrs = curl_slist_append(hdrs, "Content-Type: application/json"); - curl_easy_setopt(curl, CURLOPT_URL, "http://localhost:11434/api/embeddings"); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, hdrs); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, engram_embed_write_cb); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &resp); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5L); /* short timeout — don't block activation */ - curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); - CURLcode rc = curl_easy_perform(curl); - curl_slist_free_all(hdrs); - curl_easy_cleanup(curl); - free(body); - if (rc != CURLE_OK || !resp) { free(resp); return 0; } - const char* ep = strstr(resp, "\"embedding\""); - if (!ep) { free(resp); return 0; } - ep += strlen("\"embedding\""); - while (*ep && *ep != '[') ep++; - float* vec = NULL; - uint32_t dim = engram_parse_float_array(ep, &vec); - free(resp); - if (dim == 0) return 0; - *vec_out = vec; - return dim; -} - static float engram_cosine_sim(const float* a, const float* b, uint32_t dim) { if (!a || !b || dim == 0) return 0.0f; double dot = 0.0, na = 0.0, nb = 0.0; diff --git a/lang/releases/v1.0.0-20260501/el_runtime.h b/lang/releases/v1.0.0-20260501/el_runtime.h index c8ea749..2c6a390 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.h +++ b/lang/releases/v1.0.0-20260501/el_runtime.h @@ -430,10 +430,22 @@ el_val_t str_to_float(el_val_t s); el_val_t math_sqrt(el_val_t f); el_val_t math_log(el_val_t f); el_val_t math_ln(el_val_t f); +el_val_t math_exp(el_val_t f); el_val_t math_sin(el_val_t f); el_val_t math_cos(el_val_t f); el_val_t math_pi(void); +/* ── Float arithmetic builtins (correct IEEE 754 via bit-cast round-trip) ─── */ +el_val_t float_add(el_val_t a, el_val_t b); +el_val_t float_sub(el_val_t a, el_val_t b); +el_val_t float_mul(el_val_t a, el_val_t b); +el_val_t float_div(el_val_t a, el_val_t b); +el_val_t float_gt(el_val_t a, el_val_t b); +el_val_t float_lt(el_val_t a, el_val_t b); +el_val_t float_eq(el_val_t a, el_val_t b); +el_val_t float_gte(el_val_t a, el_val_t b); +el_val_t float_lte(el_val_t a, el_val_t b); + /* ── String additions ────────────────────────────────────────────────────── */ el_val_t str_index_of(el_val_t s, el_val_t sub); @@ -493,6 +505,7 @@ el_val_t str_join(el_val_t list, el_val_t sep); /* alias of list_joi el_val_t list_push(el_val_t list, el_val_t elem); el_val_t list_push_front(el_val_t list, el_val_t elem); +el_val_t list_set(el_val_t list, el_val_t index, el_val_t value); el_val_t list_join(el_val_t list, el_val_t sep); el_val_t list_range(el_val_t start, el_val_t end);