From ab6b52a0b4455ea1b8015266716b86a8d54177a9 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Sat, 18 Jul 2026 08:48:04 -0500 Subject: [PATCH] self-review 2026-07-18: fix soul SIGABRT double-free + engram route scoping sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. engram_neighbors_json (release runtime): BFS frontier/visited strings were el_strdup'd (arena-tracked) but manually freed, so el_request_end() double-freed every one — SIGABRT in http_worker under load (2 prod crashes today via /api/neuron/session/begin and /api/neuron/graph; reproduced and verified fixed with ASAN). Introduced when porting from the dev runtime, which correctly uses plain strdup. Third instance of the arena-vs-manual-free class (after EngramNode 07-15 and idmap keys 07-16). 2. server.el: let-in-if scoping sweep — defaults assigned inside if-blocks never mutated the outer binding, so /api/search and /api/activate always ran with q="", created nodes got node_type=""/salience=0.0, edges got relation=""/weight=0.0, and save/load with no path hit engram_save(""). Rewritten to the let-if-else expression form. /api/activate now also rejects empty queries instead of wiping carried WM weights. 3. engram_activate: retrieval reinforcement (ACT-R base-level learning) — nodes promoted to WM that survive both capacity caps now get last_activated/activation_count updated, so frequently retrieved memories decay slower than abandoned ones. Scoped to promoted-only to avoid flattening dampening across BFS fan-out. --- engram/dist/engram.c | 199 ++++----- engram/src/server.el | 169 ++++--- lang/releases/v1.0.0-20260501/el_runtime.c | 488 ++++++++++++++++++--- 3 files changed, 651 insertions(+), 205 deletions(-) diff --git a/engram/dist/engram.c b/engram/dist/engram.c index 18dc9e6..23a325f 100644 --- a/engram/dist/engram.c +++ b/engram/dist/engram.c @@ -20,16 +20,19 @@ el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body); el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body); el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body); el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body); -el_val_t route_create_ise(el_val_t method, el_val_t path, el_val_t body); -el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body); el_val_t route_save(el_val_t method, el_val_t path, el_val_t body); el_val_t route_load(el_val_t method, el_val_t path, el_val_t body); el_val_t route_health(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_emit_ise(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_capture_knowledge(el_val_t method, el_val_t path, el_val_t body); el_val_t check_auth_ok(el_val_t method, el_val_t body); el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body); +el_val_t bind_raw; el_val_t bind_str; el_val_t port; +el_val_t data_dir_raw; el_val_t data_dir; el_val_t snapshot_path; @@ -112,14 +115,10 @@ el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body) { el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body) { el_val_t content = json_get_string(body, EL_STR("content")); - el_val_t node_type = json_get_string(body, EL_STR("node_type")); - if (str_eq(node_type, EL_STR(""))) { - node_type = EL_STR("Memory"); - } - el_val_t salience = json_get_float(body, EL_STR("salience")); - if (salience == el_from_float(0.0)) { - salience = el_from_float(0.5); - } + el_val_t nt_raw = json_get_string(body, EL_STR("node_type")); + el_val_t node_type = ({ el_val_t _if_result_1 = 0; if (str_eq(nt_raw, EL_STR(""))) { _if_result_1 = (EL_STR("Memory")); } else { _if_result_1 = (nt_raw); } _if_result_1; }); + el_val_t sal_raw = json_get_float(body, EL_STR("salience")); + el_val_t salience = ({ el_val_t _if_result_2 = 0; if ((sal_raw == el_from_float(0.0))) { _if_result_2 = (el_from_float(0.5)); } else { _if_result_2 = (sal_raw); } _if_result_2; }); el_val_t id = engram_node(content, node_type, salience); return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"id\":\""), id), EL_STR("\",\"content\":\"")), content), EL_STR("\",\"node_type\":\"")), node_type), EL_STR("\"}")); return 0; @@ -146,10 +145,8 @@ el_val_t route_scan_nodes(el_val_t method, el_val_t path, el_val_t body) { } el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body) { - el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR")); - if (str_eq(dir, EL_STR(""))) { - dir = EL_STR("/tmp/engram"); - } + el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); + el_val_t dir = ({ el_val_t _if_result_3 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_3 = (EL_STR("/tmp/engram")); } else { _if_result_3 = (dir_raw); } _if_result_3; }); el_val_t snap_path = el_str_concat(dir, EL_STR("/snapshot.json")); engram_save(snap_path); el_val_t snap = fs_read(snap_path); @@ -165,36 +162,22 @@ el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body) { } el_val_t route_search(el_val_t method, el_val_t path, el_val_t body) { - el_val_t q = EL_STR(""); - if (str_eq(method, EL_STR("GET"))) { - q = query_param(path, EL_STR("q")); - } else { - q = json_get_string(body, EL_STR("query")); - } - el_val_t limit = query_int(path, EL_STR("limit"), 20); - if (limit == 0) { - limit = json_get_int(body, EL_STR("limit")); - } - if (limit == 0) { - limit = 20; - } + el_val_t q = ({ el_val_t _if_result_4 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_4 = (query_param(path, EL_STR("q"))); } else { _if_result_4 = (json_get_string(body, EL_STR("query"))); } _if_result_4; }); + el_val_t lim_url = query_int(path, EL_STR("limit"), 0); + el_val_t lim_body = json_get_int(body, EL_STR("limit")); + el_val_t lim_either = ({ el_val_t _if_result_5 = 0; if ((lim_url > 0)) { _if_result_5 = (lim_url); } else { _if_result_5 = (lim_body); } _if_result_5; }); + el_val_t limit = ({ el_val_t _if_result_6 = 0; if ((lim_either > 0)) { _if_result_6 = (lim_either); } else { _if_result_6 = (20); } _if_result_6; }); return engram_search_json(q, limit); return 0; } el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body) { - el_val_t q = EL_STR(""); - el_val_t depth = 3; - if (str_eq(method, EL_STR("GET"))) { - q = query_param(path, EL_STR("q")); - depth = query_int(path, EL_STR("depth"), 3); - } else { - q = json_get_string(body, EL_STR("query")); - el_val_t bd = json_get_int(body, EL_STR("depth")); - if (bd > 0) { - depth = bd; - } + el_val_t q = ({ el_val_t _if_result_7 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_7 = (query_param(path, EL_STR("q"))); } else { _if_result_7 = (json_get_string(body, EL_STR("query"))); } _if_result_7; }); + if (str_eq(q, EL_STR(""))) { + return err_json(EL_STR("missing query")); } + el_val_t d_raw = ({ el_val_t _if_result_8 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_8 = (query_int(path, EL_STR("depth"), 3)); } else { _if_result_8 = (json_get_int(body, EL_STR("depth"))); } _if_result_8; }); + el_val_t depth = ({ el_val_t _if_result_9 = 0; if ((d_raw > 0)) { _if_result_9 = (d_raw); } else { _if_result_9 = (3); } _if_result_9; }); return el_str_concat(el_str_concat(EL_STR("{\"results\":"), engram_activate_json(q, depth)), EL_STR("}")); return 0; } @@ -202,14 +185,10 @@ el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body) { el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body) { el_val_t from_id = json_get_string(body, EL_STR("from_id")); el_val_t to_id = json_get_string(body, EL_STR("to_id")); - el_val_t relation = json_get_string(body, EL_STR("relation")); - if (str_eq(relation, EL_STR(""))) { - relation = EL_STR("associates"); - } - el_val_t weight = json_get_float(body, EL_STR("weight")); - if (weight == el_from_float(0.0)) { - weight = el_from_float(0.5); - } + el_val_t rel_raw = json_get_string(body, EL_STR("relation")); + el_val_t relation = ({ el_val_t _if_result_10 = 0; if (str_eq(rel_raw, EL_STR(""))) { _if_result_10 = (EL_STR("associates")); } else { _if_result_10 = (rel_raw); } _if_result_10; }); + el_val_t w_raw = json_get_float(body, EL_STR("weight")); + el_val_t weight = ({ el_val_t _if_result_11 = 0; if ((w_raw == el_from_float(0.0))) { _if_result_11 = (el_from_float(0.5)); } else { _if_result_11 = (w_raw); } _if_result_11; }); engram_connect(from_id, to_id, weight, relation); return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"from_id\":\""), from_id), EL_STR("\",\"to_id\":\"")), to_id), EL_STR("\",\"relation\":\"")), relation), EL_STR("\"}")); return 0; @@ -245,25 +224,35 @@ el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body) { return 0; } -el_val_t route_create_ise(el_val_t method, el_val_t path, el_val_t body) { - el_val_t content = json_get_string(body, EL_STR("content")); - if (str_eq(content, EL_STR(""))) { - return err_json(EL_STR("missing content")); - } - el_val_t sal = el_from_float(0.3); - el_val_t imp = el_from_float(0.3); - el_val_t conf = el_from_float(0.8); - el_val_t id = engram_node_full(content, EL_STR("InternalStateEvent"), EL_STR("state-event"), sal, imp, conf, EL_STR("Episodic"), EL_STR("[\"internal-state\",\"InternalStateEvent\"]")); - return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\"}")); +el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) { + el_val_t p_raw = json_get_string(body, EL_STR("path")); + el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); + el_val_t dir = ({ el_val_t _if_result_12 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_12 = (EL_STR("/tmp/engram")); } else { _if_result_12 = (dir_raw); } _if_result_12; }); + el_val_t p = ({ el_val_t _if_result_13 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_13 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_13 = (p_raw); } _if_result_13; }); + engram_save(p); + return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"path\":\""), p), EL_STR("\"}")); + return 0; +} + +el_val_t route_load(el_val_t method, el_val_t path, el_val_t body) { + el_val_t p_raw = json_get_string(body, EL_STR("path")); + el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); + el_val_t dir = ({ el_val_t _if_result_14 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_14 = (EL_STR("/tmp/engram")); } else { _if_result_14 = (dir_raw); } _if_result_14; }); + el_val_t p = ({ el_val_t _if_result_15 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_15 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_15 = (p_raw); } _if_result_15; }); + engram_load(p); + return ok_json(); + return 0; +} + +el_val_t route_health(el_val_t method, el_val_t path, el_val_t body) { + return EL_STR("{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}"); return 0; } el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body) { - el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR")); - if (str_eq(dir, EL_STR(""))) { - dir = EL_STR("/tmp/engram"); - } - el_val_t snap_path = el_str_concat(dir, EL_STR("/sync-export.json")); + el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); + el_val_t dir = ({ el_val_t _if_result_16 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_16 = (EL_STR("/tmp/engram")); } else { _if_result_16 = (dir_raw); } _if_result_16; }); + el_val_t snap_path = el_str_concat(dir, EL_STR("/snapshot.json")); engram_save(snap_path); el_val_t snap = fs_read(snap_path); if (str_eq(snap, EL_STR(""))) { @@ -273,36 +262,49 @@ el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body) { return 0; } -el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) { - el_val_t p = json_get_string(body, EL_STR("path")); - if (str_eq(p, EL_STR(""))) { - el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR")); - if (str_eq(dir, EL_STR(""))) { - dir = EL_STR("/tmp/engram"); - } - p = el_str_concat(dir, EL_STR("/snapshot.json")); +el_val_t route_emit_ise(el_val_t method, el_val_t path, el_val_t body) { + el_val_t content = json_get_string(body, EL_STR("content")); + if (str_eq(content, EL_STR(""))) { + return err_json(EL_STR("missing content")); } - engram_save(p); - return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"path\":\""), p), EL_STR("\"}")); + el_val_t sal = el_from_float(0.3); + el_val_t imp = el_from_float(0.3); + el_val_t conf = el_from_float(0.8); + el_val_t id = engram_node_full(content, EL_STR("InternalStateEvent"), EL_STR("state-event"), sal, imp, conf, EL_STR("Episodic"), EL_STR("[\"internal-state\",\"InternalStateEvent\"]")); + el_val_t ret_raw = env(EL_STR("ENGRAM_ISE_RETENTION_MS")); + el_val_t ret_ms = ({ el_val_t _if_result_17 = 0; if (str_eq(ret_raw, EL_STR(""))) { _if_result_17 = (172800000); } else { _if_result_17 = (str_to_int(ret_raw)); } _if_result_17; }); + el_val_t pruned = engram_prune_telemetry(ret_ms); + return el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\",\"pruned\":")), int_to_str(pruned)), EL_STR("}")); return 0; } -el_val_t route_load(el_val_t method, el_val_t path, el_val_t body) { - el_val_t p = json_get_string(body, EL_STR("path")); - if (str_eq(p, EL_STR(""))) { - el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR")); - if (str_eq(dir, EL_STR(""))) { - dir = EL_STR("/tmp/engram"); - } - p = el_str_concat(dir, EL_STR("/snapshot.json")); +el_val_t route_capture_knowledge(el_val_t method, el_val_t path, el_val_t body) { + el_val_t content = json_get_string(body, EL_STR("content")); + if (str_eq(content, EL_STR(""))) { + return err_json(EL_STR("missing content")); } - engram_load(p); - return ok_json(); - return 0; -} - -el_val_t route_health(el_val_t method, el_val_t path, el_val_t body) { - return EL_STR("{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}"); + el_val_t title = json_get_string(body, EL_STR("title")); + el_val_t label = ({ el_val_t _if_result_18 = 0; if (str_eq(title, EL_STR(""))) { _if_result_18 = (str_slice(content, 0, 60)); } else { _if_result_18 = (title); } _if_result_18; }); + el_val_t category_raw = json_get_string(body, EL_STR("category")); + el_val_t category = ({ el_val_t _if_result_19 = 0; if (str_eq(category_raw, EL_STR(""))) { _if_result_19 = (EL_STR("other")); } else { _if_result_19 = (category_raw); } _if_result_19; }); + el_val_t ktier_raw = json_get_string(body, EL_STR("tier")); + el_val_t ktier = ({ el_val_t _if_result_20 = 0; if (str_eq(ktier_raw, EL_STR(""))) { _if_result_20 = (EL_STR("note")); } else { _if_result_20 = (ktier_raw); } _if_result_20; }); + el_val_t project = json_get_string(body, EL_STR("project")); + el_val_t tags_raw = json_get_raw(body, EL_STR("tags")); + el_val_t tags_base = ({ el_val_t _if_result_21 = 0; if (str_eq(tags_raw, EL_STR(""))) { _if_result_21 = (EL_STR("[]")); } else { _if_result_21 = (tags_raw); } _if_result_21; }); + el_val_t base_len = str_len(tags_base); + el_val_t head = str_slice(tags_base, 0, (base_len - 1)); + el_val_t sep = ({ el_val_t _if_result_22 = 0; if (str_eq(head, EL_STR("["))) { _if_result_22 = (EL_STR("")); } else { _if_result_22 = (EL_STR(",")); } _if_result_22; }); + el_val_t safe_cat = str_replace(category, EL_STR("\""), EL_STR("'")); + el_val_t safe_tier = str_replace(ktier, EL_STR("\""), EL_STR("'")); + el_val_t safe_proj = str_replace(project, EL_STR("\""), EL_STR("'")); + el_val_t proj_tag = ({ el_val_t _if_result_23 = 0; if (str_eq(safe_proj, EL_STR(""))) { _if_result_23 = (EL_STR("")); } else { _if_result_23 = (el_str_concat(el_str_concat(EL_STR(",\"project:"), safe_proj), EL_STR("\""))); } _if_result_23; }); + el_val_t tags = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(head, sep), EL_STR("\"category:")), safe_cat), EL_STR("\",\"tier:")), safe_tier), EL_STR("\"")), proj_tag), EL_STR("]")); + el_val_t sal = el_from_float(0.5); + el_val_t imp = el_from_float(0.5); + el_val_t conf = el_from_float(0.9); + el_val_t id = engram_node_full(content, EL_STR("Knowledge"), label, sal, imp, conf, EL_STR("Semantic"), tags); + return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\"}")); return 0; } @@ -329,12 +331,15 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) { return route_health(method, path, body); } } - if (str_eq(method, EL_STR("POST")) && str_starts_with(clean, EL_STR("/api/neuron/state-events"))) { - return route_create_ise(method, path, body); + if (str_eq(method, EL_STR("POST")) && str_eq(clean, EL_STR("/api/neuron/state-events"))) { + return route_emit_ise(method, path, body); } if (!check_auth_ok(method, body)) { return err_json(EL_STR("unauthorized")); } + if (str_eq(method, EL_STR("POST")) && str_eq(clean, EL_STR("/api/neuron/knowledge/capture"))) { + return route_capture_knowledge(method, path, body); + } if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/stats")) || str_eq(clean, EL_STR("/stats")))) { return route_stats(method, path, body); } @@ -374,30 +379,26 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) { if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/strengthen")) || str_eq(clean, EL_STR("/strengthen")))) { return route_strengthen(method, path, body); } - if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/sync")) || str_eq(clean, EL_STR("/sync")))) { - return route_sync(method, path, body); - } if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/save")) || str_eq(clean, EL_STR("/save")))) { return route_save(method, path, body); } if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/load")) || str_eq(clean, EL_STR("/load")))) { return route_load(method, path, body); } + if (str_eq(method, EL_STR("GET")) && str_eq(clean, EL_STR("/api/sync"))) { + return route_sync(method, path, body); + } return el_str_concat(el_str_concat(EL_STR("{\"error\":\"not found\",\"path\":\""), clean), EL_STR("\"}")); return 0; } int main(int _argc, char** _argv) { el_runtime_init_args(_argc, _argv); - bind_str = env(EL_STR("ENGRAM_BIND")); - if (str_eq(bind_str, EL_STR(""))) { - bind_str = EL_STR(":8742"); - } + bind_raw = env(EL_STR("ENGRAM_BIND")); + bind_str = ({ el_val_t _if_result_24 = 0; if (str_eq(bind_raw, EL_STR(""))) { _if_result_24 = (EL_STR(":8742")); } else { _if_result_24 = (bind_raw); } _if_result_24; }); port = parse_port(bind_str); - data_dir = env(EL_STR("ENGRAM_DATA_DIR")); - if (str_eq(data_dir, EL_STR(""))) { - data_dir = EL_STR("/tmp/engram"); - } + data_dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); + data_dir = ({ el_val_t _if_result_25 = 0; if (str_eq(data_dir_raw, EL_STR(""))) { _if_result_25 = (EL_STR("/tmp/engram")); } else { _if_result_25 = (data_dir_raw); } _if_result_25; }); snapshot_path = el_str_concat(data_dir, EL_STR("/snapshot.json")); engram_load(snapshot_path); println(EL_STR("[engram] runtime-native graph engine")); diff --git a/engram/src/server.el b/engram/src/server.el index e676d32..da54fc6 100644 --- a/engram/src/server.el +++ b/engram/src/server.el @@ -76,12 +76,21 @@ fn route_stats(method: String, path: String, body: String) -> String { engram_stats_json() } +// (2026-07-18 self-review) Scoping sweep: `let` inside an if-block creates an +// inner scope only — it does NOT mutate the outer binding (documented with +// evidence in awareness.el, 2026-05-25). Every default/reassignment below used +// that broken pattern, so defaults never applied: nodes were created with +// node_type="" and salience=0.0, /api/search and /api/activate ALWAYS ran with +// q="" regardless of input, edges defaulted to relation=""/weight=0.0, and +// save/load with no "path" hit engram_save(""). Rewritten to the +// `let x = if cond { a } else { b }` expression form (the pattern the newer +// routes route_emit_ise/route_capture_knowledge already use correctly). fn route_create_node(method: String, path: String, body: String) -> String { let content: String = json_get_string(body, "content") - let node_type: String = json_get_string(body, "node_type") - if str_eq(node_type, "") { let node_type = "Memory" } - let salience: Float = json_get_float(body, "salience") - if salience == 0.0 { let salience = 0.5 } + let nt_raw: String = json_get_string(body, "node_type") + let node_type: String = if str_eq(nt_raw, "") { "Memory" } else { nt_raw } + let sal_raw: Float = json_get_float(body, "salience") + let salience: Float = if sal_raw == 0.0 { 0.5 } else { sal_raw } let id: String = engram_node(content, node_type, salience) "{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\"}" } @@ -107,8 +116,8 @@ fn route_scan_nodes(method: String, path: String, body: String) -> String { // runtime keeps in lockstep with the in-memory graph. Live against the // running graph, not a stale export. fn route_scan_edges(method: String, path: String, body: String) -> String { - let dir: String = env("ENGRAM_DATA_DIR") - if str_eq(dir, "") { let dir = "/tmp/engram" } + let dir_raw: String = env("ENGRAM_DATA_DIR") + let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw } let snap_path: String = dir + "/snapshot.json" engram_save(snap_path) let snap: String = fs_read(snap_path) @@ -122,39 +131,32 @@ fn route_scan_edges(method: String, path: String, body: String) -> String { } fn route_search(method: String, path: String, body: String) -> String { - let q: String = "" - if str_eq(method, "GET") { - let q = query_param(path, "q") - } else { - let q = json_get_string(body, "query") - } - let limit: Int = query_int(path, "limit", 20) - if limit == 0 { let limit = json_get_int(body, "limit") } - if limit == 0 { let limit = 20 } + let q: String = if str_eq(method, "GET") { query_param(path, "q") } else { json_get_string(body, "query") } + let lim_url: Int = query_int(path, "limit", 0) + let lim_body: Int = json_get_int(body, "limit") + let lim_either: Int = if lim_url > 0 { lim_url } else { lim_body } + let limit: Int = if lim_either > 0 { lim_either } else { 20 } return engram_search_json(q, limit) } fn route_activate(method: String, path: String, body: String) -> String { - let q: String = "" - let depth: Int = 3 - if str_eq(method, "GET") { - let q = query_param(path, "q") - let depth = query_int(path, "depth", 3) - } else { - let q = json_get_string(body, "query") - let bd: Int = json_get_int(body, "depth") - if bd > 0 { let depth = bd } - } + let q: String = if str_eq(method, "GET") { query_param(path, "q") } else { json_get_string(body, "query") } + // Guard: engram_activate with an empty query matches zero seeds, which + // zeroes ALL carried working-memory weights (documented in awareness.el + // perceive()). Never let an empty activation through to wipe WM. + if str_eq(q, "") { return err_json("missing query") } + let d_raw: Int = if str_eq(method, "GET") { query_int(path, "depth", 3) } else { json_get_int(body, "depth") } + let depth: Int = if d_raw > 0 { d_raw } else { 3 } return "{\"results\":" + engram_activate_json(q, depth) + "}" } fn route_create_edge(method: String, path: String, body: String) -> String { let from_id: String = json_get_string(body, "from_id") let to_id: String = json_get_string(body, "to_id") - let relation: String = json_get_string(body, "relation") - if str_eq(relation, "") { let relation = "associates" } - let weight: Float = json_get_float(body, "weight") - if weight == 0.0 { let weight = 0.5 } + let rel_raw: String = json_get_string(body, "relation") + let relation: String = if str_eq(rel_raw, "") { "associates" } else { rel_raw } + let w_raw: Float = json_get_float(body, "weight") + let weight: Float = if w_raw == 0.0 { 0.5 } else { w_raw } engram_connect(from_id, to_id, weight, relation) "{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + relation + "\"}" } @@ -181,23 +183,19 @@ fn route_forget(method: String, path: String, body: String) -> String { } fn route_save(method: String, path: String, body: String) -> String { - let p: String = json_get_string(body, "path") - if str_eq(p, "") { - let dir: String = env("ENGRAM_DATA_DIR") - if str_eq(dir, "") { let dir = "/tmp/engram" } - let p = dir + "/snapshot.json" - } + let p_raw: String = json_get_string(body, "path") + let dir_raw: String = env("ENGRAM_DATA_DIR") + let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw } + let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw } engram_save(p) "{\"ok\":true,\"path\":\"" + p + "\"}" } fn route_load(method: String, path: String, body: String) -> String { - let p: String = json_get_string(body, "path") - if str_eq(p, "") { - let dir: String = env("ENGRAM_DATA_DIR") - if str_eq(dir, "") { let dir = "/tmp/engram" } - let p = dir + "/snapshot.json" - } + let p_raw: String = json_get_string(body, "path") + let dir_raw: String = env("ENGRAM_DATA_DIR") + let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw } + let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw } engram_load(p) ok_json() } @@ -219,8 +217,8 @@ fn route_health(method: String, path: String, body: String) -> String { // (it skips nodes already present by ID). Auth-exempt: same-host internal call. // (2026-06-27 self-review: added this route to fix silent 10-min sync failures) fn route_sync(method: String, path: String, body: String) -> String { - let dir: String = env("ENGRAM_DATA_DIR") - if str_eq(dir, "") { let dir = "/tmp/engram" } + let dir_raw: String = env("ENGRAM_DATA_DIR") + let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw } let snap_path: String = dir + "/snapshot.json" engram_save(snap_path) let snap: String = fs_read(snap_path) @@ -241,10 +239,20 @@ fn route_sync(method: String, path: String, body: String) -> String { // // Salience/importance set to match engram_node_full ISE defaults used by the // in-process fallback path in awareness.el (salience=0.3, importance=0.3, -// confidence=0.8, tier=Episodic). High temporal_decay_rate (1.617) — ISEs -// are inherently transient; they should decay faster than structural knowledge. +// confidence=0.8, tier=Episodic). // (2026-06-26 self-review: added this route after discovering ise_post was // silently failing — the soul posts here but the endpoint didn't exist.) +// +// Retention (2026-07-16 self-review): an earlier comment here claimed ISEs +// got temporal_decay_rate=1.617 — that was never implemented (engram_node_full +// hardcodes 0.0), and per-node decay only dampens activation anyway; it never +// removes nodes. By 2026-07-16 ISEs were 75% of the store (10,175 of 13,522 +// nodes, ~4,300/day, unbounded). ISEs are already WM-excluded in +// engram_activate, so the fix is retention, not decay: every insert calls +// engram_prune_telemetry(), a single O(nodes+edges) compaction pass that +// removes ISEs older than ENGRAM_ISE_RETENTION_MS (default 48h), protecting +// "session-start" labels and self_review events as durable history. At +// ~3 ISEs/min this bounds telemetry at ~8.6k nodes instead of growing forever. fn route_emit_ise(method: String, path: String, body: String) -> String { let content: String = json_get_string(body, "content") if str_eq(content, "") { return err_json("missing content") } @@ -256,6 +264,63 @@ fn route_emit_ise(method: String, path: String, body: String) -> String { sal, imp, conf, "Episodic", "[\"internal-state\",\"InternalStateEvent\"]" ) + let ret_raw: String = env("ENGRAM_ISE_RETENTION_MS") + let ret_ms: Int = if str_eq(ret_raw, "") { 172800000 } else { str_to_int(ret_raw) } + let pruned: Int = engram_prune_telemetry(ret_ms) + "{\"ok\":true,\"id\":\"" + id + "\",\"pruned\":" + int_to_str(pruned) + "}" +} + +// ── Knowledge capture ───────────────────────────────────────────────────────── +// +// route_capture_knowledge — direct Knowledge-node capture over HTTP. +// +// Endpoint: POST /api/neuron/knowledge/capture (auth required: "_auth" in body) +// Body: {"content": "...", "title": "...", "category": "...", +// "tier": "note|lesson|canonical", "tags": [...], "project": "...", +// "_auth": ""} +// +// WHY (2026-07-15 self-review): the world-ingestor integrator was designed +// against this endpoint (its MCP-unavailable fallback), but the route never +// existed — every direct push 404'd, and because the auth gate ran before +// routing, the failure surfaced as {"error":"unauthorized"} and was +// misdiagnosed for two weeks while world knowledge silently dropped. +// POST /api/nodes was no substitute: it discards label/tags/tier, which +// makes captured knowledge invisible to tag-scoped search and curiosity. +// +// The incoming knowledge tier (note/lesson/canonical) is preserved as a +// "tier:" tag rather than mapped onto Engram's cognitive tiers — Knowledge +// nodes land in Semantic (stable reference), and the epistemic tier stays +// queryable without inventing a lossy mapping. +fn route_capture_knowledge(method: String, path: String, body: String) -> String { + let content: String = json_get_string(body, "content") + if str_eq(content, "") { return err_json("missing content") } + let title: String = json_get_string(body, "title") + let label: String = if str_eq(title, "") { str_slice(content, 0, 60) } else { title } + let category_raw: String = json_get_string(body, "category") + let category: String = if str_eq(category_raw, "") { "other" } else { category_raw } + let ktier_raw: String = json_get_string(body, "tier") + let ktier: String = if str_eq(ktier_raw, "") { "note" } else { ktier_raw } + let project: String = json_get_string(body, "project") + let tags_raw: String = json_get_raw(body, "tags") + let tags_base: String = if str_eq(tags_raw, "") { "[]" } else { tags_raw } + // Merge category/tier/project markers into the tag array. Search matches + // against the tags string, so these make captures findable by facet. + let base_len: Int = str_len(tags_base) + let head: String = str_slice(tags_base, 0, base_len - 1) + let sep: String = if str_eq(head, "[") { "" } else { "," } + let safe_cat: String = str_replace(category, "\"", "'") + let safe_tier: String = str_replace(ktier, "\"", "'") + let safe_proj: String = str_replace(project, "\"", "'") + let proj_tag: String = if str_eq(safe_proj, "") { "" } else { ",\"project:" + safe_proj + "\"" } + let tags: String = head + sep + "\"category:" + safe_cat + "\",\"tier:" + safe_tier + "\"" + proj_tag + "]" + let sal: Float = 0.5 + let imp: Float = 0.5 + let conf: Float = 0.9 + let id: String = engram_node_full( + content, "Knowledge", label, + sal, imp, conf, + "Semantic", tags + ) "{\"ok\":true,\"id\":\"" + id + "\"}" } @@ -295,6 +360,12 @@ fn handle_request(method: String, path: String, body: String) -> String { return err_json("unauthorized") } + // Knowledge capture (auth enforced above; the world-ingestor integrator + // and any headless session without MCP push knowledge through this) + if str_eq(method, "POST") && str_eq(clean, "/api/neuron/knowledge/capture") { + return route_capture_knowledge(method, path, body) + } + // Stats if str_eq(method, "GET") && (str_eq(clean, "/api/stats") || str_eq(clean, "/stats")) { return route_stats(method, path, body) @@ -362,13 +433,13 @@ fn handle_request(method: String, path: String, body: String) -> String { // ── Entry ───────────────────────────────────────────────────────────────────── -let bind_str: String = env("ENGRAM_BIND") -if str_eq(bind_str, "") { let bind_str = ":8742" } +let bind_raw: String = env("ENGRAM_BIND") +let bind_str: String = if str_eq(bind_raw, "") { ":8742" } else { bind_raw } let port: Int = parse_port(bind_str) // On startup, try to load any existing snapshot (best effort). -let data_dir: String = env("ENGRAM_DATA_DIR") -if str_eq(data_dir, "") { let data_dir = "/tmp/engram" } +let data_dir_raw: String = env("ENGRAM_DATA_DIR") +let data_dir: String = if str_eq(data_dir_raw, "") { "/tmp/engram" } else { data_dir_raw } let snapshot_path: String = data_dir + "/snapshot.json" engram_load(snapshot_path) diff --git a/lang/releases/v1.0.0-20260501/el_runtime.c b/lang/releases/v1.0.0-20260501/el_runtime.c index f4f605f..bbbba43 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.c +++ b/lang/releases/v1.0.0-20260501/el_runtime.c @@ -102,6 +102,47 @@ void el_request_end(void) { _tl_arena.count = 0; } +/* ── Scoped arena for CLI use ─────────────────────────────────────────────── * + * CLI programs never call el_request_start/end, so all strdup allocations are + * permanent. el_arena_push/pop let the compiler free intermediate strings + * after each compilation unit. Ported verbatim from el-compiler/runtime on + * 2026-07-17: the soul daemon's awareness loop arena-scopes each tick with + * these, and they were present only in the dev runtime copy. + * + * el_arena_push() — activates the arena if not already active, saves the + * current arena count as a mark, and returns it as an el_val_t Int. + * el_arena_pop(mark) — frees all strings allocated since the push mark and + * resets the count. If count reaches 0, deactivates the arena. + */ +#define EL_ARENA_SCOPE_DEPTH 32 +static _Thread_local size_t _tl_arena_scope[EL_ARENA_SCOPE_DEPTH]; +static _Thread_local int _tl_arena_scope_depth = 0; + +el_val_t el_arena_push(void) { + if (!_tl_arena_active) { + _tl_arena_active = 1; + } + if (_tl_arena_scope_depth < EL_ARENA_SCOPE_DEPTH) { + _tl_arena_scope[_tl_arena_scope_depth++] = _tl_arena.count; + } + return (el_val_t)(int64_t)_tl_arena.count; +} + +el_val_t el_arena_pop(el_val_t mark) { + size_t save = (size_t)(int64_t)mark; + if (save > _tl_arena.count) save = 0; + for (size_t i = save; i < _tl_arena.count; i++) { + if (_tl_arena.ptrs[i]) { + free(_tl_arena.ptrs[i]); + _tl_arena.ptrs[i] = NULL; + } + } + _tl_arena.count = save; + if (_tl_arena_scope_depth > 0) _tl_arena_scope_depth--; + if (save == 0) _tl_arena_active = 0; + return 0; +} + /* Persistent allocation — bypasses the arena (state_set, engram internals). */ static char* el_strdup_persist(const char* s) { if (!s) return strdup(""); @@ -1518,6 +1559,86 @@ void http_serve(el_val_t port, el_val_t handler) { close(sock); } +/* ── http_serve_async — non-blocking HTTP server ─────────────────────────── */ +/* Runs the accept loop in a background pthread, returns immediately so the + * calling EL script can continue (e.g. to run an awareness loop). + * Ported verbatim from el-compiler/runtime on 2026-07-17: the soul daemon + * (soul.el) builds against this release runtime and calls http_serve_async, + * which was present only in the dev runtime copy. + * + * El signature: http_serve_async(port, handler) -> Void */ + +typedef struct { int sock; } HttpServeAsyncArg; + +static void* _http_serve_async_loop(void* raw) { + HttpServeAsyncArg* a = (HttpServeAsyncArg*)raw; + int sock = a->sock; + free(a); + while (1) { + struct sockaddr_in6 cli; + socklen_t clen = sizeof(cli); + int cfd = accept(sock, (struct sockaddr*)&cli, &clen); + if (cfd < 0) { + if (errno == EINTR) continue; + perror("accept"); break; + } + pthread_mutex_lock(&_http_conn_mu); + while (_http_conn_active >= HTTP_MAX_CONNS) { + pthread_cond_wait(&_http_conn_cv, &_http_conn_mu); + } + _http_conn_active++; + pthread_mutex_unlock(&_http_conn_mu); + HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg)); + if (!arg) { close(cfd); continue; } + arg->fd = cfd; + pthread_t tid; + if (pthread_create(&tid, NULL, http_worker, arg) != 0) { + close(cfd); free(arg); + pthread_mutex_lock(&_http_conn_mu); + _http_conn_active--; + pthread_cond_signal(&_http_conn_cv); + pthread_mutex_unlock(&_http_conn_mu); + continue; + } + pthread_detach(tid); + } + close(sock); + return NULL; +} + +void http_serve_async(el_val_t port, el_val_t handler) { + const char* hname = EL_CSTR(handler); + if (hname && looks_like_string(handler)) { + http_set_handler(handler); + } + int p = (int)port; + if (p <= 0 || p > 65535) { fprintf(stderr, "http_serve_async: invalid port %d\n", p); return; } + int sock = socket(AF_INET6, SOCK_STREAM, 0); + if (sock < 0) { perror("socket"); return; } + int yes = 1; int no = 0; + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &no, sizeof(no)); + struct sockaddr_in6 addr; + memset(&addr, 0, sizeof(addr)); + addr.sin6_family = AF_INET6; + addr.sin6_addr = in6addr_any; + addr.sin6_port = htons((uint16_t)p); + if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) { + perror("bind"); close(sock); return; + } + if (listen(sock, 64) < 0) { perror("listen"); close(sock); return; } + fprintf(stderr, "[http] async listening on [::]:%d (dual-stack)\n", p); + HttpServeAsyncArg* a = malloc(sizeof(HttpServeAsyncArg)); + if (!a) { close(sock); return; } + a->sock = sock; + pthread_t tid; + if (pthread_create(&tid, NULL, _http_serve_async_loop, a) != 0) { + perror("pthread_create"); free(a); close(sock); return; + } + pthread_detach(tid); + /* Returns immediately — caller can now run awareness_run() or any loop. */ +} + /* ── HTTP server v2 — request headers + structured response ──────────────── */ /* * v2 widens the handler signature from @@ -5926,7 +6047,16 @@ static void engram_idmap_put(EngramStore* g, const char* id, int64_t idx) { } /* Use tombstone slot if found (avoids growing used count unnecessarily) */ if (tomb_slot != SIZE_MAX) slot = tomb_slot; - g->id_map[slot].key = el_strdup(id); + /* MUST be el_strdup_persist: idmap keys outlive the request/tick arena. + * (2026-07-16 self-review) This was el_strdup (arena-tracked): every node + * created inside an HTTP request left its idmap key DANGLING as soon as + * el_request_end() freed the arena — subsequent lookups strcmp'd freed + * memory, and any idmap_free/rebuild in a later request double-freed it + * (SIGABRT in http_worker; found via ASAN when engram_prune_telemetry + * triggered an in-request rebuild). Same allocation-discipline class as + * the 2026-07-15 EngramNode fix — see the store-persistent comment above + * engram_new_id(). */ + g->id_map[slot].key = el_strdup_persist(id); g->id_map[slot].idx = idx; g->id_map_used++; } @@ -6102,10 +6232,23 @@ static void engram_grow_edges(void) { } /* Build a fresh UUID string. Reuses uuid_new but takes the underlying char*. */ +/* ── Store-persistent allocation discipline ───────────────────────────────── + * (2026-07-15 self-review) EngramNode/EngramEdge string fields OUTLIVE the + * request/tick arena they were created in. The 2026-07-13 leak-fix made + * el_strdup arena-tracked, which silently turned every node created inside + * an HTTP request (route_emit_ise, route_create_node, knowledge capture) or + * inside the soul's per-tick arena (engram_load_merge in the refresh cycle) + * into a bag of dangling pointers the moment the arena popped: readback by + * id returned {}, type-filtered scans skipped them, search returned request + * memory reused as node content, and snapshots persisted garbage ("numeric + * tier strings"). Everything written into the store must go through + * el_strdup_persist / el_strbuf_persist (plain malloc — free() in + * engram_forget/evolve remains valid). Arena-tracked el_strdup remains + * correct for RETURN values handed back to EL code. */ static char* engram_new_id(void) { el_val_t v = uuid_new(); const char* s = EL_CSTR(v); - return el_strdup(s ? s : ""); + return el_strdup_persist(s ? s : ""); } /* Convert a node into an ElMap of its fields. */ @@ -6166,12 +6309,12 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) { n->id = engram_new_id(); const char* c = EL_CSTR(content); const char* nt = EL_CSTR(node_type); - n->content = el_strdup(c ? c : ""); - n->node_type = el_strdup(nt && *nt ? nt : "Memory"); - n->label = engram_first_n_chars(c, 60); - n->tier = el_strdup("Working"); - n->tags = el_strdup(""); - n->metadata = el_strdup("{}"); + n->content = el_strdup_persist(c ? c : ""); + n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory"); + n->label = el_strdup_persist(engram_first_n_chars(c, 60)); + n->tier = el_strdup_persist("Working"); + n->tags = el_strdup_persist(""); + n->metadata = el_strdup_persist("{}"); n->salience = engram_decode_score(salience); if (n->salience <= 0.0 || n->salience > 1.0) n->salience = 0.5; n->importance = 0.5; @@ -6203,12 +6346,12 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label, const char* lb = EL_CSTR(label); const char* ti = EL_CSTR(tier); const char* tg = EL_CSTR(tags); - n->content = el_strdup(c ? c : ""); - n->node_type = el_strdup(nt && *nt ? nt : "Memory"); - n->label = el_strdup(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : "")); - n->tier = el_strdup(ti && *ti ? ti : "Working"); - n->tags = el_strdup(tg ? tg : ""); - n->metadata = el_strdup("{}"); + n->content = el_strdup_persist(c ? c : ""); + n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory"); + n->label = el_strdup_persist(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : "")); + n->tier = el_strdup_persist(ti && *ti ? ti : "Working"); + n->tags = el_strdup_persist(tg ? tg : ""); + n->metadata = el_strdup_persist("{}"); n->salience = engram_decode_score(salience); n->importance = engram_decode_score(importance); n->confidence = engram_decode_score(confidence); @@ -6259,20 +6402,20 @@ el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t labe const char* lb = EL_CSTR(label); const char* tg = EL_CSTR(tags); const char* st = EL_CSTR(status); - n->content = el_strdup(c ? c : ""); - n->node_type = el_strdup(nt && *nt ? nt : "Memory"); - n->label = el_strdup(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : "")); - n->tier = el_strdup("Working"); - n->tags = el_strdup(tg ? tg : ""); + n->content = el_strdup_persist(c ? c : ""); + n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory"); + n->label = el_strdup_persist(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : "")); + n->tier = el_strdup_persist("Working"); + n->tags = el_strdup_persist(tg ? tg : ""); if (st && *st) { /* Minimal metadata payload: {"status":"..."}. Keep it cheap so * callers using `status` don't pay JSON parse cost on every read. */ size_t sl = strlen(st) + 16; - char* meta = el_strbuf(sl); + char* meta = el_strbuf_persist(sl); snprintf(meta, sl, "{\"status\":\"%s\"}", st); n->metadata = meta; } else { - n->metadata = el_strdup("{}"); + n->metadata = el_strdup_persist("{}"); } n->salience = engram_decode_score(salience); n->importance = engram_decode_score(certainty); @@ -6468,6 +6611,114 @@ el_val_t engram_node_count(void) { return (el_val_t)engram_get()->node_count; } +/* ── Telemetry retention ──────────────────────────────────────────────────── + * (2026-07-16 self-review) InternalStateEvent nodes are append-only telemetry + * (heartbeat, curiosity_scan, engram_sync) written ~3/min by the awareness + * loop. Nothing ever removed them: by July 16 they were 10,175 of 13,522 + * nodes — 75% of the store was telemetry. They are already force-excluded + * from WM promotion (engram_activate), so their only effect was store bloat, + * snapshot bloat, and lexical-search noise. + * + * engram_prune_telemetry(older_than_ms) batch-removes ISE nodes whose + * created_at is older than now - older_than_ms, EXCEPT durable markers: + * - label "session-start" (boot history) + * - content containing "self_review" (daily review trail) + * Unlike repeated engram_forget (O(n) shift each), this is a single + * compaction pass over nodes plus one pass over edges, with one idmap + * rebuild — O(nodes + edges) total, safe to call on every ISE insert. + * Returns the number of nodes removed. */ + +/* FNV-1a hash for the removed-id set used by the edge sweep. */ +static uint64_t eg_fnv1a(const char* s) { + uint64_t h = 1469598103934665603ULL; + while (*s) { h ^= (unsigned char)*s++; h *= 1099511628211ULL; } + return h; +} + +el_val_t engram_prune_telemetry(el_val_t older_than_ms) { + int64_t horizon = (int64_t)older_than_ms; + if (horizon <= 0) horizon = 172800000; /* default 48h */ + EngramStore* g = engram_get(); + int64_t cutoff = engram_now_ms() - horizon; + + /* Pass 1: mark. Collect ids of prunable nodes (ownership transferred — + * strings freed after the edge sweep). */ + int64_t cap = 0; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + if (n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0 && + n->created_at < cutoff) cap++; + } + if (cap == 0) return 0; + + char** removed_ids = malloc((size_t)cap * sizeof(char*)); + if (!removed_ids) return 0; + int64_t removed = 0, w = 0; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + int prunable = + n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0 && + n->created_at < cutoff && + !(n->label && strcmp(n->label, "session-start") == 0) && + !(n->content && strstr(n->content, "self_review")); + if (prunable && removed < cap) { + removed_ids[removed++] = n->id; /* keep id for edge sweep */ + free(n->content); free(n->node_type); free(n->label); + free(n->tier); free(n->tags); free(n->metadata); + } else { + if (w != i) g->nodes[w] = g->nodes[i]; + w++; + } + } + g->node_count = w; + if (removed == 0) { free(removed_ids); return 0; } + + /* Removed-id hash set (open addressing, power-of-two >= 2*removed). */ + size_t set_cap = 16; + while (set_cap < (size_t)removed * 2) set_cap <<= 1; + const char** set = calloc(set_cap, sizeof(char*)); + if (set) { + for (int64_t i = 0; i < removed; i++) { + size_t slot = eg_fnv1a(removed_ids[i]) & (set_cap - 1); + while (set[slot]) slot = (slot + 1) & (set_cap - 1); + set[slot] = removed_ids[i]; + } + } + /* Pass 2: drop edges incident to any removed node (defensive — ISEs + * currently have no edges, but callers may connect them later). */ + if (set) { + int64_t ew = 0; + for (int64_t r = 0; r < g->edge_count; r++) { + EngramEdge* e = &g->edges[r]; + int incident = 0; + const char* ends[2] = { e->from_id, e->to_id }; + for (int k = 0; k < 2 && !incident; k++) { + if (!ends[k]) continue; + size_t slot = eg_fnv1a(ends[k]) & (set_cap - 1); + while (set[slot]) { + if (strcmp(set[slot], ends[k]) == 0) { incident = 1; break; } + slot = (slot + 1) & (set_cap - 1); + } + } + if (incident) { + free(e->id); free(e->from_id); free(e->to_id); + free(e->relation); free(e->metadata); + } else { + if (ew != r) g->edges[ew] = g->edges[r]; + ew++; + } + } + g->edge_count = ew; + free(set); + } + for (int64_t i = 0; i < removed; i++) free(removed_ids[i]); + free(removed_ids); + + engram_idmap_rebuild(g); + engram_adj_free(g); + return (el_val_t)removed; +} + static int istr_contains(const char* hay, const char* needle) { if (!hay || !needle || !*needle) return 0; size_t nl = strlen(needle); @@ -6552,10 +6803,10 @@ void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t EngramEdge* e = &g->edges[g->edge_count]; memset(e, 0, sizeof(*e)); e->id = engram_new_id(); - e->from_id = el_strdup(f); - e->to_id = el_strdup(t); - e->relation = el_strdup(r && *r ? r : "associate"); - e->metadata = el_strdup("{}"); + e->from_id = el_strdup_persist(f); + e->to_id = el_strdup_persist(t); + e->relation = el_strdup_persist(r && *r ? r : "associate"); + e->metadata = el_strdup_persist("{}"); e->weight = engram_decode_score(weight); if (e->weight <= 0.0 || e->weight > 1.0) e->weight = 0.5; e->confidence = 1.0; @@ -6853,6 +7104,14 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { } for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; + /* InternalStateEvent nodes are observability-only telemetry — never + * seed activation from them. Their JSON payloads contain common words + * ("memory", "context", ...) that lexically match almost any query, + * turning telemetry into a spreading-activation ignition source. They + * are already excluded from WM promotion in pass 2; exclude them from + * seeding here too. */ + if (n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0) + continue; if (istr_contains(n->content, q) || istr_contains(n->label, q) || istr_contains(n->tags, q)) { @@ -6929,11 +7188,22 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { if (oi < 0 || oi >= g->node_count) continue; EngramEdge* e = &g->edges[ei]; EngramNode* on = &g->nodes[oi]; + /* Never propagate INTO InternalStateEvent nodes. They are already + * barred from WM promotion (pass 2) and from seeding (above), but + * as high-degree hubs they still relayed activation across the + * graph. Skipping here keeps them out of the frontier entirely. */ + if (on->node_type && strcmp(on->node_type, "InternalStateEvent") == 0) + continue; double tbonus = engram_temporal_proximity_bonus(on->created_at, seed_epoch); double tdecay = engram_temporal_decay(on, now_ms); double dampen = engram_activation_dampen(on); double new_act = f.act * e->weight * SPREAD_DECAY * (1.0 + tbonus) * tdecay * dampen; + /* Firing threshold per classic spreading-activation: sub-threshold + * activation neither updates the target nor enqueues it, so weak + * signals die out instead of flooding the whole graph with tiny + * nonzero background activation. */ + if (new_act < 0.02) continue; if (!reached[oi] || new_act > best_bg[oi]) { best_bg[oi] = new_act; best_hops[oi] = new_hops; @@ -7174,6 +7444,31 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { } } + /* ── Retrieval reinforcement (2026-07-18 self-review) ─────────────────── + * ACT-R base-level learning: retrieval strengthens memory. Before this, + * NOTHING in engram_activate updated last_activated/activation_count — + * only the rarely-called engram_strengthen did. Consequence: temporal + * decay aged every node from its last explicit strengthen (usually + * creation), so frequently-retrieved memories decayed identically to + * abandoned ones, and engram_activation_dampen() saw a frozen count. + * + * Scope deliberately narrow: reinforce ONLY nodes promoted to WM in THIS + * call that survived both capacity caps (wm_weights[i] > 0 = promoted this + * call; working_memory_weight > 0 = survived Pass 4/5 eviction). BFS + * fan-out touches thousands of nodes per curiosity scan — reinforcing all + * of them would flatten dampening and freeze decay globally. Promotion to + * working memory is the analog of actual retrieval (executive access), + * matching ACT-R where only completed retrievals add a base-level + * presentation. Carry-over nodes (reached[i]==0) are NOT reinforced: they + * persist by decay, they were not re-retrieved. */ + for (int64_t i = 0; i < g->node_count; i++) { + if (!reached[i] || wm_weights[i] <= 0.0) continue; + EngramNode* n = &g->nodes[i]; + if (n->working_memory_weight <= 0.0) continue; /* evicted by cap */ + n->last_activated = now_ms; + n->activation_count++; + } + /* ── Collect all background-activated nodes for the return value ──── * Callers see both layers. Context compilation uses only promoted nodes * (working_memory_weight > 0). Sort: promoted first by wm_weight desc, @@ -7331,12 +7626,21 @@ el_val_t engram_save(el_val_t path) { /* Helper: extract a string field from a JSON object substring. */ static char* eg_get_str_field(const char* obj, const char* key) { + /* Returns a STORE-PERSISTENT string: callers assign the result directly + * into EngramNode/EngramEdge fields, and engram_load_merge runs inside + * the soul's per-tick arena. jp_parse_string_raw's success buffer is + * plain malloc (persist-safe, returned as-is); the empty/error returns + * were arena-tracked el_strdup("") — those dangled after the tick arena + * popped and free()ing them in callers was a latent double-free. */ const char* p = json_find_key(obj, key); - if (!p) return el_strdup(""); - if (*p != '"') return el_strdup(""); + if (!p) return el_strdup_persist(""); + if (*p != '"') return el_strdup_persist(""); JsonParser jp = { .p = p, .end = p + strlen(p), .err = 0 }; char* out = jp_parse_string_raw(&jp); - if (jp.err) { free(out); return el_strdup(""); } + /* *p == '"' is guaranteed above, so jp_parse_string_raw took its malloc + * path (its arena-tracked early-return only fires on a non-'"' start) — + * free(out) on error is safe here. */ + if (jp.err) { free(out); return el_strdup_persist(""); } return out; } @@ -7358,6 +7662,48 @@ static const char* eg_skip_ws(const char* p) { return p; } +/* eg_enforce_wm_cap_on_load — clamp a snapshot-restored working-memory + * population to ENGRAM_WM_CAP. + * + * WHY (2026-07-15 self-review): engram_load restored working_memory_weight + * verbatim from the snapshot with no cap. Pass 4/5 cap enforcement only runs + * inside engram_activate — so a snapshot frozen in the pre-2026-06-30 era + * (87-111 promoted nodes, all at the old 0.25 breakthrough floor) reloaded + * as-is on every boot, and heartbeats faithfully reported the stale + * population (wm_active 87-111, wm_avg pinned at exactly 0.25) forever. + * The cap must hold at every entry point that materializes WM, not just + * the activation path. Same top-K-by-weight logic as engram_activate + * Pass 5 (Cowan 2001: WM capacity is global). */ +static void eg_enforce_wm_cap_on_load(EngramStore* g) { + int64_t wm_count = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > 0.0) wm_count++; + } + if (wm_count <= ENGRAM_WM_CAP) return; + double* vals = malloc((size_t)wm_count * sizeof(double)); + if (!vals) return; /* skip on OOM — over cap this boot, no corruption */ + int64_t vi = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > 0.0) + vals[vi++] = g->nodes[i].working_memory_weight; + } + qsort(vals, (size_t)wm_count, sizeof(double), engram_cmp_double_desc); + double cutoff = vals[ENGRAM_WM_CAP - 1]; + free(vals); + int64_t above = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > cutoff) above++; + } + int64_t slots_at_cutoff = ENGRAM_WM_CAP - above; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + if (n->working_memory_weight <= 0.0) continue; + if (n->working_memory_weight > cutoff) continue; + if (slots_at_cutoff > 0) { slots_at_cutoff--; continue; } + n->working_memory_weight = 0.0; /* evict: over cap at load */ + } +} + el_val_t engram_load(el_val_t path) { const char* p = EL_CSTR(path); if (!p || !*p) return 0; @@ -7412,7 +7758,7 @@ el_val_t engram_load(el_val_t path) { nn->tier = eg_get_str_field(obj, "tier"); nn->tags = eg_get_str_field(obj, "tags"); nn->metadata = eg_get_str_field(obj, "metadata"); - if (!nn->metadata || !*nn->metadata) { free(nn->metadata); nn->metadata = el_strdup("{}"); } + if (!nn->metadata || !*nn->metadata) { free(nn->metadata); nn->metadata = el_strdup_persist("{}"); } nn->salience = eg_get_num_field(obj, "salience"); nn->importance = eg_get_num_field(obj, "importance"); nn->confidence = eg_get_num_field(obj, "confidence"); @@ -7424,6 +7770,14 @@ el_val_t engram_load(el_val_t path) { nn->updated_at = eg_get_int_field(obj, "updated_at"); nn->background_activation = eg_get_num_field(obj, "background_activation"); nn->working_memory_weight = eg_get_num_field(obj, "working_memory_weight"); + /* Launder persisted WM weights across restarts: snapshots carry + * legacy breakthrough-floor 0.25 weights (pinned by the old + * suppression-breakthrough path) and nothing else ever launders + * them. Halve on every boot so genuine working-memory state has + * continuity across a restart while stale pinned weights decay + * out over successive boots; sub-0.05 residue drops to zero. */ + nn->working_memory_weight *= 0.5; + if (nn->working_memory_weight < 0.05) nn->working_memory_weight = 0.0; nn->suppression_count = (int32_t)eg_get_int_field(obj, "suppression_count"); /* layer_id defaults to ENGRAM_LAYER_DEFAULT (core-identity) * for snapshots that predate the layered schema. We can't @@ -7466,7 +7820,7 @@ el_val_t engram_load(el_val_t path) { ee->to_id = eg_get_str_field(obj, "to_id"); ee->relation = eg_get_str_field(obj, "relation"); ee->metadata = eg_get_str_field(obj, "metadata"); - if (!ee->metadata || !*ee->metadata) { free(ee->metadata); ee->metadata = el_strdup("{}"); } + if (!ee->metadata || !*ee->metadata) { free(ee->metadata); ee->metadata = el_strdup_persist("{}"); } ee->weight = eg_get_num_field(obj, "weight"); ee->confidence = eg_get_num_field(obj, "confidence"); ee->created_at = eg_get_int_field(obj, "created_at"); @@ -7541,6 +7895,9 @@ el_val_t engram_load(el_val_t path) { } } } + /* WM cap discipline applies to every entry point that materializes WM, + * including snapshot restore (see eg_enforce_wm_cap_on_load). */ + eg_enforce_wm_cap_on_load(g); free(data); return 1; } @@ -7586,9 +7943,13 @@ el_val_t engram_load_merge(el_val_t path) { char* obj = malloc(n + 1); memcpy(obj, nodes_p, n); obj[n] = '\0'; char* nid = eg_get_str_field(obj, "id"); - int already = (nid && *nid && engram_find_node(nid) != NULL); + /* Nodes with an empty/unparseable id can never dedup against + * the idmap, so without this guard they were re-added on EVERY + * merge cycle — unbounded store growth. Skip them entirely. */ + int has_id = (nid && *nid); + int already = (has_id && engram_find_node(nid) != NULL); free(nid); - if (!already) { + if (has_id && !already) { engram_grow_nodes(); EngramNode* nn = &g->nodes[g->node_count]; memset(nn, 0, sizeof(*nn)); @@ -7609,9 +7970,11 @@ el_val_t engram_load_merge(el_val_t path) { nn->created_at = eg_get_int_field(obj, "created_at"); nn->updated_at = eg_get_int_field(obj, "updated_at"); nn->background_activation = eg_get_num_field(obj, "background_activation"); - nn->working_memory_weight = eg_get_num_field(obj, "working_memory_weight"); - if (!isfinite(nn->working_memory_weight) || nn->working_memory_weight < 0.0 || nn->working_memory_weight > 1.0) - nn->working_memory_weight = 0.0; + /* Nodes arriving via merge were never part of THIS store's + * working memory — importing the source store's WM weight + * would inject foreign (often legacy 0.25 breakthrough- + * floor) weights into local WM every sync cycle. */ + nn->working_memory_weight = 0.0; nn->suppression_count = (int32_t)eg_get_int_field(obj, "suppression_count"); if (json_find_key(obj, "layer_id")) { nn->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); @@ -7694,6 +8057,9 @@ el_val_t engram_load_merge(el_val_t path) { } } + /* Merged nodes can carry snapshot WM weights too — hold the cap here + * as well (see eg_enforce_wm_cap_on_load). */ + eg_enforce_wm_cap_on_load(g); free(data); return (el_val_t)added_nodes; } @@ -7860,8 +8226,16 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t di free(frontier); free(frontier_h); free(visited); jb_putc(&b, ']'); return el_wrap_str(b.buf); } - frontier[fc] = el_strdup(sid); frontier_h[fc] = 0; fc++; - visited[vc++] = el_strdup(sid); + /* MUST be el_strdup_persist: this function frees frontier/visited strings + * manually (lines below). el_strdup would ALSO register them in the + * per-request arena, so el_request_end() double-freed every one of them + * at the end of the HTTP request — SIGABRT in http_worker under load. + * (2026-07-18 self-review; reproduced via ASAN on /api/neuron/session/begin + * and /api/neuron/graph. Same allocation-discipline class as the + * 2026-07-15 EngramNode and 2026-07-16 idmap-key fixes: never mix arena + * tracking with manual free.) */ + frontier[fc] = el_strdup_persist(sid); frontier_h[fc] = 0; fc++; + visited[vc++] = el_strdup_persist(sid); int first = 1; while (fc > 0) { @@ -7890,8 +8264,8 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t di char tmp[64]; snprintf(tmp, sizeof(tmp), ",\"hops\":%lld}", (long long)(h + 1)); jb_puts(&b, tmp); first = 0; - if (vc < 1024) visited[vc++] = el_strdup(peer); - if (fc < 1024 && h + 1 < depth) { frontier[fc] = el_strdup(peer); frontier_h[fc] = h + 1; fc++; } + if (vc < 1024) visited[vc++] = el_strdup_persist(peer); + if (fc < 1024 && h + 1 < depth) { frontier[fc] = el_strdup_persist(peer); frontier_h[fc] = h + 1; fc++; } } free(cur); } @@ -8612,13 +8986,13 @@ static int64_t dharma_find_or_create_relation_edge(const char* peer_base, int cr engram_grow_nodes(); EngramNode* n = &g->nodes[g->node_count]; memset(n, 0, sizeof(*n)); - n->id = el_strdup(self_id); - n->content = el_strdup(_el_cgi_dharma_id ? _el_cgi_dharma_id : "(self)"); - n->node_type = el_strdup("DharmaSelf"); - n->label = el_strdup("dharma:self"); - n->tier = el_strdup("Working"); - n->tags = el_strdup("dharma"); - n->metadata = el_strdup("{}"); + n->id = el_strdup_persist(self_id); + n->content = el_strdup_persist(_el_cgi_dharma_id ? _el_cgi_dharma_id : "(self)"); + n->node_type = el_strdup_persist("DharmaSelf"); + n->label = el_strdup_persist("dharma:self"); + n->tier = el_strdup_persist("Working"); + n->tags = el_strdup_persist("dharma"); + n->metadata = el_strdup_persist("{}"); n->salience = 1.0; n->importance = 1.0; n->confidence = 1.0; int64_t now = engram_now_ms(); n->created_at = now; n->updated_at = now; n->last_activated = now; @@ -8629,13 +9003,13 @@ static int64_t dharma_find_or_create_relation_edge(const char* peer_base, int cr engram_grow_nodes(); EngramNode* n = &g->nodes[g->node_count]; memset(n, 0, sizeof(*n)); - n->id = el_strdup(peer_node); - n->content = el_strdup(peer_base); - n->node_type = el_strdup("DharmaPeer"); - n->label = el_strdup(peer_node); - n->tier = el_strdup("Working"); - n->tags = el_strdup("dharma"); - n->metadata = el_strdup("{}"); + n->id = el_strdup_persist(peer_node); + n->content = el_strdup_persist(peer_base); + n->node_type = el_strdup_persist("DharmaPeer"); + n->label = el_strdup_persist(peer_node); + n->tier = el_strdup_persist("Working"); + n->tags = el_strdup_persist("dharma"); + n->metadata = el_strdup_persist("{}"); n->salience = 0.5; n->importance = 0.5; n->confidence = 1.0; int64_t now = engram_now_ms(); n->created_at = now; n->updated_at = now; n->last_activated = now; @@ -8647,10 +9021,10 @@ static int64_t dharma_find_or_create_relation_edge(const char* peer_base, int cr EngramEdge* e = &g->edges[g->edge_count]; memset(e, 0, sizeof(*e)); e->id = engram_new_id(); - e->from_id = el_strdup(self_id); - e->to_id = el_strdup(peer_node); - e->relation = el_strdup("dharma-relation"); - e->metadata = el_strdup("{}"); + e->from_id = el_strdup_persist(self_id); + e->to_id = el_strdup_persist(peer_node); + e->relation = el_strdup_persist("dharma-relation"); + e->metadata = el_strdup_persist("{}"); e->weight = 0.0; e->confidence = 1.0; int64_t now = engram_now_ms();