diff --git a/engram/dist/engram b/engram/dist/engram index 0c99cb6..988e3cd 100755 Binary files a/engram/dist/engram and b/engram/dist/engram differ diff --git a/engram/dist/engram.c b/engram/dist/engram.c index 18dc9e6..f43fca5 100644 --- a/engram/dist/engram.c +++ b/engram/dist/engram.c @@ -10,6 +10,7 @@ el_val_t query_param(el_val_t path, el_val_t key); el_val_t query_int(el_val_t path, el_val_t key, el_val_t default_val); el_val_t extract_id(el_val_t path, el_val_t prefix); el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body); +el_val_t persist_canonical(void); el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body); el_val_t route_get_node(el_val_t method, el_val_t path, el_val_t body); el_val_t route_scan_nodes(el_val_t method, el_val_t path, el_val_t body); @@ -20,18 +21,23 @@ el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body); el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body); el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body); el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body); -el_val_t route_create_ise(el_val_t method, el_val_t path, el_val_t body); -el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body); el_val_t route_save(el_val_t method, el_val_t path, el_val_t body); el_val_t route_load(el_val_t method, el_val_t path, el_val_t body); el_val_t route_health(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_load_merge(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_emit_ise(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_capture_knowledge(el_val_t method, el_val_t path, el_val_t body); el_val_t check_auth_ok(el_val_t method, el_val_t body); el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body); +el_val_t bind_raw; el_val_t bind_str; el_val_t port; +el_val_t data_dir_raw; el_val_t data_dir; el_val_t snapshot_path; +el_val_t boot_snap; el_val_t parse_port(el_val_t bind) { el_val_t colon = str_index_of(bind, EL_STR(":")); @@ -110,17 +116,22 @@ el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body) { return 0; } +el_val_t persist_canonical(void) { + el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); + el_val_t dir = ({ el_val_t _if_result_1 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_1 = (EL_STR("/tmp/engram")); } else { _if_result_1 = (dir_raw); } _if_result_1; }); + engram_save(el_str_concat(dir, EL_STR("/snapshot.json"))); + return 1; + return 0; +} + el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body) { el_val_t content = json_get_string(body, EL_STR("content")); - el_val_t node_type = json_get_string(body, EL_STR("node_type")); - if (str_eq(node_type, EL_STR(""))) { - node_type = EL_STR("Memory"); - } - el_val_t salience = json_get_float(body, EL_STR("salience")); - if (salience == el_from_float(0.0)) { - salience = el_from_float(0.5); - } + el_val_t nt_raw = json_get_string(body, EL_STR("node_type")); + el_val_t node_type = ({ el_val_t _if_result_2 = 0; if (str_eq(nt_raw, EL_STR(""))) { _if_result_2 = (EL_STR("Memory")); } else { _if_result_2 = (nt_raw); } _if_result_2; }); + el_val_t sal_raw = json_get_float(body, EL_STR("salience")); + el_val_t salience = ({ el_val_t _if_result_3 = 0; if ((sal_raw == el_from_float(0.0))) { _if_result_3 = (el_from_float(0.5)); } else { _if_result_3 = (sal_raw); } _if_result_3; }); el_val_t id = engram_node(content, node_type, salience); + el_val_t saved = persist_canonical(); return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"id\":\""), id), EL_STR("\",\"content\":\"")), content), EL_STR("\",\"node_type\":\"")), node_type), EL_STR("\"}")); return 0; } @@ -146,11 +157,9 @@ el_val_t route_scan_nodes(el_val_t method, el_val_t path, el_val_t body) { } el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body) { - el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR")); - if (str_eq(dir, EL_STR(""))) { - dir = EL_STR("/tmp/engram"); - } - el_val_t snap_path = el_str_concat(dir, EL_STR("/snapshot.json")); + el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); + el_val_t dir = ({ el_val_t _if_result_4 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_4 = (EL_STR("/tmp/engram")); } else { _if_result_4 = (dir_raw); } _if_result_4; }); + el_val_t snap_path = el_str_concat(dir, EL_STR("/.scan-export.json")); engram_save(snap_path); el_val_t snap = fs_read(snap_path); if (str_eq(snap, EL_STR(""))) { @@ -165,36 +174,22 @@ el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body) { } el_val_t route_search(el_val_t method, el_val_t path, el_val_t body) { - el_val_t q = EL_STR(""); - if (str_eq(method, EL_STR("GET"))) { - q = query_param(path, EL_STR("q")); - } else { - q = json_get_string(body, EL_STR("query")); - } - el_val_t limit = query_int(path, EL_STR("limit"), 20); - if (limit == 0) { - limit = json_get_int(body, EL_STR("limit")); - } - if (limit == 0) { - limit = 20; - } + el_val_t q = ({ el_val_t _if_result_5 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_5 = (query_param(path, EL_STR("q"))); } else { _if_result_5 = (json_get_string(body, EL_STR("query"))); } _if_result_5; }); + el_val_t lim_url = query_int(path, EL_STR("limit"), 0); + el_val_t lim_body = json_get_int(body, EL_STR("limit")); + el_val_t lim_either = ({ el_val_t _if_result_6 = 0; if ((lim_url > 0)) { _if_result_6 = (lim_url); } else { _if_result_6 = (lim_body); } _if_result_6; }); + el_val_t limit = ({ el_val_t _if_result_7 = 0; if ((lim_either > 0)) { _if_result_7 = (lim_either); } else { _if_result_7 = (20); } _if_result_7; }); return engram_search_json(q, limit); return 0; } el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body) { - el_val_t q = EL_STR(""); - el_val_t depth = 3; - if (str_eq(method, EL_STR("GET"))) { - q = query_param(path, EL_STR("q")); - depth = query_int(path, EL_STR("depth"), 3); - } else { - q = json_get_string(body, EL_STR("query")); - el_val_t bd = json_get_int(body, EL_STR("depth")); - if (bd > 0) { - depth = bd; - } + el_val_t q = ({ el_val_t _if_result_8 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_8 = (query_param(path, EL_STR("q"))); } else { _if_result_8 = (json_get_string(body, EL_STR("query"))); } _if_result_8; }); + if (str_eq(q, EL_STR(""))) { + return err_json(EL_STR("missing query")); } + el_val_t d_raw = ({ el_val_t _if_result_9 = 0; if (str_eq(method, EL_STR("GET"))) { _if_result_9 = (query_int(path, EL_STR("depth"), 3)); } else { _if_result_9 = (json_get_int(body, EL_STR("depth"))); } _if_result_9; }); + el_val_t depth = ({ el_val_t _if_result_10 = 0; if ((d_raw > 0)) { _if_result_10 = (d_raw); } else { _if_result_10 = (3); } _if_result_10; }); return el_str_concat(el_str_concat(EL_STR("{\"results\":"), engram_activate_json(q, depth)), EL_STR("}")); return 0; } @@ -202,15 +197,12 @@ el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body) { el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body) { el_val_t from_id = json_get_string(body, EL_STR("from_id")); el_val_t to_id = json_get_string(body, EL_STR("to_id")); - el_val_t relation = json_get_string(body, EL_STR("relation")); - if (str_eq(relation, EL_STR(""))) { - relation = EL_STR("associates"); - } - el_val_t weight = json_get_float(body, EL_STR("weight")); - if (weight == el_from_float(0.0)) { - weight = el_from_float(0.5); - } + el_val_t rel_raw = json_get_string(body, EL_STR("relation")); + el_val_t relation = ({ el_val_t _if_result_11 = 0; if (str_eq(rel_raw, EL_STR(""))) { _if_result_11 = (EL_STR("associates")); } else { _if_result_11 = (rel_raw); } _if_result_11; }); + el_val_t w_raw = json_get_float(body, EL_STR("weight")); + el_val_t weight = ({ el_val_t _if_result_12 = 0; if ((w_raw == el_from_float(0.0))) { _if_result_12 = (el_from_float(0.5)); } else { _if_result_12 = (w_raw); } _if_result_12; }); engram_connect(from_id, to_id, weight, relation); + el_val_t saved = persist_canonical(); return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"from_id\":\""), from_id), EL_STR("\",\"to_id\":\"")), to_id), EL_STR("\",\"relation\":\"")), relation), EL_STR("\"}")); return 0; } @@ -231,6 +223,7 @@ el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body) { return err_json(EL_STR("missing node_id")); } engram_strengthen(id); + el_val_t saved = persist_canonical(); return ok_json(); return 0; } @@ -241,29 +234,40 @@ el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body) { return err_json(EL_STR("missing id")); } engram_forget(id); + el_val_t saved = persist_canonical(); return ok_json(); return 0; } -el_val_t route_create_ise(el_val_t method, el_val_t path, el_val_t body) { - el_val_t content = json_get_string(body, EL_STR("content")); - if (str_eq(content, EL_STR(""))) { - return err_json(EL_STR("missing content")); - } - el_val_t sal = el_from_float(0.3); - el_val_t imp = el_from_float(0.3); - el_val_t conf = el_from_float(0.8); - el_val_t id = engram_node_full(content, EL_STR("InternalStateEvent"), EL_STR("state-event"), sal, imp, conf, EL_STR("Episodic"), EL_STR("[\"internal-state\",\"InternalStateEvent\"]")); - return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\"}")); +el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) { + el_val_t p_raw = json_get_string(body, EL_STR("path")); + el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); + el_val_t dir = ({ el_val_t _if_result_13 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_13 = (EL_STR("/tmp/engram")); } else { _if_result_13 = (dir_raw); } _if_result_13; }); + el_val_t p = ({ el_val_t _if_result_14 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_14 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_14 = (p_raw); } _if_result_14; }); + engram_save(p); + return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"path\":\""), p), EL_STR("\"}")); + return 0; +} + +el_val_t route_load(el_val_t method, el_val_t path, el_val_t body) { + el_val_t p_raw = json_get_string(body, EL_STR("path")); + el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); + el_val_t dir = ({ el_val_t _if_result_15 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_15 = (EL_STR("/tmp/engram")); } else { _if_result_15 = (dir_raw); } _if_result_15; }); + el_val_t p = ({ el_val_t _if_result_16 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_16 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_16 = (p_raw); } _if_result_16; }); + engram_load(p); + return ok_json(); + return 0; +} + +el_val_t route_health(el_val_t method, el_val_t path, el_val_t body) { + return EL_STR("{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}"); return 0; } el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body) { - el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR")); - if (str_eq(dir, EL_STR(""))) { - dir = EL_STR("/tmp/engram"); - } - el_val_t snap_path = el_str_concat(dir, EL_STR("/sync-export.json")); + el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); + el_val_t dir = ({ el_val_t _if_result_17 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_17 = (EL_STR("/tmp/engram")); } else { _if_result_17 = (dir_raw); } _if_result_17; }); + el_val_t snap_path = el_str_concat(dir, EL_STR("/.sync-export.json")); engram_save(snap_path); el_val_t snap = fs_read(snap_path); if (str_eq(snap, EL_STR(""))) { @@ -273,36 +277,68 @@ el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body) { return 0; } -el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) { +el_val_t route_load_merge(el_val_t method, el_val_t path, el_val_t body) { el_val_t p = json_get_string(body, EL_STR("path")); if (str_eq(p, EL_STR(""))) { - el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR")); - if (str_eq(dir, EL_STR(""))) { - dir = EL_STR("/tmp/engram"); - } - p = el_str_concat(dir, EL_STR("/snapshot.json")); + return err_json(EL_STR("path is required")); } - engram_save(p); - return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"path\":\""), p), EL_STR("\"}")); + if (str_eq(fs_read(p), EL_STR(""))) { + return err_json(EL_STR("file missing or empty")); + } + el_val_t before_n = engram_node_count(); + el_val_t before_e = engram_edge_count(); + engram_load_merge(p); + el_val_t added_n = (engram_node_count() - before_n); + el_val_t added_e = (engram_edge_count() - before_e); + el_val_t saved = persist_canonical(); + return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"nodes_added\":"), int_to_str(added_n)), EL_STR(",\"edges_added\":")), int_to_str(added_e)), EL_STR(",\"node_count\":")), int_to_str(engram_node_count())), EL_STR("}")); return 0; } -el_val_t route_load(el_val_t method, el_val_t path, el_val_t body) { - el_val_t p = json_get_string(body, EL_STR("path")); - if (str_eq(p, EL_STR(""))) { - el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR")); - if (str_eq(dir, EL_STR(""))) { - dir = EL_STR("/tmp/engram"); - } - p = el_str_concat(dir, EL_STR("/snapshot.json")); +el_val_t route_emit_ise(el_val_t method, el_val_t path, el_val_t body) { + el_val_t content = json_get_string(body, EL_STR("content")); + if (str_eq(content, EL_STR(""))) { + return err_json(EL_STR("missing content")); } - engram_load(p); - return ok_json(); + el_val_t sal = el_from_float(0.3); + el_val_t imp = el_from_float(0.3); + el_val_t conf = el_from_float(0.8); + el_val_t id = engram_node_full(content, EL_STR("InternalStateEvent"), EL_STR("state-event"), sal, imp, conf, EL_STR("Episodic"), EL_STR("[\"internal-state\",\"InternalStateEvent\"]")); + el_val_t ret_raw = env(EL_STR("ENGRAM_ISE_RETENTION_MS")); + el_val_t ret_ms = ({ el_val_t _if_result_18 = 0; if (str_eq(ret_raw, EL_STR(""))) { _if_result_18 = (172800000); } else { _if_result_18 = (str_to_int(ret_raw)); } _if_result_18; }); + el_val_t pruned = engram_prune_telemetry(ret_ms); + return el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\",\"pruned\":")), int_to_str(pruned)), EL_STR("}")); return 0; } -el_val_t route_health(el_val_t method, el_val_t path, el_val_t body) { - return EL_STR("{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}"); +el_val_t route_capture_knowledge(el_val_t method, el_val_t path, el_val_t body) { + el_val_t content = json_get_string(body, EL_STR("content")); + if (str_eq(content, EL_STR(""))) { + return err_json(EL_STR("missing content")); + } + el_val_t title = json_get_string(body, EL_STR("title")); + el_val_t label = ({ el_val_t _if_result_19 = 0; if (str_eq(title, EL_STR(""))) { _if_result_19 = (str_slice(content, 0, 60)); } else { _if_result_19 = (title); } _if_result_19; }); + el_val_t category_raw = json_get_string(body, EL_STR("category")); + el_val_t category = ({ el_val_t _if_result_20 = 0; if (str_eq(category_raw, EL_STR(""))) { _if_result_20 = (EL_STR("other")); } else { _if_result_20 = (category_raw); } _if_result_20; }); + el_val_t ktier_raw = json_get_string(body, EL_STR("tier")); + el_val_t ktier = ({ el_val_t _if_result_21 = 0; if (str_eq(ktier_raw, EL_STR(""))) { _if_result_21 = (EL_STR("note")); } else { _if_result_21 = (ktier_raw); } _if_result_21; }); + el_val_t project = json_get_string(body, EL_STR("project")); + el_val_t tags_raw = json_get_raw(body, EL_STR("tags")); + el_val_t tags_base = ({ el_val_t _if_result_22 = 0; if (str_eq(tags_raw, EL_STR(""))) { _if_result_22 = (EL_STR("[]")); } else { _if_result_22 = (tags_raw); } _if_result_22; }); + el_val_t base_len = str_len(tags_base); + el_val_t head = str_slice(tags_base, 0, (base_len - 1)); + el_val_t sep = ({ el_val_t _if_result_23 = 0; if (str_eq(head, EL_STR("["))) { _if_result_23 = (EL_STR("")); } else { _if_result_23 = (EL_STR(",")); } _if_result_23; }); + el_val_t safe_cat = str_replace(category, EL_STR("\""), EL_STR("'")); + el_val_t safe_tier = str_replace(ktier, EL_STR("\""), EL_STR("'")); + el_val_t safe_proj = str_replace(project, EL_STR("\""), EL_STR("'")); + el_val_t proj_tag = ({ el_val_t _if_result_24 = 0; if (str_eq(safe_proj, EL_STR(""))) { _if_result_24 = (EL_STR("")); } else { _if_result_24 = (el_str_concat(el_str_concat(EL_STR(",\"project:"), safe_proj), EL_STR("\""))); } _if_result_24; }); + el_val_t tags = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(head, sep), EL_STR("\"category:")), safe_cat), EL_STR("\",\"tier:")), safe_tier), EL_STR("\"")), proj_tag), EL_STR("]")); + el_val_t sal = el_from_float(0.5); + el_val_t imp = el_from_float(0.5); + el_val_t conf = el_from_float(0.9); + el_val_t id = engram_node_full(content, EL_STR("Knowledge"), label, sal, imp, conf, EL_STR("Semantic"), tags); + el_val_t saved = persist_canonical(); + return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\"}")); return 0; } @@ -329,12 +365,15 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) { return route_health(method, path, body); } } - if (str_eq(method, EL_STR("POST")) && str_starts_with(clean, EL_STR("/api/neuron/state-events"))) { - return route_create_ise(method, path, body); + if (str_eq(method, EL_STR("POST")) && str_eq(clean, EL_STR("/api/neuron/state-events"))) { + return route_emit_ise(method, path, body); } if (!check_auth_ok(method, body)) { return err_json(EL_STR("unauthorized")); } + if (str_eq(method, EL_STR("POST")) && str_eq(clean, EL_STR("/api/neuron/knowledge/capture"))) { + return route_capture_knowledge(method, path, body); + } if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/stats")) || str_eq(clean, EL_STR("/stats")))) { return route_stats(method, path, body); } @@ -374,32 +413,40 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) { if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/strengthen")) || str_eq(clean, EL_STR("/strengthen")))) { return route_strengthen(method, path, body); } - if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/sync")) || str_eq(clean, EL_STR("/sync")))) { - return route_sync(method, path, body); - } if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/save")) || str_eq(clean, EL_STR("/save")))) { return route_save(method, path, body); } if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/load")) || str_eq(clean, EL_STR("/load")))) { return route_load(method, path, body); } + if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/load-merge")) || str_eq(clean, EL_STR("/load-merge")))) { + return route_load_merge(method, path, body); + } + if (str_eq(method, EL_STR("GET")) && str_eq(clean, EL_STR("/api/sync"))) { + return route_sync(method, path, body); + } return el_str_concat(el_str_concat(EL_STR("{\"error\":\"not found\",\"path\":\""), clean), EL_STR("\"}")); return 0; } int main(int _argc, char** _argv) { el_runtime_init_args(_argc, _argv); - bind_str = env(EL_STR("ENGRAM_BIND")); - if (str_eq(bind_str, EL_STR(""))) { - bind_str = EL_STR(":8742"); - } + bind_raw = env(EL_STR("ENGRAM_BIND")); + bind_str = ({ el_val_t _if_result_25 = 0; if (str_eq(bind_raw, EL_STR(""))) { _if_result_25 = (EL_STR(":8742")); } else { _if_result_25 = (bind_raw); } _if_result_25; }); port = parse_port(bind_str); - data_dir = env(EL_STR("ENGRAM_DATA_DIR")); - if (str_eq(data_dir, EL_STR(""))) { - data_dir = EL_STR("/tmp/engram"); - } + data_dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); + data_dir = ({ el_val_t _if_result_26 = 0; if (str_eq(data_dir_raw, EL_STR(""))) { _if_result_26 = (EL_STR("/tmp/engram")); } else { _if_result_26 = (data_dir_raw); } _if_result_26; }); snapshot_path = el_str_concat(data_dir, EL_STR("/snapshot.json")); engram_load(snapshot_path); + boot_snap = fs_read(snapshot_path); + if (!str_eq(boot_snap, EL_STR(""))) { + if (engram_node_count() == 0) { + println(EL_STR("[engram] WARNING: snapshot.json is non-empty but load produced 0 nodes \xe2\x80\x94 preserving copy at snapshot.failed-load.json")); + fs_write(el_str_concat(data_dir, EL_STR("/snapshot.failed-load.json")), boot_snap); + } else { + fs_write(el_str_concat(data_dir, EL_STR("/snapshot.boot-backup.json")), boot_snap); + } + } println(EL_STR("[engram] runtime-native graph engine")); println(el_str_concat(EL_STR("[engram] data_dir="), data_dir)); println(el_str_concat(EL_STR("[engram] node_count="), int_to_str(engram_node_count()))); diff --git a/engram/src/server.el b/engram/src/server.el index e676d32..3372ff6 100644 --- a/engram/src/server.el +++ b/engram/src/server.el @@ -76,13 +76,43 @@ fn route_stats(method: String, path: String, body: String) -> String { engram_stats_json() } +// (2026-07-18 self-review) Scoping sweep: `let` inside an if-block creates an +// inner scope only — it does NOT mutate the outer binding (documented with +// evidence in awareness.el, 2026-05-25). Every default/reassignment below used +// that broken pattern, so defaults never applied: nodes were created with +// node_type="" and salience=0.0, /api/search and /api/activate ALWAYS ran with +// q="" regardless of input, edges defaulted to relation=""/weight=0.0, and +// save/load with no "path" hit engram_save(""). Rewritten to the +// `let x = if cond { a } else { b }` expression form (the pattern the newer +// routes route_emit_ise/route_capture_knowledge already use correctly). +// persist_canonical — save the canonical snapshot after a durable write. +// +// WHY (2026-07-22 self-review): the 2026-07-21 fix correctly stopped READ +// routes from writing the canonical snapshot.json — but nothing was left +// that saved it on WRITE. Every mutation (node create, edge create, +// knowledge capture, forget, merge) lived only in RAM until someone POSTed +// /api/save manually; a process restart silently discarded everything since +// the last manual save. Observed live: two engram restarts during the +// 2026-07-22 review reverted the store to a ~17h-old snapshot, destroying +// same-day writes. Reads must never write the canonical; writes must always +// persist it. ISE telemetry is deliberately excluded (48h-pruned, loss- +// tolerant, ~2/min — snapshotting the whole store per heartbeat is waste; +// any durable write that follows persists the pruning too). +fn persist_canonical() -> Int { + let dir_raw: String = env("ENGRAM_DATA_DIR") + let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw } + engram_save(dir + "/snapshot.json") + return 1 +} + fn route_create_node(method: String, path: String, body: String) -> String { let content: String = json_get_string(body, "content") - let node_type: String = json_get_string(body, "node_type") - if str_eq(node_type, "") { let node_type = "Memory" } - let salience: Float = json_get_float(body, "salience") - if salience == 0.0 { let salience = 0.5 } + let nt_raw: String = json_get_string(body, "node_type") + let node_type: String = if str_eq(nt_raw, "") { "Memory" } else { nt_raw } + let sal_raw: Float = json_get_float(body, "salience") + let salience: Float = if sal_raw == 0.0 { 0.5 } else { sal_raw } let id: String = engram_node(content, node_type, salience) + let saved: Int = persist_canonical() "{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\"}" } @@ -103,13 +133,14 @@ fn route_scan_nodes(method: String, path: String, body: String) -> String { } // route_scan_edges — bulk export of all edges as a JSON array. Implemented -// via engram_save → fs_read of the canonical on-disk snapshot, which the -// runtime keeps in lockstep with the in-memory graph. Live against the -// running graph, not a stale export. +// via engram_save → fs_read of a SCRATCH export path. (2026-07-21 self-review: +// previously this saved over the canonical snapshot.json on every GET — if the +// process ever booted with a partial/empty store, the first read request +// clobbered the good snapshot. Read routes must never write the canonical path.) fn route_scan_edges(method: String, path: String, body: String) -> String { - let dir: String = env("ENGRAM_DATA_DIR") - if str_eq(dir, "") { let dir = "/tmp/engram" } - let snap_path: String = dir + "/snapshot.json" + let dir_raw: String = env("ENGRAM_DATA_DIR") + let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw } + let snap_path: String = dir + "/.scan-export.json" engram_save(snap_path) let snap: String = fs_read(snap_path) if str_eq(snap, "") { return "[]" } @@ -122,40 +153,34 @@ fn route_scan_edges(method: String, path: String, body: String) -> String { } fn route_search(method: String, path: String, body: String) -> String { - let q: String = "" - if str_eq(method, "GET") { - let q = query_param(path, "q") - } else { - let q = json_get_string(body, "query") - } - let limit: Int = query_int(path, "limit", 20) - if limit == 0 { let limit = json_get_int(body, "limit") } - if limit == 0 { let limit = 20 } + let q: String = if str_eq(method, "GET") { query_param(path, "q") } else { json_get_string(body, "query") } + let lim_url: Int = query_int(path, "limit", 0) + let lim_body: Int = json_get_int(body, "limit") + let lim_either: Int = if lim_url > 0 { lim_url } else { lim_body } + let limit: Int = if lim_either > 0 { lim_either } else { 20 } return engram_search_json(q, limit) } fn route_activate(method: String, path: String, body: String) -> String { - let q: String = "" - let depth: Int = 3 - if str_eq(method, "GET") { - let q = query_param(path, "q") - let depth = query_int(path, "depth", 3) - } else { - let q = json_get_string(body, "query") - let bd: Int = json_get_int(body, "depth") - if bd > 0 { let depth = bd } - } + let q: String = if str_eq(method, "GET") { query_param(path, "q") } else { json_get_string(body, "query") } + // Guard: engram_activate with an empty query matches zero seeds, which + // zeroes ALL carried working-memory weights (documented in awareness.el + // perceive()). Never let an empty activation through to wipe WM. + if str_eq(q, "") { return err_json("missing query") } + let d_raw: Int = if str_eq(method, "GET") { query_int(path, "depth", 3) } else { json_get_int(body, "depth") } + let depth: Int = if d_raw > 0 { d_raw } else { 3 } return "{\"results\":" + engram_activate_json(q, depth) + "}" } fn route_create_edge(method: String, path: String, body: String) -> String { let from_id: String = json_get_string(body, "from_id") let to_id: String = json_get_string(body, "to_id") - let relation: String = json_get_string(body, "relation") - if str_eq(relation, "") { let relation = "associates" } - let weight: Float = json_get_float(body, "weight") - if weight == 0.0 { let weight = 0.5 } + let rel_raw: String = json_get_string(body, "relation") + let relation: String = if str_eq(rel_raw, "") { "associates" } else { rel_raw } + let w_raw: Float = json_get_float(body, "weight") + let weight: Float = if w_raw == 0.0 { 0.5 } else { w_raw } engram_connect(from_id, to_id, weight, relation) + let saved: Int = persist_canonical() "{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + relation + "\"}" } @@ -170,6 +195,7 @@ fn route_strengthen(method: String, path: String, body: String) -> String { let id: String = json_get_string(body, "node_id") if str_eq(id, "") { return err_json("missing node_id") } engram_strengthen(id) + let saved: Int = persist_canonical() ok_json() } @@ -177,27 +203,24 @@ fn route_forget(method: String, path: String, body: String) -> String { let id: String = extract_id(path, "/api/nodes/") if str_eq(id, "") { return err_json("missing id") } engram_forget(id) + let saved: Int = persist_canonical() ok_json() } fn route_save(method: String, path: String, body: String) -> String { - let p: String = json_get_string(body, "path") - if str_eq(p, "") { - let dir: String = env("ENGRAM_DATA_DIR") - if str_eq(dir, "") { let dir = "/tmp/engram" } - let p = dir + "/snapshot.json" - } + let p_raw: String = json_get_string(body, "path") + let dir_raw: String = env("ENGRAM_DATA_DIR") + let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw } + let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw } engram_save(p) "{\"ok\":true,\"path\":\"" + p + "\"}" } fn route_load(method: String, path: String, body: String) -> String { - let p: String = json_get_string(body, "path") - if str_eq(p, "") { - let dir: String = env("ENGRAM_DATA_DIR") - if str_eq(dir, "") { let dir = "/tmp/engram" } - let p = dir + "/snapshot.json" - } + let p_raw: String = json_get_string(body, "path") + let dir_raw: String = env("ENGRAM_DATA_DIR") + let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw } + let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw } engram_load(p) ok_json() } @@ -219,15 +242,36 @@ fn route_health(method: String, path: String, body: String) -> String { // (it skips nodes already present by ID). Auth-exempt: same-host internal call. // (2026-06-27 self-review: added this route to fix silent 10-min sync failures) fn route_sync(method: String, path: String, body: String) -> String { - let dir: String = env("ENGRAM_DATA_DIR") - if str_eq(dir, "") { let dir = "/tmp/engram" } - let snap_path: String = dir + "/snapshot.json" + let dir_raw: String = env("ENGRAM_DATA_DIR") + let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw } + // 2026-07-21 self-review: export to a scratch path, never the canonical + // snapshot.json — read routes must not be able to clobber the good snapshot. + let snap_path: String = dir + "/.sync-export.json" engram_save(snap_path) let snap: String = fs_read(snap_path) if str_eq(snap, "") { return "{\"nodes\":[],\"edges\":[]}" } return snap } +// route_load_merge — POST /api/load-merge {"path": "..."} — merge a snapshot +// file into the live store WITHOUT resetting it (engram_load_merge skips nodes +// already present by id). Added 2026-07-21 self-review to restore the 244 kn- +// identity Knowledge nodes lost from the snapshot lineage between 05-13 and +// 07-13. Requires an explicit path: refuses to run without one so it can never +// be triggered accidentally against a default. +fn route_load_merge(method: String, path: String, body: String) -> String { + let p: String = json_get_string(body, "path") + if str_eq(p, "") { return err_json("path is required") } + if str_eq(fs_read(p), "") { return err_json("file missing or empty") } + let before_n: Int = engram_node_count() + let before_e: Int = engram_edge_count() + engram_load_merge(p) + let added_n: Int = engram_node_count() - before_n + let added_e: Int = engram_edge_count() - before_e + let saved: Int = persist_canonical() + "{\"ok\":true,\"nodes_added\":" + int_to_str(added_n) + ",\"edges_added\":" + int_to_str(added_e) + ",\"node_count\":" + int_to_str(engram_node_count()) + "}" +} + // route_emit_ise — write an InternalStateEvent node from the soul daemon. // // Endpoint: POST /api/neuron/state-events @@ -241,10 +285,20 @@ fn route_sync(method: String, path: String, body: String) -> String { // // Salience/importance set to match engram_node_full ISE defaults used by the // in-process fallback path in awareness.el (salience=0.3, importance=0.3, -// confidence=0.8, tier=Episodic). High temporal_decay_rate (1.617) — ISEs -// are inherently transient; they should decay faster than structural knowledge. +// confidence=0.8, tier=Episodic). // (2026-06-26 self-review: added this route after discovering ise_post was // silently failing — the soul posts here but the endpoint didn't exist.) +// +// Retention (2026-07-16 self-review): an earlier comment here claimed ISEs +// got temporal_decay_rate=1.617 — that was never implemented (engram_node_full +// hardcodes 0.0), and per-node decay only dampens activation anyway; it never +// removes nodes. By 2026-07-16 ISEs were 75% of the store (10,175 of 13,522 +// nodes, ~4,300/day, unbounded). ISEs are already WM-excluded in +// engram_activate, so the fix is retention, not decay: every insert calls +// engram_prune_telemetry(), a single O(nodes+edges) compaction pass that +// removes ISEs older than ENGRAM_ISE_RETENTION_MS (default 48h), protecting +// "session-start" labels and self_review events as durable history. At +// ~3 ISEs/min this bounds telemetry at ~8.6k nodes instead of growing forever. fn route_emit_ise(method: String, path: String, body: String) -> String { let content: String = json_get_string(body, "content") if str_eq(content, "") { return err_json("missing content") } @@ -256,6 +310,64 @@ fn route_emit_ise(method: String, path: String, body: String) -> String { sal, imp, conf, "Episodic", "[\"internal-state\",\"InternalStateEvent\"]" ) + let ret_raw: String = env("ENGRAM_ISE_RETENTION_MS") + let ret_ms: Int = if str_eq(ret_raw, "") { 172800000 } else { str_to_int(ret_raw) } + let pruned: Int = engram_prune_telemetry(ret_ms) + "{\"ok\":true,\"id\":\"" + id + "\",\"pruned\":" + int_to_str(pruned) + "}" +} + +// ── Knowledge capture ───────────────────────────────────────────────────────── +// +// route_capture_knowledge — direct Knowledge-node capture over HTTP. +// +// Endpoint: POST /api/neuron/knowledge/capture (auth required: "_auth" in body) +// Body: {"content": "...", "title": "...", "category": "...", +// "tier": "note|lesson|canonical", "tags": [...], "project": "...", +// "_auth": ""} +// +// WHY (2026-07-15 self-review): the world-ingestor integrator was designed +// against this endpoint (its MCP-unavailable fallback), but the route never +// existed — every direct push 404'd, and because the auth gate ran before +// routing, the failure surfaced as {"error":"unauthorized"} and was +// misdiagnosed for two weeks while world knowledge silently dropped. +// POST /api/nodes was no substitute: it discards label/tags/tier, which +// makes captured knowledge invisible to tag-scoped search and curiosity. +// +// The incoming knowledge tier (note/lesson/canonical) is preserved as a +// "tier:" tag rather than mapped onto Engram's cognitive tiers — Knowledge +// nodes land in Semantic (stable reference), and the epistemic tier stays +// queryable without inventing a lossy mapping. +fn route_capture_knowledge(method: String, path: String, body: String) -> String { + let content: String = json_get_string(body, "content") + if str_eq(content, "") { return err_json("missing content") } + let title: String = json_get_string(body, "title") + let label: String = if str_eq(title, "") { str_slice(content, 0, 60) } else { title } + let category_raw: String = json_get_string(body, "category") + let category: String = if str_eq(category_raw, "") { "other" } else { category_raw } + let ktier_raw: String = json_get_string(body, "tier") + let ktier: String = if str_eq(ktier_raw, "") { "note" } else { ktier_raw } + let project: String = json_get_string(body, "project") + let tags_raw: String = json_get_raw(body, "tags") + let tags_base: String = if str_eq(tags_raw, "") { "[]" } else { tags_raw } + // Merge category/tier/project markers into the tag array. Search matches + // against the tags string, so these make captures findable by facet. + let base_len: Int = str_len(tags_base) + let head: String = str_slice(tags_base, 0, base_len - 1) + let sep: String = if str_eq(head, "[") { "" } else { "," } + let safe_cat: String = str_replace(category, "\"", "'") + let safe_tier: String = str_replace(ktier, "\"", "'") + let safe_proj: String = str_replace(project, "\"", "'") + let proj_tag: String = if str_eq(safe_proj, "") { "" } else { ",\"project:" + safe_proj + "\"" } + let tags: String = head + sep + "\"category:" + safe_cat + "\",\"tier:" + safe_tier + "\"" + proj_tag + "]" + let sal: Float = 0.5 + let imp: Float = 0.5 + let conf: Float = 0.9 + let id: String = engram_node_full( + content, "Knowledge", label, + sal, imp, conf, + "Semantic", tags + ) + let saved: Int = persist_canonical() "{\"ok\":true,\"id\":\"" + id + "\"}" } @@ -295,6 +407,12 @@ fn handle_request(method: String, path: String, body: String) -> String { return err_json("unauthorized") } + // Knowledge capture (auth enforced above; the world-ingestor integrator + // and any headless session without MCP push knowledge through this) + if str_eq(method, "POST") && str_eq(clean, "/api/neuron/knowledge/capture") { + return route_capture_knowledge(method, path, body) + } + // Stats if str_eq(method, "GET") && (str_eq(clean, "/api/stats") || str_eq(clean, "/stats")) { return route_stats(method, path, body) @@ -351,6 +469,9 @@ fn handle_request(method: String, path: String, body: String) -> String { if str_eq(method, "POST") && (str_eq(clean, "/api/load") || str_eq(clean, "/load")) { return route_load(method, path, body) } + if str_eq(method, "POST") && (str_eq(clean, "/api/load-merge") || str_eq(clean, "/load-merge")) { + return route_load_merge(method, path, body) + } // Sync — soul daemon periodic pull of non-ISE knowledge into in-process graph if str_eq(method, "GET") && str_eq(clean, "/api/sync") { @@ -362,16 +483,31 @@ fn handle_request(method: String, path: String, body: String) -> String { // ── Entry ───────────────────────────────────────────────────────────────────── -let bind_str: String = env("ENGRAM_BIND") -if str_eq(bind_str, "") { let bind_str = ":8742" } +let bind_raw: String = env("ENGRAM_BIND") +let bind_str: String = if str_eq(bind_raw, "") { ":8742" } else { bind_raw } let port: Int = parse_port(bind_str) // On startup, try to load any existing snapshot (best effort). -let data_dir: String = env("ENGRAM_DATA_DIR") -if str_eq(data_dir, "") { let data_dir = "/tmp/engram" } +let data_dir_raw: String = env("ENGRAM_DATA_DIR") +let data_dir: String = if str_eq(data_dir_raw, "") { "/tmp/engram" } else { data_dir_raw } let snapshot_path: String = data_dir + "/snapshot.json" engram_load(snapshot_path) +// 2026-07-21 self-review boot guard: if the snapshot file has content but the +// load produced 0 nodes, something is wrong (corrupt file / parse failure). +// Preserve the evidence and warn loudly — and since read routes no longer write +// the canonical path, a bad boot can no longer clobber the good snapshot. +let boot_snap: String = fs_read(snapshot_path) +if !str_eq(boot_snap, "") { + if engram_node_count() == 0 { + println("[engram] WARNING: snapshot.json is non-empty but load produced 0 nodes — preserving copy at snapshot.failed-load.json") + fs_write(data_dir + "/snapshot.failed-load.json", boot_snap) + } else { + // Good load: keep a boot-time backup of the snapshot as loaded. + fs_write(data_dir + "/snapshot.boot-backup.json", boot_snap) + } +} + println("[engram] runtime-native graph engine") println("[engram] data_dir=" + data_dir) println("[engram] node_count=" + int_to_str(engram_node_count())) diff --git a/lang/el-compiler/runtime/el_runtime.c b/lang/el-compiler/runtime/el_runtime.c index 2b75c6f..af0d945 100644 --- a/lang/el-compiler/runtime/el_runtime.c +++ b/lang/el-compiler/runtime/el_runtime.c @@ -82,8 +82,14 @@ static _Thread_local ElArena _tl_arena = {NULL, 0, 0}; static _Thread_local int _tl_arena_active = 0; /* Binary-safe fs_read length — set by fs_read, consumed by http_send_response. - * Allows serving PNGs and other binary files without strlen truncation. */ -static _Thread_local size_t _tl_fs_read_len = 0; + * Allows serving PNGs and other binary files without strlen truncation. + * PAIRED with the buffer pointer it describes: the length may only be applied + * to the exact buffer fs_read returned. Without the pairing, any handler that + * fs_read a file and then WRAPPED it into a larger response had that response + * truncated to the file's length (Content-Length lied AND the send stopped + * short) — the safety-contact onboarding trap, 2026-07-17. */ +static _Thread_local size_t _tl_fs_read_len = 0; +static _Thread_local const char* _tl_fs_read_buf = NULL; static void el_arena_track(char* p) { if (!_tl_arena_active || !p) return; @@ -101,6 +107,8 @@ static void el_arena_track(char* p) { void el_request_start(void) { _tl_arena.count = 0; _tl_arena_active = 1; + _tl_fs_read_len = 0; /* never let a previous request's file length */ + _tl_fs_read_buf = NULL; /* leak into this response's byte accounting */ } /* Called by http_worker after the El handler returns and the response is sent. @@ -1484,11 +1492,14 @@ static void http_send_response(int fd, const char* body) { } const char* eff_body = is_envelope ? env_body : body; - /* Use the real byte count from fs_read if available (handles binary files - * with embedded null bytes — PNG, WOFF2, etc.). Fall back to strlen for - * normal text/JSON responses where _tl_fs_read_len is 0. */ - size_t blen = (_tl_fs_read_len > 0) ? _tl_fs_read_len : strlen(eff_body); + /* Use the real byte count from fs_read ONLY when this body IS the exact + * buffer fs_read returned (binary files with embedded null bytes — PNG, + * WOFF2, etc.). Any other body — wrapped, enveloped, or derived — must be + * measured with strlen, or it is truncated/over-read to the file's size. */ + size_t blen = (_tl_fs_read_len > 0 && eff_body == _tl_fs_read_buf) + ? _tl_fs_read_len : strlen(eff_body); _tl_fs_read_len = 0; /* consume — one-shot per response */ + _tl_fs_read_buf = NULL; int head_only = _tl_http_head_only; JsonBuf hdrs; jb_init(&hdrs); @@ -1568,11 +1579,22 @@ static void* http_worker(void* arg) { const char* rs = EL_CSTR(r); /* Copy response out BEFORE arena teardown. * For binary files, _tl_fs_read_len holds the real byte count — - * use memcpy instead of strdup so null bytes are preserved. */ - size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0); + * use memcpy instead of strdup so null bytes are preserved. + * The stored length applies ONLY when the response IS the exact + * fs_read buffer; a wrapped/derived response must use strlen or + * it gets truncated (or over-read) to the file's length. */ + size_t rlen; + if (_tl_fs_read_len > 0 && rs && rs == _tl_fs_read_buf) { + rlen = _tl_fs_read_len; /* raw file bytes — binary-safe */ + } else { + rlen = rs ? strlen(rs) : 0; + _tl_fs_read_len = 0; /* hint doesn't describe this body */ + _tl_fs_read_buf = NULL; + } response = malloc(rlen + 1); if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; } else if (response) { response[0] = '\0'; } + if (_tl_fs_read_len > 0) _tl_fs_read_buf = response; /* hint follows the copy */ } else { response = el_strdup_persist("el-runtime: no http handler registered"); } @@ -1822,10 +1844,20 @@ static void* http_worker_v2(void* arg) { el_val_t hmap = http_build_headers_map(hdr_block ? hdr_block : ""); el_val_t r = h(EL_STR(dispatch_method), EL_STR(path), hmap, EL_STR(body)); const char* rs = EL_CSTR(r); - size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0); + /* Same pairing rule as the v1 worker: the fs_read length is only + * trustworthy for the exact buffer fs_read returned. */ + size_t rlen; + if (_tl_fs_read_len > 0 && rs && rs == _tl_fs_read_buf) { + rlen = _tl_fs_read_len; /* raw file bytes — binary-safe */ + } else { + rlen = rs ? strlen(rs) : 0; + _tl_fs_read_len = 0; /* hint doesn't describe this body */ + _tl_fs_read_buf = NULL; + } response = malloc(rlen + 1); if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; } else if (response) { response[0] = '\0'; } + if (_tl_fs_read_len > 0) _tl_fs_read_buf = response; /* hint follows the copy */ el_release(hmap); } else { response = el_strdup_persist( @@ -2024,6 +2056,7 @@ el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body) { el_val_t fs_read(el_val_t pathv) { const char* path = EL_CSTR(pathv); _tl_fs_read_len = 0; + _tl_fs_read_buf = NULL; if (!path) return el_wrap_str(el_strdup("")); FILE* f = fopen(path, "rb"); if (!f) return el_wrap_str(el_strdup("")); @@ -2035,6 +2068,7 @@ el_val_t fs_read(el_val_t pathv) { size_t got = fread(buf, 1, (size_t)sz, f); buf[got] = '\0'; _tl_fs_read_len = got; /* store real byte count for binary-safe send */ + _tl_fs_read_buf = buf; /* ...valid ONLY for this exact buffer */ fclose(f); return el_wrap_str(buf); } @@ -3577,8 +3611,10 @@ el_val_t json_get_raw(el_val_t json_str, el_val_t key) { const char* k = EL_CSTR(key); const char* p = json_find_key(json, k); /* Clear fs_read binary-length hint — result is a fresh null-terminated - * string, not the raw file bytes, so Content-Length must use strlen. */ + * string, not the raw file bytes, so Content-Length must use strlen. + * (Kept although the pointer pairing now makes this redundant.) */ _tl_fs_read_len = 0; + _tl_fs_read_buf = NULL; if (!p) return el_wrap_str(el_strdup("")); const char* end = json_skip_value(p); size_t n = (size_t)(end - p); diff --git a/lang/releases/v1.0.0-20260501/el_platform_win.h b/lang/releases/v1.0.0-20260501/el_platform_win.h new file mode 100644 index 0000000..1042f44 --- /dev/null +++ b/lang/releases/v1.0.0-20260501/el_platform_win.h @@ -0,0 +1,186 @@ +#ifndef EL_PLATFORM_WIN_H +#define EL_PLATFORM_WIN_H +/* + * el_platform_win.h — Windows OS-boundary shim for el_runtime.c. + * + * Branch: feat/windows-el-runtime. Included ONLY when _WIN32 is defined; the POSIX build is + * untouched. Goal: let el_runtime.c (a BSD-sockets / dlfcn / fork host) compile and link with + * mingw-w64 into a native neuron.exe, with no behavioural change to the Linux/macOS build. + * + * What it maps: + * - sockets : winsock2 (same call names: socket/bind/listen/accept/recv/send/setsockopt). + * Sockets close with closesocket() (see el_closesocket), and the stack must be + * started once with WSAStartup — done automatically via a load-time constructor. + * - dlsym : el_runtime.c uses dlsym(RTLD_DEFAULT, name) to resolve callback/tool symbols + * exported by the main module. Windows equivalent: GetProcAddress on the process + * module. Link the soul with -Wl,--export-all-symbols so the symbols are findable. + * - popen : mapped to _popen/_pclose. + * - threads : UNCHANGED. mingw-w64 ships winpthreads, so + -lpthread just work. + */ + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#include +#include +#include +#include + +/* Portable headers mingw-w64 provides (verified present). */ +#include +#include +#include +#include +#include +#include /* strcasecmp */ +#include +#include +#include +#include /* mingw-w64 provides gettimeofday here */ +#include +#include +#include +#include +#include +#include + +/* ── socket close ─────────────────────────────────────────────────────────── */ +/* Winsock closes sockets with closesocket(), not close() (close() is for file fds). The POSIX + build defines the same helper as close() so the call sites are identical across platforms. */ +static inline int el_closesocket(SOCKET s) { return closesocket(s); } + +/* ── setsockopt optval type ───────────────────────────────────────────────── */ +/* Winsock's setsockopt takes optval as (const char*); POSIX takes (const void*), so el_runtime.c + passes &int directly. GCC 14+ makes that an error under -Wincompatible-pointer-types. Wrap it so + the runtime's POSIX-style call sites compile unchanged (defined before the macro so the wrapper + itself resolves to the real winsock setsockopt). */ +static inline int el_setsockopt(SOCKET s, int level, int optname, const void* optval, int optlen) { + return setsockopt(s, level, optname, (const char*)optval, optlen); +} +#define setsockopt(s, l, o, v, n) el_setsockopt((s), (l), (o), (v), (int)(n)) + +/* ── winsock init (once, at load) ─────────────────────────────────────────── */ +static void el__win_net_init(void) { + static int inited = 0; + if (!inited) { WSADATA w; WSAStartup(MAKEWORD(2, 2), &w); inited = 1; } +} +__attribute__((constructor)) static void el__win_ctor(void) { el__win_net_init(); } + +/* ── dlsym → GetProcAddress ───────────────────────────────────────────────── */ +#ifndef RTLD_DEFAULT +#define RTLD_DEFAULT ((void*)0) +#endif +static inline void* el_win_dlsym(void* handle, const char* name) { + (void)handle; + return (void*)(uintptr_t)GetProcAddress(GetModuleHandleA(NULL), name); +} +#define dlsym(h, n) el_win_dlsym((h), (n)) + +/* ── popen / pclose ───────────────────────────────────────────────────────── */ +#define popen _popen +#define pclose _pclose + +/* ── misc POSIX → Win32 shims ─────────────────────────────────────────────── */ +#include /* _mkdir */ +#define mkdir(path, mode) _mkdir(path) /* POSIX mkdir(path,mode) → _mkdir(path) */ +#define timegm _mkgmtime /* UTC tm → time_t */ + +/* setenv/unsetenv: not in the Windows CRT; map to _putenv_s / SetEnvironmentVariable. */ +static inline int setenv(const char* name, const char* value, int overwrite) { + (void)overwrite; + return _putenv_s(name, value ? value : ""); +} +static inline int unsetenv(const char* name) { + /* _putenv_s(name, "") sets VAR="" rather than removing it. + * SetEnvironmentVariableA(name, NULL) truly deletes it from the Win32 + * env block; then we sync the CRT cache with _putenv("NAME="). */ + SetEnvironmentVariableA(name, NULL); + size_t len = strlen(name); + char *buf = (char*)malloc(len + 2); + if (!buf) return -1; + memcpy(buf, name, len); + buf[len] = '='; + buf[len + 1] = '\0'; + _putenv(buf); + free(buf); + return 0; +} + +/* nanosleep — not available in MSVC/UCRT; approximate with Sleep(). */ +static inline int el_nanosleep(const struct timespec *req, struct timespec *rem) { + (void)rem; + DWORD ms = (DWORD)((req->tv_sec * 1000ULL) + (req->tv_nsec / 1000000ULL)); + Sleep(ms ? ms : 1); + return 0; +} +#define nanosleep(req, rem) el_nanosleep((req), (rem)) + +/* localtime_r/gmtime_r: Windows offers localtime_s/gmtime_s with reversed arg order. */ +static inline struct tm* localtime_r(const time_t* t, struct tm* out) { + return localtime_s(out, t) == 0 ? out : (struct tm*)0; +} +static inline struct tm* gmtime_r(const time_t* t, struct tm* out) { + return gmtime_s(out, t) == 0 ? out : (struct tm*)0; +} + +/* ── libcurl: degradable stubs for the curl-less Windows build ─────────────── */ +/* The curl-less validation build (WITH_CURL=0) links no libcurl. el_runtime.c uses libcurl + * unconditionally for its HTTP client / LLM layer; these stubs let it compile and link so the + * runtime, HTTP *server*, graph and memory work natively on Windows. Live outbound HTTP/LLM calls + * degrade to a runtime error (curl_easy_perform returns an error) — matching the documented + * curl-less contract. When HAVE_CURL is defined (WITH_CURL=1) the real is used and + * this whole block is compiled out. POSIX never sees this header, so the POSIX build is untouched. */ +#ifndef HAVE_CURL + +typedef void CURL; +typedef int CURLcode; + +#define CURLE_OK 0 +#define CURLE_HTTP_RETURNED_ERROR 22 +#define CURL_ERROR_SIZE 256 + +/* Option ids: values are irrelevant to the no-op setopt below; kept distinct for readability. */ +#define CURLOPT_URL 10002 +#define CURLOPT_WRITEFUNCTION 20011 +#define CURLOPT_WRITEDATA 10001 +#define CURLOPT_POSTFIELDS 10015 +#define CURLOPT_POSTFIELDSIZE 120 +#define CURLOPT_POST 47 +#define CURLOPT_HTTPHEADER 10023 +#define CURLOPT_TIMEOUT_MS 155 +#define CURLOPT_NOSIGNAL 99 +#define CURLOPT_USERAGENT 10018 +#define CURLOPT_FOLLOWLOCATION 52 +#define CURLOPT_ERRORBUFFER 10010 +#define CURLOPT_CUSTOMREQUEST 10036 +#define CURLOPT_FAILONERROR 45 + +struct curl_slist { char* data; struct curl_slist* next; }; + +static inline struct curl_slist* curl_slist_append(struct curl_slist* list, const char* s) { + struct curl_slist* node = (struct curl_slist*)malloc(sizeof(struct curl_slist)); + if (!node) return list; + node->data = s ? strdup(s) : NULL; + node->next = NULL; + if (!list) return node; + struct curl_slist* p = list; + while (p->next) p = p->next; + p->next = node; + return list; +} +static inline void curl_slist_free_all(struct curl_slist* list) { + while (list) { struct curl_slist* n = list->next; free(list->data); free(list); list = n; } +} + +static inline CURL* curl_easy_init(void) { return (CURL*)malloc(1); } +static inline CURLcode curl_easy_setopt(CURL* h, int opt, ...) { (void)h; (void)opt; return CURLE_OK; } +static inline CURLcode curl_easy_perform(CURL* h) { (void)h; return 7 /* CURLE_COULDNT_CONNECT */; } +static inline void curl_easy_cleanup(CURL* h) { free(h); } +static inline const char* curl_easy_strerror(CURLcode c) { + (void)c; return "libcurl not built in (curl-less build)"; +} + +#endif /* !HAVE_CURL */ + +#endif /* EL_PLATFORM_WIN_H */ diff --git a/lang/releases/v1.0.0-20260501/el_runtime.c b/lang/releases/v1.0.0-20260501/el_runtime.c index f4f605f..4629cc2 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.c +++ b/lang/releases/v1.0.0-20260501/el_runtime.c @@ -21,6 +21,10 @@ #include "el_runtime.h" +#ifdef _WIN32 +/* Windows OS-boundary shim (winsock/dlsym/popen). Threading stays on (winpthreads). */ +#include "el_platform_win.h" +#else #include #include /* strcasecmp */ #include @@ -42,7 +46,16 @@ #include #include #include +/* On POSIX, sockets close with the same close() as files; el_platform_win.h supplies the Windows + variant. Defined here so the socket call sites are identical across platforms. */ +static inline int el_closesocket(int s) { return close(s); } +#endif +/* libcurl: present on POSIX and on the WITH_CURL Windows build; absent on the curl-less Windows + validation build, where el_platform_win.h supplies degradable stubs. On POSIX (_WIN32 undefined) + this is always taken, so the POSIX build is unchanged. */ +#if !defined(_WIN32) || defined(HAVE_CURL) #include +#endif /* ── Internal allocators ─────────────────────────────────────────────────── */ @@ -71,8 +84,14 @@ static _Thread_local ElArena _tl_arena = {NULL, 0, 0}; static _Thread_local int _tl_arena_active = 0; /* Binary-safe fs_read length — set by fs_read, consumed by http_send_response. - * Allows serving PNGs and other binary files without strlen truncation. */ -static _Thread_local size_t _tl_fs_read_len = 0; + * Allows serving PNGs and other binary files without strlen truncation. + * PAIRED with the buffer pointer it describes: the length may only be applied + * to the exact buffer fs_read returned. Without the pairing, any handler that + * fs_read a file and then WRAPPED it into a larger response had that response + * truncated to the file's length (Content-Length lied AND the send stopped + * short) — the safety-contact onboarding trap, 2026-07-17. */ +static _Thread_local size_t _tl_fs_read_len = 0; +static _Thread_local const char* _tl_fs_read_buf = NULL; static void el_arena_track(char* p) { if (!_tl_arena_active || !p) return; @@ -90,6 +109,8 @@ static void el_arena_track(char* p) { void el_request_start(void) { _tl_arena.count = 0; _tl_arena_active = 1; + _tl_fs_read_len = 0; /* never let a previous request's file length */ + _tl_fs_read_buf = NULL; /* leak into this response's byte accounting */ } /* Called by http_worker after the El handler returns and the response is sent. @@ -102,6 +123,47 @@ void el_request_end(void) { _tl_arena.count = 0; } +/* ── Scoped arena for CLI use ─────────────────────────────────────────────── * + * CLI programs never call el_request_start/end, so all strdup allocations are + * permanent. el_arena_push/pop let the compiler free intermediate strings + * after each compilation unit. Ported verbatim from el-compiler/runtime on + * 2026-07-17: the soul daemon's awareness loop arena-scopes each tick with + * these, and they were present only in the dev runtime copy. + * + * el_arena_push() — activates the arena if not already active, saves the + * current arena count as a mark, and returns it as an el_val_t Int. + * el_arena_pop(mark) — frees all strings allocated since the push mark and + * resets the count. If count reaches 0, deactivates the arena. + */ +#define EL_ARENA_SCOPE_DEPTH 32 +static _Thread_local size_t _tl_arena_scope[EL_ARENA_SCOPE_DEPTH]; +static _Thread_local int _tl_arena_scope_depth = 0; + +el_val_t el_arena_push(void) { + if (!_tl_arena_active) { + _tl_arena_active = 1; + } + if (_tl_arena_scope_depth < EL_ARENA_SCOPE_DEPTH) { + _tl_arena_scope[_tl_arena_scope_depth++] = _tl_arena.count; + } + return (el_val_t)(int64_t)_tl_arena.count; +} + +el_val_t el_arena_pop(el_val_t mark) { + size_t save = (size_t)(int64_t)mark; + if (save > _tl_arena.count) save = 0; + for (size_t i = save; i < _tl_arena.count; i++) { + if (_tl_arena.ptrs[i]) { + free(_tl_arena.ptrs[i]); + _tl_arena.ptrs[i] = NULL; + } + } + _tl_arena.count = save; + if (_tl_arena_scope_depth > 0) _tl_arena_scope_depth--; + if (save == 0) _tl_arena_active = 0; + return 0; +} + /* Persistent allocation — bypasses the arena (state_set, engram internals). */ static char* el_strdup_persist(const char* s) { if (!s) return strdup(""); @@ -1362,11 +1424,14 @@ static void http_send_response(int fd, const char* body) { } const char* eff_body = is_envelope ? env_body : body; - /* Use the real byte count from fs_read if available (handles binary files - * with embedded null bytes — PNG, WOFF2, etc.). Fall back to strlen for - * normal text/JSON responses where _tl_fs_read_len is 0. */ - size_t blen = (_tl_fs_read_len > 0) ? _tl_fs_read_len : strlen(eff_body); + /* Use the real byte count from fs_read ONLY when this body IS the exact + * buffer fs_read returned (binary files with embedded null bytes — PNG, + * WOFF2, etc.). Any other body — wrapped, enveloped, or derived — must be + * measured with strlen, or it is truncated/over-read to the file's size. */ + size_t blen = (_tl_fs_read_len > 0 && eff_body == _tl_fs_read_buf) + ? _tl_fs_read_len : strlen(eff_body); _tl_fs_read_len = 0; /* consume — one-shot per response */ + _tl_fs_read_buf = NULL; int head_only = _tl_http_head_only; JsonBuf hdrs; jb_init(&hdrs); @@ -1416,12 +1481,20 @@ static void http_send_response(int fd, const char* body) { } typedef struct { +#ifdef _WIN32 + SOCKET fd; +#else int fd; +#endif } HttpWorkerArg; static void* http_worker(void* arg) { HttpWorkerArg* a = (HttpWorkerArg*)arg; +#ifdef _WIN32 + SOCKET fd = a->fd; +#else int fd = a->fd; +#endif free(a); char *method = NULL, *path = NULL, *body = NULL; if (http_read_request(fd, &method, &path, &body, NULL) == 0) { @@ -1438,11 +1511,22 @@ static void* http_worker(void* arg) { const char* rs = EL_CSTR(r); /* Copy response out BEFORE arena teardown. * For binary files, _tl_fs_read_len holds the real byte count — - * use memcpy instead of strdup so null bytes are preserved. */ - size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0); + * use memcpy instead of strdup so null bytes are preserved. + * The stored length applies ONLY when the response IS the exact + * fs_read buffer; a wrapped/derived response must use strlen or + * it gets truncated (or over-read) to the file's length. */ + size_t rlen; + if (_tl_fs_read_len > 0 && rs && rs == _tl_fs_read_buf) { + rlen = _tl_fs_read_len; /* raw file bytes — binary-safe */ + } else { + rlen = rs ? strlen(rs) : 0; + _tl_fs_read_len = 0; /* hint doesn't describe this body */ + _tl_fs_read_buf = NULL; + } response = malloc(rlen + 1); if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; } else if (response) { response[0] = '\0'; } + if (_tl_fs_read_len > 0) _tl_fs_read_buf = response; /* hint follows the copy */ } else { response = el_strdup_persist("el-runtime: no http handler registered"); } @@ -1453,7 +1537,7 @@ static void* http_worker(void* arg) { free(response); } free(method); free(path); free(body); - close(fd); + el_closesocket(fd); /* release a slot */ pthread_mutex_lock(&_http_conn_mu); _http_conn_active--; @@ -1483,10 +1567,60 @@ void http_serve(el_val_t port, el_val_t handler) { addr.sin6_addr = in6addr_any; addr.sin6_port = htons((uint16_t)p); if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) { - perror("bind"); close(sock); return; + perror("bind"); el_closesocket(sock); return; } - if (listen(sock, 64) < 0) { perror("listen"); close(sock); return; } + if (listen(sock, 64) < 0) { perror("listen"); el_closesocket(sock); return; } fprintf(stderr, "[http] listening on [::]:%d (dual-stack)\n", p); + while (1) { + struct sockaddr_in6 cli; + socklen_t clen = sizeof(cli); +#ifdef _WIN32 + SOCKET cfd = accept(sock, (struct sockaddr*)&cli, &clen); +#else + int cfd = accept(sock, (struct sockaddr*)&cli, &clen); +#endif + if (cfd < 0) { + if (errno == EINTR) continue; + perror("accept"); break; + } + pthread_mutex_lock(&_http_conn_mu); + while (_http_conn_active >= HTTP_MAX_CONNS) { + pthread_cond_wait(&_http_conn_cv, &_http_conn_mu); + } + _http_conn_active++; + pthread_mutex_unlock(&_http_conn_mu); + HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg)); + if (!arg) { el_closesocket(cfd); continue; } + arg->fd = cfd; + pthread_t tid; + if (pthread_create(&tid, NULL, http_worker, arg) != 0) { + el_closesocket(cfd); free(arg); + pthread_mutex_lock(&_http_conn_mu); + _http_conn_active--; + pthread_cond_signal(&_http_conn_cv); + pthread_mutex_unlock(&_http_conn_mu); + continue; + } + pthread_detach(tid); + } + el_closesocket(sock); +} + +/* ── http_serve_async — non-blocking HTTP server ─────────────────────────── */ +/* Runs the accept loop in a background pthread, returns immediately so the + * calling EL script can continue (e.g. to run an awareness loop). + * Ported verbatim from el-compiler/runtime on 2026-07-17: the soul daemon + * (soul.el) builds against this release runtime and calls http_serve_async, + * which was present only in the dev runtime copy. + * + * El signature: http_serve_async(port, handler) -> Void */ + +typedef struct { int sock; } HttpServeAsyncArg; + +static void* _http_serve_async_loop(void* raw) { + HttpServeAsyncArg* a = (HttpServeAsyncArg*)raw; + int sock = a->sock; + free(a); while (1) { struct sockaddr_in6 cli; socklen_t clen = sizeof(cli); @@ -1516,6 +1650,40 @@ void http_serve(el_val_t port, el_val_t handler) { pthread_detach(tid); } close(sock); + return NULL; +} + +void http_serve_async(el_val_t port, el_val_t handler) { + const char* hname = EL_CSTR(handler); + if (hname && looks_like_string(handler)) { + http_set_handler(handler); + } + int p = (int)port; + if (p <= 0 || p > 65535) { fprintf(stderr, "http_serve_async: invalid port %d\n", p); return; } + int sock = socket(AF_INET6, SOCK_STREAM, 0); + if (sock < 0) { perror("socket"); return; } + int yes = 1; int no = 0; + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &no, sizeof(no)); + struct sockaddr_in6 addr; + memset(&addr, 0, sizeof(addr)); + addr.sin6_family = AF_INET6; + addr.sin6_addr = in6addr_any; + addr.sin6_port = htons((uint16_t)p); + if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) { + perror("bind"); close(sock); return; + } + if (listen(sock, 64) < 0) { perror("listen"); close(sock); return; } + fprintf(stderr, "[http] async listening on [::]:%d (dual-stack)\n", p); + HttpServeAsyncArg* a = malloc(sizeof(HttpServeAsyncArg)); + if (!a) { close(sock); return; } + a->sock = sock; + pthread_t tid; + if (pthread_create(&tid, NULL, _http_serve_async_loop, a) != 0) { + perror("pthread_create"); free(a); close(sock); return; + } + pthread_detach(tid); + /* Returns immediately — caller can now run awareness_run() or any loop. */ } /* ── HTTP server v2 — request headers + structured response ──────────────── */ @@ -1671,7 +1839,11 @@ static el_val_t http_build_headers_map(const char* hdr_block) { static void* http_worker_v2(void* arg) { HttpWorkerArg* a = (HttpWorkerArg*)arg; +#ifdef _WIN32 + SOCKET fd = a->fd; +#else int fd = a->fd; +#endif free(a); char *method = NULL, *path = NULL, *body = NULL, *hdr_block = NULL; if (http_read_request(fd, &method, &path, &body, &hdr_block) == 0) { @@ -1684,10 +1856,20 @@ static void* http_worker_v2(void* arg) { el_val_t hmap = http_build_headers_map(hdr_block ? hdr_block : ""); el_val_t r = h(EL_STR(dispatch_method), EL_STR(path), hmap, EL_STR(body)); const char* rs = EL_CSTR(r); - size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0); + /* Same pairing rule as the v1 worker: the fs_read length is only + * trustworthy for the exact buffer fs_read returned. */ + size_t rlen; + if (_tl_fs_read_len > 0 && rs && rs == _tl_fs_read_buf) { + rlen = _tl_fs_read_len; /* raw file bytes — binary-safe */ + } else { + rlen = rs ? strlen(rs) : 0; + _tl_fs_read_len = 0; /* hint doesn't describe this body */ + _tl_fs_read_buf = NULL; + } response = malloc(rlen + 1); if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; } else if (response) { response[0] = '\0'; } + if (_tl_fs_read_len > 0) _tl_fs_read_buf = response; /* hint follows the copy */ el_release(hmap); } else { response = el_strdup_persist( @@ -1701,7 +1883,7 @@ static void* http_worker_v2(void* arg) { free(response); } free(method); free(path); free(body); free(hdr_block); - close(fd); + el_closesocket(fd); pthread_mutex_lock(&_http_conn_mu); _http_conn_active--; pthread_cond_signal(&_http_conn_cv); @@ -1731,14 +1913,18 @@ void http_serve_v2(el_val_t port, el_val_t handler) { addr.sin6_addr = in6addr_any; addr.sin6_port = htons((uint16_t)p); if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) { - perror("bind"); close(sock); return; + perror("bind"); el_closesocket(sock); return; } - if (listen(sock, 64) < 0) { perror("listen"); close(sock); return; } + if (listen(sock, 64) < 0) { perror("listen"); el_closesocket(sock); return; } fprintf(stderr, "[http v2] listening on [::]:%d (dual-stack)\n", p); while (1) { struct sockaddr_in6 cli; socklen_t clen = sizeof(cli); +#ifdef _WIN32 + SOCKET cfd = accept(sock, (struct sockaddr*)&cli, &clen); +#else int cfd = accept(sock, (struct sockaddr*)&cli, &clen); +#endif if (cfd < 0) { if (errno == EINTR) continue; perror("accept"); break; @@ -1750,11 +1936,11 @@ void http_serve_v2(el_val_t port, el_val_t handler) { _http_conn_active++; pthread_mutex_unlock(&_http_conn_mu); HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg)); - if (!arg) { close(cfd); continue; } + if (!arg) { el_closesocket(cfd); continue; } arg->fd = cfd; pthread_t tid; if (pthread_create(&tid, NULL, http_worker_v2, arg) != 0) { - close(cfd); free(arg); + el_closesocket(cfd); free(arg); pthread_mutex_lock(&_http_conn_mu); _http_conn_active--; pthread_cond_signal(&_http_conn_cv); @@ -1763,7 +1949,7 @@ void http_serve_v2(el_val_t port, el_val_t handler) { } pthread_detach(tid); } - close(sock); + el_closesocket(sock); } /* Build the response envelope a 4-arg handler can return. We hand-write @@ -1803,6 +1989,7 @@ el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body) { el_val_t fs_read(el_val_t pathv) { const char* path = EL_CSTR(pathv); _tl_fs_read_len = 0; + _tl_fs_read_buf = NULL; if (!path) return el_wrap_str(el_strdup("")); FILE* f = fopen(path, "rb"); if (!f) return el_wrap_str(el_strdup("")); @@ -1814,6 +2001,7 @@ el_val_t fs_read(el_val_t pathv) { size_t got = fread(buf, 1, (size_t)sz, f); buf[got] = '\0'; _tl_fs_read_len = got; /* store real byte count for binary-safe send */ + _tl_fs_read_buf = buf; /* ...valid ONLY for this exact buffer */ fclose(f); return el_wrap_str(buf); } @@ -1908,6 +2096,23 @@ el_val_t exec(el_val_t cmdv) { el_val_t exec_bg(el_val_t cmdv) { const char* cmd = EL_CSTR(cmdv); if (!cmd || !*cmd) return el_wrap_str(el_strdup("")); +#ifdef _WIN32 + /* Windows: no fork/exec. Launch a detached `cmd /c ` with no console window via + CreateProcess (DETACHED_PROCESS | CREATE_NO_WINDOW). Returns the PID as a string, "" on fail. + Mirrors the POSIX branch: child runs independently, caller is not blocked. */ + char cmdline[8192]; + snprintf(cmdline, sizeof(cmdline), "cmd.exe /c %s", cmd); + STARTUPINFOA si; ZeroMemory(&si, sizeof(si)); si.cb = sizeof(si); + PROCESS_INFORMATION pi; ZeroMemory(&pi, sizeof(pi)); + BOOL ok = CreateProcessA(NULL, cmdline, NULL, NULL, FALSE, + DETACHED_PROCESS | CREATE_NO_WINDOW, NULL, NULL, &si, &pi); + if (!ok) return el_wrap_str(el_strdup("")); + char pidbuf[32]; + snprintf(pidbuf, sizeof(pidbuf), "%lu", (unsigned long)pi.dwProcessId); + CloseHandle(pi.hProcess); + CloseHandle(pi.hThread); + return el_wrap_str(el_strdup(pidbuf)); +#else pid_t pid = fork(); if (pid < 0) { /* fork failed */ @@ -1930,6 +2135,7 @@ el_val_t exec_bg(el_val_t cmdv) { char pidbuf[32]; snprintf(pidbuf, sizeof(pidbuf), "%d", (int)pid); return el_wrap_str(el_strdup(pidbuf)); +#endif } el_val_t fs_list(el_val_t pathv) { @@ -3184,8 +3390,10 @@ el_val_t json_get_raw(el_val_t json_str, el_val_t key) { const char* k = EL_CSTR(key); const char* p = json_find_key(json, k); /* Clear fs_read binary-length hint — result is a fresh null-terminated - * string, not the raw file bytes, so Content-Length must use strlen. */ + * string, not the raw file bytes, so Content-Length must use strlen. + * (Kept although the pointer pairing now makes this redundant.) */ _tl_fs_read_len = 0; + _tl_fs_read_buf = NULL; if (!p) return el_wrap_str(el_strdup("")); const char* end = json_skip_value(p); size_t n = (size_t)(end - p); @@ -4057,7 +4265,12 @@ static int _el_decompose_earth(el_caltime_t* ct, struct tm* tm_out, int* abbr_le localtime_r(&s, &tm); *tm_out = tm; if (abbr_buf && abbr_cap > 0) { + /* mingw's struct tm has no tm_zone (BSD/glibc extension); no abbrev available there. */ +#ifdef _WIN32 + const char* z_str = ""; +#else const char* z_str = tm.tm_zone ? tm.tm_zone : ""; +#endif size_t n = strlen(z_str); if (n >= abbr_cap) n = abbr_cap - 1; memcpy(abbr_buf, z_str, n); @@ -5460,17 +5673,19 @@ void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal, * 0.25 Lesson, 0.30 Belief/Entity, 0.40 Note/Memory/Working). This constant * is NOT used in engram_activate(); it matches the Canonical tier value only * by coincidence. (2026-07-01 self-review: clarified stale doc) - * ENGRAM_WM_DECAY: per-turn decay applied to working_memory_weight for - * nodes NOT re-activated in the current turn (conversational thread - * continuity: a node promoted in turn N persists with reduced weight - * into turn N+1 without re-activation cost). + * ENGRAM_WM_DECAY: SUPERSEDED (2026-07-22 self-review, bl-b17facdd) — the + * per-turn multiplicative carry-over decay was replaced by the ACT-R/ + * Petrov base-level scheme (see ENGRAM_BLL_* below). Kept for reference; + * no longer used in engram_activate. * ENGRAM_SUPPRESSION_BREAKTHROUGH: after this many consecutive suppressions * a latent node forces itself into working memory at reduced weight, * modelling the brain's "intrusive thought" / unresolved-tension surfacing. * ENGRAM_BREAKTHROUGH_WEIGHT: the reduced working_memory_weight assigned * when a suppressed node breaks through. * ENGRAM_INHIBITION_FACTOR: multiplier applied to working_memory_weight when - * an inhibitory edge fires against a node (0 = full suppress, 0.3 = partial). */ + * an inhibitory edge fires against a node (0 = full suppress; current value + * 0.1 = near-full suppression — comment previously said 0.3, which drifted + * from the actual constant below). */ #define ENGRAM_WM_THRESHOLD 0.15 #define ENGRAM_WM_DECAY 0.7 #define ENGRAM_SUPPRESSION_BREAKTHROUGH 5 @@ -5494,6 +5709,42 @@ void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal, #define ENGRAM_WM_CAP 24 #define ENGRAM_INHIBITION_FACTOR 0.1 +/* ── ACT-R / Petrov hybrid base-level learning (2026-07-22 self-review) ────── + * Replaces the per-turn multiplicative WM carry-over decay (weight *= 0.7 per + * engram_activate call) with wall-clock power-law decay over actual access + * history. The old scheme was call-rate-dependent: 10 curiosity scans in 10 + * seconds decayed a carried node as much as 10 scans across an hour, and a + * scalar weight loses access FREQUENCY entirely — a node touched 50 times + * decayed identically to one touched once. + * + * Petrov (2006) hybrid: keep the K most recent access timestamps exactly, + * approximate the older tail in closed form: + * + * B = ln( Σ_{j=1..k} t_j^(-d) + * + (n-k) · (t_n^(1-d) - t_k^(1-d)) / ((1-d) · (t_n - t_k)) ) + * + * d = 0.5 (canonical ACT-R decay), t_j = seconds since j-th recent access, + * t_k = seconds since oldest RETAINED access, t_n = seconds since first + * presentation (node creation), n = total presentations. + * + * This is the scheme Soar ships for working-memory forgetting (Derbinsky & + * Laird 2012). Calibration: a single-touch node has B(t) = -0.5·ln(t); with + * τ = -3.0 it falls below threshold at t = e^6 ≈ 403 s. Frequently-touched + * nodes accumulate Σ t^-0.5 mass and persist proportionally longer — recency + * AND frequency in one formula, which a decayed scalar cannot represent. + * + * Carry-over weight above threshold is shaped by the ACT-R retrieval- + * probability logistic P = σ((B − τ)/s), s = 0.4, applied to the weight the + * node held at promotion (wm_anchor) — NOT re-multiplied per call, so the + * carried weight is a pure function of wall-clock time regardless of how + * often engram_activate runs. + * Sources: alexpetrov.com/pub/iccm06 · Soar cli/cmd_wm defaults · + * arxiv.org/html/2412.05112v1 (2026-07-21 integration brief, bl-b17facdd). */ +#define ENGRAM_BLL_K 10 +#define ENGRAM_BLL_D 0.5 +#define ENGRAM_BLL_TAU (-3.0) +#define ENGRAM_BLL_S 0.4 + /* qsort comparator — descending double, used by WM cap enforcement. */ static int engram_cmp_double_desc(const void* a, const void* b) { double da = *(const double*)a; @@ -5578,6 +5829,15 @@ static double engram_type_threshold(const char* node_type, const char* tier) { if (node_type) { if (strcmp(node_type, "Belief") == 0) return 0.30; if (strcmp(node_type, "Entity") == 0) return 0.30; + /* Knowledge nodes (captureKnowledge, world-ingestor) at non-Canonical/ + * non-Lesson tiers (Semantic/Episodic/Procedural) previously fell + * through to the 0.40 note default — same bar as ephemeral notes — + * so curated knowledge mostly entered WM via breakthrough suppression + * (visible as wm weights pinned near the breakthrough floor) instead + * of natural promotion. Placed AFTER the tier checks so Canonical + * (0.15) and Lesson (0.25) still win. Ported from the dev-line fix + * (2026-06-13 self-review). (2026-07-19 self-review) */ + if (strcmp(node_type, "Knowledge") == 0) return 0.20; } return 0.40; /* Note / Memory / Working (most nodes) */ } @@ -5621,8 +5881,75 @@ typedef struct EngramNode { * created via engram_node / engram_node_full and for snapshots that * predate the layered schema. */ uint32_t layer_id; + /* ACT-R base-level learning state (2026-07-22 self-review, ENGRAM_BLL_*). + * access_ts: ring buffer of the most recent access timestamps (ms). + * access_head: next write slot. access_filled: valid entries (≤ K). + * wm_anchor: the WM weight the node held when last promoted; carry-over + * decay is computed from this anchor as a pure function of wall-clock + * time. All fields zero via memset in creation/load paths; snapshots + * without them degrade gracefully to the optimized-form approximation + * in engram_bll_base_level. */ + int64_t access_ts[ENGRAM_BLL_K]; + int32_t access_head; + int32_t access_filled; + double wm_anchor; } EngramNode; +/* Record an access (ACT-R "presentation") into the base-level ring buffer. */ +static void engram_bll_record_access(EngramNode* n, int64_t now_ms) { + n->access_ts[n->access_head] = now_ms; + n->access_head = (n->access_head + 1) % ENGRAM_BLL_K; + if (n->access_filled < ENGRAM_BLL_K) n->access_filled++; +} + +/* Petrov hybrid base-level activation B in nats. Exact sum over the retained + * ring, closed-form tail for older presentations. Falls back to the ACT-R + * optimized-learning approximation B = ln(n/(1−d)) − d·ln(L) for nodes with + * no retained access history (legacy snapshots). */ +static double engram_bll_base_level(const EngramNode* n, int64_t now_ms) { + /* Total presentations: creation counts as the first, retrievals add. */ + double n_total = (double)n->activation_count + 1.0; + double t_life = (double)(now_ms - n->created_at) / 1000.0; + if (t_life < 1.0) t_life = 1.0; + if (n->access_filled == 0) { + return log(n_total / (1.0 - ENGRAM_BLL_D)) + - ENGRAM_BLL_D * log(t_life); + } + double sum = 0.0; + double t_oldest_ring = 0.0; /* seconds since oldest retained access */ + for (int32_t j = 0; j < n->access_filled; j++) { + double t = (double)(now_ms - n->access_ts[j]) / 1000.0; + if (t < 1.0) t = 1.0; + sum += pow(t, -ENGRAM_BLL_D); + if (t > t_oldest_ring) t_oldest_ring = t; + } + /* Closed-form tail for the (n − k) presentations older than the ring. + * With d = 0.5 this reduces to 2·(n−k)/(√t_n + √t_k). */ + double n_older = n_total - (double)n->access_filled; + if (n_older > 0.0 && t_life > t_oldest_ring) { + sum += n_older * (pow(t_life, 1.0 - ENGRAM_BLL_D) + - pow(t_oldest_ring, 1.0 - ENGRAM_BLL_D)) + / ((1.0 - ENGRAM_BLL_D) * (t_life - t_oldest_ring)); + } + if (sum <= 0.0) return -99.0; + return log(sum); +} + +/* Parse a persisted "access_ts" comma list (chronological order) into the + * ring buffer. Tolerates absence, empty strings, and stray whitespace. */ +static void engram_bll_parse_access(EngramNode* nn, const char* s) { + if (!s) return; + const char* p = s; + while (*p) { + char* endp = NULL; + long long v = strtoll(p, &endp, 10); + if (endp == p) break; + if (v > 0) engram_bll_record_access(nn, (int64_t)v); + p = endp; + while (*p == ',' || *p == ' ') p++; + } +} + typedef struct EngramEdge { char* id; char* from_id; @@ -5926,7 +6253,16 @@ static void engram_idmap_put(EngramStore* g, const char* id, int64_t idx) { } /* Use tombstone slot if found (avoids growing used count unnecessarily) */ if (tomb_slot != SIZE_MAX) slot = tomb_slot; - g->id_map[slot].key = el_strdup(id); + /* MUST be el_strdup_persist: idmap keys outlive the request/tick arena. + * (2026-07-16 self-review) This was el_strdup (arena-tracked): every node + * created inside an HTTP request left its idmap key DANGLING as soon as + * el_request_end() freed the arena — subsequent lookups strcmp'd freed + * memory, and any idmap_free/rebuild in a later request double-freed it + * (SIGABRT in http_worker; found via ASAN when engram_prune_telemetry + * triggered an in-request rebuild). Same allocation-discipline class as + * the 2026-07-15 EngramNode fix — see the store-persistent comment above + * engram_new_id(). */ + g->id_map[slot].key = el_strdup_persist(id); g->id_map[slot].idx = idx; g->id_map_used++; } @@ -6102,10 +6438,23 @@ static void engram_grow_edges(void) { } /* Build a fresh UUID string. Reuses uuid_new but takes the underlying char*. */ +/* ── Store-persistent allocation discipline ───────────────────────────────── + * (2026-07-15 self-review) EngramNode/EngramEdge string fields OUTLIVE the + * request/tick arena they were created in. The 2026-07-13 leak-fix made + * el_strdup arena-tracked, which silently turned every node created inside + * an HTTP request (route_emit_ise, route_create_node, knowledge capture) or + * inside the soul's per-tick arena (engram_load_merge in the refresh cycle) + * into a bag of dangling pointers the moment the arena popped: readback by + * id returned {}, type-filtered scans skipped them, search returned request + * memory reused as node content, and snapshots persisted garbage ("numeric + * tier strings"). Everything written into the store must go through + * el_strdup_persist / el_strbuf_persist (plain malloc — free() in + * engram_forget/evolve remains valid). Arena-tracked el_strdup remains + * correct for RETURN values handed back to EL code. */ static char* engram_new_id(void) { el_val_t v = uuid_new(); const char* s = EL_CSTR(v); - return el_strdup(s ? s : ""); + return el_strdup_persist(s ? s : ""); } /* Convert a node into an ElMap of its fields. */ @@ -6130,6 +6479,10 @@ static el_val_t engram_node_to_map(const EngramNode* n) { m = el_map_set(m, EL_STR(el_strdup("working_memory_weight")), el_from_float(n->working_memory_weight)); m = el_map_set(m, EL_STR(el_strdup("suppression_count")), (el_val_t)n->suppression_count); m = el_map_set(m, EL_STR(el_strdup("layer_id")), (el_val_t)(int64_t)n->layer_id); + /* Observability (2026-07-22): expose the current ACT-R base-level and + * promotion anchor so heartbeat ISEs / API consumers can see decay state. */ + m = el_map_set(m, EL_STR(el_strdup("wm_anchor")), el_from_float(n->wm_anchor)); + m = el_map_set(m, EL_STR(el_strdup("base_level")), el_from_float(engram_bll_base_level(n, engram_now_ms()))); return m; } @@ -6166,12 +6519,12 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) { n->id = engram_new_id(); const char* c = EL_CSTR(content); const char* nt = EL_CSTR(node_type); - n->content = el_strdup(c ? c : ""); - n->node_type = el_strdup(nt && *nt ? nt : "Memory"); - n->label = engram_first_n_chars(c, 60); - n->tier = el_strdup("Working"); - n->tags = el_strdup(""); - n->metadata = el_strdup("{}"); + n->content = el_strdup_persist(c ? c : ""); + n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory"); + n->label = el_strdup_persist(engram_first_n_chars(c, 60)); + n->tier = el_strdup_persist("Working"); + n->tags = el_strdup_persist(""); + n->metadata = el_strdup_persist("{}"); n->salience = engram_decode_score(salience); if (n->salience <= 0.0 || n->salience > 1.0) n->salience = 0.5; n->importance = 0.5; @@ -6203,12 +6556,12 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label, const char* lb = EL_CSTR(label); const char* ti = EL_CSTR(tier); const char* tg = EL_CSTR(tags); - n->content = el_strdup(c ? c : ""); - n->node_type = el_strdup(nt && *nt ? nt : "Memory"); - n->label = el_strdup(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : "")); - n->tier = el_strdup(ti && *ti ? ti : "Working"); - n->tags = el_strdup(tg ? tg : ""); - n->metadata = el_strdup("{}"); + n->content = el_strdup_persist(c ? c : ""); + n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory"); + n->label = el_strdup_persist(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : "")); + n->tier = el_strdup_persist(ti && *ti ? ti : "Working"); + n->tags = el_strdup_persist(tg ? tg : ""); + n->metadata = el_strdup_persist("{}"); n->salience = engram_decode_score(salience); n->importance = engram_decode_score(importance); n->confidence = engram_decode_score(confidence); @@ -6259,20 +6612,20 @@ el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t labe const char* lb = EL_CSTR(label); const char* tg = EL_CSTR(tags); const char* st = EL_CSTR(status); - n->content = el_strdup(c ? c : ""); - n->node_type = el_strdup(nt && *nt ? nt : "Memory"); - n->label = el_strdup(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : "")); - n->tier = el_strdup("Working"); - n->tags = el_strdup(tg ? tg : ""); + n->content = el_strdup_persist(c ? c : ""); + n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory"); + n->label = el_strdup_persist(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : "")); + n->tier = el_strdup_persist("Working"); + n->tags = el_strdup_persist(tg ? tg : ""); if (st && *st) { /* Minimal metadata payload: {"status":"..."}. Keep it cheap so * callers using `status` don't pay JSON parse cost on every read. */ size_t sl = strlen(st) + 16; - char* meta = el_strbuf(sl); + char* meta = el_strbuf_persist(sl); snprintf(meta, sl, "{\"status\":\"%s\"}", st); n->metadata = meta; } else { - n->metadata = el_strdup("{}"); + n->metadata = el_strdup_persist("{}"); } n->salience = engram_decode_score(salience); n->importance = engram_decode_score(certainty); @@ -6425,6 +6778,8 @@ void engram_strengthen(el_val_t node_id) { n->activation_count++; n->last_activated = engram_now_ms(); n->updated_at = n->last_activated; + /* Explicit strengthen is a presentation too (2026-07-22 self-review). */ + engram_bll_record_access(n, n->last_activated); } void engram_forget(el_val_t node_id) { @@ -6468,6 +6823,114 @@ el_val_t engram_node_count(void) { return (el_val_t)engram_get()->node_count; } +/* ── Telemetry retention ──────────────────────────────────────────────────── + * (2026-07-16 self-review) InternalStateEvent nodes are append-only telemetry + * (heartbeat, curiosity_scan, engram_sync) written ~3/min by the awareness + * loop. Nothing ever removed them: by July 16 they were 10,175 of 13,522 + * nodes — 75% of the store was telemetry. They are already force-excluded + * from WM promotion (engram_activate), so their only effect was store bloat, + * snapshot bloat, and lexical-search noise. + * + * engram_prune_telemetry(older_than_ms) batch-removes ISE nodes whose + * created_at is older than now - older_than_ms, EXCEPT durable markers: + * - label "session-start" (boot history) + * - content containing "self_review" (daily review trail) + * Unlike repeated engram_forget (O(n) shift each), this is a single + * compaction pass over nodes plus one pass over edges, with one idmap + * rebuild — O(nodes + edges) total, safe to call on every ISE insert. + * Returns the number of nodes removed. */ + +/* FNV-1a hash for the removed-id set used by the edge sweep. */ +static uint64_t eg_fnv1a(const char* s) { + uint64_t h = 1469598103934665603ULL; + while (*s) { h ^= (unsigned char)*s++; h *= 1099511628211ULL; } + return h; +} + +el_val_t engram_prune_telemetry(el_val_t older_than_ms) { + int64_t horizon = (int64_t)older_than_ms; + if (horizon <= 0) horizon = 172800000; /* default 48h */ + EngramStore* g = engram_get(); + int64_t cutoff = engram_now_ms() - horizon; + + /* Pass 1: mark. Collect ids of prunable nodes (ownership transferred — + * strings freed after the edge sweep). */ + int64_t cap = 0; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + if (n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0 && + n->created_at < cutoff) cap++; + } + if (cap == 0) return 0; + + char** removed_ids = malloc((size_t)cap * sizeof(char*)); + if (!removed_ids) return 0; + int64_t removed = 0, w = 0; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + int prunable = + n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0 && + n->created_at < cutoff && + !(n->label && strcmp(n->label, "session-start") == 0) && + !(n->content && strstr(n->content, "self_review")); + if (prunable && removed < cap) { + removed_ids[removed++] = n->id; /* keep id for edge sweep */ + free(n->content); free(n->node_type); free(n->label); + free(n->tier); free(n->tags); free(n->metadata); + } else { + if (w != i) g->nodes[w] = g->nodes[i]; + w++; + } + } + g->node_count = w; + if (removed == 0) { free(removed_ids); return 0; } + + /* Removed-id hash set (open addressing, power-of-two >= 2*removed). */ + size_t set_cap = 16; + while (set_cap < (size_t)removed * 2) set_cap <<= 1; + const char** set = calloc(set_cap, sizeof(char*)); + if (set) { + for (int64_t i = 0; i < removed; i++) { + size_t slot = eg_fnv1a(removed_ids[i]) & (set_cap - 1); + while (set[slot]) slot = (slot + 1) & (set_cap - 1); + set[slot] = removed_ids[i]; + } + } + /* Pass 2: drop edges incident to any removed node (defensive — ISEs + * currently have no edges, but callers may connect them later). */ + if (set) { + int64_t ew = 0; + for (int64_t r = 0; r < g->edge_count; r++) { + EngramEdge* e = &g->edges[r]; + int incident = 0; + const char* ends[2] = { e->from_id, e->to_id }; + for (int k = 0; k < 2 && !incident; k++) { + if (!ends[k]) continue; + size_t slot = eg_fnv1a(ends[k]) & (set_cap - 1); + while (set[slot]) { + if (strcmp(set[slot], ends[k]) == 0) { incident = 1; break; } + slot = (slot + 1) & (set_cap - 1); + } + } + if (incident) { + free(e->id); free(e->from_id); free(e->to_id); + free(e->relation); free(e->metadata); + } else { + if (ew != r) g->edges[ew] = g->edges[r]; + ew++; + } + } + g->edge_count = ew; + free(set); + } + for (int64_t i = 0; i < removed; i++) free(removed_ids[i]); + free(removed_ids); + + engram_idmap_rebuild(g); + engram_adj_free(g); + return (el_val_t)removed; +} + static int istr_contains(const char* hay, const char* needle) { if (!hay || !needle || !*needle) return 0; size_t nl = strlen(needle); @@ -6477,6 +6940,78 @@ static int istr_contains(const char* hay, const char* needle) { return 0; } +/* ── Tokenized query matching ─────────────────────────────────────────── + * The engram query surface (search / activate / goal-bias) historically + * matched the ENTIRE raw query string as a single case-insensitive + * substring via istr_contains(field, q). That is Ctrl-F, not search: + * a multi-word query like "windows msi signing" only matched a node whose + * text contained that exact contiguous run, so real multi-word queries + * returned zero. istr_contains stays as the per-TOKEN primitive; these + * helpers split the query on whitespace and match ANY token, then rank by + * how many DISTINCT tokens a node covers. Single-token queries are a strict + * special case (score is 0 or 1) so single-word callers never regress. + * (Ported 2026-07-19 from the el-compiler runtime copy, where the 2026-07-14 + * fix landed but never reached this release runtime — the copy the engram + * binary actually builds against.) */ +#define ENGRAM_MAX_QTOKENS 32 +#define ENGRAM_QTOK_LEN 256 + +/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct + * (case-insensitive) tokens. Returns the token count. Over-long tokens are + * truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */ +static int engram_tokenize_query(const char* q, + char toks[][ENGRAM_QTOK_LEN], int maxtok) { + int n = 0; + if (!q) return 0; + const char* p = q; + while (*p && n < maxtok) { + while (*p && isspace((unsigned char)*p)) p++; + if (!*p) break; + char buf[ENGRAM_QTOK_LEN]; + size_t tl = 0; + while (*p && !isspace((unsigned char)*p)) { + if (tl < sizeof(buf) - 1) buf[tl++] = *p; + p++; + } + buf[tl] = '\0'; + if (tl == 0) continue; + int dup = 0; + for (int s = 0; s < n; s++) { + if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; } + } + if (dup) continue; + memcpy(toks[n], buf, tl + 1); + n++; + } + return n; +} + +/* Count how many of the ntok distinct query tokens appear (case-insensitive) + * in the node's content, label, or tags. 0 == no match. */ +static int engram_node_match_score(const EngramNode* n, + char toks[][ENGRAM_QTOK_LEN], int ntok) { + int score = 0; + for (int t = 0; t < ntok; t++) { + if (istr_contains(n->content, toks[t]) || + istr_contains(n->label, toks[t]) || + istr_contains(n->tags, toks[t])) + score++; + } + return score; +} + +/* Rank entry: distinct-token match count (primary, desc) then salience + * (tiebreak, desc). */ +typedef struct { int64_t idx; int score; double salience; } EngramRankEntry; +static int engram_rank_cmp(const void* a, const void* b) { + const EngramRankEntry* ea = (const EngramRankEntry*)a; + const EngramRankEntry* eb = (const EngramRankEntry*)b; + if (ea->score != eb->score) return eb->score - ea->score; /* desc */ + if (ea->salience < eb->salience) return 1; + if (ea->salience > eb->salience) return -1; + return 0; +} + el_val_t engram_search(el_val_t query, el_val_t limit) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); @@ -6484,32 +7019,55 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { if (lim <= 0) lim = 100; el_val_t lst = el_list_empty(); if (!q || !*q) return lst; - int64_t found = 0; - for (int64_t i = 0; i < g->node_count && found < lim; i++) { + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS); + if (ntok == 0) return lst; + EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry)); + if (!hits) return lst; + int64_t nhits = 0; + for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; /* Filter transparent layers: nodes whose layer is `transparent=1` * shape output but are invisible to introspection ("what do you * know about yourself"). They still surface via engram_activate * + engram_compile_layered_json — that's the legitimate path. */ if (engram_layer_is_transparent(n->layer_id)) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { - lst = el_list_append(lst, engram_node_to_map(n)); - found++; + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0) { + hits[nhits].idx = i; + hits[nhits].score = sc; + hits[nhits].salience = n->salience; + nhits++; } } + /* Rank by distinct tokens matched (desc) then salience (desc), then cap. */ + qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); + int64_t end = nhits < lim ? nhits : lim; + for (int64_t k = 0; k < end; k++) { + lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx])); + } + free(hits); return lst; } -/* Sort node indices by salience desc (small N, insertion sort is fine). */ +/* Sort node indices by salience desc, tie-break created_at desc (newest + * first). The tie-break matters for telemetry: InternalStateEvent nodes all + * share salience 0.3, so before this the listing routes returned them in + * store order — OLDEST first — and any limited query (/api/nodes?...&limit=N) + * silently returned a stale window. A 41-hour-old heartbeat series read as a + * live outage during the 2026-07-22 self-review. Newest-first ties make + * limited scans return the recent window consumers actually want. + * (Small N, insertion sort is fine.) */ static void engram_sort_indices_by_salience(int64_t* arr, int64_t n, const EngramNode* nodes) { for (int64_t i = 1; i < n; i++) { int64_t key = arr[i]; double ks = nodes[key].salience; + int64_t kc = nodes[key].created_at; int64_t j = i - 1; - while (j >= 0 && nodes[arr[j]].salience < ks) { + while (j >= 0 && (nodes[arr[j]].salience < ks || + (nodes[arr[j]].salience == ks && + nodes[arr[j]].created_at < kc))) { arr[j + 1] = arr[j]; j--; } @@ -6552,10 +7110,10 @@ void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t EngramEdge* e = &g->edges[g->edge_count]; memset(e, 0, sizeof(*e)); e->id = engram_new_id(); - e->from_id = el_strdup(f); - e->to_id = el_strdup(t); - e->relation = el_strdup(r && *r ? r : "associate"); - e->metadata = el_strdup("{}"); + e->from_id = el_strdup_persist(f); + e->to_id = el_strdup_persist(t); + e->relation = el_strdup_persist(r && *r ? r : "associate"); + e->metadata = el_strdup_persist("{}"); e->weight = engram_decode_score(weight); if (e->weight <= 0.0 || e->weight > 1.0) e->weight = 0.5; e->confidence = 1.0; @@ -6746,7 +7304,7 @@ static double engram_temporal_proximity_bonus(int64_t node_created, * * Working memory persistence (turn continuity): * Nodes promoted in the previous turn retain a decayed working_memory_weight - * (weight *= ENGRAM_WM_DECAY) without needing re-activation. This models + * (ACT-R base-level carry-over, 2026-07-22) without needing re-activation. This models * conversational thread continuity — once a topic is in working memory, * it persists slightly into the next turn. * @@ -6776,10 +7334,15 @@ static double engram_temporal_proximity_bonus(int64_t node_created, static double engram_goal_bias(const EngramNode* n, const char* query) { if (!query || !*query) return 1.0; double bias = 1.0; - /* Direct lexical overlap: node content/label/tags share text with query. */ - if (istr_contains(n->content, query) || istr_contains(n->label, query) || - istr_contains(n->tags, query)) { - bias += 0.5; + /* Direct lexical overlap, graded by token coverage: a node covering all + * query tokens gets the full +0.5; partial coverage gets a proportional + * share. Single-token queries → full +0.5 on match, identical to before. + * (2026-07-19 port of the 2026-07-14 tokenized-search fix) */ + { + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(query, toks, ENGRAM_MAX_QTOKENS); + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0 && ntok > 0) bias += 0.5 * ((double)sc / (double)ntok); } /* Node-type resonance with query intent. */ int technical_query = istr_contains(query, "code") || @@ -6790,7 +7353,17 @@ static double engram_goal_bias(const EngramNode* n, const char* query) { istr_contains(query, "build") || istr_contains(query, "system") || istr_contains(query, "design") || - istr_contains(query, "architecture"); + istr_contains(query, "architecture") || + /* Curiosity-scan seeds: without these, idle-loop + * activation queries ("decision pattern lesson", + * "memory knowledge context") produce no goal-bias + * differentiation at all. Ported from dev-line fix + * d53516b (2026-06-14). (2026-07-19 self-review) */ + istr_contains(query, "knowledge") || + istr_contains(query, "pattern") || + istr_contains(query, "decision") || + istr_contains(query, "memory") || + istr_contains(query, "lesson"); int personal_query = istr_contains(query, "feel") || istr_contains(query, "emotion") || istr_contains(query, "remember") || @@ -6800,7 +7373,14 @@ static double engram_goal_bias(const EngramNode* n, const char* query) { if (n->node_type) { int is_knowledge = (strcmp(n->node_type, "Belief") == 0) || (strcmp(n->node_type, "DharmaSelf") == 0) || - (strcmp(n->node_type, "Safety") == 0); + (strcmp(n->node_type, "Safety") == 0) || + /* The primary knowledge-capture type was absent + * from its own bias class: captureKnowledge() and + * the world ingestor write node_type "Knowledge", + * which competed at neutral bias on technical + * queries. Ported from dev-line fix d53516b. + * (2026-07-19 self-review) */ + (strcmp(n->node_type, "Knowledge") == 0); int is_personal = (strcmp(n->node_type, "Memory") == 0) || (strcmp(n->node_type, "Entity") == 0); if (technical_query && is_knowledge) bias += 0.3; @@ -6851,14 +7431,34 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { if (!seeds) { free(best_bg); free(best_hops); free(reached); return out; } + /* Tokenize once: a node seeds if it matches ANY query token, and its seed + * activation is scaled by token coverage (fraction of distinct query + * tokens it contains) so a node matching all words seeds more strongly + * than one matching a single word. Single-word queries → coverage 1.0, + * identical to the prior whole-query behavior. Before this, the soul's + * rotating 3-word curiosity seeds ("working project active") activated + * ZERO nodes almost every scan — idle cognition firing blanks. + * (2026-07-19 port of the 2026-07-14 tokenized-search fix; NOTE the + * el-compiler copy of this fix dropped the ISE seed exclusion below — + * kept here deliberately, do not "sync" it away.) */ + char qtoks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int qntok = engram_tokenize_query(q, qtoks, ENGRAM_MAX_QTOKENS); for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { + /* InternalStateEvent nodes are observability-only telemetry — never + * seed activation from them. Their JSON payloads contain common words + * ("memory", "context", ...) that lexically match almost any query, + * turning telemetry into a spreading-activation ignition source. They + * are already excluded from WM promotion in pass 2; exclude them from + * seeding here too. */ + if (n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0) + continue; + int msc = engram_node_match_score(n, qtoks, qntok); + if (msc > 0) { double tdecay = engram_temporal_decay(n, now_ms); double dampen = engram_activation_dampen(n); - double act = n->salience * tdecay * dampen; + double cover = qntok > 0 ? (double)msc / (double)qntok : 1.0; + double act = n->salience * tdecay * dampen * cover; seeds[seed_count].idx = i; seeds[seed_count].act = act; seeds[seed_count].created_at = n->created_at; @@ -6868,12 +7468,20 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { reached[i] = 1; } } - /* Compute mean seed created_at for temporal proximity bonus. */ + /* Compute mean seed created_at for temporal proximity bonus. + * Was a running pairwise average — seed_epoch = (seed_epoch + t_s)/2 — + * which is NOT the arithmetic mean: it exponentially over-weights the + * later seeds (last seed gets weight 1/2, second-to-last 1/4, ...), so + * the temporal-proximity bonus skewed toward whichever seeds happened + * to sit later in the scan order. True mean via int64 sum: ms epochs + * (~1.8e12) times any plausible seed_count stays far below INT64_MAX. + * (2026-07-19 self-review) */ int64_t seed_epoch = 0; if (seed_count > 0) { - seed_epoch = seeds[0].created_at; - for (int64_t s = 1; s < seed_count; s++) - seed_epoch = (seed_epoch + seeds[s].created_at) / 2; + int64_t epoch_sum = 0; + for (int64_t s = 0; s < seed_count; s++) + epoch_sum += seeds[s].created_at; + seed_epoch = epoch_sum / seed_count; } typedef struct { int64_t idx; int64_t hops; double act; } Frontier; Frontier* fr = malloc((size_t)(g->node_count * (max_depth + 1)) * sizeof(Frontier) + 16 * sizeof(Frontier)); @@ -6929,11 +7537,22 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { if (oi < 0 || oi >= g->node_count) continue; EngramEdge* e = &g->edges[ei]; EngramNode* on = &g->nodes[oi]; + /* Never propagate INTO InternalStateEvent nodes. They are already + * barred from WM promotion (pass 2) and from seeding (above), but + * as high-degree hubs they still relayed activation across the + * graph. Skipping here keeps them out of the frontier entirely. */ + if (on->node_type && strcmp(on->node_type, "InternalStateEvent") == 0) + continue; double tbonus = engram_temporal_proximity_bonus(on->created_at, seed_epoch); double tdecay = engram_temporal_decay(on, now_ms); double dampen = engram_activation_dampen(on); double new_act = f.act * e->weight * SPREAD_DECAY * (1.0 + tbonus) * tdecay * dampen; + /* Firing threshold per classic spreading-activation: sub-threshold + * activation neither updates the target nor enqueues it, so weak + * signals die out instead of flooding the whole graph with tiny + * nonzero background activation. */ + if (new_act < 0.02) continue; if (!reached[oi] || new_act > best_bg[oi]) { best_bg[oi] = new_act; best_hops[oi] = new_hops; @@ -7119,12 +7738,34 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { for (int64_t i = 0; i < g->node_count; i++) { if (!reached[i] && g->nodes[i].working_memory_weight > 0.0) { /* Carry-over decay: node held WM weight from prior activation but - * the current query's BFS fan-out did not reach it. Apply decay - * rather than zero so recently-active context persists. */ - double decayed = g->nodes[i].working_memory_weight * ENGRAM_WM_DECAY; - g->nodes[i].working_memory_weight = (decayed < 0.01) ? 0.0 : decayed; + * the current query's BFS fan-out did not reach it. + * + * 2026-07-22 self-review (bl-b17facdd): replaced the per-call + * multiplicative decay (weight *= ENGRAM_WM_DECAY) with the + * ACT-R/Petrov base-level scheme. The old form was call-rate- + * dependent — carried context died in seconds under rapid + * curiosity scans and lingered for hours under quiet loops. + * Now: hard-evict when base-level < τ (Soar-style forgetting); + * above τ, shape the weight held at promotion (wm_anchor) by the + * retrieval-probability logistic. Pure function of wall-clock + * time — idempotent no matter how often activate is called. */ + EngramNode* cn = &g->nodes[i]; + double anchor = (cn->wm_anchor > 0.0) ? cn->wm_anchor + : cn->working_memory_weight; + double B = engram_bll_base_level(cn, now_ms); + if (B < ENGRAM_BLL_TAU) { + cn->working_memory_weight = 0.0; + } else { + double keep = 1.0 / (1.0 + exp(-(B - ENGRAM_BLL_TAU) + / ENGRAM_BLL_S)); + double w = anchor * keep; + cn->working_memory_weight = (w < 0.01) ? 0.0 : w; + } } else { g->nodes[i].working_memory_weight = wm_weights[i]; + /* Anchor the promotion weight: carry-over decay above computes + * from this fixed point rather than compounding per call. */ + if (wm_weights[i] > 0.0) g->nodes[i].wm_anchor = wm_weights[i]; } } @@ -7174,6 +7815,34 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { } } + /* ── Retrieval reinforcement (2026-07-18 self-review) ─────────────────── + * ACT-R base-level learning: retrieval strengthens memory. Before this, + * NOTHING in engram_activate updated last_activated/activation_count — + * only the rarely-called engram_strengthen did. Consequence: temporal + * decay aged every node from its last explicit strengthen (usually + * creation), so frequently-retrieved memories decayed identically to + * abandoned ones, and engram_activation_dampen() saw a frozen count. + * + * Scope deliberately narrow: reinforce ONLY nodes promoted to WM in THIS + * call that survived both capacity caps (wm_weights[i] > 0 = promoted this + * call; working_memory_weight > 0 = survived Pass 4/5 eviction). BFS + * fan-out touches thousands of nodes per curiosity scan — reinforcing all + * of them would flatten dampening and freeze decay globally. Promotion to + * working memory is the analog of actual retrieval (executive access), + * matching ACT-R where only completed retrievals add a base-level + * presentation. Carry-over nodes (reached[i]==0) are NOT reinforced: they + * persist by decay, they were not re-retrieved. */ + for (int64_t i = 0; i < g->node_count; i++) { + if (!reached[i] || wm_weights[i] <= 0.0) continue; + EngramNode* n = &g->nodes[i]; + if (n->working_memory_weight <= 0.0) continue; /* evicted by cap */ + n->last_activated = now_ms; + n->activation_count++; + /* Base-level presentation: WM promotion is the retrieval event. + * (2026-07-22 self-review — feeds engram_bll_base_level.) */ + engram_bll_record_access(n, now_ms); + } + /* ── Collect all background-activated nodes for the return value ──── * Callers see both layers. Context compilation uses only promoted nodes * (working_memory_weight > 0). Sort: promoted first by wm_weight desc, @@ -7255,6 +7924,23 @@ static void engram_emit_node_json(JsonBuf* b, const EngramNode* n) { snprintf(tmp, sizeof(tmp), ",\"working_memory_weight\":%g", n->working_memory_weight); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"suppression_count\":%d", n->suppression_count); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"layer_id\":%u", n->layer_id); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"wm_anchor\":%g", n->wm_anchor); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"base_level\":%g", + engram_bll_base_level(n, engram_now_ms())); jb_puts(b, tmp); + /* Base-level access history: chronological (oldest→newest) compact + * string. Loaders replay it through engram_bll_record_access; absent + * field = empty ring (optimized-form fallback). (2026-07-22) */ + if (n->access_filled > 0) { + jb_puts(b, ",\"access_ts\":\""); + for (int32_t j = 0; j < n->access_filled; j++) { + int32_t idx = (n->access_head - n->access_filled + j + + 2 * ENGRAM_BLL_K) % ENGRAM_BLL_K; + snprintf(tmp, sizeof(tmp), "%s%lld", j ? "," : "", + (long long)n->access_ts[idx]); + jb_puts(b, tmp); + } + jb_putc(b, '"'); + } jb_putc(b, '}'); } @@ -7331,12 +8017,21 @@ el_val_t engram_save(el_val_t path) { /* Helper: extract a string field from a JSON object substring. */ static char* eg_get_str_field(const char* obj, const char* key) { + /* Returns a STORE-PERSISTENT string: callers assign the result directly + * into EngramNode/EngramEdge fields, and engram_load_merge runs inside + * the soul's per-tick arena. jp_parse_string_raw's success buffer is + * plain malloc (persist-safe, returned as-is); the empty/error returns + * were arena-tracked el_strdup("") — those dangled after the tick arena + * popped and free()ing them in callers was a latent double-free. */ const char* p = json_find_key(obj, key); - if (!p) return el_strdup(""); - if (*p != '"') return el_strdup(""); + if (!p) return el_strdup_persist(""); + if (*p != '"') return el_strdup_persist(""); JsonParser jp = { .p = p, .end = p + strlen(p), .err = 0 }; char* out = jp_parse_string_raw(&jp); - if (jp.err) { free(out); return el_strdup(""); } + /* *p == '"' is guaranteed above, so jp_parse_string_raw took its malloc + * path (its arena-tracked early-return only fires on a non-'"' start) — + * free(out) on error is safe here. */ + if (jp.err) { free(out); return el_strdup_persist(""); } return out; } @@ -7358,6 +8053,48 @@ static const char* eg_skip_ws(const char* p) { return p; } +/* eg_enforce_wm_cap_on_load — clamp a snapshot-restored working-memory + * population to ENGRAM_WM_CAP. + * + * WHY (2026-07-15 self-review): engram_load restored working_memory_weight + * verbatim from the snapshot with no cap. Pass 4/5 cap enforcement only runs + * inside engram_activate — so a snapshot frozen in the pre-2026-06-30 era + * (87-111 promoted nodes, all at the old 0.25 breakthrough floor) reloaded + * as-is on every boot, and heartbeats faithfully reported the stale + * population (wm_active 87-111, wm_avg pinned at exactly 0.25) forever. + * The cap must hold at every entry point that materializes WM, not just + * the activation path. Same top-K-by-weight logic as engram_activate + * Pass 5 (Cowan 2001: WM capacity is global). */ +static void eg_enforce_wm_cap_on_load(EngramStore* g) { + int64_t wm_count = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > 0.0) wm_count++; + } + if (wm_count <= ENGRAM_WM_CAP) return; + double* vals = malloc((size_t)wm_count * sizeof(double)); + if (!vals) return; /* skip on OOM — over cap this boot, no corruption */ + int64_t vi = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > 0.0) + vals[vi++] = g->nodes[i].working_memory_weight; + } + qsort(vals, (size_t)wm_count, sizeof(double), engram_cmp_double_desc); + double cutoff = vals[ENGRAM_WM_CAP - 1]; + free(vals); + int64_t above = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > cutoff) above++; + } + int64_t slots_at_cutoff = ENGRAM_WM_CAP - above; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + if (n->working_memory_weight <= 0.0) continue; + if (n->working_memory_weight > cutoff) continue; + if (slots_at_cutoff > 0) { slots_at_cutoff--; continue; } + n->working_memory_weight = 0.0; /* evict: over cap at load */ + } +} + el_val_t engram_load(el_val_t path) { const char* p = EL_CSTR(path); if (!p || !*p) return 0; @@ -7412,7 +8149,7 @@ el_val_t engram_load(el_val_t path) { nn->tier = eg_get_str_field(obj, "tier"); nn->tags = eg_get_str_field(obj, "tags"); nn->metadata = eg_get_str_field(obj, "metadata"); - if (!nn->metadata || !*nn->metadata) { free(nn->metadata); nn->metadata = el_strdup("{}"); } + if (!nn->metadata || !*nn->metadata) { free(nn->metadata); nn->metadata = el_strdup_persist("{}"); } nn->salience = eg_get_num_field(obj, "salience"); nn->importance = eg_get_num_field(obj, "importance"); nn->confidence = eg_get_num_field(obj, "confidence"); @@ -7424,6 +8161,14 @@ el_val_t engram_load(el_val_t path) { nn->updated_at = eg_get_int_field(obj, "updated_at"); nn->background_activation = eg_get_num_field(obj, "background_activation"); nn->working_memory_weight = eg_get_num_field(obj, "working_memory_weight"); + /* Launder persisted WM weights across restarts: snapshots carry + * legacy breakthrough-floor 0.25 weights (pinned by the old + * suppression-breakthrough path) and nothing else ever launders + * them. Halve on every boot so genuine working-memory state has + * continuity across a restart while stale pinned weights decay + * out over successive boots; sub-0.05 residue drops to zero. */ + nn->working_memory_weight *= 0.5; + if (nn->working_memory_weight < 0.05) nn->working_memory_weight = 0.0; nn->suppression_count = (int32_t)eg_get_int_field(obj, "suppression_count"); /* layer_id defaults to ENGRAM_LAYER_DEFAULT (core-identity) * for snapshots that predate the layered schema. We can't @@ -7434,6 +8179,13 @@ el_val_t engram_load(el_val_t path) { } else { nn->layer_id = ENGRAM_LAYER_DEFAULT; } + /* Base-level state (2026-07-22): absent fields leave the ring + * empty (memset) → optimized-form fallback. */ + nn->wm_anchor = eg_get_num_field(obj, "wm_anchor"); + { + char* ats = eg_get_str_field(obj, "access_ts"); + if (ats) { engram_bll_parse_access(nn, ats); free(ats); } + } int64_t load_idx = g->node_count; g->node_count++; if (nn->id && *nn->id) engram_idmap_put(g, nn->id, load_idx); @@ -7466,7 +8218,7 @@ el_val_t engram_load(el_val_t path) { ee->to_id = eg_get_str_field(obj, "to_id"); ee->relation = eg_get_str_field(obj, "relation"); ee->metadata = eg_get_str_field(obj, "metadata"); - if (!ee->metadata || !*ee->metadata) { free(ee->metadata); ee->metadata = el_strdup("{}"); } + if (!ee->metadata || !*ee->metadata) { free(ee->metadata); ee->metadata = el_strdup_persist("{}"); } ee->weight = eg_get_num_field(obj, "weight"); ee->confidence = eg_get_num_field(obj, "confidence"); ee->created_at = eg_get_int_field(obj, "created_at"); @@ -7541,6 +8293,9 @@ el_val_t engram_load(el_val_t path) { } } } + /* WM cap discipline applies to every entry point that materializes WM, + * including snapshot restore (see eg_enforce_wm_cap_on_load). */ + eg_enforce_wm_cap_on_load(g); free(data); return 1; } @@ -7586,9 +8341,13 @@ el_val_t engram_load_merge(el_val_t path) { char* obj = malloc(n + 1); memcpy(obj, nodes_p, n); obj[n] = '\0'; char* nid = eg_get_str_field(obj, "id"); - int already = (nid && *nid && engram_find_node(nid) != NULL); + /* Nodes with an empty/unparseable id can never dedup against + * the idmap, so without this guard they were re-added on EVERY + * merge cycle — unbounded store growth. Skip them entirely. */ + int has_id = (nid && *nid); + int already = (has_id && engram_find_node(nid) != NULL); free(nid); - if (!already) { + if (has_id && !already) { engram_grow_nodes(); EngramNode* nn = &g->nodes[g->node_count]; memset(nn, 0, sizeof(*nn)); @@ -7609,15 +8368,24 @@ el_val_t engram_load_merge(el_val_t path) { nn->created_at = eg_get_int_field(obj, "created_at"); nn->updated_at = eg_get_int_field(obj, "updated_at"); nn->background_activation = eg_get_num_field(obj, "background_activation"); - nn->working_memory_weight = eg_get_num_field(obj, "working_memory_weight"); - if (!isfinite(nn->working_memory_weight) || nn->working_memory_weight < 0.0 || nn->working_memory_weight > 1.0) - nn->working_memory_weight = 0.0; + /* Nodes arriving via merge were never part of THIS store's + * working memory — importing the source store's WM weight + * would inject foreign (often legacy 0.25 breakthrough- + * floor) weights into local WM every sync cycle. */ + nn->working_memory_weight = 0.0; nn->suppression_count = (int32_t)eg_get_int_field(obj, "suppression_count"); if (json_find_key(obj, "layer_id")) { nn->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); } else { nn->layer_id = ENGRAM_LAYER_DEFAULT; } + /* Base-level history merges with the node (2026-07-22); + * wm_anchor stays 0 — WM state is local-only (see the + * working_memory_weight comment above). */ + { + char* ats = eg_get_str_field(obj, "access_ts"); + if (ats) { engram_bll_parse_access(nn, ats); free(ats); } + } int64_t merge_idx = g->node_count; g->node_count++; added_nodes++; @@ -7694,6 +8462,9 @@ el_val_t engram_load_merge(el_val_t path) { } } + /* Merged nodes can carry snapshot WM weights too — hold the cap here + * as well (see eg_enforce_wm_cap_on_load). */ + eg_enforce_wm_cap_on_load(g); free(data); return (el_val_t)added_nodes; } @@ -7750,19 +8521,36 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { JsonBuf b; jb_init(&b); jb_putc(&b, '['); int first = 1; - int64_t found = 0; if (q && *q) { - for (int64_t i = 0; i < g->node_count && found < lim; i++) { - EngramNode* n = &g->nodes[i]; - /* Filter transparent layers — same as engram_search. */ - if (engram_layer_is_transparent(n->layer_id)) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { - if (!first) jb_putc(&b, ','); - engram_emit_node_json(&b, n); - first = 0; - found++; + /* Tokenized + ranked, same scheme as engram_search: match ANY query + * token, rank by distinct-token coverage then salience, cap at lim. + * (2026-07-19 port of the 2026-07-14 tokenized-search fix) */ + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS); + if (ntok > 0) { + EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry)); + if (hits) { + int64_t nhits = 0; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + /* Filter transparent layers — same as engram_search. */ + if (engram_layer_is_transparent(n->layer_id)) continue; + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0) { + hits[nhits].idx = i; + hits[nhits].score = sc; + hits[nhits].salience = n->salience; + nhits++; + } + } + qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); + int64_t end = nhits < lim ? nhits : lim; + for (int64_t k = 0; k < end; k++) { + if (!first) jb_putc(&b, ','); + engram_emit_node_json(&b, &g->nodes[hits[k].idx]); + first = 0; + } + free(hits); } } } @@ -7860,8 +8648,16 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t di free(frontier); free(frontier_h); free(visited); jb_putc(&b, ']'); return el_wrap_str(b.buf); } - frontier[fc] = el_strdup(sid); frontier_h[fc] = 0; fc++; - visited[vc++] = el_strdup(sid); + /* MUST be el_strdup_persist: this function frees frontier/visited strings + * manually (lines below). el_strdup would ALSO register them in the + * per-request arena, so el_request_end() double-freed every one of them + * at the end of the HTTP request — SIGABRT in http_worker under load. + * (2026-07-18 self-review; reproduced via ASAN on /api/neuron/session/begin + * and /api/neuron/graph. Same allocation-discipline class as the + * 2026-07-15 EngramNode and 2026-07-16 idmap-key fixes: never mix arena + * tracking with manual free.) */ + frontier[fc] = el_strdup_persist(sid); frontier_h[fc] = 0; fc++; + visited[vc++] = el_strdup_persist(sid); int first = 1; while (fc > 0) { @@ -7890,8 +8686,8 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t di char tmp[64]; snprintf(tmp, sizeof(tmp), ",\"hops\":%lld}", (long long)(h + 1)); jb_puts(&b, tmp); first = 0; - if (vc < 1024) visited[vc++] = el_strdup(peer); - if (fc < 1024 && h + 1 < depth) { frontier[fc] = el_strdup(peer); frontier_h[fc] = h + 1; fc++; } + if (vc < 1024) visited[vc++] = el_strdup_persist(peer); + if (fc < 1024 && h + 1 < depth) { frontier[fc] = el_strdup_persist(peer); frontier_h[fc] = h + 1; fc++; } } free(cur); } @@ -8612,13 +9408,13 @@ static int64_t dharma_find_or_create_relation_edge(const char* peer_base, int cr engram_grow_nodes(); EngramNode* n = &g->nodes[g->node_count]; memset(n, 0, sizeof(*n)); - n->id = el_strdup(self_id); - n->content = el_strdup(_el_cgi_dharma_id ? _el_cgi_dharma_id : "(self)"); - n->node_type = el_strdup("DharmaSelf"); - n->label = el_strdup("dharma:self"); - n->tier = el_strdup("Working"); - n->tags = el_strdup("dharma"); - n->metadata = el_strdup("{}"); + n->id = el_strdup_persist(self_id); + n->content = el_strdup_persist(_el_cgi_dharma_id ? _el_cgi_dharma_id : "(self)"); + n->node_type = el_strdup_persist("DharmaSelf"); + n->label = el_strdup_persist("dharma:self"); + n->tier = el_strdup_persist("Working"); + n->tags = el_strdup_persist("dharma"); + n->metadata = el_strdup_persist("{}"); n->salience = 1.0; n->importance = 1.0; n->confidence = 1.0; int64_t now = engram_now_ms(); n->created_at = now; n->updated_at = now; n->last_activated = now; @@ -8629,13 +9425,13 @@ static int64_t dharma_find_or_create_relation_edge(const char* peer_base, int cr engram_grow_nodes(); EngramNode* n = &g->nodes[g->node_count]; memset(n, 0, sizeof(*n)); - n->id = el_strdup(peer_node); - n->content = el_strdup(peer_base); - n->node_type = el_strdup("DharmaPeer"); - n->label = el_strdup(peer_node); - n->tier = el_strdup("Working"); - n->tags = el_strdup("dharma"); - n->metadata = el_strdup("{}"); + n->id = el_strdup_persist(peer_node); + n->content = el_strdup_persist(peer_base); + n->node_type = el_strdup_persist("DharmaPeer"); + n->label = el_strdup_persist(peer_node); + n->tier = el_strdup_persist("Working"); + n->tags = el_strdup_persist("dharma"); + n->metadata = el_strdup_persist("{}"); n->salience = 0.5; n->importance = 0.5; n->confidence = 1.0; int64_t now = engram_now_ms(); n->created_at = now; n->updated_at = now; n->last_activated = now; @@ -8647,10 +9443,10 @@ static int64_t dharma_find_or_create_relation_edge(const char* peer_base, int cr EngramEdge* e = &g->edges[g->edge_count]; memset(e, 0, sizeof(*e)); e->id = engram_new_id(); - e->from_id = el_strdup(self_id); - e->to_id = el_strdup(peer_node); - e->relation = el_strdup("dharma-relation"); - e->metadata = el_strdup("{}"); + e->from_id = el_strdup_persist(self_id); + e->to_id = el_strdup_persist(peer_node); + e->relation = el_strdup_persist("dharma-relation"); + e->metadata = el_strdup_persist("{}"); e->weight = 0.0; e->confidence = 1.0; int64_t now = engram_now_ms(); diff --git a/lang/releases/v1.0.0-20260501/el_runtime.h b/lang/releases/v1.0.0-20260501/el_runtime.h index a3dd683..c3dad8e 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.h +++ b/lang/releases/v1.0.0-20260501/el_runtime.h @@ -758,6 +758,18 @@ el_val_t trace_span_start(el_val_t name); el_val_t trace_span_end(el_val_t span_handle); el_val_t emit_event(el_val_t name, el_val_t duration_ms); +/* ── Runtime symbols required by the soul modules ──────────────────────────── */ +/* All implemented in el_runtime.c but omitted from this release header; the soul dist modules + * reference them directly, so the public header must export them. Declarations only — mirrors the + * mainline el_runtime.h and is platform-independent (no behavioural change to the POSIX build). */ +typedef el_val_t (*http_handler_fn)(el_val_t method, el_val_t path, el_val_t body); +typedef el_val_t (*http_handler4_fn)(el_val_t method, el_val_t path, el_val_t body, el_val_t headers); +el_val_t el_arena_push(void); +el_val_t el_arena_pop(el_val_t mark); +void http_serve_async(el_val_t port, el_val_t handler); +el_val_t engram_get_node_by_label(el_val_t label); +el_val_t engram_prune_telemetry(el_val_t older_than_ms); + #ifdef __cplusplus } #endif