diff --git a/engram/dist/engram.c b/engram/dist/engram.c index 4f61634..78c8663 100644 --- a/engram/dist/engram.c +++ b/engram/dist/engram.c @@ -10,6 +10,7 @@ el_val_t query_param(el_val_t path, el_val_t key); el_val_t query_int(el_val_t path, el_val_t key, el_val_t default_val); el_val_t extract_id(el_val_t path, el_val_t prefix); el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_act_stats(el_val_t method, el_val_t path, el_val_t body); el_val_t persist_canonical(void); el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body); el_val_t route_get_node(el_val_t method, el_val_t path, el_val_t body); @@ -18,6 +19,7 @@ el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body); el_val_t route_search(el_val_t method, el_val_t path, el_val_t body); el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body); el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_create_edges_batch(el_val_t method, el_val_t path, el_val_t body); el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body); el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body); el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body); @@ -118,6 +120,11 @@ el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body) { return 0; } +el_val_t route_act_stats(el_val_t method, el_val_t path, el_val_t body) { + return engram_act_stats_json(); + return 0; +} + el_val_t persist_canonical(void) { el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); el_val_t dir = ({ el_val_t _if_result_1 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_1 = (EL_STR("/tmp/engram")); } else { _if_result_1 = (dir_raw); } _if_result_1; }); @@ -218,6 +225,41 @@ el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body) { return 0; } +el_val_t route_create_edges_batch(el_val_t method, el_val_t path, el_val_t body) { + el_val_t arr = json_get_raw(body, EL_STR("edges")); + if (str_eq(arr, EL_STR(""))) { + return err_json(EL_STR("missing edges array")); + } + el_val_t n = json_array_len(arr); + if (n == 0) { + return EL_STR("{\"ok\":true,\"accepted\":0,\"skipped\":0}"); + } + el_val_t i = 0; + el_val_t accepted = 0; + el_val_t skipped = 0; + while (i < n) { + el_val_t item = json_array_get(arr, i); + el_val_t from_id = json_get_string(item, EL_STR("from_id")); + el_val_t to_id = json_get_string(item, EL_STR("to_id")); + if (str_eq(from_id, EL_STR("")) || str_eq(to_id, EL_STR(""))) { + skipped = (skipped + 1); + } else { + el_val_t rel_raw = json_get_string(item, EL_STR("relation")); + el_val_t relation = ({ el_val_t _if_result_17 = 0; if (str_eq(rel_raw, EL_STR(""))) { _if_result_17 = (EL_STR("associates")); } else { _if_result_17 = (rel_raw); } _if_result_17; }); + el_val_t w_present = json_get_raw(item, EL_STR("weight")); + el_val_t weight = ({ el_val_t _if_result_18 = 0; if (str_eq(w_present, EL_STR(""))) { _if_result_18 = (el_from_float(0.5)); } else { _if_result_18 = (json_get_float(item, EL_STR("weight"))); } _if_result_18; }); + engram_connect(from_id, to_id, weight, relation); + accepted = (accepted + 1); + } + i = (i + 1); + } + if (accepted > 0) { + el_val_t saved = persist_canonical(); + } + return el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"accepted\":"), int_to_str(accepted)), EL_STR(",\"skipped\":")), int_to_str(skipped)), EL_STR("}")); + return 0; +} + el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body) { el_val_t id = extract_id(path, EL_STR("/api/neighbors/")); if (str_eq(id, EL_STR(""))) { @@ -253,8 +295,8 @@ el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body) { el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) { el_val_t p_raw = json_get_string(body, EL_STR("path")); el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); - el_val_t dir = ({ el_val_t _if_result_17 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_17 = (EL_STR("/tmp/engram")); } else { _if_result_17 = (dir_raw); } _if_result_17; }); - el_val_t p = ({ el_val_t _if_result_18 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_18 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_18 = (p_raw); } _if_result_18; }); + el_val_t dir = ({ el_val_t _if_result_19 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_19 = (EL_STR("/tmp/engram")); } else { _if_result_19 = (dir_raw); } _if_result_19; }); + el_val_t p = ({ el_val_t _if_result_20 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_20 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_20 = (p_raw); } _if_result_20; }); engram_save(p); return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"path\":\""), p), EL_STR("\"}")); return 0; @@ -263,8 +305,8 @@ el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) { el_val_t route_load(el_val_t method, el_val_t path, el_val_t body) { el_val_t p_raw = json_get_string(body, EL_STR("path")); el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); - el_val_t dir = ({ el_val_t _if_result_19 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_19 = (EL_STR("/tmp/engram")); } else { _if_result_19 = (dir_raw); } _if_result_19; }); - el_val_t p = ({ el_val_t _if_result_20 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_20 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_20 = (p_raw); } _if_result_20; }); + el_val_t dir = ({ el_val_t _if_result_21 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_21 = (EL_STR("/tmp/engram")); } else { _if_result_21 = (dir_raw); } _if_result_21; }); + el_val_t p = ({ el_val_t _if_result_22 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_22 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_22 = (p_raw); } _if_result_22; }); engram_load(p); return ok_json(); return 0; @@ -288,7 +330,7 @@ el_val_t route_embed_backfill(el_val_t method, el_val_t path, el_val_t body) { el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body) { el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); - el_val_t dir = ({ el_val_t _if_result_21 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_21 = (EL_STR("/tmp/engram")); } else { _if_result_21 = (dir_raw); } _if_result_21; }); + el_val_t dir = ({ el_val_t _if_result_23 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_23 = (EL_STR("/tmp/engram")); } else { _if_result_23 = (dir_raw); } _if_result_23; }); el_val_t snap_path = el_str_concat(dir, EL_STR("/.sync-export.json")); engram_save(snap_path); el_val_t snap = fs_read(snap_path); @@ -327,7 +369,7 @@ el_val_t route_emit_ise(el_val_t method, el_val_t path, el_val_t body) { el_val_t conf = el_from_float(0.8); el_val_t id = engram_node_full(content, EL_STR("InternalStateEvent"), EL_STR("state-event"), sal, imp, conf, EL_STR("Episodic"), EL_STR("[\"internal-state\",\"InternalStateEvent\"]")); el_val_t ret_raw = env(EL_STR("ENGRAM_ISE_RETENTION_MS")); - el_val_t ret_ms = ({ el_val_t _if_result_22 = 0; if (str_eq(ret_raw, EL_STR(""))) { _if_result_22 = (172800000); } else { _if_result_22 = (str_to_int(ret_raw)); } _if_result_22; }); + el_val_t ret_ms = ({ el_val_t _if_result_24 = 0; if (str_eq(ret_raw, EL_STR(""))) { _if_result_24 = (172800000); } else { _if_result_24 = (str_to_int(ret_raw)); } _if_result_24; }); el_val_t pruned = engram_prune_telemetry(ret_ms); return el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\",\"pruned\":")), int_to_str(pruned)), EL_STR("}")); return 0; @@ -339,21 +381,21 @@ el_val_t route_capture_knowledge(el_val_t method, el_val_t path, el_val_t body) return err_json(EL_STR("missing content")); } el_val_t title = json_get_string(body, EL_STR("title")); - el_val_t label = ({ el_val_t _if_result_23 = 0; if (str_eq(title, EL_STR(""))) { _if_result_23 = (str_slice(content, 0, 60)); } else { _if_result_23 = (title); } _if_result_23; }); + el_val_t label = ({ el_val_t _if_result_25 = 0; if (str_eq(title, EL_STR(""))) { _if_result_25 = (str_slice(content, 0, 60)); } else { _if_result_25 = (title); } _if_result_25; }); el_val_t category_raw = json_get_string(body, EL_STR("category")); - el_val_t category = ({ el_val_t _if_result_24 = 0; if (str_eq(category_raw, EL_STR(""))) { _if_result_24 = (EL_STR("other")); } else { _if_result_24 = (category_raw); } _if_result_24; }); + el_val_t category = ({ el_val_t _if_result_26 = 0; if (str_eq(category_raw, EL_STR(""))) { _if_result_26 = (EL_STR("other")); } else { _if_result_26 = (category_raw); } _if_result_26; }); el_val_t ktier_raw = json_get_string(body, EL_STR("tier")); - el_val_t ktier = ({ el_val_t _if_result_25 = 0; if (str_eq(ktier_raw, EL_STR(""))) { _if_result_25 = (EL_STR("note")); } else { _if_result_25 = (ktier_raw); } _if_result_25; }); + el_val_t ktier = ({ el_val_t _if_result_27 = 0; if (str_eq(ktier_raw, EL_STR(""))) { _if_result_27 = (EL_STR("note")); } else { _if_result_27 = (ktier_raw); } _if_result_27; }); el_val_t project = json_get_string(body, EL_STR("project")); el_val_t tags_raw = json_get_raw(body, EL_STR("tags")); - el_val_t tags_base = ({ el_val_t _if_result_26 = 0; if (str_eq(tags_raw, EL_STR(""))) { _if_result_26 = (EL_STR("[]")); } else { _if_result_26 = (tags_raw); } _if_result_26; }); + el_val_t tags_base = ({ el_val_t _if_result_28 = 0; if (str_eq(tags_raw, EL_STR(""))) { _if_result_28 = (EL_STR("[]")); } else { _if_result_28 = (tags_raw); } _if_result_28; }); el_val_t base_len = str_len(tags_base); el_val_t head = str_slice(tags_base, 0, (base_len - 1)); - el_val_t sep = ({ el_val_t _if_result_27 = 0; if (str_eq(head, EL_STR("["))) { _if_result_27 = (EL_STR("")); } else { _if_result_27 = (EL_STR(",")); } _if_result_27; }); + el_val_t sep = ({ el_val_t _if_result_29 = 0; if (str_eq(head, EL_STR("["))) { _if_result_29 = (EL_STR("")); } else { _if_result_29 = (EL_STR(",")); } _if_result_29; }); el_val_t safe_cat = str_replace(category, EL_STR("\""), EL_STR("'")); el_val_t safe_tier = str_replace(ktier, EL_STR("\""), EL_STR("'")); el_val_t safe_proj = str_replace(project, EL_STR("\""), EL_STR("'")); - el_val_t proj_tag = ({ el_val_t _if_result_28 = 0; if (str_eq(safe_proj, EL_STR(""))) { _if_result_28 = (EL_STR("")); } else { _if_result_28 = (el_str_concat(el_str_concat(EL_STR(",\"project:"), safe_proj), EL_STR("\""))); } _if_result_28; }); + el_val_t proj_tag = ({ el_val_t _if_result_30 = 0; if (str_eq(safe_proj, EL_STR(""))) { _if_result_30 = (EL_STR("")); } else { _if_result_30 = (el_str_concat(el_str_concat(EL_STR(",\"project:"), safe_proj), EL_STR("\""))); } _if_result_30; }); el_val_t tags = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(head, sep), EL_STR("\"category:")), safe_cat), EL_STR("\",\"tier:")), safe_tier), EL_STR("\"")), proj_tag), EL_STR("]")); el_val_t sal = el_from_float(0.5); el_val_t imp = el_from_float(0.5); @@ -413,6 +455,9 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) { if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/stats")) || str_eq(clean, EL_STR("/stats")))) { return route_stats(method, path, body); } + if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/act-stats")) || str_eq(clean, EL_STR("/act-stats")))) { + return route_act_stats(method, path, body); + } if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/nodes")) || str_eq(clean, EL_STR("/nodes")))) { return route_create_node(method, path, body); } @@ -431,6 +476,9 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) { if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/edges")) || str_eq(clean, EL_STR("/edges")))) { return route_create_edge(method, path, body); } + if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/edges/batch")) || str_eq(clean, EL_STR("/edges/batch")))) { + return route_create_edges_batch(method, path, body); + } if (str_eq(method, EL_STR("GET")) && str_starts_with(clean, EL_STR("/api/neighbors/"))) { return route_neighbors(method, path, body); } @@ -474,10 +522,10 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) { int main(int _argc, char** _argv) { el_runtime_init_args(_argc, _argv); bind_raw = env(EL_STR("ENGRAM_BIND")); - bind_str = ({ el_val_t _if_result_29 = 0; if (str_eq(bind_raw, EL_STR(""))) { _if_result_29 = (EL_STR(":8742")); } else { _if_result_29 = (bind_raw); } _if_result_29; }); + bind_str = ({ el_val_t _if_result_31 = 0; if (str_eq(bind_raw, EL_STR(""))) { _if_result_31 = (EL_STR(":8742")); } else { _if_result_31 = (bind_raw); } _if_result_31; }); port = parse_port(bind_str); data_dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); - data_dir = ({ el_val_t _if_result_30 = 0; if (str_eq(data_dir_raw, EL_STR(""))) { _if_result_30 = (EL_STR("/tmp/engram")); } else { _if_result_30 = (data_dir_raw); } _if_result_30; }); + data_dir = ({ el_val_t _if_result_32 = 0; if (str_eq(data_dir_raw, EL_STR(""))) { _if_result_32 = (EL_STR("/tmp/engram")); } else { _if_result_32 = (data_dir_raw); } _if_result_32; }); snapshot_path = el_str_concat(data_dir, EL_STR("/snapshot.json")); engram_load(snapshot_path); boot_snap = fs_read(snapshot_path); diff --git a/engram/src/server.el b/engram/src/server.el index 004a2d1..e6303cd 100644 --- a/engram/src/server.el +++ b/engram/src/server.el @@ -234,6 +234,55 @@ fn route_create_edge(method: String, path: String, body: String) -> String { "{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + relation + "\"}" } +// route_create_edges_batch — POST /api/edges/batch {"edges":[{from_id,to_id,relation,weight}, ...]} +// +// WHY THIS EXISTS (2026-08-07 self-review). persist_canonical() writes the +// FULL canonical snapshot — 60MB at current graph size — and route_create_edge +// calls it once per edge. That is correct for the interactive one-edge case and +// ruinous for any bulk write: the soul's Hebbian consolidation path delivers +// ~14 associations per 8-minute heartbeat, which through the single-edge route +// would be ~840MB of disk writes per beat, ~150GB/day, to persist 14 edges. +// +// The fix is not to weaken durability — it is to make the unit of durability +// the BATCH. Connect every edge, then snapshot exactly once. Same guarantee +// (nothing acknowledged is lost to a restart), 1/N the writes. Empty or +// malformed entries are skipped rather than aborting the batch: a consolidation +// payload is best-effort by design, and one bad id should not cost the other 13. +// +// Returns the accepted count so the caller can tell delivery from silence. +fn route_create_edges_batch(method: String, path: String, body: String) -> String { + let arr: String = json_get_raw(body, "edges") + if str_eq(arr, "") { return err_json("missing edges array") } + let n: Int = json_array_len(arr) + if n == 0 { return "{\"ok\":true,\"accepted\":0,\"skipped\":0}" } + let i: Int = 0 + let accepted: Int = 0 + let skipped: Int = 0 + while i < n { + let item: String = json_array_get(arr, i) + let from_id: String = json_get_string(item, "from_id") + let to_id: String = json_get_string(item, "to_id") + if str_eq(from_id, "") || str_eq(to_id, "") { + let skipped = skipped + 1 + } else { + let rel_raw: String = json_get_string(item, "relation") + let relation: String = if str_eq(rel_raw, "") { "associates" } else { rel_raw } + let w_present: String = json_get_raw(item, "weight") + let weight: Float = if str_eq(w_present, "") { 0.5 } else { json_get_float(item, "weight") } + engram_connect(from_id, to_id, weight, relation) + let accepted = accepted + 1 + } + let i = i + 1 + } + // ONE snapshot for the whole batch — the entire point of this route. + // Skip it when nothing was accepted: an all-malformed payload must not + // trigger a 60MB write. + if accepted > 0 { + let saved: Int = persist_canonical() + } + return "{\"ok\":true,\"accepted\":" + int_to_str(accepted) + ",\"skipped\":" + int_to_str(skipped) + "}" +} + fn route_neighbors(method: String, path: String, body: String) -> String { let id: String = extract_id(path, "/api/neighbors/") if str_eq(id, "") { return err_json("missing id") } @@ -546,6 +595,13 @@ fn handle_request(method: String, path: String, body: String) -> String { if str_eq(method, "POST") && (str_eq(clean, "/api/edges") || str_eq(clean, "/edges")) { return route_create_edge(method, path, body) } + // Batch edge write — one snapshot for the whole payload. Must be tested + // BEFORE nothing else claims it; the exact-match on "/api/edges" above + // does not catch "/api/edges/batch", so order is not load-bearing here, + // but keeping the two adjacent keeps them from drifting apart. + if str_eq(method, "POST") && (str_eq(clean, "/api/edges/batch") || str_eq(clean, "/edges/batch")) { + return route_create_edges_batch(method, path, body) + } if str_eq(method, "GET") && str_starts_with(clean, "/api/neighbors/") { return route_neighbors(method, path, body) } diff --git a/lang/releases/v1.0.0-20260501/el_runtime.c b/lang/releases/v1.0.0-20260501/el_runtime.c index aea3a9c..a15bb15 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.c +++ b/lang/releases/v1.0.0-20260501/el_runtime.c @@ -6250,6 +6250,79 @@ static void engram_bll_parse_access(EngramNode* nn, const char* s) { * learn real associations, not enough to drown what was authored. */ #define ENGRAM_HEBB_LINK_MAX_FRAC 0.05 +/* ── Systems consolidation: the durable write-back queue (2026-08-07) ──────── + * + * THE DEFECT THIS CLOSES. Yesterday's eligibility-trace fix made Hebbian + * learning numerically real: hebb_max went 0.000799 → 0.4725 and 1,198 + * `hebbian-associate` edges formed in 23h48m. Today's census found where they + * went: nowhere. Measured on the live system — + * + * soul daemon (pid 3269, in-process graph): 42,426 edges, 1,198 hebbian + * engram server (:8742, the persistent store): 41,213 edges, 49 hebbian + * + * Two processes, two graphs, one direction of travel. The soul pulls from the + * server every 10 min via GET /api/sync and merges. It never pushes. And it + * cannot fall back on saving its own copy: `soul_snapshot_path` is set only + * inside `if is_genesis && safe_to_seed` in soul.el, and safe_to_seed is + * unconditionally false whenever ENGRAM_URL is set (it is, in the launchd + * plist) — because the HTTP server owns persistence and a soul that wrote + * snapshot.json would clobber it. That guard is correct. The consequence was + * not: mem_save() has never once executed, so every association the soul + * learns lives in RAM until the process dies. + * + * The soul is the ONLY process that runs idle cognition — curiosity scans + * every ~8 min, around the clock. It is where essentially all co-activation + * happens. So the system's entire capacity to grow its own structure was + * pointed at a volatile store. 1,198 associations/day, discarded at restart, + * every day, silently. The mechanism worked and the learning still evaporated. + * + * WHY A QUEUE AND NOT A SAVE. The fix is not to let the soul write the + * snapshot — that reintroduces the clobber the guard exists to prevent. It is + * to make consolidation a MESSAGE, not a file: the fast volatile store hands + * each newly-formed association to the slow durable store, one edge at a time, + * over the API the server already exposes (POST /api/edges). This is the + * hippocampal→neocortical split the rest of this file is already modeled on. + * Fast store learns online and forgets; slow store receives what survived the + * threshold and keeps it. Only edges that already cleared ENGRAM_HEBB_LINK_MIN + * are enqueued, so what crosses the process boundary is what earned it. + * + * SHAPE. Fixed 512-slot ring, overwrite-oldest. 512 is ~18x the observed + * formation rate per drain interval (14 links per 8-min heartbeat), so the + * queue only saturates when the writer is down — and when it is, keeping the + * freshest associations is the right loss. Drops are counted, not silent: + * a consolidation path that quietly discards is the failure mode this whole + * entry exists to correct. Enqueue is strdup'd because g->edges may realloc + * and node ids may be freed by later pruning; the queue owns its copies. */ +#define ENGRAM_HEBB_WB_SLOTS 512 + +typedef struct { char* a; char* b; double w; double hebb; } EgHebbWB; +static EgHebbWB _eg_hebb_wb[ENGRAM_HEBB_WB_SLOTS]; +static int _eg_hebb_wb_head = 0; /* index of the oldest live entry */ +static int _eg_hebb_wb_len = 0; +static int64_t _eg_hebb_wb_dropped = 0; /* lost to a full queue, cumulative */ +static int64_t _eg_hebb_wb_drained = 0; /* handed to the durable store, cum. */ + +static void eg_hebb_wb_push(const char* a, const char* b, double w, double h) { + if (!a || !b) return; + int slot; + if (_eg_hebb_wb_len >= ENGRAM_HEBB_WB_SLOTS) { + slot = _eg_hebb_wb_head; + free(_eg_hebb_wb[slot].a); + free(_eg_hebb_wb[slot].b); + _eg_hebb_wb_head = (_eg_hebb_wb_head + 1) % ENGRAM_HEBB_WB_SLOTS; + _eg_hebb_wb_dropped++; + } else { + slot = (_eg_hebb_wb_head + _eg_hebb_wb_len) % ENGRAM_HEBB_WB_SLOTS; + _eg_hebb_wb_len++; + } + _eg_hebb_wb[slot].a = strdup(a); + _eg_hebb_wb[slot].b = strdup(b); + _eg_hebb_wb[slot].w = w; + _eg_hebb_wb[slot].hebb = h; + /* strdup failure leaves a NULL id; the drain skips those rather than + * emitting a malformed edge. */ +} + typedef struct { char* a; char* b; double score; } EgHebbCand; static EgHebbCand _eg_hebb_cand[ENGRAM_HEBB_CAND_SLOTS]; static int64_t _eg_hebb_links_formed = 0; @@ -9314,6 +9387,13 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { _eg_hebb_links_formed++; hebb_edge_total++; formed++; + /* Hand the association to the durable store. In the soul + * daemon this local edge is the ONLY copy and dies with the + * process — see the ENGRAM_HEBB_WB_SLOTS block. Enqueue + * from ne->from_id/to_id rather than c->a/c->b: the slot is + * cleared on the next line and the edge now owns the ids. */ + eg_hebb_wb_push(ne->from_id, ne->to_id, + ne->weight, ne->hebb); eg_hebb_slot_clear(c); /* the edge is the record now */ } } @@ -10457,7 +10537,10 @@ el_val_t engram_act_stats_json(void) { if (_eg_hebb_cand[i].score > hebb_cand_max) hebb_cand_max = _eg_hebb_cand[i].score; } - char buf[512]; + /* 768, not 512: the write-back gauges added 2026-08-07 push the worst-case + * rendering past the old bound, and snprintf would truncate the JSON into + * an unparseable tail rather than fail loudly. */ + char buf[768]; /* ctx_cos (2026-07-29): cos(query, context centroid) at the LAST * activate call, measured before the query was folded in. ~1.0 = * context aligned with current query; low = divergence (expected at @@ -10470,6 +10553,8 @@ el_val_t engram_act_stats_json(void) { "\"hebb_edges\":%lld,\"hebb_max\":%.4f,\"hebb_mass\":%.3f," "\"hebb_cands\":%d,\"hebb_cand_max\":%.4f,\"hebb_links\":%lld," "\"hebb_warm\":%d," + "\"hebb_wb_pending\":%d,\"hebb_wb_drained\":%lld," + "\"hebb_wb_dropped\":%lld," "\"dup_seeds\":%lld,\"dup_wm\":%lld,\"dup_wm_global\":%lld}", (long long)_eg_act_wm_evicted, (long long)_eg_act_breakthroughs, @@ -10478,11 +10563,65 @@ el_val_t engram_act_stats_json(void) { (long long)hebb_edges, hebb_max, hebb_mass, hebb_cands, hebb_cand_max, (long long)_eg_hebb_links_formed, _eg_act_hebb_warm, + _eg_hebb_wb_len, (long long)_eg_hebb_wb_drained, + (long long)_eg_hebb_wb_dropped, (long long)_eg_act_dup_seeds, (long long)_eg_act_dup_wm, (long long)_eg_act_dup_wm_global); return el_wrap_str(el_strdup(buf)); } +/* engram_hebb_drain_json — pop up to `max` newly-formed Hebbian associations + * off the write-back queue and return them as a JSON array: + * + * [{"from_id":"...","to_id":"...","weight":0.15,"hebb":0.31}, ...] + * + * Draining is DESTRUCTIVE: entries returned here are gone from the queue. The + * caller owns delivery from that point on. That is deliberate — the alternative + * (peek, deliver, ack) needs a second round trip and a retry ledger to be + * correct, and the payload is an association that will re-form from live + * co-activation if it genuinely matters. Losing one is cheap; a queue that + * silently refills forever because acks never land is not. + * + * Empty queue returns "[]". See the ENGRAM_HEBB_WB_SLOTS block for why this + * exists at all: the process that does the learning is not the process that + * owns persistence. (2026-08-07 self-review.) */ +el_val_t engram_hebb_drain_json(el_val_t max_v) { + int64_t max_n = (int64_t)max_v; + if (max_n <= 0) max_n = 64; + if (max_n > ENGRAM_HEBB_WB_SLOTS) max_n = ENGRAM_HEBB_WB_SLOTS; + JsonBuf b; jb_init(&b); + jb_puts(&b, "["); + int emitted = 0; + while (_eg_hebb_wb_len > 0 && emitted < (int)max_n) { + EgHebbWB* e = &_eg_hebb_wb[_eg_hebb_wb_head]; + if (e->a && e->b) { + if (emitted > 0) jb_puts(&b, ","); + /* relation is emitted here, not stamped on by the caller: the + * payload should be postable to /api/edges/batch verbatim. A + * consumer that has to rewrite the JSON to make it valid is a + * consumer that will eventually rewrite it wrong. */ + jb_puts(&b, "{\"relation\":\"hebbian-associate\",\"from_id\":"); + jb_emit_escaped(&b, e->a); + jb_puts(&b, ",\"to_id\":"); + jb_emit_escaped(&b, e->b); + char tmp[96]; + snprintf(tmp, sizeof(tmp), ",\"weight\":%.6g,\"hebb\":%.6g}", + e->w, e->hebb); + jb_puts(&b, tmp); + emitted++; + _eg_hebb_wb_drained++; + } + /* Free and advance whether or not the entry rendered — a NULL id is a + * strdup failure at push time, not a retryable condition. */ + free(e->a); free(e->b); + e->a = NULL; e->b = NULL; + _eg_hebb_wb_head = (_eg_hebb_wb_head + 1) % ENGRAM_HEBB_WB_SLOTS; + _eg_hebb_wb_len--; + } + jb_puts(&b, "]"); + return el_wrap_str(b.buf ? b.buf : el_strdup("[]")); +} + /* engram_cosine_sim — cosine similarity between two nodes' embeddings. * Returns a float in [-1, 1], or -2.0 when either node is missing or not * yet embedded. Exposed so EL code (and the introspection API) can probe diff --git a/lang/releases/v1.0.0-20260501/el_runtime.h b/lang/releases/v1.0.0-20260501/el_runtime.h index 88da992..97ec897 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.h +++ b/lang/releases/v1.0.0-20260501/el_runtime.h @@ -619,6 +619,11 @@ el_val_t engram_activate_json(el_val_t query, el_val_t depth); el_val_t engram_stats_json(void); el_val_t engram_act_stats_json(void); el_val_t engram_cosine_sim(el_val_t id_a, el_val_t id_b); +/* Destructively pop up to `max` newly-formed Hebbian associations as a JSON + * array of {from_id,to_id,weight,hebb}. The learning process (soul daemon) is + * not the process that owns persistence (engram HTTP server); this is how a + * self-formed association crosses that boundary. (2026-08-07 self-review.) */ +el_val_t engram_hebb_drain_json(el_val_t max); /* Document frequency of a term across node labels — term-specificity signal * for curiosity seed selection. (2026-08-03 self-review.) */ el_val_t engram_label_df(el_val_t term);