self-review 2026-08-07: learning that cannot outlive the process is not learning
Yesterday's eligibility-trace fix made Hebbian consolidation numerically real: hebb_max 0.000799 -> 0.4725, and 1,198 hebbian-associate edges formed in 23h48m. This morning's census found where they went: nowhere. soul daemon (in-process graph): 42,426 edges, 1,198 hebbian engram server (:8742, durable): 41,213 edges, 49 hebbian Two processes, two graphs, one direction of travel. The soul pulls from the server every 10 min (GET /api/sync) and never pushes. It cannot fall back on saving its own copy either: soul.el sets soul_snapshot_path only inside `if is_genesis && safe_to_seed`, and safe_to_seed is unconditionally false whenever ENGRAM_URL is set -- because the server owns persistence and a soul writing snapshot.json would clobber it. That guard is correct. The consequence was not: mem_save() has never once executed. The soul is the ONLY process running idle cognition, so it is where essentially all co-activation happens -- and it was throwing away every association it learned, every restart, silently. The mechanism worked and the learning still evaporated. Consolidation is now a message, not a file. Fast volatile store hands each newly-formed association to the slow durable store over the API the server already exposes; only edges past ENGRAM_HEBB_LINK_MIN are ever queued, so what crosses the process boundary already earned it. - el_runtime.c: 512-slot overwrite-oldest write-back ring; enqueue at edge formation; engram_hebb_drain_json() pops a postable JSON batch. Drops and drains are counted, not silent -- a consolidation path that quietly discards is the exact failure this entry exists to correct. - server.el: POST /api/edges/batch. persist_canonical() writes the full 60MB snapshot per call, and route_create_edge calls it per edge -- correct for one interactive edge, ruinous for bulk (~840MB/beat to persist 14 associations). Batch connects all, snapshots once. Same durability, 1/N the writes. - act-stats: hebb_wb_pending / _drained / _dropped. pending climbing with drained flat = drain not called; drained climbing with sent 0 = POST refused. Both failure modes are now visible in the stream instead of in an autopsy. Verified live: batch route accepts valid entries, skips malformed ones without aborting the batch, and enforces _auth. All 1,256 learned associations are now in the canonical store; the soul booted at 42,431 edges with hebb_max 0.4941 carried across the restart for the first time.
This commit is contained in:
Vendored
+62
-14
@@ -10,6 +10,7 @@ el_val_t query_param(el_val_t path, el_val_t key);
|
||||
el_val_t query_int(el_val_t path, el_val_t key, el_val_t default_val);
|
||||
el_val_t extract_id(el_val_t path, el_val_t prefix);
|
||||
el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_act_stats(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t persist_canonical(void);
|
||||
el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_get_node(el_val_t method, el_val_t path, el_val_t body);
|
||||
@@ -18,6 +19,7 @@ el_val_t route_scan_edges(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_search(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_activate(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_create_edges_batch(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body);
|
||||
el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body);
|
||||
@@ -118,6 +120,11 @@ el_val_t route_stats(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_act_stats(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return engram_act_stats_json();
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t persist_canonical(void) {
|
||||
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
el_val_t dir = ({ el_val_t _if_result_1 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_1 = (EL_STR("/tmp/engram")); } else { _if_result_1 = (dir_raw); } _if_result_1; });
|
||||
@@ -218,6 +225,41 @@ el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_create_edges_batch(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t arr = json_get_raw(body, EL_STR("edges"));
|
||||
if (str_eq(arr, EL_STR(""))) {
|
||||
return err_json(EL_STR("missing edges array"));
|
||||
}
|
||||
el_val_t n = json_array_len(arr);
|
||||
if (n == 0) {
|
||||
return EL_STR("{\"ok\":true,\"accepted\":0,\"skipped\":0}");
|
||||
}
|
||||
el_val_t i = 0;
|
||||
el_val_t accepted = 0;
|
||||
el_val_t skipped = 0;
|
||||
while (i < n) {
|
||||
el_val_t item = json_array_get(arr, i);
|
||||
el_val_t from_id = json_get_string(item, EL_STR("from_id"));
|
||||
el_val_t to_id = json_get_string(item, EL_STR("to_id"));
|
||||
if (str_eq(from_id, EL_STR("")) || str_eq(to_id, EL_STR(""))) {
|
||||
skipped = (skipped + 1);
|
||||
} else {
|
||||
el_val_t rel_raw = json_get_string(item, EL_STR("relation"));
|
||||
el_val_t relation = ({ el_val_t _if_result_17 = 0; if (str_eq(rel_raw, EL_STR(""))) { _if_result_17 = (EL_STR("associates")); } else { _if_result_17 = (rel_raw); } _if_result_17; });
|
||||
el_val_t w_present = json_get_raw(item, EL_STR("weight"));
|
||||
el_val_t weight = ({ el_val_t _if_result_18 = 0; if (str_eq(w_present, EL_STR(""))) { _if_result_18 = (el_from_float(0.5)); } else { _if_result_18 = (json_get_float(item, EL_STR("weight"))); } _if_result_18; });
|
||||
engram_connect(from_id, to_id, weight, relation);
|
||||
accepted = (accepted + 1);
|
||||
}
|
||||
i = (i + 1);
|
||||
}
|
||||
if (accepted > 0) {
|
||||
el_val_t saved = persist_canonical();
|
||||
}
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"accepted\":"), int_to_str(accepted)), EL_STR(",\"skipped\":")), int_to_str(skipped)), EL_STR("}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t id = extract_id(path, EL_STR("/api/neighbors/"));
|
||||
if (str_eq(id, EL_STR(""))) {
|
||||
@@ -253,8 +295,8 @@ el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t p_raw = json_get_string(body, EL_STR("path"));
|
||||
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
el_val_t dir = ({ el_val_t _if_result_17 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_17 = (EL_STR("/tmp/engram")); } else { _if_result_17 = (dir_raw); } _if_result_17; });
|
||||
el_val_t p = ({ el_val_t _if_result_18 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_18 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_18 = (p_raw); } _if_result_18; });
|
||||
el_val_t dir = ({ el_val_t _if_result_19 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_19 = (EL_STR("/tmp/engram")); } else { _if_result_19 = (dir_raw); } _if_result_19; });
|
||||
el_val_t p = ({ el_val_t _if_result_20 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_20 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_20 = (p_raw); } _if_result_20; });
|
||||
engram_save(p);
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"path\":\""), p), EL_STR("\"}"));
|
||||
return 0;
|
||||
@@ -263,8 +305,8 @@ el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t route_load(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t p_raw = json_get_string(body, EL_STR("path"));
|
||||
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
el_val_t dir = ({ el_val_t _if_result_19 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_19 = (EL_STR("/tmp/engram")); } else { _if_result_19 = (dir_raw); } _if_result_19; });
|
||||
el_val_t p = ({ el_val_t _if_result_20 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_20 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_20 = (p_raw); } _if_result_20; });
|
||||
el_val_t dir = ({ el_val_t _if_result_21 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_21 = (EL_STR("/tmp/engram")); } else { _if_result_21 = (dir_raw); } _if_result_21; });
|
||||
el_val_t p = ({ el_val_t _if_result_22 = 0; if (str_eq(p_raw, EL_STR(""))) { _if_result_22 = (el_str_concat(dir, EL_STR("/snapshot.json"))); } else { _if_result_22 = (p_raw); } _if_result_22; });
|
||||
engram_load(p);
|
||||
return ok_json();
|
||||
return 0;
|
||||
@@ -288,7 +330,7 @@ el_val_t route_embed_backfill(el_val_t method, el_val_t path, el_val_t body) {
|
||||
|
||||
el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
el_val_t dir = ({ el_val_t _if_result_21 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_21 = (EL_STR("/tmp/engram")); } else { _if_result_21 = (dir_raw); } _if_result_21; });
|
||||
el_val_t dir = ({ el_val_t _if_result_23 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_23 = (EL_STR("/tmp/engram")); } else { _if_result_23 = (dir_raw); } _if_result_23; });
|
||||
el_val_t snap_path = el_str_concat(dir, EL_STR("/.sync-export.json"));
|
||||
engram_save(snap_path);
|
||||
el_val_t snap = fs_read(snap_path);
|
||||
@@ -327,7 +369,7 @@ el_val_t route_emit_ise(el_val_t method, el_val_t path, el_val_t body) {
|
||||
el_val_t conf = el_from_float(0.8);
|
||||
el_val_t id = engram_node_full(content, EL_STR("InternalStateEvent"), EL_STR("state-event"), sal, imp, conf, EL_STR("Episodic"), EL_STR("[\"internal-state\",\"InternalStateEvent\"]"));
|
||||
el_val_t ret_raw = env(EL_STR("ENGRAM_ISE_RETENTION_MS"));
|
||||
el_val_t ret_ms = ({ el_val_t _if_result_22 = 0; if (str_eq(ret_raw, EL_STR(""))) { _if_result_22 = (172800000); } else { _if_result_22 = (str_to_int(ret_raw)); } _if_result_22; });
|
||||
el_val_t ret_ms = ({ el_val_t _if_result_24 = 0; if (str_eq(ret_raw, EL_STR(""))) { _if_result_24 = (172800000); } else { _if_result_24 = (str_to_int(ret_raw)); } _if_result_24; });
|
||||
el_val_t pruned = engram_prune_telemetry(ret_ms);
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\",\"pruned\":")), int_to_str(pruned)), EL_STR("}"));
|
||||
return 0;
|
||||
@@ -339,21 +381,21 @@ el_val_t route_capture_knowledge(el_val_t method, el_val_t path, el_val_t body)
|
||||
return err_json(EL_STR("missing content"));
|
||||
}
|
||||
el_val_t title = json_get_string(body, EL_STR("title"));
|
||||
el_val_t label = ({ el_val_t _if_result_23 = 0; if (str_eq(title, EL_STR(""))) { _if_result_23 = (str_slice(content, 0, 60)); } else { _if_result_23 = (title); } _if_result_23; });
|
||||
el_val_t label = ({ el_val_t _if_result_25 = 0; if (str_eq(title, EL_STR(""))) { _if_result_25 = (str_slice(content, 0, 60)); } else { _if_result_25 = (title); } _if_result_25; });
|
||||
el_val_t category_raw = json_get_string(body, EL_STR("category"));
|
||||
el_val_t category = ({ el_val_t _if_result_24 = 0; if (str_eq(category_raw, EL_STR(""))) { _if_result_24 = (EL_STR("other")); } else { _if_result_24 = (category_raw); } _if_result_24; });
|
||||
el_val_t category = ({ el_val_t _if_result_26 = 0; if (str_eq(category_raw, EL_STR(""))) { _if_result_26 = (EL_STR("other")); } else { _if_result_26 = (category_raw); } _if_result_26; });
|
||||
el_val_t ktier_raw = json_get_string(body, EL_STR("tier"));
|
||||
el_val_t ktier = ({ el_val_t _if_result_25 = 0; if (str_eq(ktier_raw, EL_STR(""))) { _if_result_25 = (EL_STR("note")); } else { _if_result_25 = (ktier_raw); } _if_result_25; });
|
||||
el_val_t ktier = ({ el_val_t _if_result_27 = 0; if (str_eq(ktier_raw, EL_STR(""))) { _if_result_27 = (EL_STR("note")); } else { _if_result_27 = (ktier_raw); } _if_result_27; });
|
||||
el_val_t project = json_get_string(body, EL_STR("project"));
|
||||
el_val_t tags_raw = json_get_raw(body, EL_STR("tags"));
|
||||
el_val_t tags_base = ({ el_val_t _if_result_26 = 0; if (str_eq(tags_raw, EL_STR(""))) { _if_result_26 = (EL_STR("[]")); } else { _if_result_26 = (tags_raw); } _if_result_26; });
|
||||
el_val_t tags_base = ({ el_val_t _if_result_28 = 0; if (str_eq(tags_raw, EL_STR(""))) { _if_result_28 = (EL_STR("[]")); } else { _if_result_28 = (tags_raw); } _if_result_28; });
|
||||
el_val_t base_len = str_len(tags_base);
|
||||
el_val_t head = str_slice(tags_base, 0, (base_len - 1));
|
||||
el_val_t sep = ({ el_val_t _if_result_27 = 0; if (str_eq(head, EL_STR("["))) { _if_result_27 = (EL_STR("")); } else { _if_result_27 = (EL_STR(",")); } _if_result_27; });
|
||||
el_val_t sep = ({ el_val_t _if_result_29 = 0; if (str_eq(head, EL_STR("["))) { _if_result_29 = (EL_STR("")); } else { _if_result_29 = (EL_STR(",")); } _if_result_29; });
|
||||
el_val_t safe_cat = str_replace(category, EL_STR("\""), EL_STR("'"));
|
||||
el_val_t safe_tier = str_replace(ktier, EL_STR("\""), EL_STR("'"));
|
||||
el_val_t safe_proj = str_replace(project, EL_STR("\""), EL_STR("'"));
|
||||
el_val_t proj_tag = ({ el_val_t _if_result_28 = 0; if (str_eq(safe_proj, EL_STR(""))) { _if_result_28 = (EL_STR("")); } else { _if_result_28 = (el_str_concat(el_str_concat(EL_STR(",\"project:"), safe_proj), EL_STR("\""))); } _if_result_28; });
|
||||
el_val_t proj_tag = ({ el_val_t _if_result_30 = 0; if (str_eq(safe_proj, EL_STR(""))) { _if_result_30 = (EL_STR("")); } else { _if_result_30 = (el_str_concat(el_str_concat(EL_STR(",\"project:"), safe_proj), EL_STR("\""))); } _if_result_30; });
|
||||
el_val_t tags = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(head, sep), EL_STR("\"category:")), safe_cat), EL_STR("\",\"tier:")), safe_tier), EL_STR("\"")), proj_tag), EL_STR("]"));
|
||||
el_val_t sal = el_from_float(0.5);
|
||||
el_val_t imp = el_from_float(0.5);
|
||||
@@ -413,6 +455,9 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
|
||||
if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/stats")) || str_eq(clean, EL_STR("/stats")))) {
|
||||
return route_stats(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/act-stats")) || str_eq(clean, EL_STR("/act-stats")))) {
|
||||
return route_act_stats(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/nodes")) || str_eq(clean, EL_STR("/nodes")))) {
|
||||
return route_create_node(method, path, body);
|
||||
}
|
||||
@@ -431,6 +476,9 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/edges")) || str_eq(clean, EL_STR("/edges")))) {
|
||||
return route_create_edge(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/edges/batch")) || str_eq(clean, EL_STR("/edges/batch")))) {
|
||||
return route_create_edges_batch(method, path, body);
|
||||
}
|
||||
if (str_eq(method, EL_STR("GET")) && str_starts_with(clean, EL_STR("/api/neighbors/"))) {
|
||||
return route_neighbors(method, path, body);
|
||||
}
|
||||
@@ -474,10 +522,10 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
|
||||
int main(int _argc, char** _argv) {
|
||||
el_runtime_init_args(_argc, _argv);
|
||||
bind_raw = env(EL_STR("ENGRAM_BIND"));
|
||||
bind_str = ({ el_val_t _if_result_29 = 0; if (str_eq(bind_raw, EL_STR(""))) { _if_result_29 = (EL_STR(":8742")); } else { _if_result_29 = (bind_raw); } _if_result_29; });
|
||||
bind_str = ({ el_val_t _if_result_31 = 0; if (str_eq(bind_raw, EL_STR(""))) { _if_result_31 = (EL_STR(":8742")); } else { _if_result_31 = (bind_raw); } _if_result_31; });
|
||||
port = parse_port(bind_str);
|
||||
data_dir_raw = env(EL_STR("ENGRAM_DATA_DIR"));
|
||||
data_dir = ({ el_val_t _if_result_30 = 0; if (str_eq(data_dir_raw, EL_STR(""))) { _if_result_30 = (EL_STR("/tmp/engram")); } else { _if_result_30 = (data_dir_raw); } _if_result_30; });
|
||||
data_dir = ({ el_val_t _if_result_32 = 0; if (str_eq(data_dir_raw, EL_STR(""))) { _if_result_32 = (EL_STR("/tmp/engram")); } else { _if_result_32 = (data_dir_raw); } _if_result_32; });
|
||||
snapshot_path = el_str_concat(data_dir, EL_STR("/snapshot.json"));
|
||||
engram_load(snapshot_path);
|
||||
boot_snap = fs_read(snapshot_path);
|
||||
|
||||
@@ -234,6 +234,55 @@ fn route_create_edge(method: String, path: String, body: String) -> String {
|
||||
"{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + relation + "\"}"
|
||||
}
|
||||
|
||||
// route_create_edges_batch — POST /api/edges/batch {"edges":[{from_id,to_id,relation,weight}, ...]}
|
||||
//
|
||||
// WHY THIS EXISTS (2026-08-07 self-review). persist_canonical() writes the
|
||||
// FULL canonical snapshot — 60MB at current graph size — and route_create_edge
|
||||
// calls it once per edge. That is correct for the interactive one-edge case and
|
||||
// ruinous for any bulk write: the soul's Hebbian consolidation path delivers
|
||||
// ~14 associations per 8-minute heartbeat, which through the single-edge route
|
||||
// would be ~840MB of disk writes per beat, ~150GB/day, to persist 14 edges.
|
||||
//
|
||||
// The fix is not to weaken durability — it is to make the unit of durability
|
||||
// the BATCH. Connect every edge, then snapshot exactly once. Same guarantee
|
||||
// (nothing acknowledged is lost to a restart), 1/N the writes. Empty or
|
||||
// malformed entries are skipped rather than aborting the batch: a consolidation
|
||||
// payload is best-effort by design, and one bad id should not cost the other 13.
|
||||
//
|
||||
// Returns the accepted count so the caller can tell delivery from silence.
|
||||
fn route_create_edges_batch(method: String, path: String, body: String) -> String {
|
||||
let arr: String = json_get_raw(body, "edges")
|
||||
if str_eq(arr, "") { return err_json("missing edges array") }
|
||||
let n: Int = json_array_len(arr)
|
||||
if n == 0 { return "{\"ok\":true,\"accepted\":0,\"skipped\":0}" }
|
||||
let i: Int = 0
|
||||
let accepted: Int = 0
|
||||
let skipped: Int = 0
|
||||
while i < n {
|
||||
let item: String = json_array_get(arr, i)
|
||||
let from_id: String = json_get_string(item, "from_id")
|
||||
let to_id: String = json_get_string(item, "to_id")
|
||||
if str_eq(from_id, "") || str_eq(to_id, "") {
|
||||
let skipped = skipped + 1
|
||||
} else {
|
||||
let rel_raw: String = json_get_string(item, "relation")
|
||||
let relation: String = if str_eq(rel_raw, "") { "associates" } else { rel_raw }
|
||||
let w_present: String = json_get_raw(item, "weight")
|
||||
let weight: Float = if str_eq(w_present, "") { 0.5 } else { json_get_float(item, "weight") }
|
||||
engram_connect(from_id, to_id, weight, relation)
|
||||
let accepted = accepted + 1
|
||||
}
|
||||
let i = i + 1
|
||||
}
|
||||
// ONE snapshot for the whole batch — the entire point of this route.
|
||||
// Skip it when nothing was accepted: an all-malformed payload must not
|
||||
// trigger a 60MB write.
|
||||
if accepted > 0 {
|
||||
let saved: Int = persist_canonical()
|
||||
}
|
||||
return "{\"ok\":true,\"accepted\":" + int_to_str(accepted) + ",\"skipped\":" + int_to_str(skipped) + "}"
|
||||
}
|
||||
|
||||
fn route_neighbors(method: String, path: String, body: String) -> String {
|
||||
let id: String = extract_id(path, "/api/neighbors/")
|
||||
if str_eq(id, "") { return err_json("missing id") }
|
||||
@@ -546,6 +595,13 @@ fn handle_request(method: String, path: String, body: String) -> String {
|
||||
if str_eq(method, "POST") && (str_eq(clean, "/api/edges") || str_eq(clean, "/edges")) {
|
||||
return route_create_edge(method, path, body)
|
||||
}
|
||||
// Batch edge write — one snapshot for the whole payload. Must be tested
|
||||
// BEFORE nothing else claims it; the exact-match on "/api/edges" above
|
||||
// does not catch "/api/edges/batch", so order is not load-bearing here,
|
||||
// but keeping the two adjacent keeps them from drifting apart.
|
||||
if str_eq(method, "POST") && (str_eq(clean, "/api/edges/batch") || str_eq(clean, "/edges/batch")) {
|
||||
return route_create_edges_batch(method, path, body)
|
||||
}
|
||||
if str_eq(method, "GET") && str_starts_with(clean, "/api/neighbors/") {
|
||||
return route_neighbors(method, path, body)
|
||||
}
|
||||
|
||||
@@ -6250,6 +6250,79 @@ static void engram_bll_parse_access(EngramNode* nn, const char* s) {
|
||||
* learn real associations, not enough to drown what was authored. */
|
||||
#define ENGRAM_HEBB_LINK_MAX_FRAC 0.05
|
||||
|
||||
/* ── Systems consolidation: the durable write-back queue (2026-08-07) ────────
|
||||
*
|
||||
* THE DEFECT THIS CLOSES. Yesterday's eligibility-trace fix made Hebbian
|
||||
* learning numerically real: hebb_max went 0.000799 → 0.4725 and 1,198
|
||||
* `hebbian-associate` edges formed in 23h48m. Today's census found where they
|
||||
* went: nowhere. Measured on the live system —
|
||||
*
|
||||
* soul daemon (pid 3269, in-process graph): 42,426 edges, 1,198 hebbian
|
||||
* engram server (:8742, the persistent store): 41,213 edges, 49 hebbian
|
||||
*
|
||||
* Two processes, two graphs, one direction of travel. The soul pulls from the
|
||||
* server every 10 min via GET /api/sync and merges. It never pushes. And it
|
||||
* cannot fall back on saving its own copy: `soul_snapshot_path` is set only
|
||||
* inside `if is_genesis && safe_to_seed` in soul.el, and safe_to_seed is
|
||||
* unconditionally false whenever ENGRAM_URL is set (it is, in the launchd
|
||||
* plist) — because the HTTP server owns persistence and a soul that wrote
|
||||
* snapshot.json would clobber it. That guard is correct. The consequence was
|
||||
* not: mem_save() has never once executed, so every association the soul
|
||||
* learns lives in RAM until the process dies.
|
||||
*
|
||||
* The soul is the ONLY process that runs idle cognition — curiosity scans
|
||||
* every ~8 min, around the clock. It is where essentially all co-activation
|
||||
* happens. So the system's entire capacity to grow its own structure was
|
||||
* pointed at a volatile store. 1,198 associations/day, discarded at restart,
|
||||
* every day, silently. The mechanism worked and the learning still evaporated.
|
||||
*
|
||||
* WHY A QUEUE AND NOT A SAVE. The fix is not to let the soul write the
|
||||
* snapshot — that reintroduces the clobber the guard exists to prevent. It is
|
||||
* to make consolidation a MESSAGE, not a file: the fast volatile store hands
|
||||
* each newly-formed association to the slow durable store, one edge at a time,
|
||||
* over the API the server already exposes (POST /api/edges). This is the
|
||||
* hippocampal→neocortical split the rest of this file is already modeled on.
|
||||
* Fast store learns online and forgets; slow store receives what survived the
|
||||
* threshold and keeps it. Only edges that already cleared ENGRAM_HEBB_LINK_MIN
|
||||
* are enqueued, so what crosses the process boundary is what earned it.
|
||||
*
|
||||
* SHAPE. Fixed 512-slot ring, overwrite-oldest. 512 is ~18x the observed
|
||||
* formation rate per drain interval (14 links per 8-min heartbeat), so the
|
||||
* queue only saturates when the writer is down — and when it is, keeping the
|
||||
* freshest associations is the right loss. Drops are counted, not silent:
|
||||
* a consolidation path that quietly discards is the failure mode this whole
|
||||
* entry exists to correct. Enqueue is strdup'd because g->edges may realloc
|
||||
* and node ids may be freed by later pruning; the queue owns its copies. */
|
||||
#define ENGRAM_HEBB_WB_SLOTS 512
|
||||
|
||||
typedef struct { char* a; char* b; double w; double hebb; } EgHebbWB;
|
||||
static EgHebbWB _eg_hebb_wb[ENGRAM_HEBB_WB_SLOTS];
|
||||
static int _eg_hebb_wb_head = 0; /* index of the oldest live entry */
|
||||
static int _eg_hebb_wb_len = 0;
|
||||
static int64_t _eg_hebb_wb_dropped = 0; /* lost to a full queue, cumulative */
|
||||
static int64_t _eg_hebb_wb_drained = 0; /* handed to the durable store, cum. */
|
||||
|
||||
static void eg_hebb_wb_push(const char* a, const char* b, double w, double h) {
|
||||
if (!a || !b) return;
|
||||
int slot;
|
||||
if (_eg_hebb_wb_len >= ENGRAM_HEBB_WB_SLOTS) {
|
||||
slot = _eg_hebb_wb_head;
|
||||
free(_eg_hebb_wb[slot].a);
|
||||
free(_eg_hebb_wb[slot].b);
|
||||
_eg_hebb_wb_head = (_eg_hebb_wb_head + 1) % ENGRAM_HEBB_WB_SLOTS;
|
||||
_eg_hebb_wb_dropped++;
|
||||
} else {
|
||||
slot = (_eg_hebb_wb_head + _eg_hebb_wb_len) % ENGRAM_HEBB_WB_SLOTS;
|
||||
_eg_hebb_wb_len++;
|
||||
}
|
||||
_eg_hebb_wb[slot].a = strdup(a);
|
||||
_eg_hebb_wb[slot].b = strdup(b);
|
||||
_eg_hebb_wb[slot].w = w;
|
||||
_eg_hebb_wb[slot].hebb = h;
|
||||
/* strdup failure leaves a NULL id; the drain skips those rather than
|
||||
* emitting a malformed edge. */
|
||||
}
|
||||
|
||||
typedef struct { char* a; char* b; double score; } EgHebbCand;
|
||||
static EgHebbCand _eg_hebb_cand[ENGRAM_HEBB_CAND_SLOTS];
|
||||
static int64_t _eg_hebb_links_formed = 0;
|
||||
@@ -9314,6 +9387,13 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
|
||||
_eg_hebb_links_formed++;
|
||||
hebb_edge_total++;
|
||||
formed++;
|
||||
/* Hand the association to the durable store. In the soul
|
||||
* daemon this local edge is the ONLY copy and dies with the
|
||||
* process — see the ENGRAM_HEBB_WB_SLOTS block. Enqueue
|
||||
* from ne->from_id/to_id rather than c->a/c->b: the slot is
|
||||
* cleared on the next line and the edge now owns the ids. */
|
||||
eg_hebb_wb_push(ne->from_id, ne->to_id,
|
||||
ne->weight, ne->hebb);
|
||||
eg_hebb_slot_clear(c); /* the edge is the record now */
|
||||
}
|
||||
}
|
||||
@@ -10457,7 +10537,10 @@ el_val_t engram_act_stats_json(void) {
|
||||
if (_eg_hebb_cand[i].score > hebb_cand_max)
|
||||
hebb_cand_max = _eg_hebb_cand[i].score;
|
||||
}
|
||||
char buf[512];
|
||||
/* 768, not 512: the write-back gauges added 2026-08-07 push the worst-case
|
||||
* rendering past the old bound, and snprintf would truncate the JSON into
|
||||
* an unparseable tail rather than fail loudly. */
|
||||
char buf[768];
|
||||
/* ctx_cos (2026-07-29): cos(query, context centroid) at the LAST
|
||||
* activate call, measured before the query was folded in. ~1.0 =
|
||||
* context aligned with current query; low = divergence (expected at
|
||||
@@ -10470,6 +10553,8 @@ el_val_t engram_act_stats_json(void) {
|
||||
"\"hebb_edges\":%lld,\"hebb_max\":%.4f,\"hebb_mass\":%.3f,"
|
||||
"\"hebb_cands\":%d,\"hebb_cand_max\":%.4f,\"hebb_links\":%lld,"
|
||||
"\"hebb_warm\":%d,"
|
||||
"\"hebb_wb_pending\":%d,\"hebb_wb_drained\":%lld,"
|
||||
"\"hebb_wb_dropped\":%lld,"
|
||||
"\"dup_seeds\":%lld,\"dup_wm\":%lld,\"dup_wm_global\":%lld}",
|
||||
(long long)_eg_act_wm_evicted,
|
||||
(long long)_eg_act_breakthroughs,
|
||||
@@ -10478,11 +10563,65 @@ el_val_t engram_act_stats_json(void) {
|
||||
(long long)hebb_edges, hebb_max, hebb_mass,
|
||||
hebb_cands, hebb_cand_max, (long long)_eg_hebb_links_formed,
|
||||
_eg_act_hebb_warm,
|
||||
_eg_hebb_wb_len, (long long)_eg_hebb_wb_drained,
|
||||
(long long)_eg_hebb_wb_dropped,
|
||||
(long long)_eg_act_dup_seeds, (long long)_eg_act_dup_wm,
|
||||
(long long)_eg_act_dup_wm_global);
|
||||
return el_wrap_str(el_strdup(buf));
|
||||
}
|
||||
|
||||
/* engram_hebb_drain_json — pop up to `max` newly-formed Hebbian associations
|
||||
* off the write-back queue and return them as a JSON array:
|
||||
*
|
||||
* [{"from_id":"...","to_id":"...","weight":0.15,"hebb":0.31}, ...]
|
||||
*
|
||||
* Draining is DESTRUCTIVE: entries returned here are gone from the queue. The
|
||||
* caller owns delivery from that point on. That is deliberate — the alternative
|
||||
* (peek, deliver, ack) needs a second round trip and a retry ledger to be
|
||||
* correct, and the payload is an association that will re-form from live
|
||||
* co-activation if it genuinely matters. Losing one is cheap; a queue that
|
||||
* silently refills forever because acks never land is not.
|
||||
*
|
||||
* Empty queue returns "[]". See the ENGRAM_HEBB_WB_SLOTS block for why this
|
||||
* exists at all: the process that does the learning is not the process that
|
||||
* owns persistence. (2026-08-07 self-review.) */
|
||||
el_val_t engram_hebb_drain_json(el_val_t max_v) {
|
||||
int64_t max_n = (int64_t)max_v;
|
||||
if (max_n <= 0) max_n = 64;
|
||||
if (max_n > ENGRAM_HEBB_WB_SLOTS) max_n = ENGRAM_HEBB_WB_SLOTS;
|
||||
JsonBuf b; jb_init(&b);
|
||||
jb_puts(&b, "[");
|
||||
int emitted = 0;
|
||||
while (_eg_hebb_wb_len > 0 && emitted < (int)max_n) {
|
||||
EgHebbWB* e = &_eg_hebb_wb[_eg_hebb_wb_head];
|
||||
if (e->a && e->b) {
|
||||
if (emitted > 0) jb_puts(&b, ",");
|
||||
/* relation is emitted here, not stamped on by the caller: the
|
||||
* payload should be postable to /api/edges/batch verbatim. A
|
||||
* consumer that has to rewrite the JSON to make it valid is a
|
||||
* consumer that will eventually rewrite it wrong. */
|
||||
jb_puts(&b, "{\"relation\":\"hebbian-associate\",\"from_id\":");
|
||||
jb_emit_escaped(&b, e->a);
|
||||
jb_puts(&b, ",\"to_id\":");
|
||||
jb_emit_escaped(&b, e->b);
|
||||
char tmp[96];
|
||||
snprintf(tmp, sizeof(tmp), ",\"weight\":%.6g,\"hebb\":%.6g}",
|
||||
e->w, e->hebb);
|
||||
jb_puts(&b, tmp);
|
||||
emitted++;
|
||||
_eg_hebb_wb_drained++;
|
||||
}
|
||||
/* Free and advance whether or not the entry rendered — a NULL id is a
|
||||
* strdup failure at push time, not a retryable condition. */
|
||||
free(e->a); free(e->b);
|
||||
e->a = NULL; e->b = NULL;
|
||||
_eg_hebb_wb_head = (_eg_hebb_wb_head + 1) % ENGRAM_HEBB_WB_SLOTS;
|
||||
_eg_hebb_wb_len--;
|
||||
}
|
||||
jb_puts(&b, "]");
|
||||
return el_wrap_str(b.buf ? b.buf : el_strdup("[]"));
|
||||
}
|
||||
|
||||
/* engram_cosine_sim — cosine similarity between two nodes' embeddings.
|
||||
* Returns a float in [-1, 1], or -2.0 when either node is missing or not
|
||||
* yet embedded. Exposed so EL code (and the introspection API) can probe
|
||||
|
||||
@@ -619,6 +619,11 @@ el_val_t engram_activate_json(el_val_t query, el_val_t depth);
|
||||
el_val_t engram_stats_json(void);
|
||||
el_val_t engram_act_stats_json(void);
|
||||
el_val_t engram_cosine_sim(el_val_t id_a, el_val_t id_b);
|
||||
/* Destructively pop up to `max` newly-formed Hebbian associations as a JSON
|
||||
* array of {from_id,to_id,weight,hebb}. The learning process (soul daemon) is
|
||||
* not the process that owns persistence (engram HTTP server); this is how a
|
||||
* self-formed association crosses that boundary. (2026-08-07 self-review.) */
|
||||
el_val_t engram_hebb_drain_json(el_val_t max);
|
||||
/* Document frequency of a term across node labels — term-specificity signal
|
||||
* for curiosity seed selection. (2026-08-03 self-review.) */
|
||||
el_val_t engram_label_df(el_val_t term);
|
||||
|
||||
Reference in New Issue
Block a user