neuron-technologies/el
2
0
Fork 0
Code Issues 1 Pull Requests 6 Actions Packages Projects Releases 5 Wiki Activity

self-review 2026-05-28: checkpoint ISE decay fix + auto-linking for MCP nodes

Browse Source 
Three changes:

1. Fix checkpoint ISE temporal_decay_rate: engram_emit_ise_internal was
   hardcoded to 0.0 (global 168h default) instead of 2.310 (Working-tier
   48h). Result: checkpoint ISEs accumulated at 3.5x intended rate.

2. Raise CHECKPOINT_INTERVAL 1→10: checkpoint ISE fires on every single
   node write, producing 2:1 checkpoint:content ratio in ISE stream.
   MCP routes still call engram_write_binary_el explicitly after each
   important write, so no knowledge durability is lost.

3. Add auto_link_content_node to server.el: route_neuron_memory and
   route_neuron_knowledge_capture were creating nodes with zero edges —
   invisible to BFS traversal, only reachable via lexical/semantic seed.
   New helper runs BM25 over top-20 results, skips ISE nodes (which
   dominate the 14K-node corpus), connects up to 3 related nodes.
This commit is contained in:
Will Anderson
2026-05-28 08:42:41 -05:00
parent 8f922e68b3
commit a000599bfe
4 changed files with 144 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engram/dist/engram
Vendored
BIN
View File
Binary file not shown.
engram/dist/engram.c
Vendored
+59 -8
View File
@@ -6,6 +6,7 @@ el_val_t bm25_tokenize(el_val_t text);
el_val_t bm25_count_term(el_val_t term, el_val_t doc_tokens);
el_val_t bm25_score_doc(el_val_t doc_content, el_val_t query_tokens, el_val_t corpus_size, el_val_t avg_doc_len);
el_val_t bm25_search_json(el_val_t query, el_val_t limit);
el_val_t auto_link_content_node(el_val_t node_id, el_val_t content);
el_val_t parse_port(el_val_t bind);
el_val_t ok_json(void);
el_val_t err_json(el_val_t msg);
@@ -217,6 +218,54 @@ el_val_t bm25_search_json(el_val_t query, el_val_t limit) {
return 0;
}
el_val_t auto_link_content_node(el_val_t node_id, el_val_t content) {
el_val_t clen = str_len(content);
if (clen < 20) {
return 0;
}
el_val_t sp1 = str_index_of(content, EL_STR(" "));
el_val_t w1end = ({ el_val_t _if_result_3 = 0; if ((sp1 < 0)) { _if_result_3 = (clen); } else { _if_result_3 = (sp1); } _if_result_3; });
el_val_t word1 = str_slice(content, 0, w1end);
state_set(EL_STR("aln_term"), EL_STR(""));
if (str_len(word1) >= 5) {
state_set(EL_STR("aln_term"), word1);
}
if (str_eq(state_get(EL_STR("aln_term")), EL_STR(""))) {
if (sp1 >= 0) {
el_val_t rest = str_slice(content, (sp1 + 1), clen);
el_val_t sp2 = str_index_of(rest, EL_STR(" "));
el_val_t w2end = ({ el_val_t _if_result_4 = 0; if ((sp2 < 0)) { _if_result_4 = (str_len(rest)); } else { _if_result_4 = (sp2); } _if_result_4; });
el_val_t word2 = str_slice(rest, 0, w2end);
if (str_len(word2) >= 5) {
state_set(EL_STR("aln_term"), word2);
}
}
}
el_val_t search_term = state_get(EL_STR("aln_term"));
if (str_eq(search_term, EL_STR(""))) {
return 0;
}
el_val_t results = bm25_search_json(search_term, 20);
el_val_t n = json_array_len(results);
state_set(EL_STR("aln_linked"), EL_STR("0"));
el_val_t i = 0;
while (i < n) {
el_val_t linked_so_far = str_to_int(state_get(EL_STR("aln_linked")));
if (linked_so_far < 3) {
el_val_t elem = json_array_get(results, i);
el_val_t rid = json_get_string(elem, EL_STR("id"));
el_val_t rtype = json_get_string(elem, EL_STR("node_type"));
if ((!str_eq(rtype, EL_STR("InternalStateEvent")) && !str_eq(rid, EL_STR(""))) && !str_eq(rid, node_id)) {
engram_connect(node_id, rid, el_from_float(0.6), EL_STR("related"));
state_set(EL_STR("aln_linked"), int_to_str((linked_so_far + 1)));
}
}
i = (i + 1);
}
return str_to_int(state_get(EL_STR("aln_linked")));
return 0;
}
el_val_t parse_port(el_val_t bind) {
el_val_t colon = str_index_of(bind, EL_STR(":"));
if (colon < 0) {
@@ -549,7 +598,7 @@ el_val_t route_neuron_session_begin(el_val_t method, el_val_t path, el_val_t bod
el_val_t route_neuron_ctx(el_val_t method, el_val_t path, el_val_t body) {
el_val_t results = engram_activate_json(EL_STR("architecture decision memory"), 2);
el_val_t n = json_array_len(results);
el_val_t limit = ({ el_val_t _if_result_3 = 0; if ((n > 10)) { _if_result_3 = (10); } else { _if_result_3 = (n); } _if_result_3; });
el_val_t limit = ({ el_val_t _if_result_5 = 0; if ((n > 10)) { _if_result_5 = (10); } else { _if_result_5 = (n); } _if_result_5; });
el_val_t ctx = EL_STR("Recent working memory:\n");
el_val_t i = 0;
el_val_t ctx_body = EL_STR("");
@@ -558,7 +607,7 @@ el_val_t route_neuron_ctx(el_val_t method, el_val_t path, el_val_t body) {
el_val_t label = json_get_string(elem, EL_STR("label"));
el_val_t content = json_get_string(elem, EL_STR("content"));
el_val_t clen = str_len(content);
el_val_t snippet = ({ el_val_t _if_result_4 = 0; if ((clen > 200)) { _if_result_4 = (str_slice(content, 0, 200)); } else { _if_result_4 = (content); } _if_result_4; });
el_val_t snippet = ({ el_val_t _if_result_6 = 0; if ((clen > 200)) { _if_result_6 = (str_slice(content, 0, 200)); } else { _if_result_6 = (content); } _if_result_6; });
ctx_body = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(ctx_body, EL_STR("- [")), label), EL_STR("]: ")), snippet), EL_STR("\n"));
i = (i + 1);
}
@@ -607,13 +656,14 @@ el_val_t route_neuron_memory(el_val_t method, el_val_t path, el_val_t body) {
}
}
el_val_t id = engram_node_full(content, node_type, label, el_from_float(0.5), el_from_float(0.5), el_from_float(1.0), tier, tags_str);
el_val_t auto_linked = auto_link_content_node(id, content);
el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR"));
if (str_eq(dir, EL_STR(""))) {
dir = EL_STR("/tmp/engram");
}
el_val_t db_path = el_str_concat(dir, EL_STR("/engram.db"));
engram_write_binary_el(db_path);
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\",\"content\":\"")), str_replace(str_replace(content, EL_STR("\\"), EL_STR("\\\\")), EL_STR("\""), EL_STR("\\\""))), EL_STR("\"}"));
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\",\"auto_linked\":")), int_to_str(auto_linked)), EL_STR(",\"content\":\"")), str_replace(str_replace(content, EL_STR("\\"), EL_STR("\\\\")), EL_STR("\""), EL_STR("\\\""))), EL_STR("\"}"));
return 0;
}
@@ -655,13 +705,14 @@ el_val_t route_neuron_knowledge_capture(el_val_t method, el_val_t path, el_val_t
}
}
el_val_t id = engram_node_full(content, EL_STR("Knowledge"), title, el_from_float(0.7), el_from_float(0.7), el_from_float(1.0), tier, tags_str);
el_val_t auto_linked = auto_link_content_node(id, content);
el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR"));
if (str_eq(dir, EL_STR(""))) {
dir = EL_STR("/tmp/engram");
}
el_val_t db_path = el_str_concat(dir, EL_STR("/engram.db"));
engram_write_binary_el(db_path);
return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\"}"));
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\",\"auto_linked\":")), int_to_str(auto_linked)), EL_STR("}"));
return 0;
}
@@ -751,7 +802,7 @@ el_val_t route_neuron_recall(el_val_t method, el_val_t path, el_val_t body) {
if (limit == 0) {
limit = 20;
}
el_val_t q = ({ el_val_t _if_result_5 = 0; if (str_eq(query, EL_STR(""))) { _if_result_5 = (chain); } else { _if_result_5 = (query); } _if_result_5; });
el_val_t q = ({ el_val_t _if_result_7 = 0; if (str_eq(query, EL_STR(""))) { _if_result_7 = (chain); } else { _if_result_7 = (query); } _if_result_7; });
if (str_eq(q, EL_STR(""))) {
return engram_scan_nodes_json(limit, 0);
}
@@ -822,9 +873,9 @@ el_val_t route_neuron_config(el_val_t method, el_val_t path, el_val_t body) {
el_val_t route_neuron_state_events(el_val_t method, el_val_t path, el_val_t body) {
if (str_eq(method, EL_STR("GET"))) {
el_val_t limit_str = query_param(path, EL_STR("limit"));
el_val_t limit = ({ el_val_t _if_result_6 = 0; if (str_eq(limit_str, EL_STR(""))) { _if_result_6 = (50); } else { _if_result_6 = (str_to_int(limit_str)); } _if_result_6; });
el_val_t limit = ({ el_val_t _if_result_8 = 0; if (str_eq(limit_str, EL_STR(""))) { _if_result_8 = (50); } else { _if_result_8 = (str_to_int(limit_str)); } _if_result_8; });
el_val_t offset_str = query_param(path, EL_STR("offset"));
el_val_t offset = ({ el_val_t _if_result_7 = 0; if (str_eq(offset_str, EL_STR(""))) { _if_result_7 = (0); } else { _if_result_7 = (str_to_int(offset_str)); } _if_result_7; });
el_val_t offset = ({ el_val_t _if_result_9 = 0; if (str_eq(offset_str, EL_STR(""))) { _if_result_9 = (0); } else { _if_result_9 = (str_to_int(offset_str)); } _if_result_9; });
return engram_scan_nodes_by_type_json(EL_STR("InternalStateEvent"), limit, offset);
}
el_val_t content = json_get_string(body, EL_STR("content"));
@@ -832,7 +883,7 @@ el_val_t route_neuron_state_events(el_val_t method, el_val_t path, el_val_t body
content = body;
}
el_val_t event_label = json_get_string(content, EL_STR("event"));
el_val_t label = ({ el_val_t _if_result_8 = 0; if (str_eq(event_label, EL_STR(""))) { _if_result_8 = (EL_STR("state-event")); } else { _if_result_8 = (event_label); } _if_result_8; });
el_val_t label = ({ el_val_t _if_result_10 = 0; if (str_eq(event_label, EL_STR(""))) { _if_result_10 = (EL_STR("state-event")); } else { _if_result_10 = (event_label); } _if_result_10; });
el_val_t id = engram_node_full(content, EL_STR("InternalStateEvent"), label, el_from_float(0.3), el_from_float(0.3), el_from_float(1.0), EL_STR("Working"), EL_STR("internal-state"));
return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\"}"));
return 0;
engram/src/server.el
+80 -2
View File
@@ -227,6 +227,76 @@ fn bm25_search_json(query: String, limit: Int) -> String {
out + "]"
}
// ── Auto-linking ─────────────────────────────────────────────────────────────
//
// auto_link_content_node — link a newly-created Knowledge or Memory node to
// semantically related non-ISE nodes via BM25 search.
//
// Problem it solves: route_neuron_memory and route_neuron_knowledge_capture
// both call engram_node_full directly, creating nodes with zero edges. With
// 14K+ ISEs dominating the corpus, BFS traversal contributes nothing — every
// query relies solely on lexical/semantic seed matching. Auto-linking builds
// explicit "related" edges so activated knowledge nodes fan out to connected
// neighbors during BFS.
//
// Design choices:
// - BM25 (not substring search): ranks by relevance, not just occurrence
// - Skip InternalStateEvent nodes: ISEs dominate the corpus and are not
// useful link targets for knowledge/memory nodes
// - Up to 3 edges per node: enough to build graph structure without over-linking
// - weight=0.6: moderately strong; causal edges (field-validated at 2.0) are
// much stronger, so these "related" edges don't flood activation paths
// - state_set for linked counter: EL `let` in nested if-blocks creates inner
// scope only; state_set persists across block boundaries (2026-05-25 lesson)
//
// (2026-05-28 self-review)
fn auto_link_content_node(node_id: String, content: String) -> Int {
let clen: Int = str_len(content)
if clen < 20 { return 0 }
// Find search term: first word >= 5 chars, or second word.
let sp1: Int = str_index_of(content, " ")
let w1end: Int = if sp1 < 0 { clen } else { sp1 }
let word1: String = str_slice(content, 0, w1end)
state_set("aln_term", "")
if str_len(word1) >= 5 {
state_set("aln_term", word1)
}
if str_eq(state_get("aln_term"), "") {
if sp1 >= 0 {
let rest: String = str_slice(content, sp1 + 1, clen)
let sp2: Int = str_index_of(rest, " ")
let w2end: Int = if sp2 < 0 { str_len(rest) } else { sp2 }
let word2: String = str_slice(rest, 0, w2end)
if str_len(word2) >= 5 {
state_set("aln_term", word2)
}
}
}
let search_term: String = state_get("aln_term")
if str_eq(search_term, "") { return 0 }
// BM25 over top-20 results; skip ISE nodes; connect up to 3.
let results: String = bm25_search_json(search_term, 20)
let n: Int = json_array_len(results)
state_set("aln_linked", "0")
let i: Int = 0
while i < n {
let linked_so_far: Int = str_to_int(state_get("aln_linked"))
if linked_so_far < 3 {
let elem: String = json_array_get(results, i)
let rid: String = json_get_string(elem, "id")
let rtype: String = json_get_string(elem, "node_type")
if !str_eq(rtype, "InternalStateEvent") && !str_eq(rid, "") && !str_eq(rid, node_id) {
engram_connect(node_id, rid, 0.6, "related")
state_set("aln_linked", int_to_str(linked_so_far + 1))
}
}
let i = i + 1
}
return str_to_int(state_get("aln_linked"))
}
// ── Helpers ───────────────────────────────────────────────────────────────────
fn parse_port(bind: String) -> Int {
@@ -565,13 +635,18 @@ fn route_neuron_memory(method: String, path: String, body: String) -> String {
let id: String = engram_node_full(content, node_type, label, 0.5, 0.5, 1.0, tier, tags_str)
// Auto-link to related non-ISE nodes so this memory is reachable via BFS traversal.
// Without this, MCP-created nodes arrive with zero edges and are invisible to
// graph spread during activation (only lexical/semantic seed matching finds them).
let auto_linked: Int = auto_link_content_node(id, content)
// Checkpoint after write
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let db_path: String = dir + "/engram.db"
engram_write_binary_el(db_path)
"{\"ok\":true,\"id\":\"" + id + "\",\"content\":\"" + str_replace(str_replace(content, "\\", "\\\\"), "\"", "\\\"") + "\"}"
"{\"ok\":true,\"id\":\"" + id + "\",\"auto_linked\":" + int_to_str(auto_linked) + ",\"content\":\"" + str_replace(str_replace(content, "\\", "\\\\"), "\"", "\\\"") + "\"}"
}
// route_neuron_knowledge_capture — create a Knowledge node
@@ -611,13 +686,16 @@ fn route_neuron_knowledge_capture(method: String, path: String, body: String) ->
let id: String = engram_node_full(content, "Knowledge", title, 0.7, 0.7, 1.0, tier, tags_str)
// Auto-link to related non-ISE nodes for BFS reachability (same rationale as route_neuron_memory).
let auto_linked: Int = auto_link_content_node(id, content)
// Checkpoint
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let db_path: String = dir + "/engram.db"
engram_write_binary_el(db_path)
"{\"ok\":true,\"id\":\"" + id + "\"}"
"{\"ok\":true,\"id\":\"" + id + "\",\"auto_linked\":" + int_to_str(auto_linked) + "}"
}
// route_neuron_knowledge_evolve — create updated node (evolution via new node)
lang/releases/v1.0.0-20260501/el_runtime.c
+5 -2
View File
@@ -5768,7 +5768,7 @@ static uint8_t g_user_pub[1568];
static uint8_t g_user_priv[3168];
static int g_keys_loaded = 0;
static int64_t g_writes_since_checkpoint = 0;
#define ENGRAM_CHECKPOINT_INTERVAL 1 /* save binary on every write */
#define ENGRAM_CHECKPOINT_INTERVAL 10 /* save binary every 10 writes (MCP routes still save explicitly) */
/* Initialize the five canonical layers on a fresh store. Called once from
* engram_get(). Layer ids 0..4 are reserved; runtime-injected imprint/suit
@@ -8198,7 +8198,10 @@ static void engram_emit_ise_internal(const char* content, const char* label) {
n->salience = 0.3f;
n->importance = 0.5f;
n->confidence = 1.0f;
n->temporal_decay_rate = 0.0;
/* Working-tier decay: 48h half-life (lambda=2.31). Previously 0.0 which
* incorrectly used the 168h global default, causing checkpoint ISEs to
* accumulate at ~3.5x their intended rate. (2026-05-28 self-review) */
n->temporal_decay_rate = 2.310;
n->activation_count = 0;
int64_t now = engram_now_ms();
n->last_activated = now;