neuron-technologies/el
2
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 5 Wiki Activity

self-review 2026-06-30: WM cap, breakthrough floor, ISE exclusion + route

Browse Source 
Port critical WM fixes from self-review 2026-06-26 branch (f7bd99a) that were
never merged to HEAD. Running binary had these fixes; source did not — rebuild
would have silently regressed all three improvements.

1. ENGRAM_BREAKTHROUGH_WEIGHT 0.25→0.10
   With 0.25, naturally-promoted nodes (threshold ≥0.15) decayed below the
   breakthrough floor within one activation call and lost their WM slot to
   fresh breakthrough candidates. All 524/525 WM nodes were at floor = useless.
   Invariant: BREAKTHROUGH_WEIGHT < min(type_thresholds = 0.15 Canonical).

2. ENGRAM_WM_CAP=24 with Pass 4 (per-call) + Pass 5 (global) enforcement
   Without cap, broad curiosity seeds promote 500+ nodes simultaneously.
   wm_avg_weight collapses, goal-bias differentiation is lost. Verified:
   "knowledge" query now promotes exactly 24 nodes (was 525). Cowan (2001)
   cognitive basis: WM capacity ~4 chunks; 24 allows rich multi-topic context.

3. ISE exclusion from WM (Pass 2 guard)
   InternalStateEvent JSON content ("knowledge", "memory", etc.) triggered
   lexical seeding → suppression accumulation → breakthrough at floor. ISEs
   are observability-only and must never surface in context compilation.
   suppression_count cleared so ISEs never build toward breakthrough.

4. route_create_ise importance fix (0.5→0.3)
   Corrects mismatch between HTTP route and awareness.el in-process fallback.
   Also adds body comment clarifying auth-exempt rationale.

SYNAPSE (arXiv 2601.02744) validates WM cap design and ISE exclusion principle.
Next priority: cosine similarity seeding to complement lexical BFS.
This commit is contained in:
Will Anderson
2026-06-30 08:48:19 -05:00
parent 35c189759c
commit da116b2884
4 changed files with 333 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engram/dist/engram
Vendored Executable
BIN
View File
Binary file not shown.
engram/dist/engram.c
Vendored
+38 -2
View File
@@ -20,6 +20,8 @@ el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_create_ise(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_save(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_load(el_val_t method, el_val_t path, el_val_t body);
el_val_t route_health(el_val_t method, el_val_t path, el_val_t body);
@@ -115,7 +117,7 @@ el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body) {
node_type = EL_STR("Memory");
}
el_val_t salience = json_get_float(body, EL_STR("salience"));
if (str_eq(salience, el_from_float(0.0))) {
if (salience == el_from_float(0.0)) {
salience = el_from_float(0.5);
}
el_val_t id = engram_node(content, node_type, salience);
@@ -205,7 +207,7 @@ el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body) {
relation = EL_STR("associates");
}
el_val_t weight = json_get_float(body, EL_STR("weight"));
if (str_eq(weight, el_from_float(0.0))) {
if (weight == el_from_float(0.0)) {
weight = el_from_float(0.5);
}
engram_connect(from_id, to_id, weight, relation);
@@ -243,6 +245,34 @@ el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body) {
return 0;
}
el_val_t route_create_ise(el_val_t method, el_val_t path, el_val_t body) {
el_val_t content = json_get_string(body, EL_STR("content"));
if (str_eq(content, EL_STR(""))) {
return err_json(EL_STR("missing content"));
}
el_val_t sal = el_from_float(0.3);
el_val_t imp = el_from_float(0.3);
el_val_t conf = el_from_float(0.8);
el_val_t id = engram_node_full(content, EL_STR("InternalStateEvent"), EL_STR("state-event"), sal, imp, conf, EL_STR("Episodic"), EL_STR("[\"internal-state\",\"InternalStateEvent\"]"));
return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\"}"));
return 0;
}
el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body) {
el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR"));
if (str_eq(dir, EL_STR(""))) {
dir = EL_STR("/tmp/engram");
}
el_val_t snap_path = el_str_concat(dir, EL_STR("/sync-export.json"));
engram_save(snap_path);
el_val_t snap = fs_read(snap_path);
if (str_eq(snap, EL_STR(""))) {
return EL_STR("{\"nodes\":[],\"edges\":[]}");
}
return snap;
return 0;
}
el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) {
el_val_t p = json_get_string(body, EL_STR("path"));
if (str_eq(p, EL_STR(""))) {
@@ -299,6 +329,9 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
return route_health(method, path, body);
}
}
if (str_eq(method, EL_STR("POST")) && str_starts_with(clean, EL_STR("/api/neuron/state-events"))) {
return route_create_ise(method, path, body);
}
if (!check_auth_ok(method, body)) {
return err_json(EL_STR("unauthorized"));
}
@@ -341,6 +374,9 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/strengthen")) || str_eq(clean, EL_STR("/strengthen")))) {
return route_strengthen(method, path, body);
}
if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/sync")) || str_eq(clean, EL_STR("/sync")))) {
return route_sync(method, path, body);
}
if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/save")) || str_eq(clean, EL_STR("/save")))) {
return route_save(method, path, body);
}
engram/src/server.el
+50
View File
@@ -180,6 +180,43 @@ fn route_forget(method: String, path: String, body: String) -> String {
ok_json()
}
// route_create_ise — POST /api/neuron/state-events
// Creates an InternalStateEvent node from a JSON body with a "content" field.
// Returns {"ok":true,"id":"<uuid>"}. Used by the soul daemon's ise_post() to
// record internal state transitions in the authoritative Engram store.
// This route was in the original server.el but was lost during a refactor;
// its absence would break ISE recording on the next Engram restart.
// (Restored 2026-06-30 self-review)
// importance=0.3 matches awareness.el in-process fallback (engram_node_full ISE
// defaults). Original had 0.5 which was a mismatch. (Corrected 2026-06-30)
fn route_create_ise(method: String, path: String, body: String) -> String {
let content: String = json_get_string(body, "content")
if str_eq(content, "") { return err_json("missing content") }
let sal: Float = 0.3
let imp: Float = 0.3
let conf: Float = 0.8
let id: String = engram_node_full(content, "InternalStateEvent", "state-event",
sal, imp, conf, "Episodic", "[\"internal-state\",\"InternalStateEvent\"]")
"{\"ok\":true,\"id\":\"" + id + "\"}"
}
// route_sync — GET /api/sync
// Returns the full graph snapshot as JSON (nodes + edges), suitable for loading
// via engram_load_merge. Used by the soul daemon's periodic refresh cycle to
// keep its in-process Engram store in sync with this authoritative HTTP store.
// Saves a temporary snapshot to avoid holding a large in-memory string while
// streaming — caller reads the file through the HTTP response body.
// (Restored 2026-06-30 self-review)
fn route_sync(method: String, path: String, body: String) -> String {
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let snap_path: String = dir + "/sync-export.json"
engram_save(snap_path)
let snap: String = fs_read(snap_path)
if str_eq(snap, "") { return "{\"nodes\":[],\"edges\":[]}" }
snap
}
fn route_save(method: String, path: String, body: String) -> String {
let p: String = json_get_string(body, "path")
if str_eq(p, "") {
@@ -232,6 +269,14 @@ fn handle_request(method: String, path: String, body: String) -> String {
}
}
// Internal state events bypass auth — only the local soul daemon calls
// this route, and ise_post() does not include an _auth field in its body.
// Placing this before the auth gate preserves the old binary's behavior
// and ensures the soul daemon can always write ISEs.
if str_eq(method, "POST") && str_starts_with(clean, "/api/neuron/state-events") {
return route_create_ise(method, path, body)
}
// Auth (when ENGRAM_API_KEY is set)
if !check_auth_ok(method, body) {
return err_json("unauthorized")
@@ -286,6 +331,11 @@ fn handle_request(method: String, path: String, body: String) -> String {
return route_strengthen(method, path, body)
}
// Sync — soul daemon fetches here for periodic in-process graph refresh
if str_eq(method, "GET") && (str_eq(clean, "/api/sync") || str_eq(clean, "/sync")) {
return route_sync(method, path, body)
}
// Persistence
if str_eq(method, "POST") && (str_eq(clean, "/api/save") || str_eq(clean, "/save")) {
return route_save(method, path, body)
lang/releases/v1.0.0-20260501/el_runtime.c
+245 -3
View File
@@ -5470,9 +5470,35 @@ void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal,
#define ENGRAM_WM_THRESHOLD 0.15
#define ENGRAM_WM_DECAY 0.7
#define ENGRAM_SUPPRESSION_BREAKTHROUGH 5
#define ENGRAM_BREAKTHROUGH_WEIGHT 0.25
/* ENGRAM_BREAKTHROUGH_WEIGHT: lowered 0.25→0.10 (2026-06-30 self-review, porting
* fix from self-review 2026-06-26 branch). With 0.25, Knowledge nodes (threshold
* 0.15) promoted at ~0.21 decay in one call to ~0.147, fall below the 0.25 floor,
* and immediately lose their WM slot to fresh breakthrough candidates at 0.25.
* Natural promotion was invisible: live data showed 524/525 WM nodes at 0.25
* breakthrough floor. With 0.10, all per-type thresholds (minimum 0.15 Canonical)
* exceed the floor, so naturally-promoted nodes survive multiple decay cycles.
* Invariant maintained: BREAKTHROUGH_WEIGHT < min(type_thresholds). */
#define ENGRAM_BREAKTHROUGH_WEIGHT 0.10
/* ENGRAM_WM_CAP: hard limit on concurrent working-memory nodes (2026-06-30
* self-review, porting fix from self-review 2026-06-26 branch). Without this,
* broad curiosity seeds like "knowledge" promote 500+ nodes simultaneously —
* wm_avg_weight collapses to the breakthrough floor, goal-bias differentiation
* is lost, and heartbeat ISEs show useless WM composition data. Cognitive
* basis: WM capacity is ~4 chunks (Cowan 2001); 24 allows richer multi-topic
* context while preventing flooding. Enforced in Pass 4 (per-call) and Pass 5
* (global across prior-promoted nodes). */
#define ENGRAM_WM_CAP 24
#define ENGRAM_INHIBITION_FACTOR 0.1
/* qsort comparator — descending double, used by WM cap enforcement. */
static int engram_cmp_double_desc(const void* a, const void* b) {
double da = *(const double*)a;
double db = *(const double*)b;
if (da > db) return -1;
if (da < db) return 1;
return 0;
}
/* ── Layered consciousness architecture ──────────────────────────────────────
*
* The engram graph is stratified into LAYERS that gate which suppressions
@@ -6659,6 +6685,19 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
for (int64_t i = 0; i < g->node_count; i++) {
if (!reached[i] || best_bg[i] <= 0.0) continue;
EngramNode* n = &g->nodes[i];
/* InternalStateEvent nodes are observability-only — never admit to WM.
* Their JSON content (curiosity seeds, heartbeat payloads) contains common
* words that trigger lexical seeding (e.g. "knowledge" in curiosity ISEs),
* leading to repeated suppression and eventual breakthrough at the floor.
* ISEs surfacing in context compilation are noise, not signal. Clear their
* suppression_count so they don't build toward breakthrough, then skip.
* (2026-06-30 self-review: porting fix from 2026-06-26 branch; SYNAPSE
* paper confirms WM should hold only semantically relevant content.) */
if (n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0) {
n->suppression_count = 0;
wm_weights[i] = 0.0;
continue;
}
/* Per-type threshold: safety nodes break through more easily. */
double type_threshold = engram_type_threshold(n->node_type, n->tier);
/* Goal bias weights the node's relevance to current intent. */
@@ -6710,9 +6749,123 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
n->suppression_count = 0;
}
/* Persist working_memory_weight (post Pass 3) to node store. */
/* ── PASS 4: WM capacity cap (per-call) ─────────────────────────────────
* Enforce ENGRAM_WM_CAP as a hard upper bound on nodes promoted in this
* activation call. Without this, broad curiosity seeds like "knowledge"
* promote 500+ nodes simultaneously — wm_avg_weight collapses to the
* breakthrough floor, goal-bias differentiation is lost, and working memory
* becomes useless. (Ported from 2026-06-26 self-review branch; observed
* 525 promoted for "knowledge", 524 at breakthrough floor 0.25, 1 natural.) */
{
int64_t cap_count = 0;
for (int64_t i = 0; i < g->node_count; i++) {
if (wm_weights[i] > 0.0) cap_count++;
}
if (cap_count > ENGRAM_WM_CAP) {
double* cap_vals = malloc((size_t)cap_count * sizeof(double));
if (cap_vals) {
int64_t ci = 0;
for (int64_t i = 0; i < g->node_count; i++) {
if (wm_weights[i] > 0.0) cap_vals[ci++] = wm_weights[i];
}
qsort(cap_vals, (size_t)cap_count, sizeof(double),
engram_cmp_double_desc);
/* cap_vals[ENGRAM_WM_CAP-1] is the lowest weight that still
* fits inside the cap when sorted descending. */
double cutoff = cap_vals[ENGRAM_WM_CAP - 1];
free(cap_vals);
/* Count strictly above cutoff to handle ties correctly. */
int64_t above = 0;
for (int64_t i = 0; i < g->node_count; i++) {
if (wm_weights[i] > cutoff) above++;
}
int64_t at_cutoff_slots = ENGRAM_WM_CAP - above;
/* Evict nodes that don't make the cut. */
for (int64_t i = 0; i < g->node_count; i++) {
if (wm_weights[i] <= 0.0) continue; /* not promoted */
if (wm_weights[i] > cutoff) continue; /* above cutoff */
if (at_cutoff_slots > 0) {
at_cutoff_slots--;
continue; /* fills a slot */
}
wm_weights[i] = 0.0; /* over cap: evict */
}
}
/* If malloc failed, skip cap — WM unbounded this call, no corruption. */
}
}
/* Persist working_memory_weight (post Pass 4) to node store.
*
* Conversational thread continuity (ENGRAM_WM_DECAY):
* Nodes promoted in a previous turn but NOT reached by the current BFS
* fan-out retain a decayed weight rather than being zeroed. This models
* the brain's ability to maintain recent context across successive turns
* without requiring explicit re-activation. A node that was relevant one
* query ago stays weakly present in working memory; a node from two
* queries ago retains 0.7² ≈ 0.49 of its original weight; after ~5 quiet
* turns it falls below 0.01 and is effectively evicted (set to 0.0).
*
* NOTE: this was documented in the ENGRAM_WM_DECAY constant comment since
* the two-layer architecture was introduced, but was never implemented —
* unreached nodes were always zeroed unconditionally. Fixed 2026-06-30
* self-review. */
for (int64_t i = 0; i < g->node_count; i++) {
g->nodes[i].working_memory_weight = wm_weights[i];
if (!reached[i] && g->nodes[i].working_memory_weight > 0.0) {
/* Carry-over decay: node held WM weight from prior activation but
* the current query's BFS fan-out did not reach it. Apply decay
* rather than zero so recently-active context persists. */
double decayed = g->nodes[i].working_memory_weight * ENGRAM_WM_DECAY;
g->nodes[i].working_memory_weight = (decayed < 0.01) ? 0.0 : decayed;
} else {
g->nodes[i].working_memory_weight = wm_weights[i];
}
}
/* ── PASS 5: Global WM cap enforcement ───────────────────────────────────
* Pass 4 capped this call's new candidates. But nodes already in WM from
* prior calls retain their persisted working_memory_weight (via the decay
* carry-over above). Over multiple activation calls total WM can grow well
* above ENGRAM_WM_CAP. This pass enforces the cap globally across ALL
* nodes in the store, keeping only the top ENGRAM_WM_CAP by current weight.
* Correct cognitive model: WM capacity is global (Cowan 2001); more recent
* activations outcompete older decayed ones. (Ported from 2026-06-26
* self-review branch.) */
{
int64_t global_wm_count = 0;
for (int64_t i = 0; i < g->node_count; i++) {
if (g->nodes[i].working_memory_weight > 0.0) global_wm_count++;
}
if (global_wm_count > ENGRAM_WM_CAP) {
double* gvals = malloc((size_t)global_wm_count * sizeof(double));
if (gvals) {
int64_t gi = 0;
for (int64_t i = 0; i < g->node_count; i++) {
if (g->nodes[i].working_memory_weight > 0.0)
gvals[gi++] = g->nodes[i].working_memory_weight;
}
qsort(gvals, (size_t)global_wm_count, sizeof(double),
engram_cmp_double_desc);
double gcutoff = gvals[ENGRAM_WM_CAP - 1];
free(gvals);
int64_t gabove = 0;
for (int64_t i = 0; i < g->node_count; i++) {
if (g->nodes[i].working_memory_weight > gcutoff) gabove++;
}
int64_t gslots_at_cutoff = ENGRAM_WM_CAP - gabove;
for (int64_t i = 0; i < g->node_count; i++) {
EngramNode* n = &g->nodes[i];
if (n->working_memory_weight <= 0.0) continue;
if (n->working_memory_weight > gcutoff) continue;
if (gslots_at_cutoff > 0) {
gslots_at_cutoff--;
continue; /* fills a slot */
}
n->working_memory_weight = 0.0; /* evict: over global cap */
}
}
/* If malloc failed, skip — WM over cap this call, no data corruption. */
}
}
/* ── Collect all background-activated nodes for the return value ────
@@ -7298,6 +7451,95 @@ el_val_t engram_activate_json(el_val_t query, el_val_t depth) {
return el_wrap_str(b.buf);
}
/* ── Working memory introspection helpers ────────────────────────────────────
*
* These three functions give the soul daemon visibility into WM composition
* without re-running activation. Used in heartbeat ISEs and curiosity scans.
* Ported from el-compiler/runtime to releases/v1.0.0-20260501 on 2026-06-30
* self-review (they were missing from the release build, breaking soul daemon
* compilation). */
el_val_t engram_wm_count(void) {
EngramStore* g = engram_get();
int64_t count = 0;
for (int64_t i = 0; i < g->node_count; i++) {
if (g->nodes[i].working_memory_weight > 0.0) count++;
}
return (el_val_t)count;
}
/* Average working_memory_weight across all promoted nodes (wm > 0).
* Returns the float bit-pattern via el_from_float so EL can use it with
* float_to_str / float_gt. Returns 0.0 when no nodes are promoted.
* Useful in heartbeat ISEs to distinguish "many weak activations" from
* "few strong activations". Added 2026-06-04 self-review. */
el_val_t engram_wm_avg_weight(void) {
EngramStore* g = engram_get();
double sum = 0.0;
int64_t count = 0;
for (int64_t i = 0; i < g->node_count; i++) {
double w = g->nodes[i].working_memory_weight;
/* Skip corrupt/out-of-range values so a single bad snapshot node
* doesn't produce a garbage average. */
if (w > 0.0 && w <= 1.0 && isfinite(w)) { sum += w; count++; }
}
double avg = (count > 0) ? (sum / (double)count) : 0.0;
return el_from_float(avg);
}
/* engram_wm_top_json — return top N working-memory nodes (by wm weight) as a
* compact JSON array for ISE heartbeat reporting.
* Each element: {"label":"...","node_type":"...","tier":"...","wm":0.42}
* InternalStateEvent nodes are excluded — they're observation artifacts that
* would bury substantive WM content. Added 2026-06-05 self-review. */
el_val_t engram_wm_top_json(el_val_t n_v) {
int64_t top_n = (int64_t)n_v;
if (top_n <= 0) top_n = 10;
if (top_n > 50) top_n = 50;
EngramStore* g = engram_get();
int64_t* idx = malloc((size_t)(g->node_count + 1) * sizeof(int64_t));
if (!idx) return el_wrap_str(el_strdup("[]"));
int64_t mc = 0;
for (int64_t i = 0; i < g->node_count; i++) {
if (g->nodes[i].working_memory_weight > 0.0) {
const char* nt = g->nodes[i].node_type;
if (nt && strcmp(nt, "InternalStateEvent") == 0) continue;
idx[mc++] = i;
}
}
/* Insertion-sort descending by wm weight (mc is typically small). */
for (int64_t i = 1; i < mc; i++) {
int64_t key = idx[i];
double kw = g->nodes[key].working_memory_weight;
int64_t j = i;
while (j > 0 && g->nodes[idx[j-1]].working_memory_weight < kw) {
idx[j] = idx[j-1]; j--;
}
idx[j] = key;
}
int64_t emit = mc < top_n ? mc : top_n;
JsonBuf b; jb_init(&b);
jb_putc(&b, '[');
for (int64_t k = 0; k < emit; k++) {
EngramNode* n = &g->nodes[idx[k]];
if (k > 0) jb_putc(&b, ',');
jb_putc(&b, '{');
jb_puts(&b, "\"label\":");
jb_emit_escaped(&b, n->label ? n->label : "");
jb_puts(&b, ",\"node_type\":");
jb_emit_escaped(&b, n->node_type ? n->node_type : "");
jb_puts(&b, ",\"tier\":");
jb_emit_escaped(&b, n->tier ? n->tier : "");
char tmp[48];
snprintf(tmp, sizeof(tmp), ",\"wm\":%.3f", n->working_memory_weight);
jb_puts(&b, tmp);
jb_putc(&b, '}');
}
free(idx);
jb_putc(&b, ']');
return el_wrap_str(b.buf);
}
el_val_t engram_stats_json(void) {
EngramStore* g = engram_get();
char buf[128];