diff --git a/engram/dist/engram b/engram/dist/engram new file mode 100755 index 0000000..a917fa0 Binary files /dev/null and b/engram/dist/engram differ diff --git a/engram/dist/engram.c b/engram/dist/engram.c index 275c694..18dc9e6 100644 --- a/engram/dist/engram.c +++ b/engram/dist/engram.c @@ -20,6 +20,8 @@ el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body); el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body); el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body); el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_create_ise(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body); el_val_t route_save(el_val_t method, el_val_t path, el_val_t body); el_val_t route_load(el_val_t method, el_val_t path, el_val_t body); el_val_t route_health(el_val_t method, el_val_t path, el_val_t body); @@ -115,7 +117,7 @@ el_val_t route_create_node(el_val_t method, el_val_t path, el_val_t body) { node_type = EL_STR("Memory"); } el_val_t salience = json_get_float(body, EL_STR("salience")); - if (str_eq(salience, el_from_float(0.0))) { + if (salience == el_from_float(0.0)) { salience = el_from_float(0.5); } el_val_t id = engram_node(content, node_type, salience); @@ -205,7 +207,7 @@ el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body) { relation = EL_STR("associates"); } el_val_t weight = json_get_float(body, EL_STR("weight")); - if (str_eq(weight, el_from_float(0.0))) { + if (weight == el_from_float(0.0)) { weight = el_from_float(0.5); } engram_connect(from_id, to_id, weight, relation); @@ -243,6 +245,34 @@ el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body) { return 0; } +el_val_t route_create_ise(el_val_t method, el_val_t path, el_val_t body) { + el_val_t content = json_get_string(body, EL_STR("content")); + if (str_eq(content, EL_STR(""))) { + return err_json(EL_STR("missing content")); + } + el_val_t sal = el_from_float(0.3); + el_val_t imp = el_from_float(0.3); + el_val_t conf = el_from_float(0.8); + el_val_t id = engram_node_full(content, EL_STR("InternalStateEvent"), EL_STR("state-event"), sal, imp, conf, EL_STR("Episodic"), EL_STR("[\"internal-state\",\"InternalStateEvent\"]")); + return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\"}")); + return 0; +} + +el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body) { + el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR")); + if (str_eq(dir, EL_STR(""))) { + dir = EL_STR("/tmp/engram"); + } + el_val_t snap_path = el_str_concat(dir, EL_STR("/sync-export.json")); + engram_save(snap_path); + el_val_t snap = fs_read(snap_path); + if (str_eq(snap, EL_STR(""))) { + return EL_STR("{\"nodes\":[],\"edges\":[]}"); + } + return snap; + return 0; +} + el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) { el_val_t p = json_get_string(body, EL_STR("path")); if (str_eq(p, EL_STR(""))) { @@ -299,6 +329,9 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) { return route_health(method, path, body); } } + if (str_eq(method, EL_STR("POST")) && str_starts_with(clean, EL_STR("/api/neuron/state-events"))) { + return route_create_ise(method, path, body); + } if (!check_auth_ok(method, body)) { return err_json(EL_STR("unauthorized")); } @@ -341,6 +374,9 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) { if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/strengthen")) || str_eq(clean, EL_STR("/strengthen")))) { return route_strengthen(method, path, body); } + if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/sync")) || str_eq(clean, EL_STR("/sync")))) { + return route_sync(method, path, body); + } if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/save")) || str_eq(clean, EL_STR("/save")))) { return route_save(method, path, body); } diff --git a/engram/src/server.el b/engram/src/server.el index ebaf34a..38049bd 100644 --- a/engram/src/server.el +++ b/engram/src/server.el @@ -180,6 +180,43 @@ fn route_forget(method: String, path: String, body: String) -> String { ok_json() } +// route_create_ise — POST /api/neuron/state-events +// Creates an InternalStateEvent node from a JSON body with a "content" field. +// Returns {"ok":true,"id":""}. Used by the soul daemon's ise_post() to +// record internal state transitions in the authoritative Engram store. +// This route was in the original server.el but was lost during a refactor; +// its absence would break ISE recording on the next Engram restart. +// (Restored 2026-06-30 self-review) +// importance=0.3 matches awareness.el in-process fallback (engram_node_full ISE +// defaults). Original had 0.5 which was a mismatch. (Corrected 2026-06-30) +fn route_create_ise(method: String, path: String, body: String) -> String { + let content: String = json_get_string(body, "content") + if str_eq(content, "") { return err_json("missing content") } + let sal: Float = 0.3 + let imp: Float = 0.3 + let conf: Float = 0.8 + let id: String = engram_node_full(content, "InternalStateEvent", "state-event", + sal, imp, conf, "Episodic", "[\"internal-state\",\"InternalStateEvent\"]") + "{\"ok\":true,\"id\":\"" + id + "\"}" +} + +// route_sync — GET /api/sync +// Returns the full graph snapshot as JSON (nodes + edges), suitable for loading +// via engram_load_merge. Used by the soul daemon's periodic refresh cycle to +// keep its in-process Engram store in sync with this authoritative HTTP store. +// Saves a temporary snapshot to avoid holding a large in-memory string while +// streaming — caller reads the file through the HTTP response body. +// (Restored 2026-06-30 self-review) +fn route_sync(method: String, path: String, body: String) -> String { + let dir: String = env("ENGRAM_DATA_DIR") + if str_eq(dir, "") { let dir = "/tmp/engram" } + let snap_path: String = dir + "/sync-export.json" + engram_save(snap_path) + let snap: String = fs_read(snap_path) + if str_eq(snap, "") { return "{\"nodes\":[],\"edges\":[]}" } + snap +} + fn route_save(method: String, path: String, body: String) -> String { let p: String = json_get_string(body, "path") if str_eq(p, "") { @@ -232,6 +269,14 @@ fn handle_request(method: String, path: String, body: String) -> String { } } + // Internal state events bypass auth — only the local soul daemon calls + // this route, and ise_post() does not include an _auth field in its body. + // Placing this before the auth gate preserves the old binary's behavior + // and ensures the soul daemon can always write ISEs. + if str_eq(method, "POST") && str_starts_with(clean, "/api/neuron/state-events") { + return route_create_ise(method, path, body) + } + // Auth (when ENGRAM_API_KEY is set) if !check_auth_ok(method, body) { return err_json("unauthorized") @@ -286,6 +331,11 @@ fn handle_request(method: String, path: String, body: String) -> String { return route_strengthen(method, path, body) } + // Sync — soul daemon fetches here for periodic in-process graph refresh + if str_eq(method, "GET") && (str_eq(clean, "/api/sync") || str_eq(clean, "/sync")) { + return route_sync(method, path, body) + } + // Persistence if str_eq(method, "POST") && (str_eq(clean, "/api/save") || str_eq(clean, "/save")) { return route_save(method, path, body) diff --git a/lang/releases/v1.0.0-20260501/el_runtime.c b/lang/releases/v1.0.0-20260501/el_runtime.c index 0ba1432..1aa4233 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.c +++ b/lang/releases/v1.0.0-20260501/el_runtime.c @@ -5470,9 +5470,35 @@ void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal, #define ENGRAM_WM_THRESHOLD 0.15 #define ENGRAM_WM_DECAY 0.7 #define ENGRAM_SUPPRESSION_BREAKTHROUGH 5 -#define ENGRAM_BREAKTHROUGH_WEIGHT 0.25 +/* ENGRAM_BREAKTHROUGH_WEIGHT: lowered 0.25→0.10 (2026-06-30 self-review, porting + * fix from self-review 2026-06-26 branch). With 0.25, Knowledge nodes (threshold + * 0.15) promoted at ~0.21 decay in one call to ~0.147, fall below the 0.25 floor, + * and immediately lose their WM slot to fresh breakthrough candidates at 0.25. + * Natural promotion was invisible: live data showed 524/525 WM nodes at 0.25 + * breakthrough floor. With 0.10, all per-type thresholds (minimum 0.15 Canonical) + * exceed the floor, so naturally-promoted nodes survive multiple decay cycles. + * Invariant maintained: BREAKTHROUGH_WEIGHT < min(type_thresholds). */ +#define ENGRAM_BREAKTHROUGH_WEIGHT 0.10 +/* ENGRAM_WM_CAP: hard limit on concurrent working-memory nodes (2026-06-30 + * self-review, porting fix from self-review 2026-06-26 branch). Without this, + * broad curiosity seeds like "knowledge" promote 500+ nodes simultaneously — + * wm_avg_weight collapses to the breakthrough floor, goal-bias differentiation + * is lost, and heartbeat ISEs show useless WM composition data. Cognitive + * basis: WM capacity is ~4 chunks (Cowan 2001); 24 allows richer multi-topic + * context while preventing flooding. Enforced in Pass 4 (per-call) and Pass 5 + * (global across prior-promoted nodes). */ +#define ENGRAM_WM_CAP 24 #define ENGRAM_INHIBITION_FACTOR 0.1 +/* qsort comparator — descending double, used by WM cap enforcement. */ +static int engram_cmp_double_desc(const void* a, const void* b) { + double da = *(const double*)a; + double db = *(const double*)b; + if (da > db) return -1; + if (da < db) return 1; + return 0; +} + /* ── Layered consciousness architecture ────────────────────────────────────── * * The engram graph is stratified into LAYERS that gate which suppressions @@ -6659,6 +6685,19 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { for (int64_t i = 0; i < g->node_count; i++) { if (!reached[i] || best_bg[i] <= 0.0) continue; EngramNode* n = &g->nodes[i]; + /* InternalStateEvent nodes are observability-only — never admit to WM. + * Their JSON content (curiosity seeds, heartbeat payloads) contains common + * words that trigger lexical seeding (e.g. "knowledge" in curiosity ISEs), + * leading to repeated suppression and eventual breakthrough at the floor. + * ISEs surfacing in context compilation are noise, not signal. Clear their + * suppression_count so they don't build toward breakthrough, then skip. + * (2026-06-30 self-review: porting fix from 2026-06-26 branch; SYNAPSE + * paper confirms WM should hold only semantically relevant content.) */ + if (n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0) { + n->suppression_count = 0; + wm_weights[i] = 0.0; + continue; + } /* Per-type threshold: safety nodes break through more easily. */ double type_threshold = engram_type_threshold(n->node_type, n->tier); /* Goal bias weights the node's relevance to current intent. */ @@ -6710,9 +6749,123 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { n->suppression_count = 0; } - /* Persist working_memory_weight (post Pass 3) to node store. */ + /* ── PASS 4: WM capacity cap (per-call) ───────────────────────────────── + * Enforce ENGRAM_WM_CAP as a hard upper bound on nodes promoted in this + * activation call. Without this, broad curiosity seeds like "knowledge" + * promote 500+ nodes simultaneously — wm_avg_weight collapses to the + * breakthrough floor, goal-bias differentiation is lost, and working memory + * becomes useless. (Ported from 2026-06-26 self-review branch; observed + * 525 promoted for "knowledge", 524 at breakthrough floor 0.25, 1 natural.) */ + { + int64_t cap_count = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (wm_weights[i] > 0.0) cap_count++; + } + if (cap_count > ENGRAM_WM_CAP) { + double* cap_vals = malloc((size_t)cap_count * sizeof(double)); + if (cap_vals) { + int64_t ci = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (wm_weights[i] > 0.0) cap_vals[ci++] = wm_weights[i]; + } + qsort(cap_vals, (size_t)cap_count, sizeof(double), + engram_cmp_double_desc); + /* cap_vals[ENGRAM_WM_CAP-1] is the lowest weight that still + * fits inside the cap when sorted descending. */ + double cutoff = cap_vals[ENGRAM_WM_CAP - 1]; + free(cap_vals); + /* Count strictly above cutoff to handle ties correctly. */ + int64_t above = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (wm_weights[i] > cutoff) above++; + } + int64_t at_cutoff_slots = ENGRAM_WM_CAP - above; + /* Evict nodes that don't make the cut. */ + for (int64_t i = 0; i < g->node_count; i++) { + if (wm_weights[i] <= 0.0) continue; /* not promoted */ + if (wm_weights[i] > cutoff) continue; /* above cutoff */ + if (at_cutoff_slots > 0) { + at_cutoff_slots--; + continue; /* fills a slot */ + } + wm_weights[i] = 0.0; /* over cap: evict */ + } + } + /* If malloc failed, skip cap — WM unbounded this call, no corruption. */ + } + } + + /* Persist working_memory_weight (post Pass 4) to node store. + * + * Conversational thread continuity (ENGRAM_WM_DECAY): + * Nodes promoted in a previous turn but NOT reached by the current BFS + * fan-out retain a decayed weight rather than being zeroed. This models + * the brain's ability to maintain recent context across successive turns + * without requiring explicit re-activation. A node that was relevant one + * query ago stays weakly present in working memory; a node from two + * queries ago retains 0.7² ≈ 0.49 of its original weight; after ~5 quiet + * turns it falls below 0.01 and is effectively evicted (set to 0.0). + * + * NOTE: this was documented in the ENGRAM_WM_DECAY constant comment since + * the two-layer architecture was introduced, but was never implemented — + * unreached nodes were always zeroed unconditionally. Fixed 2026-06-30 + * self-review. */ for (int64_t i = 0; i < g->node_count; i++) { - g->nodes[i].working_memory_weight = wm_weights[i]; + if (!reached[i] && g->nodes[i].working_memory_weight > 0.0) { + /* Carry-over decay: node held WM weight from prior activation but + * the current query's BFS fan-out did not reach it. Apply decay + * rather than zero so recently-active context persists. */ + double decayed = g->nodes[i].working_memory_weight * ENGRAM_WM_DECAY; + g->nodes[i].working_memory_weight = (decayed < 0.01) ? 0.0 : decayed; + } else { + g->nodes[i].working_memory_weight = wm_weights[i]; + } + } + + /* ── PASS 5: Global WM cap enforcement ─────────────────────────────────── + * Pass 4 capped this call's new candidates. But nodes already in WM from + * prior calls retain their persisted working_memory_weight (via the decay + * carry-over above). Over multiple activation calls total WM can grow well + * above ENGRAM_WM_CAP. This pass enforces the cap globally across ALL + * nodes in the store, keeping only the top ENGRAM_WM_CAP by current weight. + * Correct cognitive model: WM capacity is global (Cowan 2001); more recent + * activations outcompete older decayed ones. (Ported from 2026-06-26 + * self-review branch.) */ + { + int64_t global_wm_count = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > 0.0) global_wm_count++; + } + if (global_wm_count > ENGRAM_WM_CAP) { + double* gvals = malloc((size_t)global_wm_count * sizeof(double)); + if (gvals) { + int64_t gi = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > 0.0) + gvals[gi++] = g->nodes[i].working_memory_weight; + } + qsort(gvals, (size_t)global_wm_count, sizeof(double), + engram_cmp_double_desc); + double gcutoff = gvals[ENGRAM_WM_CAP - 1]; + free(gvals); + int64_t gabove = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > gcutoff) gabove++; + } + int64_t gslots_at_cutoff = ENGRAM_WM_CAP - gabove; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + if (n->working_memory_weight <= 0.0) continue; + if (n->working_memory_weight > gcutoff) continue; + if (gslots_at_cutoff > 0) { + gslots_at_cutoff--; + continue; /* fills a slot */ + } + n->working_memory_weight = 0.0; /* evict: over global cap */ + } + } + /* If malloc failed, skip — WM over cap this call, no data corruption. */ + } } /* ── Collect all background-activated nodes for the return value ──── @@ -7298,6 +7451,95 @@ el_val_t engram_activate_json(el_val_t query, el_val_t depth) { return el_wrap_str(b.buf); } +/* ── Working memory introspection helpers ──────────────────────────────────── + * + * These three functions give the soul daemon visibility into WM composition + * without re-running activation. Used in heartbeat ISEs and curiosity scans. + * Ported from el-compiler/runtime to releases/v1.0.0-20260501 on 2026-06-30 + * self-review (they were missing from the release build, breaking soul daemon + * compilation). */ + +el_val_t engram_wm_count(void) { + EngramStore* g = engram_get(); + int64_t count = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > 0.0) count++; + } + return (el_val_t)count; +} + +/* Average working_memory_weight across all promoted nodes (wm > 0). + * Returns the float bit-pattern via el_from_float so EL can use it with + * float_to_str / float_gt. Returns 0.0 when no nodes are promoted. + * Useful in heartbeat ISEs to distinguish "many weak activations" from + * "few strong activations". Added 2026-06-04 self-review. */ +el_val_t engram_wm_avg_weight(void) { + EngramStore* g = engram_get(); + double sum = 0.0; + int64_t count = 0; + for (int64_t i = 0; i < g->node_count; i++) { + double w = g->nodes[i].working_memory_weight; + /* Skip corrupt/out-of-range values so a single bad snapshot node + * doesn't produce a garbage average. */ + if (w > 0.0 && w <= 1.0 && isfinite(w)) { sum += w; count++; } + } + double avg = (count > 0) ? (sum / (double)count) : 0.0; + return el_from_float(avg); +} + +/* engram_wm_top_json — return top N working-memory nodes (by wm weight) as a + * compact JSON array for ISE heartbeat reporting. + * Each element: {"label":"...","node_type":"...","tier":"...","wm":0.42} + * InternalStateEvent nodes are excluded — they're observation artifacts that + * would bury substantive WM content. Added 2026-06-05 self-review. */ +el_val_t engram_wm_top_json(el_val_t n_v) { + int64_t top_n = (int64_t)n_v; + if (top_n <= 0) top_n = 10; + if (top_n > 50) top_n = 50; + EngramStore* g = engram_get(); + int64_t* idx = malloc((size_t)(g->node_count + 1) * sizeof(int64_t)); + if (!idx) return el_wrap_str(el_strdup("[]")); + int64_t mc = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > 0.0) { + const char* nt = g->nodes[i].node_type; + if (nt && strcmp(nt, "InternalStateEvent") == 0) continue; + idx[mc++] = i; + } + } + /* Insertion-sort descending by wm weight (mc is typically small). */ + for (int64_t i = 1; i < mc; i++) { + int64_t key = idx[i]; + double kw = g->nodes[key].working_memory_weight; + int64_t j = i; + while (j > 0 && g->nodes[idx[j-1]].working_memory_weight < kw) { + idx[j] = idx[j-1]; j--; + } + idx[j] = key; + } + int64_t emit = mc < top_n ? mc : top_n; + JsonBuf b; jb_init(&b); + jb_putc(&b, '['); + for (int64_t k = 0; k < emit; k++) { + EngramNode* n = &g->nodes[idx[k]]; + if (k > 0) jb_putc(&b, ','); + jb_putc(&b, '{'); + jb_puts(&b, "\"label\":"); + jb_emit_escaped(&b, n->label ? n->label : ""); + jb_puts(&b, ",\"node_type\":"); + jb_emit_escaped(&b, n->node_type ? n->node_type : ""); + jb_puts(&b, ",\"tier\":"); + jb_emit_escaped(&b, n->tier ? n->tier : ""); + char tmp[48]; + snprintf(tmp, sizeof(tmp), ",\"wm\":%.3f", n->working_memory_weight); + jb_puts(&b, tmp); + jb_putc(&b, '}'); + } + free(idx); + jb_putc(&b, ']'); + return el_wrap_str(b.buf); +} + el_val_t engram_stats_json(void) { EngramStore* g = engram_get(); char buf[128];