diff --git a/engram/dist/engram b/engram/dist/engram index 2236bee..1d7defa 100755 Binary files a/engram/dist/engram and b/engram/dist/engram differ diff --git a/engram/dist/engram.c b/engram/dist/engram.c index 9c3ffce..00aa3cc 100644 --- a/engram/dist/engram.c +++ b/engram/dist/engram.c @@ -24,6 +24,7 @@ el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body); el_val_t route_save(el_val_t method, el_val_t path, el_val_t body); el_val_t route_load(el_val_t method, el_val_t path, el_val_t body); el_val_t route_health(el_val_t method, el_val_t path, el_val_t body); +el_val_t route_embed_backfill(el_val_t method, el_val_t path, el_val_t body); el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body); el_val_t route_load_merge(el_val_t method, el_val_t path, el_val_t body); el_val_t route_emit_ise(el_val_t method, el_val_t path, el_val_t body); @@ -271,6 +272,17 @@ el_val_t route_health(el_val_t method, el_val_t path, el_val_t body) { return 0; } +el_val_t route_embed_backfill(el_val_t method, el_val_t path, el_val_t body) { + el_val_t n = query_int(path, EL_STR("n"), 32); + el_val_t result = engram_embed_backfill(n); + el_val_t done = json_get_float(result, EL_STR("embedded")); + if (done > el_from_float(0.0)) { + el_val_t saved = persist_canonical(); + } + return result; + return 0; +} + el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body) { el_val_t dir_raw = env(EL_STR("ENGRAM_DATA_DIR")); el_val_t dir = ({ el_val_t _if_result_20 = 0; if (str_eq(dir_raw, EL_STR(""))) { _if_result_20 = (EL_STR("/tmp/engram")); } else { _if_result_20 = (dir_raw); } _if_result_20; }); @@ -432,6 +444,9 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) { if (str_eq(method, EL_STR("GET")) && str_eq(clean, EL_STR("/api/sync"))) { return route_sync(method, path, body); } + if (str_eq(clean, EL_STR("/api/embed-backfill"))) { + return route_embed_backfill(method, path, body); + } return el_str_concat(el_str_concat(EL_STR("{\"error\":\"not found\",\"path\":\""), clean), EL_STR("\"}")); return 0; } diff --git a/engram/src/server.el b/engram/src/server.el index 5e72136..6689ed8 100644 --- a/engram/src/server.el +++ b/engram/src/server.el @@ -246,6 +246,28 @@ fn route_health(method: String, path: String, body: String) -> String { "{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}" } +// route_embed_backfill — GET/POST /api/embed-backfill?n=48 +// +// (2026-07-25 self-review) The lazy embedding backfill runs only inside +// engram_activate, and nothing in production calls /api/activate on this +// store — the soul's curiosity loop activates its own in-process graph. +// After a restart from a snapshot without vectors, embedded_count stalled +// at 93/12175 and would never recover. This route lets the soul's +// heartbeat pump the backfill explicitly (48/min clears a 12k backlog in +// ~4h). Persists the canonical snapshot whenever new vectors were +// generated — the 2026-07-25 regression happened precisely because 3747 +// in-RAM embeddings were never snapshotted before a restart. Self- +// limiting: once coverage is full, embedded=0 and no save occurs. +fn route_embed_backfill(method: String, path: String, body: String) -> String { + let n: Int = query_int(path, "n", 32) + let result: String = engram_embed_backfill(n) + let done: Float = json_get_float(result, "embedded") + if done > 0.0 { + let saved: Int = persist_canonical() + } + return result +} + // route_sync — return a snapshot of non-ISE/non-Working nodes for the soul daemon // to merge into its in-process graph via engram_load_merge. // @@ -495,6 +517,11 @@ fn handle_request(method: String, path: String, body: String) -> String { return route_sync(method, path, body) } + // Embedding backfill — pumped by the soul heartbeat (2026-07-25) + if str_eq(clean, "/api/embed-backfill") { + return route_embed_backfill(method, path, body) + } + "{\"error\":\"not found\",\"path\":\"" + clean + "\"}" } diff --git a/lang/releases/v1.0.0-20260501/el_runtime.c b/lang/releases/v1.0.0-20260501/el_runtime.c index cf31d3c..04f0948 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.c +++ b/lang/releases/v1.0.0-20260501/el_runtime.c @@ -5680,6 +5680,24 @@ void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal, #define ENGRAM_BLL_TAU (-3.0) #define ENGRAM_BLL_S 0.4 +/* Short-term inhibition-of-return (2026-07-25 self-review). + * Lebiere & Best 2009 ("Balancing Long-Term Reinforcement and Short-Term + * Inhibition", CogSci): subtract ln(1 + (t_n/t_s)^-d_s) from activation, + * where t_n = time since the MOST RECENT access. With their best-fit + * d_s = 1.0 the exp of the subtraction reduces to the clean multiplier + * m(t_n) = t_n / (t_n + t_s) + * applied to raw_wm in Pass 2. Immediately after a promotion the node's + * score is crushed (m → 0), then self-heals as a power law — producing an + * emergent round-robin over WM candidates instead of winner-take-all. + * This replaces the non-decaying suppression_count as the damping + * mechanism (that counter never entered the score at all — it only ever + * pushed nodes TOWARD surfacing via breakthrough; kept for that role). + * t_s = 4× the ~30 s curiosity-scan interval per the paper's guidance + * (t_s ≈ peak re-occurrence lag). A node promoted every scan holds + * m ≈ 0.2 until it loses its slot; after ~2 min unretrieved, m ≥ 0.5. + * Source: act-r.psy.cmu.edu/.../894Cogsci09-Lebiere-Best.pdf */ +#define ENGRAM_STI_TS 120.0 + /* qsort comparator — descending double, used by WM cap enforcement. */ static int engram_cmp_double_desc(const void* a, const void* b) { double da = *(const double*)a; @@ -7830,6 +7848,23 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { raw_wm += ENGRAM_EMBED_WM_WEIGHT * (cosq[i] - ENGRAM_EMBED_S0) / (1.0 - ENGRAM_EMBED_S0); } + /* Short-term inhibition-of-return (2026-07-25, Lebiere-Best): + * damp by t_n/(t_n + t_s) where t_n = seconds since the most + * recent recorded access (WM promotion / strengthen). Applied + * after the cosine term so the WHOLE score rotates — a node that + * just held a WM slot yields it even to semantically weaker + * competitors, and recovers as t_n grows. access_ts is recorded + * at promotion, so persistent WM residents self-inhibit. Nodes + * with no access history (never promoted) are uninhibited. + * Layer-0 override in Pass 3 still floors safety nodes. */ + if (n->access_filled > 0) { + int32_t sti_last = (n->access_head + ENGRAM_BLL_K - 1) + % ENGRAM_BLL_K; + double sti_tn = (double)(now_ms - n->access_ts[sti_last]) + / 1000.0; + if (sti_tn < 0.1) sti_tn = 0.1; + raw_wm *= sti_tn / (sti_tn + ENGRAM_STI_TS); + } /* Threshold gate: must exceed per-type threshold to enter working * memory. Type threshold replaces the old flat 0.2 filter. */ if (raw_wm >= type_threshold) { @@ -9080,6 +9115,43 @@ el_val_t engram_cosine_sim(el_val_t id_a, el_val_t id_b) { return el_from_float(eg_cosine(a->emb, b->emb, a->emb_dim)); } +/* engram_embed_backfill — explicitly drive the lazy embedding backfill. + * (2026-07-25 self-review.) The per-activate backfill (8 nodes/call) only + * runs inside engram_activate, and on the authoritative HTTP store nothing + * calls /api/activate in production — so embedded_count stalled at ~93 of + * 12k after a restart from a snapshot without vectors. This entry point + * lets the server expose a route the soul's heartbeat can pump. Embeds up + * to `count` eligible un-embedded nodes (clamped to [1, 64]), newest + * first, respecting the same eligibility rules and circuit breaker as the + * lazy path. Returns JSON: {"embedded":k,"embedded_count":m,"node_count":c}. + * k = 0 with the breaker open or embedder down — never blocks past the + * per-fetch timeout chain. */ +el_val_t engram_embed_backfill(el_val_t count) { + EngramStore* g = engram_get(); + int64_t want = (int64_t)EL_INT(count); + if (want < 1) want = 1; + if (want > 64) want = 64; + int64_t done = 0; + for (int64_t i = g->node_count - 1; i >= 0 && done < want; i--) { + EngramNode* n = &g->nodes[i]; + if (n->emb || !eg_embed_eligible(n)) continue; + int32_t d = 0; + float* v = eg_embed_fetch(n->content, &d); + if (!v) break; /* embedder down / breaker open — stop this call */ + n->emb = v; n->emb_dim = d; + done++; + } + int64_t total = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].emb) total++; + } + char buf[160]; + snprintf(buf, sizeof(buf), + "{\"embedded\":%lld,\"embedded_count\":%lld,\"node_count\":%lld}", + (long long)done, (long long)total, (long long)g->node_count); + return el_wrap_str(el_strdup(buf)); +} + /* engram_list_layers_json — serialized counterpart of engram_list_layers. * Returns a JSON array, sorted by activation_priority ascending. */ el_val_t engram_list_layers_json(void) { diff --git a/lang/releases/v1.0.0-20260501/el_runtime.h b/lang/releases/v1.0.0-20260501/el_runtime.h index 87ac86b..a35d1ae 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.h +++ b/lang/releases/v1.0.0-20260501/el_runtime.h @@ -618,6 +618,7 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t d el_val_t engram_activate_json(el_val_t query, el_val_t depth); el_val_t engram_stats_json(void); el_val_t engram_cosine_sim(el_val_t id_a, el_val_t id_b); +el_val_t engram_embed_backfill(el_val_t count); el_val_t engram_list_layers_json(void); /* Working memory introspection — count, mean weight, and top-N snapshot. * Ported from el-compiler/runtime on 2026-06-30 self-review. */