From 7f3d6ed8cd5006a22d8893bf226c11950c986f52 Mon Sep 17 00:00:00 2001 From: "will.anderson" Date: Mon, 3 Aug 2026 11:07:38 -0500 Subject: [PATCH] ci: update vendored el-runtime to complete v1.0.0-20260501 The runtime vendored alongside the CI pin was the Jul-21 snapshot, which predates two builtins the reconciled ship-soul now calls: - http_delete_json (boot-counter HTTP write-back, awareness/memory self-review) - engram_act_stats_json (heartbeat activation observability) Compiling dist/soul.c against the stale runtime fails with implicit-declaration errors. Vendor the current release runtime (identical to the one the soul was gate-verified against: verify-soul-contract PASS, genesis boots clean, full safety-contact) so the CI Linux soul is byte-for-byte the verified soul. --- .../el-runtime/v1.0.0-20260501/el_runtime.c | 1168 ++++++++++++++++- .../el-runtime/v1.0.0-20260501/el_runtime.h | 7 + 2 files changed, 1129 insertions(+), 46 deletions(-) diff --git a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c index b452dbf..4cc00dd 100644 --- a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c +++ b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c @@ -788,9 +788,12 @@ static long el_http_timeout_ms(void) { return parsed; } -/* Internal: do a libcurl request; takes optional body/headers, optional method override. */ -static el_val_t http_do(const char* method, const char* url, const char* body, - struct curl_slist* extra_headers) { +/* Internal: do a libcurl request; takes optional body/headers, optional method + * override, and an optional timeout override (0 = use EL_HTTP_TIMEOUT_MS). + * The override exists for the embedding path: activation must never wait the + * full 60s default on a wedged Ollama. (2026-07-24 self-review) */ +static el_val_t http_do_t(const char* method, const char* url, const char* body, + struct curl_slist* extra_headers, long timeout_ms) { if (!url || !*url) return http_error_json("empty url"); CURL* c = curl_easy_init(); if (!c) return http_error_json("curl_easy_init failed"); @@ -800,7 +803,8 @@ static el_val_t http_do(const char* method, const char* url, const char* body, curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_write_cb); curl_easy_setopt(c, CURLOPT_WRITEDATA, &rb); curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, el_http_timeout_ms()); + curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, + timeout_ms > 0 ? timeout_ms : el_http_timeout_ms()); curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf); curl_easy_setopt(c, CURLOPT_USERAGENT, "el-runtime/1.0"); @@ -811,6 +815,13 @@ static el_val_t http_do(const char* method, const char* url, const char* body, curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)(body ? strlen(body) : 0)); } else if (method && strcmp(method, "DELETE") == 0) { curl_easy_setopt(c, CURLOPT_CUSTOMREQUEST, "DELETE"); + /* DELETE with a body (2026-07-24): the engram server authenticates + * mutating requests via an "_auth" field in the JSON body, so EL + * code must be able to send DELETE + body. Absent body → unchanged. */ + if (body && *body) { + curl_easy_setopt(c, CURLOPT_POSTFIELDS, body); + curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)strlen(body)); + } } CURLcode rc = curl_easy_perform(c); curl_easy_cleanup(c); @@ -822,6 +833,12 @@ static el_val_t http_do(const char* method, const char* url, const char* body, return el_wrap_str(rb.data); } +/* Legacy entry point: default timeout. */ +static el_val_t http_do(const char* method, const char* url, const char* body, + struct curl_slist* extra_headers) { + return http_do_t(method, url, body, extra_headers, 0); +} + el_val_t http_get(el_val_t url) { return http_do("GET", EL_CSTR(url), NULL, NULL); } @@ -895,6 +912,16 @@ el_val_t http_delete(el_val_t url) { return http_do("DELETE", EL_CSTR(url), NULL, NULL); } +/* DELETE with a JSON body — required by the engram server's body-based + * "_auth" scheme for mutating requests. (2026-07-24 self-review) */ +el_val_t http_delete_json(el_val_t url, el_val_t json_body) { + struct curl_slist* h = NULL; + h = curl_slist_append(h, "Content-Type: application/json"); + el_val_t r = http_do("DELETE", EL_CSTR(url), EL_CSTR(json_body), h); + curl_slist_free_all(h); + return r; +} + /* ── HTTP → file streaming ──────────────────────────────────────────────── * * Why this exists: el_val_t strings are NUL-terminated by convention, so @@ -5581,10 +5608,10 @@ void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal, * 0.25 Lesson, 0.30 Belief/Entity, 0.40 Note/Memory/Working). This constant * is NOT used in engram_activate(); it matches the Canonical tier value only * by coincidence. (2026-07-01 self-review: clarified stale doc) - * ENGRAM_WM_DECAY: per-turn decay applied to working_memory_weight for - * nodes NOT re-activated in the current turn (conversational thread - * continuity: a node promoted in turn N persists with reduced weight - * into turn N+1 without re-activation cost). + * ENGRAM_WM_DECAY: SUPERSEDED (2026-07-22 self-review, bl-b17facdd) — the + * per-turn multiplicative carry-over decay was replaced by the ACT-R/ + * Petrov base-level scheme (see ENGRAM_BLL_* below). Kept for reference; + * no longer used in engram_activate. * ENGRAM_SUPPRESSION_BREAKTHROUGH: after this many consecutive suppressions * a latent node forces itself into working memory at reduced weight, * modelling the brain's "intrusive thought" / unresolved-tension surfacing. @@ -5606,6 +5633,43 @@ void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal, * exceed the floor, so naturally-promoted nodes survive multiple decay cycles. * Invariant maintained: BREAKTHROUGH_WEIGHT < min(type_thresholds). */ #define ENGRAM_BREAKTHROUGH_WEIGHT 0.10 +/* ENGRAM_BREAKTHROUGH_BUDGET / ENGRAM_BREAKTHROUGH_COOLDOWN (2026-08-02 + * self-review): the breakthrough path was an unbounded, self-resetting loop. + * Every reached node failing its type threshold incremented suppression_count; + * on the 5th failure it was force-promoted at exactly 0.10 AND had its counter + * reset to 0 — so it re-entered the identical cycle immediately. Because + * BREAKTHROUGH_WEIGHT (0.10) > WM_FLOOR (0.05), all of them cleared the + * absolute floor and entered the rank contest tied at 0.10, where all but a + * handful were evicted by ENGRAM_WM_CAP. Evicted nodes get no access_ts + * record (Pass 6 skips wm_weights<=0), so the short-term inhibition-of-return + * damper at ENGRAM_STI_TS never applied to them and they were re-suppressed + * completely unmarked. Steady state: N_suppressed/5 breakthroughs per call, + * essentially all of them evicted the same call. + * + * Live telemetry 2026-08-02 (boot 19, uptime 23h48m): breakthroughs_delta + * 661–903 and wm_evicted_delta 485–717 PER 60s heartbeat against wm_active + * pinned at 22–24. At ~4 activate calls/minute that is ~165–225 breakthroughs + * per call — ~825–1125 nodes cycling in 5-call lockstep. Working memory was + * not remembering; it was thrashing, and the churn drowned genuine promotion. + * + * Two bounds, both required: + * BUDGET — at most WM_CAP/4 (6) intrusive thoughts may surface per call. + * Breakthrough is meant to be an occasional intrusive thought, + * not a stampede; it can never again exceed a quarter of WM. + * COOLDOWN — on breakthrough, suppression_count is set NEGATIVE rather than + * 0, so the node needs COOLDOWN+SUPPRESSION_BREAKTHROUGH further + * suppressions before it may surface again (~60 calls ≈ 15 min at + * the current cadence, vs 5 calls ≈ 75s before). This is + * inhibition-of-return applied to the breakthrough path, matching + * the STI damper already applied to the natural path. Stored in + * the existing int32_t field — serialized as %d and parsed via + * eg_get_int_field, so negative values round-trip through + * snapshots without a struct or format change. + * When the budget or cooldown blocks a breakthrough, suppression_count is NOT + * reset — it saturates, so a starved node surfaces on a later call rather than + * restarting its climb. */ +#define ENGRAM_BREAKTHROUGH_BUDGET (ENGRAM_WM_CAP / 4) +#define ENGRAM_BREAKTHROUGH_COOLDOWN 55 /* ENGRAM_WM_CAP: hard limit on concurrent working-memory nodes (2026-06-30 * self-review, porting fix from self-review 2026-06-26 branch). Without this, * broad curiosity seeds like "knowledge" promote 500+ nodes simultaneously — @@ -5615,8 +5679,100 @@ void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal, * context while preventing flooding. Enforced in Pass 4 (per-call) and Pass 5 * (global across prior-promoted nodes). */ #define ENGRAM_WM_CAP 24 +/* ENGRAM_WM_FLOOR: absolute admission floor for a working-memory slot + * (2026-07-30 self-review). Before this, Pass 4/Pass 5/load-cap only ever + * trimmed the WM population down TO the cap and never below it — rank-based + * eviction guarantees the cap is filled whenever ≥24 nodes hold any nonzero + * weight, so wm_active was pinned at 24/24 and wm_saturated:1 was + * definitionally true on every heartbeat (carried no information). Soar's WM + * forgetting (Derbinsky & Laird, ICCM 2012) removes elements by comparing + * activation to an ABSOLUTE threshold θ, independent of how many other + * elements exist — fill below capacity is a reachable, meaningful state + * ("low cognitive load"). This floor is the weight-domain analogue of that θ: + * any slot whose weight sinks below it is dropped even when WM is under cap. + * Value 0.05 = the lowest per-type promotion threshold (Safety/DharmaSelf in + * engram_type_threshold) and the existing boot-time launder floor, and sits + * below ENGRAM_BREAKTHROUGH_WEIGHT (0.10) so intrusive-thought breakthroughs + * still surface. Applied: Pass 4, carry-over, Pass 5, load-cap. */ +#define ENGRAM_WM_FLOOR 0.05 #define ENGRAM_INHIBITION_FACTOR 0.1 +/* ── ACT-R / Petrov hybrid base-level learning (2026-07-22 self-review) ────── + * Replaces the per-turn multiplicative WM carry-over decay (weight *= 0.7 per + * engram_activate call) with wall-clock power-law decay over actual access + * history. The old scheme was call-rate-dependent: 10 curiosity scans in 10 + * seconds decayed a carried node as much as 10 scans across an hour, and a + * scalar weight loses access FREQUENCY entirely — a node touched 50 times + * decayed identically to one touched once. + * + * Petrov (2006) hybrid: keep the K most recent access timestamps exactly, + * approximate the older tail in closed form: + * + * B = ln( Σ_{j=1..k} t_j^(-d) + * + (n-k) · (t_n^(1-d) - t_k^(1-d)) / ((1-d) · (t_n - t_k)) ) + * + * d = 0.5 (canonical ACT-R decay), t_j = seconds since j-th recent access, + * t_k = seconds since oldest RETAINED access, t_n = seconds since first + * presentation (node creation), n = total presentations. + * + * This is the scheme Soar ships for working-memory forgetting (Derbinsky & + * Laird 2012). Calibration: a single-touch node has B(t) = -0.5·ln(t); with + * τ = -3.0 it falls below threshold at t = e^6 ≈ 403 s. Frequently-touched + * nodes accumulate Σ t^-0.5 mass and persist proportionally longer — recency + * AND frequency in one formula, which a decayed scalar cannot represent. + * + * Carry-over weight above threshold is shaped by the ACT-R retrieval- + * probability logistic P = σ((B − τ)/s), s = 0.4, applied to the weight the + * node held at promotion (wm_anchor) — NOT re-multiplied per call, so the + * carried weight is a pure function of wall-clock time regardless of how + * often engram_activate runs. + * Sources: alexpetrov.com/pub/iccm06 · Soar cli/cmd_wm defaults · + * arxiv.org/html/2412.05112v1 (2026-07-21 integration brief, bl-b17facdd). */ +#define ENGRAM_BLL_K 10 +#define ENGRAM_BLL_D 0.5 +#define ENGRAM_BLL_TAU (-3.0) +#define ENGRAM_BLL_S 0.4 + +/* Short-term inhibition-of-return (2026-07-25 self-review). + * Lebiere & Best 2009 ("Balancing Long-Term Reinforcement and Short-Term + * Inhibition", CogSci): subtract ln(1 + (t_n/t_s)^-d_s) from activation, + * where t_n = time since the MOST RECENT access. With their best-fit + * d_s = 1.0 the exp of the subtraction reduces to the clean multiplier + * m(t_n) = t_n / (t_n + t_s) + * applied to raw_wm in Pass 2. Immediately after a promotion the node's + * score is crushed (m → 0), then self-heals as a power law — producing an + * emergent round-robin over WM candidates instead of winner-take-all. + * This replaces the non-decaying suppression_count as the damping + * mechanism (that counter never entered the score at all — it only ever + * pushed nodes TOWARD surfacing via breakthrough; kept for that role). + * t_s = 4× the ~30 s curiosity-scan interval per the paper's guidance + * (t_s ≈ peak re-occurrence lag). A node promoted every scan holds + * m ≈ 0.2 until it loses its slot; after ~2 min unretrieved, m ≥ 0.5. + * Source: act-r.psy.cmu.edu/.../894Cogsci09-Lebiere-Best.pdf */ +#define ENGRAM_STI_TS 120.0 + +/* Carry-over occupancy inhibition (2026-07-26 self-review). + * The STI multiplier above only runs in the REACHED branch of Pass 2. + * A node carried over WITHOUT being reached (Pass 4½ below) kept + * w = wm_anchor * keep, and for a node whose base-level was inflated + * during the pre-07-25 unconditional-reinforcement era, keep ≈ 1.0 for + * days — the anchor weight is re-emitted verbatim forever. Observed + * live: wm_top0_streak = 1407 heartbeats (~23 h), one node frozen at + * its anchor 0.589 while every reached candidate rotated at the 0.10 + * breakthrough floor. Deterministic argmax over a quasi-static score + * fixates regardless of any inhibition applied only to the reached set + * (Morita et al. 2021, citing Lebiere & Best 2009). + * Fix: key inhibition on OCCUPANCY, not retrieval recency — multiply + * the carried weight by the same d_s=1 closed form over hold time + * m(t_h) = t_c / (t_c + t_h), t_h = seconds since last_activated + * (carry-over nodes are deliberately never reinforced, so + * last_activated marks when the node last EARNED its slot). Power-law + * self-healing: a node re-reached by any future activation is + * re-scored fresh in Pass 2 and re-anchors. t_c = 3600 s → a carried + * node keeps ~92% after 5 min, 50% after 1 h, ~4% after 23 h. Unlike + * m(t_n), this cannot saturate to no-op while the node camps. */ +#define ENGRAM_CARRY_TC 3600.0 + /* qsort comparator — descending double, used by WM cap enforcement. */ static int engram_cmp_double_desc(const void* a, const void* b) { double da = *(const double*)a; @@ -5753,8 +5909,329 @@ typedef struct EngramNode { * created via engram_node / engram_node_full and for snapshots that * predate the layered schema. */ uint32_t layer_id; + /* ACT-R base-level learning state (2026-07-22 self-review, ENGRAM_BLL_*). + * access_ts: ring buffer of the most recent access timestamps (ms). + * access_head: next write slot. access_filled: valid entries (≤ K). + * wm_anchor: the WM weight the node held when last promoted; carry-over + * decay is computed from this anchor as a pure function of wall-clock + * time. All fields zero via memset in creation/load paths; snapshots + * without them degrade gracefully to the optimized-form approximation + * in engram_bll_base_level. */ + int64_t access_ts[ENGRAM_BLL_K]; + int32_t access_head; + int32_t access_filled; + double wm_anchor; + /* Semantic embedding (2026-07-24 self-review, bl-b2d1c944). + * emb: malloc'd float vector (nomic-embed-text, 768-dim) or NULL. + * emb_dim: vector length; 0 = not embedded. Populated lazily by the + * backfill pass in engram_activate — NOT on the node-create hot path, + * so bulk sync imports never block on Ollama. Freed in engram_forget / + * engram_prune_telemetry; shift-copies move the pointer intact. */ + float* emb; + int32_t emb_dim; } EngramNode; +/* Record an access (ACT-R "presentation") into the base-level ring buffer. */ +static void engram_bll_record_access(EngramNode* n, int64_t now_ms) { + n->access_ts[n->access_head] = now_ms; + n->access_head = (n->access_head + 1) % ENGRAM_BLL_K; + if (n->access_filled < ENGRAM_BLL_K) n->access_filled++; +} + +/* Petrov hybrid base-level activation B in nats. Exact sum over the retained + * ring, closed-form tail for older presentations. Falls back to the ACT-R + * optimized-learning approximation B = ln(n/(1−d)) − d·ln(L) for nodes with + * no retained access history (legacy snapshots). */ +static double engram_bll_base_level(const EngramNode* n, int64_t now_ms) { + /* Total presentations: creation counts as the first, retrievals add. */ + double n_total = (double)n->activation_count + 1.0; + double t_life = (double)(now_ms - n->created_at) / 1000.0; + if (t_life < 1.0) t_life = 1.0; + if (n->access_filled == 0) { + return log(n_total / (1.0 - ENGRAM_BLL_D)) + - ENGRAM_BLL_D * log(t_life); + } + double sum = 0.0; + double t_oldest_ring = 0.0; /* seconds since oldest retained access */ + for (int32_t j = 0; j < n->access_filled; j++) { + double t = (double)(now_ms - n->access_ts[j]) / 1000.0; + if (t < 1.0) t = 1.0; + sum += pow(t, -ENGRAM_BLL_D); + if (t > t_oldest_ring) t_oldest_ring = t; + } + /* Closed-form tail for the (n − k) presentations older than the ring. + * With d = 0.5 this reduces to 2·(n−k)/(√t_n + √t_k). */ + double n_older = n_total - (double)n->access_filled; + if (n_older > 0.0 && t_life > t_oldest_ring) { + sum += n_older * (pow(t_life, 1.0 - ENGRAM_BLL_D) + - pow(t_oldest_ring, 1.0 - ENGRAM_BLL_D)) + / ((1.0 - ENGRAM_BLL_D) * (t_life - t_oldest_ring)); + } + if (sum <= 0.0) return -99.0; + return log(sum); +} + +/* Parse a persisted "access_ts" comma list (chronological order) into the + * ring buffer. Tolerates absence, empty strings, and stray whitespace. */ +static void engram_bll_parse_access(EngramNode* nn, const char* s) { + if (!s) return; + const char* p = s; + while (*p) { + char* endp = NULL; + long long v = strtoll(p, &endp, 10); + if (endp == p) break; + if (v > 0) engram_bll_record_access(nn, (int64_t)v); + p = endp; + while (*p == ',' || *p == ' ') p++; + } +} + +/* ── Embedding-based activation (2026-07-24 self-review, bl-b2d1c944) ────── + * Closes the gap named in every heartbeat since 2026-06-30: "no embedding + * call is made during activation. The seed-finding loop uses istr_contains + * only." Design per the 2026-07-21 integration brief: + * - Lazy backfill: engram_activate embeds up to BACKFILL_PER_CALL + * un-embedded, non-telemetry nodes per call, newest first. No latency + * on node-create paths; a fresh node is embedded within ~1 scan cycle. + * - Semantic seeding (HippoRAG pattern, use similarity twice): the query + * is embedded, the top-K nodes by cosine ≥ SEED_MIN join the seed set + * with initial activation = the similarity itself. + * - Additive WM term: raw_wm += W * relu((cos − S0)/(1 − S0)). Shift-and- + * floor at S0 because nomic-embed scores unrelated pairs 0.4–0.5; raw + * cosine in a weighted sum is a constant bias, not a signal. + * - Circuit breaker: 3 consecutive Ollama failures → stop trying for 5 + * minutes. Activation NEVER blocks on a dead embedder beyond timeout. + */ +#define ENGRAM_EMBED_S0 0.45 +#define ENGRAM_EMBED_WM_WEIGHT 0.20 +#define ENGRAM_EMBED_SEED_K 8 +#define ENGRAM_EMBED_SEED_MIN 0.60 +#define ENGRAM_EMBED_BACKFILL_PER_CALL 8 +/* ENGRAM_QGATE_FLOOR: minimum propagation multiplier for an EMBEDDED target + * node with zero/negative query similarity. Query-aware spreading gate + * (arXiv:2606.30133) adapted for partial embedding coverage — see the + * propagation loop in engram_activate. 0.25 damps semantically unrelated + * branches ~4x without severing them. Unembedded targets are ungated. */ +#define ENGRAM_QGATE_FLOOR 0.25 +#define ENGRAM_EMBED_MAX_CHARS 2000 +#define ENGRAM_EMBED_TIMEOUT_MS 4000L +#define ENGRAM_EMBED_BREAKER_LIMIT 3 +#define ENGRAM_EMBED_BREAKER_COOLDOWN_MS 300000 + +/* ── Context centroid (2026-07-29 self-review) ─────────────────────────────── + * Closes the last gap from the 2026-07-21 decay/embedding brief: cosine was + * computed against the per-call query embedding ONLY, so every curiosity scan + * was semantically memoryless — the 4 rotating seed phrases fully determined + * what ignited, with zero continuity from what the system actually touched. + * + * Mechanism (brief spec): a running EMA centroid over touch embeddings, + * c ← normalize(μ·c + (1−μ)·e_touch), μ = ENGRAM_CTX_MU + * where a "touch" is (a) the query embedding each activate call and (b) the + * embeddings of up to ENGRAM_CTX_TOUCH_MAX top WM-promoted survivors of that + * call — promotion is the retrieval event (same rule as BLL reinforcement). + * + * Feedback-loop guard: scoring does NOT use the raw centroid. The lit + * failure mode of centroid memories (EMA of your own outputs → runaway + * attractor; cf. the wm_top0_streak=1407 freeze this store already hit) is + * bounded by scoring against a query-dominant blend: + * e_eff = normalize(α·e_q + (1−α)·c), α = ENGRAM_CTX_QALPHA + * so the exogenous rotating seeds always contribute the majority of the + * scoring direction; the centroid is a context tint, not the signal. + * + * Observability: _eg_act_ctx_cos = cos(e_q, c) BEFORE the query is blended + * in. ~1.0 → centroid aligned with current query; low → context and query + * have diverged (expected at domain-rotation boundaries); -2.0 → no centroid + * yet / embedder down. Exposed via engram_act_stats_json → heartbeat ISE so + * drift is diagnosable from telemetry. In-memory only: context is + * short-term by definition, a restart legitimately starts cold. */ +#define ENGRAM_CTX_MU 0.90 +#define ENGRAM_CTX_QALPHA 0.65 +#define ENGRAM_CTX_TOUCH_MAX 8 + +static float* _eg_ctx_c = NULL; +static int32_t _eg_ctx_dim = 0; +static double _eg_act_ctx_cos = -2.0; + +static int _eg_embed_consec_fail = 0; +static int64_t _eg_embed_breaker_until = 0; + +/* ── Activation observability counters (2026-07-27 self-review) ────────────── + * The executive-filter pathologies this system has repeatedly debugged by + * inference (WM flooded with breakthrough-floor nodes, silent cap evictions, + * embedder wedged behind the breaker) were all invisible: no ISE, no stats + * field. engram_act_stats_json() exposes them so the soul heartbeat can emit + * them. + * + * CUMULATIVE CONTRACT (2026-07-31 self-review): these are monotonic totals + * for the process lifetime, like pulse/sync_added_total — NOT per-call. The + * original per-call reset meant engram_act_stats_json only reported the LAST + * activate call, and the 60s heartbeat (2 curiosity activates per 30s in + * between) missed nearly every eviction/breakthrough event. Consumers wanting + * rates keep the previous reading and diff. Restart legitimately resets to 0. */ +static int64_t _eg_act_breakthroughs = 0; /* forced promotions at the floor, cumulative */ +static int64_t _eg_act_wm_evicted = 0; /* ALL WM evictions, cumulative (see below) */ +/* 2026-08-02 self-review: this counted only the three Pass 4 / Pass 5 floor + * and rank paths. The two carry-over eviction paths (base-level below τ, and + * decayed weight below WM_FLOOR) were silent, so the reported eviction rate + * was an undercount of unknown magnitude — which mattered precisely while + * diagnosing the breakthrough storm. All five paths now increment. */ + +static int64_t engram_now_ms(void); /* defined in the store section below */ + +static const char* eg_embed_url(void) { + const char* s = getenv("EL_EMBED_URL"); + return (s && *s) ? s : "http://localhost:11434/api/embeddings"; +} +static const char* eg_embed_model(void) { + const char* s = getenv("EL_EMBED_MODEL"); + return (s && *s) ? s : "nomic-embed-text"; +} + +/* Cosine similarity over raw (unnormalized — nomic emits magnitudes >1) + * float vectors. Returns -2.0 on dim mismatch / null input so callers can + * distinguish "orthogonal" (0.0) from "not comparable". */ +static double eg_cosine(const float* a, const float* b, int32_t dim) { + if (!a || !b || dim <= 0) return -2.0; + double dot = 0.0, na = 0.0, nb = 0.0; + for (int32_t i = 0; i < dim; i++) { + dot += (double)a[i] * (double)b[i]; + na += (double)a[i] * (double)a[i]; + nb += (double)b[i] * (double)b[i]; + } + if (na <= 0.0 || nb <= 0.0) return -2.0; + return dot / (sqrt(na) * sqrt(nb)); +} + +/* eg_ctx_blend — fold one touch embedding into the context centroid: + * c ← normalize(μ·c + (1−μ)·e). Initializes the centroid (normalized copy) + * on first touch or dim change; silently skips degenerate vectors. */ +static void eg_ctx_blend(const float* e, int32_t dim) { + if (!e || dim <= 0) return; + double ne = 0.0; + for (int32_t i = 0; i < dim; i++) ne += (double)e[i] * (double)e[i]; + if (ne <= 0.0) return; + ne = sqrt(ne); + if (!_eg_ctx_c || _eg_ctx_dim != dim) { + float* c = malloc((size_t)dim * sizeof(float)); + if (!c) return; + for (int32_t i = 0; i < dim; i++) c[i] = (float)((double)e[i] / ne); + free(_eg_ctx_c); + _eg_ctx_c = c; + _eg_ctx_dim = dim; + return; + } + double nc = 0.0; + for (int32_t i = 0; i < dim; i++) { + double v = ENGRAM_CTX_MU * (double)_eg_ctx_c[i] + + (1.0 - ENGRAM_CTX_MU) * ((double)e[i] / ne); + _eg_ctx_c[i] = (float)v; + nc += v * v; + } + if (nc > 0.0) { + nc = sqrt(nc); + for (int32_t i = 0; i < dim; i++) + _eg_ctx_c[i] = (float)((double)_eg_ctx_c[i] / nc); + } +} + +/* Fetch an embedding from Ollama. Returns malloc'd float[dim] or NULL. + * Truncates input to ENGRAM_EMBED_MAX_CHARS and JSON-escapes it. Honors the + * circuit breaker; a NULL return is always safe to ignore (fail-soft). */ +static float* eg_embed_fetch(const char* text, int32_t* out_dim) { + *out_dim = 0; + if (!text || !*text) return NULL; + int64_t now = engram_now_ms(); + if (now < _eg_embed_breaker_until) return NULL; + /* Build request body with escaped, truncated prompt. */ + size_t tlen = strlen(text); + if (tlen > ENGRAM_EMBED_MAX_CHARS) tlen = ENGRAM_EMBED_MAX_CHARS; + char* esc = malloc(tlen * 6 + 1); + if (!esc) return NULL; + size_t w = 0; + for (size_t i = 0; i < tlen; i++) { + unsigned char c = (unsigned char)text[i]; + if (c == '"' || c == '\\') { esc[w++] = '\\'; esc[w++] = (char)c; } + else if (c == '\n') { esc[w++] = '\\'; esc[w++] = 'n'; } + else if (c == '\r') { esc[w++] = '\\'; esc[w++] = 'r'; } + else if (c == '\t') { esc[w++] = '\\'; esc[w++] = 't'; } + else if (c < 0x20) { w += (size_t)snprintf(esc + w, 7, "\\u%04x", c); } + else esc[w++] = (char)c; + } + esc[w] = '\0'; + size_t blen = w + strlen(eg_embed_model()) + 64; + char* body = malloc(blen); + if (!body) { free(esc); return NULL; } + snprintf(body, blen, "{\"model\":\"%s\",\"prompt\":\"%s\"}", + eg_embed_model(), esc); + free(esc); + struct curl_slist* h = curl_slist_append(NULL, "Content-Type: application/json"); + el_val_t resp = http_do_t("POST", eg_embed_url(), body, h, + ENGRAM_EMBED_TIMEOUT_MS); + curl_slist_free_all(h); + free(body); + const char* r = EL_CSTR(resp); + const char* arr = r ? strstr(r, "\"embedding\"") : NULL; + if (!arr) { + if (++_eg_embed_consec_fail >= ENGRAM_EMBED_BREAKER_LIMIT) { + _eg_embed_breaker_until = now + ENGRAM_EMBED_BREAKER_COOLDOWN_MS; + _eg_embed_consec_fail = 0; + } + return NULL; + } + arr = strchr(arr, '['); + if (!arr) return NULL; + arr++; + int32_t cap = 1024, dim = 0; + float* v = malloc((size_t)cap * sizeof(float)); + if (!v) return NULL; + const char* p = arr; + while (*p && *p != ']') { + char* endp = NULL; + double d = strtod(p, &endp); + if (endp == p) break; + if (dim >= cap) { break; } /* >1024 dims: refuse, model mismatch */ + v[dim++] = (float)d; + p = endp; + while (*p == ',' || *p == ' ' || *p == '\n') p++; + } + if (dim < 8) { free(v); return NULL; } /* junk response */ + _eg_embed_consec_fail = 0; + *out_dim = dim; + return v; +} + +/* Node types that never receive embeddings: pure telemetry and structural + * plumbing. Everything else (Knowledge, Memory, BacklogItem, Entity, ...) + * is eligible. */ +static int eg_embed_eligible(const EngramNode* n) { + if (!n->content || strlen(n->content) < 8) return 0; + if (!n->node_type) return 1; + if (strcmp(n->node_type, "InternalStateEvent") == 0) return 0; + if (strcmp(n->node_type, "Tag") == 0) return 0; + return 1; +} + +/* Parse a persisted comma-separated float list into node->emb. */ +static void eg_parse_emb(EngramNode* nn, const char* s) { + if (!s || !*s) return; + int32_t cap = 1024, dim = 0; + float* v = malloc((size_t)cap * sizeof(float)); + if (!v) return; + const char* p = s; + while (*p) { + char* endp = NULL; + double d = strtod(p, &endp); + if (endp == p) break; + if (dim >= cap) break; + v[dim++] = (float)d; + p = endp; + while (*p == ',' || *p == ' ') p++; + } + if (dim < 8) { free(v); return; } + nn->emb = v; + nn->emb_dim = dim; +} + typedef struct EngramEdge { char* id; char* from_id; @@ -6284,12 +6761,19 @@ static el_val_t engram_node_to_map(const EngramNode* n) { m = el_map_set(m, EL_STR(el_strdup("working_memory_weight")), el_from_float(n->working_memory_weight)); m = el_map_set(m, EL_STR(el_strdup("suppression_count")), (el_val_t)n->suppression_count); m = el_map_set(m, EL_STR(el_strdup("layer_id")), (el_val_t)(int64_t)n->layer_id); + /* Observability (2026-07-22): expose the current ACT-R base-level and + * promotion anchor so heartbeat ISEs / API consumers can see decay state. */ + m = el_map_set(m, EL_STR(el_strdup("wm_anchor")), el_from_float(n->wm_anchor)); + m = el_map_set(m, EL_STR(el_strdup("base_level")), el_from_float(engram_bll_base_level(n, engram_now_ms()))); + /* emb_dim only — the vector itself is too large for map/API output. + * 0 = not yet embedded by the lazy backfill. (2026-07-24) */ + m = el_map_set(m, EL_STR(el_strdup("emb_dim")), (el_val_t)(int64_t)n->emb_dim); return m; } /* (Node JSON serialization is provided by `engram_emit_node_json` further * down in the persistence section — reused by the *_json builtins below.) */ -static void engram_emit_node_json(JsonBuf* b, const EngramNode* n); +static void engram_emit_node_json(JsonBuf* b, const EngramNode* n, int include_emb); static void engram_emit_edge_json(JsonBuf* b, const EngramEdge* e); /* Salience may arrive either as a float bit-pattern or as a small integer @@ -6579,6 +7063,16 @@ void engram_strengthen(el_val_t node_id) { n->activation_count++; n->last_activated = engram_now_ms(); n->updated_at = n->last_activated; + /* 2026-07-26 self-review: REMOVED the BLL access record added on + * 2026-07-22 ("explicit strengthen is a presentation too"). The + * 2026-07-25 STI multiplier reads the same access ring — so an + * explicit strengthen crushed the strengthened node's promotion + * score by ×t_n/(t_n+120) for the next ~2 minutes. The awareness + * loop strengthens exactly when a node NEWLY reaches WM top + * (novelty gating); recording an access here made that + * reinforcement self-defeating. Salience/activation_count bumps + * above carry the reinforcement; the access ring stays reserved + * for genuine retrieval events (promotions in engram_activate). */ } void engram_forget(el_val_t node_id) { @@ -6591,6 +7085,7 @@ void engram_forget(el_val_t node_id) { EngramNode* n = &g->nodes[idx]; free(n->id); free(n->content); free(n->node_type); free(n->label); free(n->tier); free(n->tags); free(n->metadata); + free(n->emb); /* Shift remaining nodes down */ for (int64_t i = idx + 1; i < g->node_count; i++) { g->nodes[i - 1] = g->nodes[i]; @@ -6676,6 +7171,7 @@ el_val_t engram_prune_telemetry(el_val_t older_than_ms) { removed_ids[removed++] = n->id; /* keep id for edge sweep */ free(n->content); free(n->node_type); free(n->label); free(n->tier); free(n->tags); free(n->metadata); + free(n->emb); } else { if (w != i) g->nodes[w] = g->nodes[i]; w++; @@ -6849,14 +7345,24 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { return lst; } -/* Sort node indices by salience desc (small N, insertion sort is fine). */ +/* Sort node indices by salience desc, tie-break created_at desc (newest + * first). The tie-break matters for telemetry: InternalStateEvent nodes all + * share salience 0.3, so before this the listing routes returned them in + * store order — OLDEST first — and any limited query (/api/nodes?...&limit=N) + * silently returned a stale window. A 41-hour-old heartbeat series read as a + * live outage during the 2026-07-22 self-review. Newest-first ties make + * limited scans return the recent window consumers actually want. + * (Small N, insertion sort is fine.) */ static void engram_sort_indices_by_salience(int64_t* arr, int64_t n, const EngramNode* nodes) { for (int64_t i = 1; i < n; i++) { int64_t key = arr[i]; double ks = nodes[key].salience; + int64_t kc = nodes[key].created_at; int64_t j = i - 1; - while (j >= 0 && nodes[arr[j]].salience < ks) { + while (j >= 0 && (nodes[arr[j]].salience < ks || + (nodes[arr[j]].salience == ks && + nodes[arr[j]].created_at < kc))) { arr[j + 1] = arr[j]; j--; } @@ -7093,7 +7599,7 @@ static double engram_temporal_proximity_bonus(int64_t node_created, * * Working memory persistence (turn continuity): * Nodes promoted in the previous turn retain a decayed working_memory_weight - * (weight *= ENGRAM_WM_DECAY) without needing re-activation. This models + * (ACT-R base-level carry-over, 2026-07-22) without needing re-activation. This models * conversational thread continuity — once a topic is in working memory, * it persists slightly into the next turn. * @@ -7202,12 +7708,106 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { int64_t now_ms = engram_now_ms(); + /* Observability counters: _eg_act_breakthroughs/_eg_act_wm_evicted are + * CUMULATIVE for the process lifetime and intentionally NOT reset here + * (2026-07-31 self-review — the old per-call reset made the 60s heartbeat + * miss nearly all events between beats; see the definition site). + * ctx_cos stays per-call: it is a gauge of THIS query vs the centroid. */ + _eg_act_ctx_cos = -2.0; + + /* ── Embedding backfill + query embedding (2026-07-24, bl-b2d1c944) ── + * Backfill: embed up to N un-embedded eligible nodes per call, newest + * first (append order ≈ creation order), so fresh content is semantic- + * searchable within one scan cycle and the historical store fills in + * gradually — ~16 nodes/min under the 30s curiosity cadence, no bulk + * hammering of Ollama, no latency on any create path. */ + { + int backfilled = 0; + for (int64_t i = g->node_count - 1; + i >= 0 && backfilled < ENGRAM_EMBED_BACKFILL_PER_CALL; i--) { + EngramNode* n = &g->nodes[i]; + if (n->emb || !eg_embed_eligible(n)) continue; + int32_t d = 0; + float* v = eg_embed_fetch(n->content, &d); + if (!v) break; /* embedder down / breaker open — stop this call */ + n->emb = v; n->emb_dim = d; + backfilled++; + } + } + /* Query embedding, cached single-slot: the curiosity loop re-issues the + * same 4 rotating phrases, so consecutive identical queries skip the + * HTTP round-trip entirely. */ + static char* _eg_qcache_text = NULL; + static float* _eg_qcache_emb = NULL; + static int32_t _eg_qcache_dim = 0; + float* q_emb = NULL; + int32_t q_dim = 0; + if (_eg_qcache_text && strcmp(_eg_qcache_text, q) == 0) { + q_emb = _eg_qcache_emb; q_dim = _eg_qcache_dim; + } else { + int32_t d = 0; + float* v = eg_embed_fetch(q, &d); + if (v) { + free(_eg_qcache_text); free(_eg_qcache_emb); + _eg_qcache_text = strdup(q); + _eg_qcache_emb = v; + _eg_qcache_dim = d; + q_emb = v; q_dim = d; + } + } + /* ── Context centroid fold-in (2026-07-29) ────────────────────────── + * Record drift BEFORE blending (cos of the query against yesterday's + * context), then fold the query in as a touch, then build the + * query-dominant effective scoring vector. See the ENGRAM_CTX_* block + * for the design and the feedback-loop guard rationale. */ + float* e_eff = NULL; + if (q_emb) { + if (_eg_ctx_c && _eg_ctx_dim == q_dim) + _eg_act_ctx_cos = eg_cosine(q_emb, _eg_ctx_c, q_dim); + eg_ctx_blend(q_emb, q_dim); + if (_eg_ctx_c && _eg_ctx_dim == q_dim) { + e_eff = malloc((size_t)q_dim * sizeof(float)); + if (e_eff) { + double nq = 0.0; + for (int32_t i = 0; i < q_dim; i++) + nq += (double)q_emb[i] * (double)q_emb[i]; + nq = (nq > 0.0) ? sqrt(nq) : 1.0; + double nn = 0.0; + for (int32_t i = 0; i < q_dim; i++) { + double v = ENGRAM_CTX_QALPHA * ((double)q_emb[i] / nq) + + (1.0 - ENGRAM_CTX_QALPHA) * (double)_eg_ctx_c[i]; + e_eff[i] = (float)v; + nn += v * v; + } + if (nn <= 0.0) { free(e_eff); e_eff = NULL; } + } + } + } + /* Per-node cosine vs the effective query (query ⊕ context centroid; + * plain query on cold start), computed once, consumed twice: semantic + * seeding below and the additive WM term in Pass 2 (use similarity + * twice, coherently — HippoRAG). cosq stays NULL when the embedder is + * unavailable; every consumer degrades to pure lexical behavior. */ + double* cosq = NULL; + if (q_emb) { + const float* qv = e_eff ? e_eff : q_emb; + cosq = calloc((size_t)g->node_count, sizeof(double)); + if (cosq) { + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + cosq[i] = (n->emb && n->emb_dim == q_dim) + ? eg_cosine(n->emb, qv, q_dim) : -2.0; + } + } + } + free(e_eff); e_eff = NULL; /* only needed to fill cosq */ + /* Per-node layer-1 tracking. */ double* best_bg = calloc((size_t)g->node_count, sizeof(double)); int64_t* best_hops = calloc((size_t)g->node_count, sizeof(int64_t)); int* reached = calloc((size_t)g->node_count, sizeof(int)); if (!best_bg || !best_hops || !reached) { - free(best_bg); free(best_hops); free(reached); return out; + free(best_bg); free(best_hops); free(reached); free(cosq); return out; } /* ── LAYER 1: broad fan-out (background activation) ───────────────── @@ -7218,7 +7818,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { SeedEntry* seeds = malloc((size_t)g->node_count * sizeof(SeedEntry)); int64_t seed_count = 0; if (!seeds) { - free(best_bg); free(best_hops); free(reached); return out; + free(best_bg); free(best_hops); free(reached); free(cosq); return out; } /* Tokenize once: a node seeds if it matches ANY query token, and its seed * activation is scaled by token coverage (fraction of distinct query @@ -7257,6 +7857,35 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { reached[i] = 1; } } + /* ── Semantic seed supplement (2026-07-24, bl-b2d1c944) ───────────── + * Top-K nodes by cosine ≥ SEED_MIN join the seed set with initial + * activation = similarity × the same decay/dampen shaping the lexical + * seeds get. This is the fix for "idle cognition firing blanks": a + * curiosity phrase like "decision pattern lesson" now ignites nodes + * that MEAN decisions and lessons, not just nodes that contain those + * literal substrings. Lexically-seeded nodes are skipped — the lexical + * path already gave them coverage-scaled activation. */ + if (cosq) { + for (int k = 0; k < ENGRAM_EMBED_SEED_K; k++) { + int64_t bi = -1; double bc = ENGRAM_EMBED_SEED_MIN; + for (int64_t i = 0; i < g->node_count; i++) { + if (reached[i]) continue; + if (cosq[i] > bc) { bc = cosq[i]; bi = i; } + } + if (bi < 0) break; + EngramNode* n = &g->nodes[bi]; + double tdecay = engram_temporal_decay(n, now_ms); + double dampen = engram_activation_dampen(n); + double act = bc * tdecay * dampen; + seeds[seed_count].idx = bi; + seeds[seed_count].act = act; + seeds[seed_count].created_at = n->created_at; + seed_count++; + best_bg[bi] = act; + best_hops[bi] = 0; + reached[bi] = 1; + } + } /* Compute mean seed created_at for temporal proximity bonus. * Was a running pairwise average — seed_epoch = (seed_epoch + t_s)/2 — * which is NOT the arithmetic mean: it exponentially over-weights the @@ -7275,7 +7904,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { typedef struct { int64_t idx; int64_t hops; double act; } Frontier; Frontier* fr = malloc((size_t)(g->node_count * (max_depth + 1)) * sizeof(Frontier) + 16 * sizeof(Frontier)); if (!fr) { - free(best_bg); free(best_hops); free(reached); free(seeds); return out; + free(best_bg); free(best_hops); free(reached); free(seeds); free(cosq); return out; } int64_t fhead = 0, ftail = 0; int64_t fcap = (int64_t)((size_t)(g->node_count * (max_depth + 1)) + 16); @@ -7335,8 +7964,31 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { double tbonus = engram_temporal_proximity_bonus(on->created_at, seed_epoch); double tdecay = engram_temporal_decay(on, now_ms); double dampen = engram_activation_dampen(on); + /* ── Query-aware propagation gate (2026-07-27 self-review) ── + * Prior behavior was "query-blind" spreading: the query chose + * the seeds, but propagation depended only on graph structure, + * so high-degree hubs relayed activation into branches with no + * semantic relation to the query. Per arXiv:2606.30133, gating + * each increment by the TARGET node's query similarity + * (sigma(v) = max(cos(e_v, e_q), 0)) prunes low-information + * branches at every hop (+3.6..+7.4 F1 over uniform spreading, + * 1.5-4.9x faster via a shrinking working set). + * + * Adaptation for partial embedding coverage: the paper skips + * unembedded targets outright, but only eligible non-ISE/Tag + * nodes carry embeddings here — a hard gate would sever purely + * lexical/structural pathways. So: embedded targets get a soft + * gate FLOOR + (1-FLOOR)*clip(cos) (dissimilar nodes damped + * ~4x, never killed); unembedded targets pass ungated (no + * information, no penalty); cosq == NULL (embedder down) means + * no gating at all — same graceful degradation as seeding. */ + double qgate = 1.0; + if (cosq && cosq[oi] > -1.5) { + double c = cosq[oi] > 0.0 ? cosq[oi] : 0.0; + qgate = ENGRAM_QGATE_FLOOR + (1.0 - ENGRAM_QGATE_FLOOR) * c; + } double new_act = f.act * e->weight * SPREAD_DECAY * (1.0 + tbonus) - * tdecay * dampen; + * tdecay * dampen * qgate; /* Firing threshold per classic spreading-activation: sub-threshold * activation neither updates the target nor enqueues it, so weak * signals die out instead of flooding the whole graph with tiny @@ -7369,7 +8021,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { double* inhibition = calloc((size_t)g->node_count, sizeof(double)); if (!inhibition) { free(best_bg); free(best_hops); free(reached); free(seeds); free(fr); - return out; + free(cosq); return out; } for (int64_t ei = 0; ei < g->edge_count; ei++) { EngramEdge* e = &g->edges[ei]; @@ -7394,8 +8046,10 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { double* wm_weights = calloc((size_t)g->node_count, sizeof(double)); if (!wm_weights) { free(best_bg); free(best_hops); free(reached); free(seeds); - free(fr); free(inhibition); return out; + free(fr); free(inhibition); free(cosq); return out; } + /* Per-call breakthrough budget (2026-08-02) — see ENGRAM_BREAKTHROUGH_BUDGET. */ + int64_t bt_budget = ENGRAM_BREAKTHROUGH_BUDGET; for (int64_t i = 0; i < g->node_count; i++) { if (!reached[i] || best_bg[i] <= 0.0) continue; EngramNode* n = &g->nodes[i]; @@ -7416,13 +8070,60 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { double type_threshold = engram_type_threshold(n->node_type, n->tier); /* Goal bias weights the node's relevance to current intent. */ double bias = engram_goal_bias(n, q); - /* Raw working memory score. */ - double raw_wm = best_bg[i] * bias * n->confidence; + /* Raw working memory score. + * Importance factor (2026-07-28 self-review): n->importance was + * stored, serialized, and clamped at creation (default 0.5) but + * never read by any scoring path — a curated importance=1.0 node + * competed identically with a throwaway note at equal activation. + * Map it to a gentle multiplier centered on the 0.5 default: + * impf = 0.5 + importance → default nodes unchanged (×1.0), + * critical (1.0) ×1.5, low (0.1) ×0.6. Nodes loaded from legacy + * snapshots with importance<=0 stay neutral rather than being + * silently suppressed. */ + double impf = (n->importance > 0.0) ? (0.5 + n->importance) : 1.0; + double raw_wm = best_bg[i] * bias * n->confidence * impf; /* Apply inhibitory suppression. Full inhibition → scale by factor. */ double inh = inhibition[i]; if (inh > 1.0) inh = 1.0; double suppress = 1.0 - (1.0 - ENGRAM_INHIBITION_FACTOR) * inh; raw_wm *= suppress; + /* Short-term inhibition-of-return (2026-07-25, Lebiere-Best): + * damp by t_n/(t_n + t_s) where t_n = seconds since the most + * recent recorded access (WM promotion / strengthen). A node that + * just held a WM slot yields it even to structurally stronger + * competitors, and recovers as t_n grows. access_ts is recorded + * at promotion, so persistent WM residents self-inhibit. Nodes + * with no access history (never promoted) are uninhibited. + * Layer-0 override in Pass 3 still floors safety nodes. */ + if (n->access_filled > 0) { + int32_t sti_last = (n->access_head + ENGRAM_BLL_K - 1) + % ENGRAM_BLL_K; + double sti_tn = (double)(now_ms - n->access_ts[sti_last]) + / 1000.0; + if (sti_tn < 0.1) sti_tn = 0.1; + raw_wm *= sti_tn / (sti_tn + ENGRAM_STI_TS); + } + /* Additive semantic-relevance term (2026-07-24, bl-b2d1c944): + * shift-and-floor at S0 — nomic-embed scores unrelated pairs + * 0.4–0.5, so raw cosine in a weighted sum would be a constant + * bias swamping the decayed base-level signal. Above S0 the term + * ramps 0 → WM_WEIGHT, breaking ties among structurally equivalent + * candidates in favor of nodes that mean what the query means. + * + * MOVED AFTER the STI damper (2026-08-02 self-review). It used to + * be added BEFORE, so the recency multiplier scaled the semantic + * term too: an incumbent re-reached 30s later took t_n/(t_n+120) + * = 0.2×, cutting the cosine term's ceiling from 0.20 to 0.04 — + * below every per-type threshold (0.15–0.40). Meaning-match was + * being punished for having been recently useful. Inhibition-of- + * return should rotate the STRUCTURAL score (what the graph + * dragged in), not the semantic one (what the query actually + * means); relevance to the current query is not stale merely + * because the node was in WM a moment ago. */ + if (cosq && cosq[i] > ENGRAM_EMBED_S0) { + raw_wm += ENGRAM_EMBED_WM_WEIGHT + * (cosq[i] - ENGRAM_EMBED_S0) / (1.0 - ENGRAM_EMBED_S0); + } /* Threshold gate: must exceed per-type threshold to enter working * memory. Type threshold replaces the old flat 0.2 filter. */ if (raw_wm >= type_threshold) { @@ -7430,13 +8131,41 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { if (n->suppression_count > 0) n->suppression_count = 0; } else { /* Node didn't make it through — increment suppression counter. - * After N consecutive suppressions: force breakthrough. */ + * After N consecutive suppressions it MAY force a breakthrough, + * subject to the per-call budget and the negative-count cooldown + * (2026-08-02 — see ENGRAM_BREAKTHROUGH_BUDGET/_COOLDOWN). */ n->suppression_count++; - if (n->suppression_count >= ENGRAM_SUPPRESSION_BREAKTHROUGH) { - wm_weights[i] = ENGRAM_BREAKTHROUGH_WEIGHT; - n->suppression_count = 0; + if (n->suppression_count >= ENGRAM_SUPPRESSION_BREAKTHROUGH + && bt_budget > 0) { + /* Graded breakthrough weight (2026-08-02): previously every + * breakthrough landed on exactly ENGRAM_BREAKTHROUGH_WEIGHT, + * so hundreds tied at 0.10 and the rank-cap tie-break at the + * cutoff degenerated to node-array index order — i.e. whoever + * was inserted earliest won, which is not a cognitive + * criterion. Scale within ±10% by how close the node came to + * its own threshold, so a near-miss outranks a node that was + * nowhere near. Stays strictly below min(type_threshold) + * (0.15) and strictly above ENGRAM_WM_FLOOR (0.05), which is + * the invariant ENGRAM_BREAKTHROUGH_WEIGHT documents. */ + double near = (type_threshold > 0.0) + ? (raw_wm / type_threshold) : 0.0; + if (near < 0.0) near = 0.0; + if (near > 1.0) near = 1.0; + wm_weights[i] = ENGRAM_BREAKTHROUGH_WEIGHT + * (0.9 + 0.2 * near); + /* Negative = cooldown. Must climb back through the cooldown + * before it can breach again. */ + n->suppression_count = -ENGRAM_BREAKTHROUGH_COOLDOWN; + bt_budget--; + _eg_act_breakthroughs++; } else { wm_weights[i] = 0.0; + /* Budget-starved or cooling down: do NOT reset the counter — + * let it saturate so the node surfaces on a later call rather + * than restarting its climb from zero. Cap the ceiling so the + * int32 cannot drift unbounded over a long uptime. */ + if (n->suppression_count > ENGRAM_SUPPRESSION_BREAKTHROUGH * 4) + n->suppression_count = ENGRAM_SUPPRESSION_BREAKTHROUGH * 4; } } } @@ -7471,6 +8200,16 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { * becomes useless. (Ported from 2026-06-26 self-review branch; observed * 525 promoted for "knowledge", 524 at breakthrough floor 0.25, 1 natural.) */ { + /* Absolute admission floor (2026-07-30): drop sub-floor candidates + * BEFORE rank-trimming, so the cap is filled only by nodes that clear + * an absolute bar — fill below ENGRAM_WM_CAP becomes reachable and + * wm_saturated becomes an informative signal. See ENGRAM_WM_FLOOR. */ + for (int64_t i = 0; i < g->node_count; i++) { + if (wm_weights[i] > 0.0 && wm_weights[i] < ENGRAM_WM_FLOOR) { + wm_weights[i] = 0.0; + _eg_act_wm_evicted++; + } + } int64_t cap_count = 0; for (int64_t i = 0; i < g->node_count; i++) { if (wm_weights[i] > 0.0) cap_count++; @@ -7503,12 +8242,26 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { continue; /* fills a slot */ } wm_weights[i] = 0.0; /* over cap: evict */ + _eg_act_wm_evicted++; } } /* If malloc failed, skip cap — WM unbounded this call, no corruption. */ } } + /* Pre-persist residency snapshot (2026-07-30): record which nodes held a + * WM slot BEFORE this call's results are written back. Used below to fold + * only NEW WM entrants into the context centroid — an incumbent that gets + * re-promoted every scan no longer re-entrenches the centroid each time, + * which was the remaining positive-feedback path in the WM→centroid→ + * e_eff→re-selection loop (fixation driver; cf. wm_top0_streak=1407 + * incident). NULL on OOM → fold falls back to previous behavior. */ + unsigned char* was_wm = malloc((size_t)g->node_count); + if (was_wm) { + for (int64_t i = 0; i < g->node_count; i++) + was_wm[i] = (g->nodes[i].working_memory_weight > 0.0) ? 1 : 0; + } + /* Persist working_memory_weight (post Pass 4) to node store. * * Conversational thread continuity (ENGRAM_WM_DECAY): @@ -7527,12 +8280,55 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { for (int64_t i = 0; i < g->node_count; i++) { if (!reached[i] && g->nodes[i].working_memory_weight > 0.0) { /* Carry-over decay: node held WM weight from prior activation but - * the current query's BFS fan-out did not reach it. Apply decay - * rather than zero so recently-active context persists. */ - double decayed = g->nodes[i].working_memory_weight * ENGRAM_WM_DECAY; - g->nodes[i].working_memory_weight = (decayed < 0.01) ? 0.0 : decayed; + * the current query's BFS fan-out did not reach it. + * + * 2026-07-22 self-review (bl-b17facdd): replaced the per-call + * multiplicative decay (weight *= ENGRAM_WM_DECAY) with the + * ACT-R/Petrov base-level scheme. The old form was call-rate- + * dependent — carried context died in seconds under rapid + * curiosity scans and lingered for hours under quiet loops. + * Now: hard-evict when base-level < τ (Soar-style forgetting); + * above τ, shape the weight held at promotion (wm_anchor) by the + * retrieval-probability logistic. Pure function of wall-clock + * time — idempotent no matter how often activate is called. */ + EngramNode* cn = &g->nodes[i]; + double anchor = (cn->wm_anchor > 0.0) ? cn->wm_anchor + : cn->working_memory_weight; + double B = engram_bll_base_level(cn, now_ms); + if (B < ENGRAM_BLL_TAU) { + cn->working_memory_weight = 0.0; + cn->wm_anchor = 0.0; /* keep anchor coherent with eviction */ + _eg_act_wm_evicted++; /* was uncounted before 2026-08-02 */ + } else { + double keep = 1.0 / (1.0 + exp(-(B - ENGRAM_BLL_TAU) + / ENGRAM_BLL_S)); + /* Occupancy inhibition (2026-07-26): decay the carried + * weight with hold time so an unreached incumbent cannot + * hold its anchor verbatim indefinitely. See + * ENGRAM_CARRY_TC comment for derivation. */ + double hold_s = (double)(now_ms - cn->last_activated) / 1000.0; + if (hold_s < 0.0) hold_s = 0.0; + double occ = ENGRAM_CARRY_TC / (ENGRAM_CARRY_TC + hold_s); + double w = anchor * keep * occ; + /* Evict floor raised 0.01 → ENGRAM_WM_FLOOR (2026-07-30): + * one consistent absolute bar across all WM entry/exit paths. */ + if (w < ENGRAM_WM_FLOOR) { + cn->working_memory_weight = 0.0; + cn->wm_anchor = 0.0; + _eg_act_wm_evicted++; /* was uncounted before 2026-08-02 */ + } else { + cn->working_memory_weight = w; + } + } } else { g->nodes[i].working_memory_weight = wm_weights[i]; + /* Anchor the promotion weight: carry-over decay above computes + * from this fixed point rather than compounding per call. + * Zero the anchor when the slot empties (2026-07-30): a stale + * anchor on an evicted node was a latent resurrection bug if the + * carry-over entry guard ever changes. */ + if (wm_weights[i] > 0.0) g->nodes[i].wm_anchor = wm_weights[i]; + else g->nodes[i].wm_anchor = 0.0; } } @@ -7546,6 +8342,19 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { * activations outcompete older decayed ones. (Ported from 2026-06-26 * self-review branch.) */ { + /* Absolute admission floor, global pass (2026-07-30): see + * ENGRAM_WM_FLOOR. Sub-floor residents are dropped even when the + * global population is under cap — this is what lets wm_active + * drain below 24 during quiet periods. */ + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* fn = &g->nodes[i]; + if (fn->working_memory_weight > 0.0 && + fn->working_memory_weight < ENGRAM_WM_FLOOR) { + fn->working_memory_weight = 0.0; + fn->wm_anchor = 0.0; + _eg_act_wm_evicted++; + } + } int64_t global_wm_count = 0; for (int64_t i = 0; i < g->node_count; i++) { if (g->nodes[i].working_memory_weight > 0.0) global_wm_count++; @@ -7576,6 +8385,8 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { continue; /* fills a slot */ } n->working_memory_weight = 0.0; /* evict: over global cap */ + n->wm_anchor = 0.0; /* keep anchor coherent */ + _eg_act_wm_evicted++; /* was uncounted before 2026-08-02 */ } } /* If malloc failed, skip — WM over cap this call, no data corruption. */ @@ -7605,6 +8416,9 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { if (n->working_memory_weight <= 0.0) continue; /* evicted by cap */ n->last_activated = now_ms; n->activation_count++; + /* Base-level presentation: WM promotion is the retrieval event. + * (2026-07-22 self-review — feeds engram_bll_base_level.) */ + engram_bll_record_access(n, now_ms); } /* ── Collect all background-activated nodes for the return value ──── @@ -7616,7 +8430,8 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { int64_t rcount = 0; if (!results) { free(best_bg); free(best_hops); free(reached); free(seeds); - free(fr); free(inhibition); free(wm_weights); return out; + free(fr); free(inhibition); free(wm_weights); free(cosq); + free(was_wm); return out; } for (int64_t i = 0; i < g->node_count; i++) { if (!reached[i]) continue; @@ -7643,6 +8458,28 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { } results[j + 1] = key; } + /* ── Context centroid: fold in the touched nodes (2026-07-29) ─────── + * Results are sorted promoted-first by wm_weight desc, so the first + * ENGRAM_CTX_TOUCH_MAX embedded entries with wm > 0 are exactly the + * strongest WM survivors of THIS call — the same "promotion is the + * retrieval event" rule the BLL reinforcement pass uses. μ=0.9 EMA + * keeps any single scan's touches a minority contribution. */ + { + int touched = 0; + for (int64_t i = 0; i < rcount && touched < ENGRAM_CTX_TOUCH_MAX; i++) { + if (results[i].wm <= 0.0) break; /* promoted block exhausted */ + EngramNode* n = &g->nodes[results[i].idx]; + if (!n->emb || n->emb_dim <= 0) continue; + /* New-entrant gate (2026-07-30): skip nodes that already held a + * WM slot before this call — incumbents must not keep pulling + * the centroid toward themselves. Fresh topical shifts (new + * entrants + the query fold at call start) steer it instead. */ + if (was_wm && was_wm[results[i].idx]) continue; + eg_ctx_blend(n->emb, n->emb_dim); + touched++; + } + } + free(was_wm); for (int64_t i = 0; i < rcount; i++) { el_val_t entry = el_map_new(0); entry = el_map_set(entry, EL_STR(el_strdup("node")), @@ -7661,12 +8498,19 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { } free(best_bg); free(best_hops); free(reached); free(seeds); free(fr); free(inhibition); free(wm_weights); free(results); + free(cosq); return out; } /* ── Engram persistence (JSON snapshot) ─────────────────────────────────── */ -static void engram_emit_node_json(JsonBuf* b, const EngramNode* n) { +/* include_emb (2026-07-31 self-review): the ~5.7KB "emb" vector belongs ONLY + * in persistence/replication output (engram_save → snapshot.json, which also + * backs /api/sync and /api/edges via scratch exports). Consumer read routes + * (/api/nodes, /api/search, activation results, neighbors, compiled context) + * were shipping it on every node — responses 10-50x oversized, blowing MCP + * token limits. Pass include_emb=1 only from engram_save. */ +static void engram_emit_node_json(JsonBuf* b, const EngramNode* n, int include_emb) { jb_putc(b, '{'); jb_puts(b, "\"id\":"); jb_emit_escaped(b, n->id ? n->id : ""); jb_puts(b, ",\"content\":"); jb_emit_escaped(b, n->content ? n->content : ""); @@ -7688,6 +8532,36 @@ static void engram_emit_node_json(JsonBuf* b, const EngramNode* n) { snprintf(tmp, sizeof(tmp), ",\"working_memory_weight\":%g", n->working_memory_weight); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"suppression_count\":%d", n->suppression_count); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"layer_id\":%u", n->layer_id); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"wm_anchor\":%g", n->wm_anchor); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"base_level\":%g", + engram_bll_base_level(n, engram_now_ms())); jb_puts(b, tmp); + /* Base-level access history: chronological (oldest→newest) compact + * string. Loaders replay it through engram_bll_record_access; absent + * field = empty ring (optimized-form fallback). (2026-07-22) */ + if (n->access_filled > 0) { + jb_puts(b, ",\"access_ts\":\""); + for (int32_t j = 0; j < n->access_filled; j++) { + int32_t idx = (n->access_head - n->access_filled + j + + 2 * ENGRAM_BLL_K) % ENGRAM_BLL_K; + snprintf(tmp, sizeof(tmp), "%s%lld", j ? "," : "", + (long long)n->access_ts[idx]); + jb_puts(b, tmp); + } + jb_putc(b, '"'); + } + /* Semantic embedding (2026-07-24): compact %.4g comma list — cosine is + * insensitive to 4-sig-fig rounding, and this keeps snapshot bloat to + * ~5KB per embedded node without a base64 codec. Absent field = not + * embedded; the lazy backfill re-embeds eventually if dropped. */ + if (include_emb && n->emb && n->emb_dim > 0) { + jb_puts(b, ",\"emb\":\""); + for (int32_t j = 0; j < n->emb_dim; j++) { + snprintf(tmp, sizeof(tmp), "%s%.4g", j ? "," : "", + (double)n->emb[j]); + jb_puts(b, tmp); + } + jb_putc(b, '"'); + } jb_putc(b, '}'); } @@ -7717,7 +8591,7 @@ el_val_t engram_save(el_val_t path) { jb_puts(&b, "{\"nodes\":["); for (int64_t i = 0; i < g->node_count; i++) { if (i > 0) jb_putc(&b, ','); - engram_emit_node_json(&b, &g->nodes[i]); + engram_emit_node_json(&b, &g->nodes[i], 1); } jb_puts(&b, "],\"edges\":["); for (int64_t i = 0; i < g->edge_count; i++) { @@ -7813,6 +8687,15 @@ static const char* eg_skip_ws(const char* p) { * the activation path. Same top-K-by-weight logic as engram_activate * Pass 5 (Cowan 2001: WM capacity is global). */ static void eg_enforce_wm_cap_on_load(EngramStore* g) { + /* Absolute admission floor (2026-07-30): see ENGRAM_WM_FLOOR. */ + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* fn = &g->nodes[i]; + if (fn->working_memory_weight > 0.0 && + fn->working_memory_weight < ENGRAM_WM_FLOOR) { + fn->working_memory_weight = 0.0; + fn->wm_anchor = 0.0; + } + } int64_t wm_count = 0; for (int64_t i = 0; i < g->node_count; i++) { if (g->nodes[i].working_memory_weight > 0.0) wm_count++; @@ -7839,6 +8722,7 @@ static void eg_enforce_wm_cap_on_load(EngramStore* g) { if (n->working_memory_weight > cutoff) continue; if (slots_at_cutoff > 0) { slots_at_cutoff--; continue; } n->working_memory_weight = 0.0; /* evict: over cap at load */ + n->wm_anchor = 0.0; /* keep anchor coherent */ } } @@ -7863,6 +8747,10 @@ el_val_t engram_load(el_val_t path) { free(g->nodes[i].id); free(g->nodes[i].content); free(g->nodes[i].node_type); free(g->nodes[i].label); free(g->nodes[i].tier); free(g->nodes[i].tags); free(g->nodes[i].metadata); + /* 2026-07-26 self-review: emb was the one heap field not freed + * here — ~3 KB leaked per embedded node per reload (~11 MB per + * reload at 3.7k embedded). forget/prune already free it. */ + free(g->nodes[i].emb); g->nodes[i].emb = NULL; g->nodes[i].emb_dim = 0; } g->node_count = 0; for (int64_t i = 0; i < g->edge_count; i++) { @@ -7915,7 +8803,7 @@ el_val_t engram_load(el_val_t path) { * continuity across a restart while stale pinned weights decay * out over successive boots; sub-0.05 residue drops to zero. */ nn->working_memory_weight *= 0.5; - if (nn->working_memory_weight < 0.05) nn->working_memory_weight = 0.0; + if (nn->working_memory_weight < ENGRAM_WM_FLOOR) nn->working_memory_weight = 0.0; nn->suppression_count = (int32_t)eg_get_int_field(obj, "suppression_count"); /* layer_id defaults to ENGRAM_LAYER_DEFAULT (core-identity) * for snapshots that predate the layered schema. We can't @@ -7926,6 +8814,17 @@ el_val_t engram_load(el_val_t path) { } else { nn->layer_id = ENGRAM_LAYER_DEFAULT; } + /* Base-level state (2026-07-22): absent fields leave the ring + * empty (memset) → optimized-form fallback. */ + nn->wm_anchor = eg_get_num_field(obj, "wm_anchor"); + { + char* ats = eg_get_str_field(obj, "access_ts"); + if (ats) { engram_bll_parse_access(nn, ats); free(ats); } + } + { + char* es = eg_get_str_field(obj, "emb"); + if (es) { eg_parse_emb(nn, es); free(es); } + } int64_t load_idx = g->node_count; g->node_count++; if (nn->id && *nn->id) engram_idmap_put(g, nn->id, load_idx); @@ -8119,6 +9018,17 @@ el_val_t engram_load_merge(el_val_t path) { } else { nn->layer_id = ENGRAM_LAYER_DEFAULT; } + /* Base-level history merges with the node (2026-07-22); + * wm_anchor stays 0 — WM state is local-only (see the + * working_memory_weight comment above). */ + { + char* ats = eg_get_str_field(obj, "access_ts"); + if (ats) { engram_bll_parse_access(nn, ats); free(ats); } + } + { + char* es = eg_get_str_field(obj, "emb"); + if (es) { eg_parse_emb(nn, es); free(es); } + } int64_t merge_idx = g->node_count; g->node_count++; added_nodes++; @@ -8214,7 +9124,7 @@ el_val_t engram_get_node_json(el_val_t id) { EngramNode* n = engram_find_node(sid); if (!n) return el_wrap_str(el_strdup("{}")); JsonBuf b; jb_init(&b); - engram_emit_node_json(&b, n); + engram_emit_node_json(&b, n, 0); return el_wrap_str(b.buf); } @@ -8239,7 +9149,7 @@ el_val_t engram_get_node_by_label(el_val_t label) { EngramNode* n = &g->nodes[i]; if (n->label && strcmp(n->label, lbl) == 0) { JsonBuf b; jb_init(&b); - engram_emit_node_json(&b, n); + engram_emit_node_json(&b, n, 0); return el_wrap_str(b.buf); } } @@ -8280,7 +9190,7 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { int64_t end = nhits < lim ? nhits : lim; for (int64_t k = 0; k < end; k++) { if (!first) jb_putc(&b, ','); - engram_emit_node_json(&b, &g->nodes[hits[k].idx]); + engram_emit_node_json(&b, &g->nodes[hits[k].idx], 0); first = 0; } free(hits); @@ -8312,7 +9222,7 @@ el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset) { int first = 1; for (int64_t i = off; i < end; i++) { if (!first) jb_putc(&b, ','); - engram_emit_node_json(&b, &g->nodes[idx[i]]); + engram_emit_node_json(&b, &g->nodes[idx[i]], 0); first = 0; } free(idx); @@ -8349,7 +9259,7 @@ el_val_t engram_scan_nodes_by_type_json(el_val_t type_v, el_val_t limit, el_val_ int first = 1; for (int64_t i = off; i < end; i++) { if (!first) jb_putc(&b, ','); - engram_emit_node_json(&b, &g->nodes[idx[i]]); + engram_emit_node_json(&b, &g->nodes[idx[i]], 0); first = 0; } free(idx); @@ -8413,7 +9323,7 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t di if (!n) continue; if (!first) jb_putc(&b, ','); jb_puts(&b, "{\"node\":"); - engram_emit_node_json(&b, n); + engram_emit_node_json(&b, n, 0); jb_puts(&b, ",\"edge\":"); engram_emit_edge_json(&b, e); char tmp[64]; snprintf(tmp, sizeof(tmp), ",\"hops\":%lld}", (long long)(h + 1)); @@ -8456,7 +9366,7 @@ el_val_t engram_activate_json(el_val_t query, el_val_t depth) { if (i > 0) jb_putc(&b, ','); jb_puts(&b, "{\"node\":"); if (n) { - engram_emit_node_json(&b, n); + engram_emit_node_json(&b, n, 0); } else { jb_puts(&b, "{}"); } @@ -8545,7 +9455,13 @@ el_val_t engram_wm_top_json(el_val_t n_v) { EngramNode* n = &g->nodes[idx[k]]; if (k > 0) jb_putc(&b, ','); jb_putc(&b, '{'); - jb_puts(&b, "\"label\":"); + /* 2026-07-26 self-review: id was never emitted here, so the + * awareness heartbeat's wm_top0_streak compared ""=="" and + * incremented unconditionally — the streak metric measured + * uptime, not fixation. */ + jb_puts(&b, "\"id\":"); + jb_emit_escaped(&b, n->id ? n->id : ""); + jb_puts(&b, ",\"label\":"); jb_emit_escaped(&b, n->label ? n->label : ""); jb_puts(&b, ",\"node_type\":"); jb_emit_escaped(&b, n->node_type ? n->node_type : ""); @@ -8563,10 +9479,170 @@ el_val_t engram_wm_top_json(el_val_t n_v) { el_val_t engram_stats_json(void) { EngramStore* g = engram_get(); - char buf[128]; + /* embedded_count: how far the lazy backfill has progressed. The single + * observable that tells the daily self-review whether semantic + * activation is actually accumulating coverage. (2026-07-24) + * + * embed_eligible_count (2026-07-27): embedded_count alone misleads — + * ~70%+ of the store is ISE/Tag/short-content nodes that are permanently + * ineligible for embedding, so raw embedded/node_count reads as "~30% + * coverage, something is broken" when eligible coverage may be complete. + * This exact misdiagnosis happened in today's self-review. Report the + * true denominator so coverage = embedded_count / embed_eligible_count. */ + int64_t embedded = 0, eligible = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].emb) embedded++; + if (eg_embed_eligible(&g->nodes[i])) eligible++; + } + char buf[256]; snprintf(buf, sizeof(buf), - "{\"node_count\":%lld,\"edge_count\":%lld,\"layer_count\":%zu}", - (long long)g->node_count, (long long)g->edge_count, g->layer_count); + "{\"node_count\":%lld,\"edge_count\":%lld,\"layer_count\":%zu," + "\"embedded_count\":%lld,\"embed_eligible_count\":%lld}", + (long long)g->node_count, (long long)g->edge_count, g->layer_count, + (long long)embedded, (long long)eligible); + return el_wrap_str(el_strdup(buf)); +} + +/* engram_act_stats_json — activation observability + embedder breaker state. + * (2026-07-27 self-review; counters made cumulative 2026-07-31.) + * wm_evicted/breakthroughs are monotonic process-lifetime totals across ALL + * engram_activate calls on this store (diff successive readings for rates; + * they reset to 0 only on restart); embed_breaker_open=1 means + * eg_embed_fetch is currently refusing calls (semantic activation silently + * degraded to lexical until the cooldown expires). The soul heartbeat folds + * this into its ISE so the pathologies are diagnosable from telemetry + * instead of inferred from wm_avg_weight hovering at the floor. */ +el_val_t engram_act_stats_json(void) { + int64_t now = engram_now_ms(); + int breaker_open = (now < _eg_embed_breaker_until) ? 1 : 0; + char buf[256]; + /* ctx_cos (2026-07-29): cos(query, context centroid) at the LAST + * activate call, measured before the query was folded in. ~1.0 = + * context aligned with current query; low = divergence (expected at + * curiosity domain-rotation boundaries); -2.0 = no centroid yet or + * embedder down. The drift gauge for the context-centroid mechanism. */ + snprintf(buf, sizeof(buf), + "{\"wm_evicted\":%lld,\"breakthroughs\":%lld," + "\"embed_breaker_open\":%d,\"embed_consec_fail\":%d," + "\"ctx_cos\":%.3f}", + (long long)_eg_act_wm_evicted, + (long long)_eg_act_breakthroughs, + breaker_open, _eg_embed_consec_fail, + _eg_act_ctx_cos); + return el_wrap_str(el_strdup(buf)); +} + +/* engram_cosine_sim — cosine similarity between two nodes' embeddings. + * Returns a float in [-1, 1], or -2.0 when either node is missing or not + * yet embedded. Exposed so EL code (and the introspection API) can probe + * semantic distance directly. (2026-07-24, bl-b2d1c944) */ +el_val_t engram_cosine_sim(el_val_t id_a, el_val_t id_b) { + EngramStore* g = engram_get(); + int64_t ia = engram_find_node_index(EL_CSTR(id_a)); + int64_t ib = engram_find_node_index(EL_CSTR(id_b)); + if (ia < 0 || ib < 0) return el_from_float(-2.0); + EngramNode* a = &g->nodes[ia]; + EngramNode* b = &g->nodes[ib]; + if (!a->emb || !b->emb || a->emb_dim != b->emb_dim) + return el_from_float(-2.0); + return el_from_float(eg_cosine(a->emb, b->emb, a->emb_dim)); +} + +/* engram_label_df — document frequency of `term` across node LABELS. + * Returns the count of nodes whose label contains term (case-insensitive), + * or the total node count for an empty/NULL term so callers treat "no term" + * as maximally unspecific (i.e. reject it). + * + * WHY THIS EXISTS (2026-08-03 self-review). The soul's auto-term extractor + * (awareness.el:auto_term_try_slot) picks a curiosity seed by taking the + * FIRST WORD of a top-WM node label. A first-word extractor has no notion of + * term quality, so three consecutive self-reviews each bolted another + * hand-curated blocklist onto it — genre words (07-23), quoted titles + * (07-25), English stopwords (07-30). Every one of those was written + * REACTIVELY, after observing a flood in the live ISE stream. The mechanism + * is whack-a-mole: the list can only ever contain floods that already + * happened. + * + * Measured live on this store (13,370 nodes) while two unanticipated floods + * were in flight and unfixed: + * "