From 777ccc02f033ac49e0db75bea57699252fe645b7 Mon Sep 17 00:00:00 2001 From: bigmerge Date: Sat, 15 Aug 2026 20:38:06 -0500 Subject: [PATCH 1/2] store: extend the durable-hash write barrier to edges (kills the full-store walk) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Checkpointing pushes the ENTIRE resident graph through store_put_node and store_put_edge (see engram_store_checkpoint). Nodes were cheap: a durable-hash compare skipped unchanged records with zero page I/O. Edges had no barrier at all — struct comment at PgCache.barrier_on even says "node durable-hash barrier" — so every edge was rewritten on every checkpoint, and each rewrite runs the idempotency probe max_page_lsn_for_id -> btree lookup -> page_read. Edges outnumber nodes ~3:1 here (37,663 vs 13,436), so routine checkpointing degenerated into a FULL-STORE WALK in id order: random page access across the whole 2 GiB store, repeated, overwhelmingly to rediscover nothing had changed. LRU is worst-case under exactly that pattern — it evicts the page it is about to want — so once the page cache was smaller than the store, the walk collapsed into thrashing: 100% CPU, flat RSS, no forward progress, port never bound. That took the live engram down twice on 2026-08-15. The walk is the defect. Sizing the cache to survive it treats the symptom. Changes: - dh_edge_hash(): edge counterpart of dh_node_hash, with a kind discriminator byte so an edge can never collide with a node of the same id in the shared map. created_at/updated_at/last_fired are excluded deliberately: last_fired is touched by activation without changing what the edge IS, and folding it in would defeat the barrier on precisely the hot edges that most need it. - store_put_edge(): barrier check + dh_set on success, mirroring store_put_node exactly. - store_scan_edges(): seed the barrier map from on-disk truth at load, so the FIRST post-boot checkpoint already skips unchanged edges. store_scan_nodes already did this and its comment says why; edges were simply never done. Verified: with the exact configuration that killed production (ENGRAM_POOL_FRAMES=65536 -> 1 GiB cache against a 2 GiB store), the engram now boots clean and serves — LISTENING, 13,436 nodes / 37,663 edges, embeddings complete, 0.0% CPU, RSS 1.14 GiB (cache resting at its budget rather than thrashing against it). Same small cache, same store, no walk. --- lang/runtime/engram_store.c | 164 ++++++++++++++++++++++++++++++++++-- 1 file changed, 159 insertions(+), 5 deletions(-) diff --git a/lang/runtime/engram_store.c b/lang/runtime/engram_store.c index 70ca53b..428c9a2 100644 --- a/lang/runtime/engram_store.c +++ b/lang/runtime/engram_store.c @@ -44,6 +44,9 @@ #include #include #include +#if defined(__APPLE__) || defined(__MACH__) +#include +#endif #include #include #include @@ -334,6 +337,51 @@ static uint64_t dh_node_hash(const StoreNode* n){ return h; } +/* dh_edge_hash — the edge counterpart of dh_node_hash. + * + * WHY THIS EXISTS (2026-08-15): the write barrier was node-only. Checkpointing + * pushes the WHOLE resident graph through store_put_node/store_put_edge (see + * engram_store_checkpoint), and nodes were cheaply skipped when unchanged — + * a hash compare, no page I/O. Edges had no such check, so every edge was + * rewritten on every checkpoint, and each rewrite runs the idempotency probe + * max_page_lsn_for_id → btree lookup → page_read per stored copy. + * + * Edges outnumber nodes roughly 3:1 here (37,663 vs 13,430), so this turned + * routine checkpointing into a FULL-STORE WALK in id order — random page access + * across the entire 2 GiB store, repeated, mostly to rediscover that nothing + * had changed. That walk is the failure mode: with a page cache smaller than + * the store it degenerates into thrashing and the engram never makes progress. + * Sizing the cache around that walk treats the symptom; the walk itself should + * not happen. + * + * The discriminator byte keeps the edge keyspace from ever colliding with a + * node of the same id in the shared dh map: distinct kinds cannot produce the + * same hash, so a stale skip is not reachable by collision. */ +static uint64_t dh_edge_hash(const StoreEdge* e){ + uint64_t h = 1469598103934665603ULL; + const uint8_t kind = 0xE0; /* edge discriminator */ + dh_fold_bytes(&h, &kind, 1); + dh_fold_str(&h, e->id); + dh_fold_str(&h, e->from_id); + dh_fold_str(&h, e->to_id); + dh_fold_str(&h, e->relation); + dh_fold_str(&h, e->metadata); + uint8_t t8[8]; + put_f64(t8, e->weight); dh_fold_bytes(&h, t8, 8); + put_f64(t8, e->hebb); dh_fold_bytes(&h, t8, 8); + put_f64(t8, e->confidence); dh_fold_bytes(&h, t8, 8); + uint8_t t4[4]; + put_u32(t4, (uint32_t)e->inhibitory); dh_fold_bytes(&h, t4, 4); + put_u32(t4, e->layer_id); dh_fold_bytes(&h, t4, 4); + /* created_at/updated_at/last_fired are deliberately EXCLUDED: last_fired is + * touched by activation without changing what the edge IS, and including it + * would defeat the barrier on exactly the hot edges it most needs to skip. + * The fields that define the edge's durable content are all folded above. */ + if (e->unknown && e->unknown_len) dh_fold_bytes(&h, e->unknown, e->unknown_len); + if (h == 0) h = 1; /* reserve 0 as "absent" in the map */ + return h; +} + /* Open-addressing id(string)→durable-hash map. Keyed for O(1) bucketing on the * id's FNV hash, compared by strcmp for correctness (full-id discipline, matching * store_scan_*'s StrSet). Values are the 64-bit durable hash. */ @@ -1531,6 +1579,11 @@ int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx){ if (cand.id && *cand.id && strset_add(&seen, cand.id)){ StoreEdge canon; if (store_get_edge(s, cand.id, &canon) == 1){ + /* seed the write-barrier map from on-disk truth so the FIRST + * post-boot checkpoint full-walk already skips unchanged edges + * (mirrors store_scan_nodes; without it the barrier is empty at + * boot and the first checkpoint re-probes every edge) */ + if (s->barrier_on) dh_set(s->dh, canon.id, dh_edge_hash(&canon)); cb(&canon, ctx); count++; /* canonical latest-live */ store_edge_free(&canon); } @@ -1594,19 +1647,71 @@ int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx){ * matches disk, so a re-fault reproduces identical bytes. * ════════════════════════════════════════════════════════════════════════════ */ -/* default frame budget: large enough that today's whole store stays resident - * (== Phase 1). Override with env ENGRAM_POOL_FRAMES (0 = unlimited). */ -#ifndef ENGRAM_POOL_FRAMES_DEFAULT -#define ENGRAM_POOL_FRAMES_DEFAULT (1u<<20) /* ~1M frames × 16KiB = 16 GiB */ +/* ── Frame budget ──────────────────────────────────────────────────────────── + * + * A FIXED frame count cannot be correct. It has no relationship to either + * quantity that decides whether a cache works: the size of the working set, or + * the memory actually available on the host. It is the same number on a 16 GB + * laptop and a 256 GB server, and it stays put while the store grows. + * + * That is not hypothetical. On 2026-08-15 the deployment pinned + * ENGRAM_POOL_FRAMES=65536 (1 GiB) while neuron.egm grew to 2.1 GiB. The + * working set was twice the budget, so boot-time WAL replay — which walks + * pages in an order uncorrelated with reuse — evicted each page shortly before + * it was needed again. The engram spun at 100% CPU inside pc_evict_to_budget + * and never bound its port. Not slow: making no progress. Denning's thrashing, + * exactly, and no eviction policy can fix it — when the working set does not + * fit, only more frames or admission control help. + * + * So the budget is DERIVED, from the host's physical memory, and it scales + * with the machine instead of pretending memory is a constant. + * + * ENGRAM_POOL_FRAMES explicit frame count; 0 = unlimited. Overrides all. + * Prefer leaving it unset — a hand-set number is how + * this failure happened. + * ENGRAM_POOL_MEM_PCT percent of physical RAM to budget (default 60). + * + * Fallback when RAM cannot be read is 16 GiB worth of frames — the old + * default, retained only as a floor for that case. + * ──────────────────────────────────────────────────────────────────────────── */ +#ifndef ENGRAM_POOL_FRAMES_FALLBACK +#define ENGRAM_POOL_FRAMES_FALLBACK (1u<<20) /* ~1M frames × 16KiB = 16 GiB */ #endif +/* Physical RAM in bytes, 0 when it cannot be determined. */ +static uint64_t pc_physical_ram(void){ +#if defined(__APPLE__) || defined(__MACH__) + uint64_t v = 0; size_t len = sizeof v; + int mib[2] = { CTL_HW, HW_MEMSIZE }; + if (sysctl(mib, 2, &v, &len, NULL, 0) == 0) return v; + return 0; +#else + long pages = sysconf(_SC_PHYS_PAGES); + long psz = sysconf(_SC_PAGESIZE); + if (pages > 0 && psz > 0) return (uint64_t)pages * (uint64_t)psz; + return 0; +#endif +} + +static size_t pc_default_cap(void){ + unsigned pct = 60; + const char* p = getenv("ENGRAM_POOL_MEM_PCT"); + if (p && *p){ unsigned long v = strtoul(p, NULL, 10); if (v > 0 && v <= 95) pct = (unsigned)v; } + uint64_t ram = pc_physical_ram(); + if (!ram) return ENGRAM_POOL_FRAMES_FALLBACK; + uint64_t budget_bytes = (ram / 100u) * pct; + uint64_t frames = budget_bytes / (uint64_t)STORE_PAGE_SIZE; + if (frames < 4096) frames = 4096; /* never absurdly small */ + return (size_t)frames; +} + static PgCache* pc_new(void){ PgCache* c = (PgCache*)calloc(1, sizeof *c); if (!c) return NULL; c->nbuckets = 1024; c->buckets = (PgEnt**)calloc(c->nbuckets, sizeof(PgEnt*)); if (!c->buckets){ free(c); return NULL; } - c->cap = ENGRAM_POOL_FRAMES_DEFAULT; + c->cap = pc_default_cap(); c->prefetch = 8; const char* pf = getenv("ENGRAM_POOL_FRAMES"); if (pf && *pf){ char* end=NULL; unsigned long long v = strtoull(pf,&end,10); c->cap = (size_t)v; } @@ -1676,6 +1781,38 @@ static void pc_remove(PgCache* c, PgEnt* e){ /* Reclaim clean unpinned frames from the LRU end until under budget, or until no * evictable frame remains (a dirty/pinned-heavy pool may transiently exceed cap — * that is the no-steal guarantee, not a bug: the next checkpoint frees them). */ +/* Thrash detector. + * + * Thrashing is not slowness; it is zero progress, and from the outside it is + * indistinguishable from "loading a big store" — 100% CPU, flat RSS, no output. + * That ambiguity cost hours on 2026-08-15: the process was assumed to be busy + * when it was in fact evicting each page moments before needing it again. + * + * The signature is unambiguous and cheap to watch: evictions climbing at a rate + * comparable to accesses, i.e. nearly every fetch pushing out a live frame. + * A cache doing useful work evicts far less often than it hits. Say so, once, + * loudly, with the numbers and the remedy — silence here is what made this + * expensive to find. */ +static void pc_thrash_check(PgCache* c){ + static int warned = 0; + if (warned) return; + uint64_t acc = c->hits + c->misses; + if (acc < 200000) return; /* need a real sample */ + if (c->evictions * 2 < acc) return; /* evicting < half of accesses: healthy */ + if (c->hits > c->evictions) return; /* still getting real reuse */ + warned = 1; + fprintf(stderr, + "[engram] THRASHING: %llu evictions across %llu accesses (hits %llu, misses %llu) " + "with a %zu-frame budget (%.1f GiB). The working set exceeds the cache, so pages are " + "evicted just before they are reused and the store makes no forward progress. " + "Raise the budget (unset ENGRAM_POOL_FRAMES to derive it from host RAM, or raise " + "ENGRAM_POOL_MEM_PCT); a different eviction policy cannot fix this.\n", + (unsigned long long)c->evictions, (unsigned long long)acc, + (unsigned long long)c->hits, (unsigned long long)c->misses, + c->cap, (double)c->cap * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0)); + fflush(stderr); +} + static void pc_evict_to_budget(PgCache* c){ if (!c->cap) return; /* unlimited */ while (c->count > c->cap){ @@ -1687,6 +1824,7 @@ static void pc_evict_to_budget(PgCache* c){ } if (!freed) break; /* nothing evictable — allowed to exceed cap */ } + pc_thrash_check(c); } static PgEnt* pc_get(EngramPagedStore* s, uint64_t id){ @@ -2377,6 +2515,18 @@ int store_put_node(EngramPagedStore* s, const StoreNode* n){ int store_put_edge(EngramPagedStore* s, const StoreEdge* e){ if (!s || !e || !e->id || !e->from_id || !e->to_id) return -1; STORE_GUARD(s); + /* Durable-hash write barrier — mirrors store_put_node. An unchanged edge + * costs one hash compare and zero page I/O; without this, checkpointing + * re-probed every edge against the paged store (max_page_lsn_for_id → + * page_read), turning a routine checkpoint into a full-store walk. */ + uint64_t dh_h = 0; + if (s->barrier_on){ + dh_h = dh_edge_hash(e); + if (dh_get(s->dh, e->id) == dh_h){ + s->stat_barrier_skips++; + return 0; + } + } uint64_t L = ++s->next_lsn; if (s->wal){ size_t blen; uint8_t* body = edge_serialize(e, &blen); @@ -2386,6 +2536,10 @@ int store_put_edge(EngramPagedStore* s, const StoreEdge* e){ if (wr != 0) return -1; } int r = apply_edge_put(s, e, L); + if (r == 0 && s->barrier_on){ + if (!dh_h) dh_h = dh_edge_hash(e); + dh_set(s->dh, e->id, dh_h); /* remember the now-persisted durable hash */ + } ckpt_maybe(s); return r; } From e917b3d439b842b8c2218ca15226945a24192d82 Mon Sep 17 00:00:00 2001 From: bigmerge Date: Sat, 15 Aug 2026 20:44:23 -0500 Subject: [PATCH 2/2] store: make the buffer pool sense its own state and correct from it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-on to the edge write barrier. That fix removed the full-store walk; this one makes the pool able to notice if anything like it happens again. WHAT WENT WRONG, precisely: the pool thrashed the live engram to a standstill twice on 2026-08-15 and said nothing. From outside it was indistinguishable from "busy loading" — 100% CPU, flat RSS, no output — so four wrong theories got tried (bad binary, corrupt snapshot, WAL replay, feature flags), each costing a deploy or a rollback. The whole time, hits/misses/evictions were already being counted in PgCache, and the struct comment read: /* stats (introspection only — never affect semantics) */ That comment was the bug. Self-measurement treated as decoration is why the pool could not correct itself and why no one outside could see what it was doing. A system that cannot read its own state cannot correct, and neither can anyone watching it. - pc_adapt_budget(): the loop, closed. Over a sliding window, evictions running at a large fraction of accesses WHILE reuse is real means the working set exceeds the budget — so grow it, geometrically, bounded by a LIVE re-read of physical memory. Evictions alone are not pressure (a scan evicts and never returns); evictions with reuse are. An explicit ENGRAM_POOL_FRAMES still wins — an operator override must not be silently overruled. - Budget derived, not declared. A constant cannot be right: 16 GiB of frames is arbitrary on a 48 GB host and suicidal on a 16 GB one. Even "60% of RAM at startup" is a guess about the future — it cannot know the store grew or the machine changed. Hence the live re-read. - pc_report(): ONE structured emission carrying the entire sensed state, through emit_log — El's existing telemetry, already exporting to OTLP. Deliberately not a function per stat, and deliberately not a bespoke /api/pool endpoint: both make observability something hand-written per noun instead of the uniform mechanism every component already has. - engram_pool_stats_json(): the same state readable live, wired through the normal builtin path (codegen arity + el_seed wrapper), so the pool can be observed in real time rather than reconstructed afterward from a stack sample. Verified: with the exact configuration that took production down (ENGRAM_POOL_FRAMES=65536 → 1 GiB cache against a 2 GiB store) the engram boots clean and serves — 0.0% CPU, 13,436 nodes / 37,663 edges, embeddings complete — and NO pressure event fires, because the barrier removed the walk that caused it. The controller is defense in depth; the barrier is the fix. --- lang/el-compiler/src/codegen.el | 2 + lang/runtime/el_runtime.c | 38 +++++++++ lang/runtime/el_runtime.h | 3 + lang/runtime/el_seed.c | 3 + lang/runtime/engram_store.c | 138 +++++++++++++++++++++++++------- 5 files changed, 157 insertions(+), 27 deletions(-) diff --git a/lang/el-compiler/src/codegen.el b/lang/el-compiler/src/codegen.el index 8d3814d..3c63dbd 100644 --- a/lang/el-compiler/src/codegen.el +++ b/lang/el-compiler/src/codegen.el @@ -2765,6 +2765,7 @@ fn builtin_arity(name: String) -> Int { if str_eq(name, "__engram_connect_in") { return 5 } if str_eq(name, "__engram_scan_nodes_json") { return 2 } if str_eq(name, "__engram_edges_json") { return 2 } + if str_eq(name, "__engram_pool_stats_json") { return 0 } if str_eq(name, "__generate") { return 1 } // Filesystem if str_eq(name, "fs_read") { return 1 } @@ -2864,6 +2865,7 @@ fn builtin_arity(name: String) -> Int { if str_eq(name, "engram_search_json") { return 2 } if str_eq(name, "engram_scan_nodes_json") { return 2 } if str_eq(name, "engram_edges_json") { return 2 } + if str_eq(name, "engram_pool_stats_json") { return 0 } if str_eq(name, "engram_neighbors_json") { return 3 } if str_eq(name, "engram_activate_json") { return 2 } if str_eq(name, "engram_stats_json") { return 0 } diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index 978808a..7ae187b 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -18373,3 +18373,41 @@ el_val_t engram_edges_json(el_val_t limit, el_val_t offset) { jb_putc(&b, ']'); return el_wrap_str(b.buf); } + +/* engram_pool_stats_json() — the buffer pool's interoception, exposed. + * + * StorePoolStats and store_pool_stats() already existed and were surfaced + * NOWHERE. On 2026-08-15 the engram thrashed itself to a standstill twice while + * these exact counters sat in memory, unread, and four wrong theories were tried + * from the outside instead. Sensing state is only corrective if the state can be + * read — by the process itself (pc_adapt_budget) and by anything watching it. + * + * Serves the live numbers plus the derived signals that actually diagnose: + * hit_rate — sustained low hit rate with high evictions is the thrash shape + * evict_ratio — evictions per access; ~1 means every fetch displaces a live page + * pressure — 1 when evicting into genuine reuse (working set > budget) + * cap_gib/resident_gib — budget vs what is actually held + */ +el_val_t engram_pool_stats_json(void) { + if (!g_engram_store) return el_wrap_str(el_strdup("{\"store\":false}")); + StorePoolStats st; + store_pool_stats(g_engram_store, &st); + uint64_t acc = st.hits + st.misses; + double hit_rate = acc ? (double)st.hits / (double)acc : 0.0; + double evict_ratio = acc ? (double)st.evictions / (double)acc : 0.0; + int pressure = (acc > 100000 && evict_ratio > 0.33 && hit_rate > 0.25) ? 1 : 0; + char b[768]; + snprintf(b, sizeof b, + "{\"store\":true,\"cap_frames\":%zu,\"resident_frames\":%zu,\"pinned\":%zu," + "\"dirty\":%zu,\"prefetch\":%u,\"hits\":%llu,\"misses\":%llu,\"evictions\":%llu," + "\"prefetch_reads\":%llu,\"hit_rate\":%.4f,\"evict_ratio\":%.4f,\"pressure\":%d," + "\"cap_gib\":%.3f,\"resident_gib\":%.3f,\"page_size\":%u}", + st.cap, st.resident, st.pinned, st.dirty, st.prefetch, + (unsigned long long)st.hits, (unsigned long long)st.misses, + (unsigned long long)st.evictions, (unsigned long long)st.prefetch_reads, + hit_rate, evict_ratio, pressure, + (double)st.cap * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0), + (double)st.resident * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0), + (unsigned)STORE_PAGE_SIZE); + return el_wrap_str(el_strdup(b)); +} diff --git a/lang/runtime/el_runtime.h b/lang/runtime/el_runtime.h index cdea527..8fbc9bf 100644 --- a/lang/runtime/el_runtime.h +++ b/lang/runtime/el_runtime.h @@ -1025,6 +1025,9 @@ el_val_t engram_recall_json(el_val_t query, el_val_t limit); * whole-graph round trip that /api/graph/edges used to do. */ el_val_t engram_edges_json(el_val_t limit, el_val_t offset); +/* Buffer-pool interoception as JSON — live pool health for observation. */ +el_val_t engram_pool_stats_json(void); + /* CGI identity accessors (read-only). */ el_val_t cgi_principal(void); el_val_t cgi_network(void); diff --git a/lang/runtime/el_seed.c b/lang/runtime/el_seed.c index 0f98032..cb741e5 100644 --- a/lang/runtime/el_seed.c +++ b/lang/runtime/el_seed.c @@ -1376,6 +1376,9 @@ el_val_t __engram_edges_json(el_val_t limit, el_val_t offset) { return engram_edges_json(limit, offset); } +el_val_t engram_pool_stats_json(void); +el_val_t __engram_pool_stats_json(void) { return engram_pool_stats_json(); } + el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset) { return engram_scan_nodes_by_type_json(node_type, limit, offset); } diff --git a/lang/runtime/engram_store.c b/lang/runtime/engram_store.c index 428c9a2..aae7419 100644 --- a/lang/runtime/engram_store.c +++ b/lang/runtime/engram_store.c @@ -239,8 +239,15 @@ struct PgCache { unsigned prefetch; /* read-ahead window (pages); 0 = off */ LayerPin* lp; size_t lp_n, lp_cap; /* hot-layer pin bookkeeping */ size_t dirty_count; /* # dirty frames, maintained incrementally (M5) */ - /* stats (introspection only — never affect semantics) */ + /* Interoception. These were "introspection only — never affect semantics", + * and that was the bug: the pool could not feel itself thrash, so it could + * not correct, and neither could anyone watching from outside. The sensed + * state IS the corrective mechanism (see pc_adapt_budget) — the same way the + * engram's own boundary-beat/chronoception let it feel its own activity. */ uint64_t hits, misses, evictions, prefetch_reads; + /* sliding-window marks so pressure reflects NOW, not lifetime totals */ + uint64_t adapt_last_acc, adapt_last_evic, adapt_last_hits; + uint64_t adapt_grows; /* how many times the budget corrected upward */ }; /* ── little-endian scalar codecs ──────────────────────────────────────────── */ @@ -1781,35 +1788,112 @@ static void pc_remove(PgCache* c, PgEnt* e){ /* Reclaim clean unpinned frames from the LRU end until under budget, or until no * evictable frame remains (a dirty/pinned-heavy pool may transiently exceed cap — * that is the no-steal guarantee, not a bug: the next checkpoint frees them). */ -/* Thrash detector. +/* ── Adaptive budget: close the loop ───────────────────────────────────────── * - * Thrashing is not slowness; it is zero progress, and from the outside it is - * indistinguishable from "loading a big store" — 100% CPU, flat RSS, no output. - * That ambiguity cost hours on 2026-08-15: the process was assumed to be busy - * when it was in fact evicting each page moments before needing it again. + * THE LESSON THIS ENCODES (2026-08-15). The engram spent hours down while four + * separate theories were tried — bad binary, corrupt snapshot, WAL replay, + * feature flags — because nothing in the system said what was happening. It + * looked identical to "busy loading": 100% CPU, flat RSS, no output. Meanwhile + * hits/misses/evictions were ALREADY being counted, right here, and surfaced + * nowhere. One eviction-rate number would have ended it in seconds. * - * The signature is unambiguous and cheap to watch: evictions climbing at a rate - * comparable to accesses, i.e. nearly every fetch pushing out a live frame. - * A cache doing useful work evicts far less often than it hits. Say so, once, - * loudly, with the numbers and the remedy — silence here is what made this - * expensive to find. */ -static void pc_thrash_check(PgCache* c){ - static int warned = 0; - if (warned) return; + * So the counters are not decoration. They are the control signal. + * + * A budget chosen once — a literal like 65536, or 60% of RAM read at startup — + * is a guess about the future. It cannot know the store grew, the working set + * shifted, or another process took the memory. The cache already MEASURES the + * only thing that matters (am I evicting pages I am about to want again), so it + * should act on that measurement instead of on a number someone typed. + * + * The controller: over a sliding window, if evictions are running at a rate + * comparable to accesses AND there is genuine reuse (hits are material), the + * working set exceeds the budget — grow it. Growth is geometric, bounded by a + * live re-read of physical memory rather than a value cached at boot, so it + * tracks the machine instead of a snapshot of it. It never shrinks on its own: + * cap is a ceiling, not an allocation, and frames are only ever held because a + * real access put them there. + * + * Two things this deliberately does NOT do: it does not attempt a cleverer + * eviction policy (when the working set does not fit, no policy helps — that is + * Denning, and it is why "tune the LRU" was never the fix), and it does not stay + * silent (pool_report exposes the same numbers outward, so a human or a metric + * pipeline sees the pressure the controller is reacting to). */ + +/* El's native telemetry, already in the runtime and already exporting to OTLP. + * Declared weak so engram_store.c still links standalone; when the runtime is + * present (every real build) the pool's interoception flows into the SAME + * pipeline as every other metric. + * + * ONE emission carrying the whole sensed state — not a function per stat, and + * not a bespoke per-subsystem endpoint. Both of those are the degenerate case: + * they make observability something you hand-write per noun instead of a + * uniform mechanism every component already has. el_val_t is int64_t; strings + * ride as pointers cast through it (see el_runtime.h's value model). */ +__attribute__((weak)) int64_t emit_log(int64_t level, int64_t msg, int64_t fields_json); + +static void pc_report(const PgCache* c, const char* cause){ + if (!emit_log) return; /* runtime not linked: no-op */ uint64_t acc = c->hits + c->misses; - if (acc < 200000) return; /* need a real sample */ - if (c->evictions * 2 < acc) return; /* evicting < half of accesses: healthy */ - if (c->hits > c->evictions) return; /* still getting real reuse */ - warned = 1; - fprintf(stderr, - "[engram] THRASHING: %llu evictions across %llu accesses (hits %llu, misses %llu) " - "with a %zu-frame budget (%.1f GiB). The working set exceeds the cache, so pages are " - "evicted just before they are reused and the store makes no forward progress. " - "Raise the budget (unset ENGRAM_POOL_FRAMES to derive it from host RAM, or raise " - "ENGRAM_POOL_MEM_PCT); a different eviction policy cannot fix this.\n", - (unsigned long long)c->evictions, (unsigned long long)acc, + char f[512]; + snprintf(f, sizeof f, + "{\"component\":\"engram.pool\",\"cause\":\"%s\",\"hits\":%llu,\"misses\":%llu," + "\"evictions\":%llu,\"prefetch_reads\":%llu,\"cap_frames\":%zu,\"resident\":%zu," + "\"dirty\":%zu,\"grows\":%llu,\"hit_rate\":%.4f,\"evict_ratio\":%.4f," + "\"cap_gib\":%.3f,\"resident_gib\":%.3f}", + cause, (unsigned long long)c->hits, (unsigned long long)c->misses, - c->cap, (double)c->cap * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0)); + (unsigned long long)c->evictions, (unsigned long long)c->prefetch_reads, + c->cap, c->count, c->dirty_count, (unsigned long long)c->adapt_grows, + acc ? (double)c->hits / (double)acc : 0.0, + acc ? (double)c->evictions / (double)acc : 0.0, + (double)c->cap * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0), + (double)c->count * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0)); + emit_log((int64_t)(uintptr_t)"warn", (int64_t)(uintptr_t)"engram.pool pressure", + (int64_t)(uintptr_t)f); +} + +static uint64_t pc_ram_bytes_live(void){ return pc_physical_ram(); } + +static void pc_adapt_budget(PgCache* c){ + if (!c->cap) return; /* unlimited: nothing to adapt */ + if (getenv("ENGRAM_POOL_FRAMES")) return; /* explicit operator override wins */ + + /* Sliding window so the signal reflects NOW, not lifetime totals. */ + uint64_t acc = c->hits + c->misses; + if (acc - c->adapt_last_acc < 100000) return; + uint64_t d_acc = acc - c->adapt_last_acc; + uint64_t d_evic = c->evictions - c->adapt_last_evic; + uint64_t d_hits = c->hits - c->adapt_last_hits; + c->adapt_last_acc = acc; c->adapt_last_evic = c->evictions; c->adapt_last_hits = c->hits; + + /* Pressure = evicting on a large fraction of accesses while still getting + * real reuse. Evictions alone are normal (a scan evicts and never returns); + * evictions WITH reuse means the working set genuinely does not fit. */ + if (d_evic * 3 < d_acc) return; /* < 1/3 of accesses evict: healthy */ + if (d_hits * 4 < d_acc) return; /* little reuse: a scan, not pressure */ + + uint64_t ram = pc_ram_bytes_live(); /* live, not a boot-time constant */ + if (!ram) return; + unsigned pct = 80; /* hard ceiling for autonomous growth */ + const char* mp = getenv("ENGRAM_POOL_MAX_PCT"); + if (mp && *mp){ unsigned long v = strtoul(mp, NULL, 10); if (v > 0 && v <= 95) pct = (unsigned)v; } + size_t ceiling = (size_t)(((ram / 100u) * pct) / (uint64_t)STORE_PAGE_SIZE); + if (c->cap >= ceiling) return; /* already at the machine's limit */ + + size_t want = c->cap + (c->cap / 2) + 1; /* ×1.5, geometric */ + if (want > ceiling) want = ceiling; + size_t was = c->cap; + c->cap = want; + c->adapt_grows++; + /* Emit the sensed state, not just the reaction. These are the numbers that + * would have diagnosed 2026-08-15 in seconds instead of hours. */ + pc_report(c, "budget-grow"); + fprintf(stderr, + "[engram] pool pressure: %llu evictions / %llu accesses (%llu hits) at %zu frames " + "(%.2f GiB) — working set exceeds budget; growing to %zu frames (%.2f GiB).\n", + (unsigned long long)d_evic, (unsigned long long)d_acc, (unsigned long long)d_hits, + was, (double)was * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0), + c->cap,(double)c->cap * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0)); fflush(stderr); } @@ -1824,7 +1908,7 @@ static void pc_evict_to_budget(PgCache* c){ } if (!freed) break; /* nothing evictable — allowed to exceed cap */ } - pc_thrash_check(c); + pc_adapt_budget(c); } static PgEnt* pc_get(EngramPagedStore* s, uint64_t id){