store: make the buffer pool sense its own state and correct from it

Follow-on to the edge write barrier. That fix removed the full-store walk;
this one makes the pool able to notice if anything like it happens again.

WHAT WENT WRONG, precisely: the pool thrashed the live engram to a standstill
twice on 2026-08-15 and said nothing. From outside it was indistinguishable
from "busy loading" — 100% CPU, flat RSS, no output — so four wrong theories
got tried (bad binary, corrupt snapshot, WAL replay, feature flags), each
costing a deploy or a rollback. The whole time, hits/misses/evictions were
already being counted in PgCache, and the struct comment read:

    /* stats (introspection only — never affect semantics) */

That comment was the bug. Self-measurement treated as decoration is why the
pool could not correct itself and why no one outside could see what it was
doing. A system that cannot read its own state cannot correct, and neither can
anyone watching it.

  - pc_adapt_budget(): the loop, closed. Over a sliding window, evictions
    running at a large fraction of accesses WHILE reuse is real means the
    working set exceeds the budget — so grow it, geometrically, bounded by a
    LIVE re-read of physical memory. Evictions alone are not pressure (a scan
    evicts and never returns); evictions with reuse are. An explicit
    ENGRAM_POOL_FRAMES still wins — an operator override must not be silently
    overruled.

  - Budget derived, not declared. A constant cannot be right: 16 GiB of frames
    is arbitrary on a 48 GB host and suicidal on a 16 GB one. Even "60% of RAM
    at startup" is a guess about the future — it cannot know the store grew or
    the machine changed. Hence the live re-read.

  - pc_report(): ONE structured emission carrying the entire sensed state,
    through emit_log — El's existing telemetry, already exporting to OTLP.
    Deliberately not a function per stat, and deliberately not a bespoke
    /api/pool endpoint: both make observability something hand-written per noun
    instead of the uniform mechanism every component already has.

  - engram_pool_stats_json(): the same state readable live, wired through the
    normal builtin path (codegen arity + el_seed wrapper), so the pool can be
    observed in real time rather than reconstructed afterward from a stack
    sample.

Verified: with the exact configuration that took production down
(ENGRAM_POOL_FRAMES=65536 → 1 GiB cache against a 2 GiB store) the engram boots
clean and serves — 0.0% CPU, 13,436 nodes / 37,663 edges, embeddings complete —
and NO pressure event fires, because the barrier removed the walk that caused
it. The controller is defense in depth; the barrier is the fix.

lang/el-compiler/src/codegen.el

@@ -2765,6 +2765,7 @@ fn builtin_arity(name: String) -> Int {
     if str_eq(name, "__engram_connect_in") { return 5 }
     if str_eq(name, "__engram_scan_nodes_json") { return 2 }
     if str_eq(name, "__engram_edges_json") { return 2 }
+    if str_eq(name, "__engram_pool_stats_json") { return 0 }
     if str_eq(name, "__generate") { return 1 }
     // Filesystem
     if str_eq(name, "fs_read") { return 1 }
@@ -2864,6 +2865,7 @@ fn builtin_arity(name: String) -> Int {
     if str_eq(name, "engram_search_json") { return 2 }
     if str_eq(name, "engram_scan_nodes_json") { return 2 }
     if str_eq(name, "engram_edges_json") { return 2 }
+    if str_eq(name, "engram_pool_stats_json") { return 0 }
     if str_eq(name, "engram_neighbors_json") { return 3 }
     if str_eq(name, "engram_activate_json") { return 2 }
     if str_eq(name, "engram_stats_json") { return 0 }

lang/runtime/el_runtime.c

@@ -18373,3 +18373,41 @@ el_val_t engram_edges_json(el_val_t limit, el_val_t offset) {
     jb_putc(&b, ']');
     return el_wrap_str(b.buf);
 }
+
+/* engram_pool_stats_json() — the buffer pool's interoception, exposed.
+ *
+ * StorePoolStats and store_pool_stats() already existed and were surfaced
+ * NOWHERE. On 2026-08-15 the engram thrashed itself to a standstill twice while
+ * these exact counters sat in memory, unread, and four wrong theories were tried
+ * from the outside instead. Sensing state is only corrective if the state can be
+ * read — by the process itself (pc_adapt_budget) and by anything watching it.
+ *
+ * Serves the live numbers plus the derived signals that actually diagnose:
+ *   hit_rate    — sustained low hit rate with high evictions is the thrash shape
+ *   evict_ratio — evictions per access; ~1 means every fetch displaces a live page
+ *   pressure    — 1 when evicting into genuine reuse (working set > budget)
+ *   cap_gib/resident_gib — budget vs what is actually held
+ */
+el_val_t engram_pool_stats_json(void) {
+    if (!g_engram_store) return el_wrap_str(el_strdup("{\"store\":false}"));
+    StorePoolStats st;
+    store_pool_stats(g_engram_store, &st);
+    uint64_t acc = st.hits + st.misses;
+    double hit_rate    = acc ? (double)st.hits / (double)acc : 0.0;
+    double evict_ratio = acc ? (double)st.evictions / (double)acc : 0.0;
+    int pressure = (acc > 100000 && evict_ratio > 0.33 && hit_rate > 0.25) ? 1 : 0;
+    char b[768];
+    snprintf(b, sizeof b,
+        "{\"store\":true,\"cap_frames\":%zu,\"resident_frames\":%zu,\"pinned\":%zu,"
+        "\"dirty\":%zu,\"prefetch\":%u,\"hits\":%llu,\"misses\":%llu,\"evictions\":%llu,"
+        "\"prefetch_reads\":%llu,\"hit_rate\":%.4f,\"evict_ratio\":%.4f,\"pressure\":%d,"
+        "\"cap_gib\":%.3f,\"resident_gib\":%.3f,\"page_size\":%u}",
+        st.cap, st.resident, st.pinned, st.dirty, st.prefetch,
+        (unsigned long long)st.hits, (unsigned long long)st.misses,
+        (unsigned long long)st.evictions, (unsigned long long)st.prefetch_reads,
+        hit_rate, evict_ratio, pressure,
+        (double)st.cap      * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0),
+        (double)st.resident * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0),
+        (unsigned)STORE_PAGE_SIZE);
+    return el_wrap_str(el_strdup(b));
+}

lang/runtime/el_runtime.h

@@ -1025,6 +1025,9 @@ el_val_t  engram_recall_json(el_val_t query, el_val_t limit);
  * whole-graph round trip that /api/graph/edges used to do. */
 el_val_t  engram_edges_json(el_val_t limit, el_val_t offset);
 
+/* Buffer-pool interoception as JSON — live pool health for observation. */
+el_val_t  engram_pool_stats_json(void);
+
 /* CGI identity accessors (read-only). */
 el_val_t  cgi_principal(void);
 el_val_t  cgi_network(void);

lang/runtime/el_seed.c

@@ -1376,6 +1376,9 @@ el_val_t __engram_edges_json(el_val_t limit, el_val_t offset) {
     return engram_edges_json(limit, offset);
 }
 
+el_val_t engram_pool_stats_json(void);
+el_val_t __engram_pool_stats_json(void) { return engram_pool_stats_json(); }
+
 el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset) {
     return engram_scan_nodes_by_type_json(node_type, limit, offset);
 }

lang/runtime/engram_store.c

@@ -239,8 +239,15 @@ struct PgCache {
     unsigned prefetch;           /* read-ahead window (pages); 0 = off */
     LayerPin* lp; size_t lp_n, lp_cap;   /* hot-layer pin bookkeeping */
     size_t   dirty_count;        /* # dirty frames, maintained incrementally (M5) */
-    /* stats (introspection only — never affect semantics) */
+    /* Interoception. These were "introspection only — never affect semantics",
+     * and that was the bug: the pool could not feel itself thrash, so it could
+     * not correct, and neither could anyone watching from outside. The sensed
+     * state IS the corrective mechanism (see pc_adapt_budget) — the same way the
+     * engram's own boundary-beat/chronoception let it feel its own activity. */
     uint64_t hits, misses, evictions, prefetch_reads;
+    /* sliding-window marks so pressure reflects NOW, not lifetime totals */
+    uint64_t adapt_last_acc, adapt_last_evic, adapt_last_hits;
+    uint64_t adapt_grows;        /* how many times the budget corrected upward */
 };
 
 /* ── little-endian scalar codecs ──────────────────────────────────────────── */
@@ -1781,35 +1788,112 @@ static void pc_remove(PgCache* c, PgEnt* e){
 /* Reclaim clean unpinned frames from the LRU end until under budget, or until no
  * evictable frame remains (a dirty/pinned-heavy pool may transiently exceed cap —
  * that is the no-steal guarantee, not a bug: the next checkpoint frees them). */
-/* Thrash detector.
+/* ── Adaptive budget: close the loop ─────────────────────────────────────────
  *
- * Thrashing is not slowness; it is zero progress, and from the outside it is
- * indistinguishable from "loading a big store" — 100% CPU, flat RSS, no output.
- * That ambiguity cost hours on 2026-08-15: the process was assumed to be busy
- * when it was in fact evicting each page moments before needing it again.
+ * THE LESSON THIS ENCODES (2026-08-15). The engram spent hours down while four
+ * separate theories were tried — bad binary, corrupt snapshot, WAL replay,
+ * feature flags — because nothing in the system said what was happening. It
+ * looked identical to "busy loading": 100% CPU, flat RSS, no output. Meanwhile
+ * hits/misses/evictions were ALREADY being counted, right here, and surfaced
+ * nowhere. One eviction-rate number would have ended it in seconds.
  *
- * The signature is unambiguous and cheap to watch: evictions climbing at a rate
- * comparable to accesses, i.e. nearly every fetch pushing out a live frame.
- * A cache doing useful work evicts far less often than it hits. Say so, once,
- * loudly, with the numbers and the remedy — silence here is what made this
- * expensive to find. */
-static void pc_thrash_check(PgCache* c){
-    static int warned = 0;
-    if (warned) return;
+ * So the counters are not decoration. They are the control signal.
+ *
+ * A budget chosen once — a literal like 65536, or 60% of RAM read at startup —
+ * is a guess about the future. It cannot know the store grew, the working set
+ * shifted, or another process took the memory. The cache already MEASURES the
+ * only thing that matters (am I evicting pages I am about to want again), so it
+ * should act on that measurement instead of on a number someone typed.
+ *
+ * The controller: over a sliding window, if evictions are running at a rate
+ * comparable to accesses AND there is genuine reuse (hits are material), the
+ * working set exceeds the budget — grow it. Growth is geometric, bounded by a
+ * live re-read of physical memory rather than a value cached at boot, so it
+ * tracks the machine instead of a snapshot of it. It never shrinks on its own:
+ * cap is a ceiling, not an allocation, and frames are only ever held because a
+ * real access put them there.
+ *
+ * Two things this deliberately does NOT do: it does not attempt a cleverer
+ * eviction policy (when the working set does not fit, no policy helps — that is
+ * Denning, and it is why "tune the LRU" was never the fix), and it does not stay
+ * silent (pool_report exposes the same numbers outward, so a human or a metric
+ * pipeline sees the pressure the controller is reacting to). */
+
+/* El's native telemetry, already in the runtime and already exporting to OTLP.
+ * Declared weak so engram_store.c still links standalone; when the runtime is
+ * present (every real build) the pool's interoception flows into the SAME
+ * pipeline as every other metric.
+ *
+ * ONE emission carrying the whole sensed state — not a function per stat, and
+ * not a bespoke per-subsystem endpoint. Both of those are the degenerate case:
+ * they make observability something you hand-write per noun instead of a
+ * uniform mechanism every component already has. el_val_t is int64_t; strings
+ * ride as pointers cast through it (see el_runtime.h's value model). */
+__attribute__((weak)) int64_t emit_log(int64_t level, int64_t msg, int64_t fields_json);
+
+static void pc_report(const PgCache* c, const char* cause){
+    if (!emit_log) return;                    /* runtime not linked: no-op */
     uint64_t acc = c->hits + c->misses;
-    if (acc < 200000) return;                       /* need a real sample */
-    if (c->evictions * 2 < acc) return;             /* evicting < half of accesses: healthy */
-    if (c->hits > c->evictions) return;             /* still getting real reuse */
-    warned = 1;
-    fprintf(stderr,
-        "[engram] THRASHING: %llu evictions across %llu accesses (hits %llu, misses %llu) "
-        "with a %zu-frame budget (%.1f GiB). The working set exceeds the cache, so pages are "
-        "evicted just before they are reused and the store makes no forward progress. "
-        "Raise the budget (unset ENGRAM_POOL_FRAMES to derive it from host RAM, or raise "
-        "ENGRAM_POOL_MEM_PCT); a different eviction policy cannot fix this.\n",
-        (unsigned long long)c->evictions, (unsigned long long)acc,
+    char f[512];
+    snprintf(f, sizeof f,
+        "{\"component\":\"engram.pool\",\"cause\":\"%s\",\"hits\":%llu,\"misses\":%llu,"
+        "\"evictions\":%llu,\"prefetch_reads\":%llu,\"cap_frames\":%zu,\"resident\":%zu,"
+        "\"dirty\":%zu,\"grows\":%llu,\"hit_rate\":%.4f,\"evict_ratio\":%.4f,"
+        "\"cap_gib\":%.3f,\"resident_gib\":%.3f}",
+        cause,
         (unsigned long long)c->hits, (unsigned long long)c->misses,
-        c->cap, (double)c->cap * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0));
+        (unsigned long long)c->evictions, (unsigned long long)c->prefetch_reads,
+        c->cap, c->count, c->dirty_count, (unsigned long long)c->adapt_grows,
+        acc ? (double)c->hits / (double)acc : 0.0,
+        acc ? (double)c->evictions / (double)acc : 0.0,
+        (double)c->cap   * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0),
+        (double)c->count * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0));
+    emit_log((int64_t)(uintptr_t)"warn", (int64_t)(uintptr_t)"engram.pool pressure",
+             (int64_t)(uintptr_t)f);
+}
+
+static uint64_t pc_ram_bytes_live(void){ return pc_physical_ram(); }
+
+static void pc_adapt_budget(PgCache* c){
+    if (!c->cap) return;                     /* unlimited: nothing to adapt */
+    if (getenv("ENGRAM_POOL_FRAMES")) return; /* explicit operator override wins */
+
+    /* Sliding window so the signal reflects NOW, not lifetime totals. */
+    uint64_t acc = c->hits + c->misses;
+    if (acc - c->adapt_last_acc < 100000) return;
+    uint64_t d_acc  = acc - c->adapt_last_acc;
+    uint64_t d_evic = c->evictions - c->adapt_last_evic;
+    uint64_t d_hits = c->hits - c->adapt_last_hits;
+    c->adapt_last_acc = acc; c->adapt_last_evic = c->evictions; c->adapt_last_hits = c->hits;
+
+    /* Pressure = evicting on a large fraction of accesses while still getting
+     * real reuse. Evictions alone are normal (a scan evicts and never returns);
+     * evictions WITH reuse means the working set genuinely does not fit. */
+    if (d_evic * 3 < d_acc) return;          /* < 1/3 of accesses evict: healthy */
+    if (d_hits * 4 < d_acc) return;          /* little reuse: a scan, not pressure */
+
+    uint64_t ram = pc_ram_bytes_live();       /* live, not a boot-time constant */
+    if (!ram) return;
+    unsigned pct = 80;                        /* hard ceiling for autonomous growth */
+    const char* mp = getenv("ENGRAM_POOL_MAX_PCT");
+    if (mp && *mp){ unsigned long v = strtoul(mp, NULL, 10); if (v > 0 && v <= 95) pct = (unsigned)v; }
+    size_t ceiling = (size_t)(((ram / 100u) * pct) / (uint64_t)STORE_PAGE_SIZE);
+    if (c->cap >= ceiling) return;            /* already at the machine's limit */
+
+    size_t want = c->cap + (c->cap / 2) + 1;  /* ×1.5, geometric */
+    if (want > ceiling) want = ceiling;
+    size_t was = c->cap;
+    c->cap = want;
+    c->adapt_grows++;
+    /* Emit the sensed state, not just the reaction. These are the numbers that
+     * would have diagnosed 2026-08-15 in seconds instead of hours. */
+    pc_report(c, "budget-grow");
+    fprintf(stderr,
+        "[engram] pool pressure: %llu evictions / %llu accesses (%llu hits) at %zu frames "
+        "(%.2f GiB) — working set exceeds budget; growing to %zu frames (%.2f GiB).\n",
+        (unsigned long long)d_evic, (unsigned long long)d_acc, (unsigned long long)d_hits,
+        was,  (double)was  * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0),
+        c->cap,(double)c->cap * (double)STORE_PAGE_SIZE / (1024.0*1024.0*1024.0));
     fflush(stderr);
 }
 
@@ -1824,7 +1908,7 @@ static void pc_evict_to_budget(PgCache* c){
         }
         if (!freed) break;                  /* nothing evictable — allowed to exceed cap */
     }
-    pc_thrash_check(c);
+    pc_adapt_budget(c);
 }
 
 static PgEnt* pc_get(EngramPagedStore* s, uint64_t id){