From 02dc12d785d3f9f1b77def283054469678535bf4 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Wed, 12 Aug 2026 15:42:49 -0500 Subject: [PATCH] engram tiered storage M4: demand-paging buffer pool (Phase 2, additive) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turn M2's write-back/no-steal cache into a bounded, demand-paged buffer pool so the paged store can exceed RAM while keeping only hot pages resident. On-disk format UNCHANGED (additive residency only; no migration). Default budget is large enough that today's store stays fully resident, so default behaviour == Phase 1. - Frame table capped at `cap` frames (env ENGRAM_POOL_FRAMES; 0 = unlimited; default 1<<20). Not-resident access faults in from neuron.egm. - LRU eviction of CLEAN, unpinned frames only. Dirty frames are never stolen (M2 no-steal / WAL durability preserved) — turned evictable by a checkpoint's pc_flush, which then trims the pool back to budget. - Pinning: superblocks (0,1) + index root/interior pages auto-pinned; explicit store_pin_page/unpin and store_pin_layer/unpin (hot WM/core layers). - Bounded sequential read-ahead on scans (env ENGRAM_PREFETCH, default 8). - Correctness rests on callers copying page bytes into local buffers and never retaining a frame pointer across another access, so evict+re-fault is safe. Gates (plain gcc, ASan/UBSan clean): M4 run_bufpool_tests.sh ...... 37 passed, 0 failed (+ ASan/UBSan: 37/0) small-pool round-trip (cap=32 vs 1599 pages, 2708 evictions): 5000 nodes + 4000 sampled edges bit-exact, crc clean, pool bounded to cap. eviction: hot set 0 re-faults, cold evicted, hit-rate 0.989; no-steal burst (cap=8) holds 309 dirty frames > cap, reads correct from dirty pages. pinning: superblocks/roots/explicit page/hot-layer(19 pages) stay resident; unpin makes them evictable. prefetch: sequential scan 511 demand-faults OFF -> 4 ON. crash-under-paging (ENGRAM_POOL_FRAMES=16): WAL replay + checkpoint-crash phases 0-4 all recover bit-exact. default pool: 0 evictions, whole store resident (== Phase 1). No regression: M1 33/0, M2 36/0, M3 parity PASS, M3.5 PASS. --- engram/test/run_bufpool_tests.sh | 22 ++ engram/test/test_bufpool.c | 496 +++++++++++++++++++++++++++++++ lang/runtime/engram_store.c | 304 ++++++++++++++++++- lang/runtime/engram_store.h | 35 +++ 4 files changed, 844 insertions(+), 13 deletions(-) create mode 100755 engram/test/run_bufpool_tests.sh create mode 100644 engram/test/test_bufpool.c diff --git a/engram/test/run_bufpool_tests.sh b/engram/test/run_bufpool_tests.sh new file mode 100755 index 0000000..2e1e887 --- /dev/null +++ b/engram/test/run_bufpool_tests.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# M4 demand-paging buffer-pool gate. Pure C (NOT elb/elc). Writes only under /tmp. +# Runs the suite twice: an -O2 correctness build and an ASan+UBSan build. +set -e +HERE="$(cd "$(dirname "$0")" && pwd)" +SRC="$HERE/../../lang/runtime/engram_store.c" +TST="$HERE/test_bufpool.c" + +echo "== compiling (gcc -O2): test_bufpool.c engram_store.c ==" +BIN="/tmp/test_bufpool.$$" +gcc -O2 -Wall -Wextra -std=c11 "$TST" "$SRC" -o "$BIN" +"$BIN"; rc=$? +rm -f "$BIN"; rm -rf /tmp/engram-bufpool-test-* +[ $rc -ne 0 ] && exit $rc + +echo +echo "== ASan+UBSan build (memory-error + UB checks; LSan unavailable on macOS) ==" +ABIN="/tmp/test_bufpool_asan.$$" +gcc -O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer -std=c11 "$TST" "$SRC" -o "$ABIN" +ASAN_OPTIONS=detect_leaks=0 UBSAN_OPTIONS=halt_on_error=1 "$ABIN"; rc=$? +rm -f "$ABIN"; rm -rf /tmp/engram-bufpool-test-* +exit $rc diff --git a/engram/test/test_bufpool.c b/engram/test/test_bufpool.c new file mode 100644 index 0000000..00564fe --- /dev/null +++ b/engram/test/test_bufpool.c @@ -0,0 +1,496 @@ +/* test_bufpool.c — M4 gate for the demand-paging BUFFER POOL (engram_store.{c,h}). + * + * Pure C. Build: gcc -O2 test_bufpool.c ../../lang/runtime/engram_store.c -o t + * Writes ONLY under a throwaway /tmp dir. Never touches ~/.neuron or live ports. + * + * Proves the M4 pool preserves every M1/M2 invariant when the pool is SMALLER + * than the store (pages evict + re-fault): small-pool round-trip correctness, + * LRU eviction policy (hot resident / cold evicted / no dirty stolen), pinned + * residency (superblocks, index roots, explicit page + hot-layer pins), bounded + * read-ahead, and crash safety (WAL replay + checkpoint-crash) under paging. + */ +#include "../../lang/runtime/engram_store.h" + +#include +#include +#include +#include +#include +#include +#include + +static int g_pass = 0, g_fail = 0; +static void ok(const char* name, int cond){ + printf(" [%s] %s\n", cond ? "PASS" : "FAIL", name); + if (cond) g_pass++; else g_fail++; +} + +static char g_dir[512]; +static void mk_dir(void){ + snprintf(g_dir, sizeof g_dir, "/tmp/engram-bufpool-test-%d", (int)getpid()); + mkdir(g_dir, 0700); +} +static void path_in(char* out, size_t cap, const char* name){ + snprintf(out, cap, "%s/%s", g_dir, name); +} + +/* ── deterministic generators (bit-exact regeneration for oracles) ─────────── */ +static uint64_t xs(uint64_t* s){ uint64_t x=*s; x^=x<<13; x^=x>>7; x^=x<<17; *s=x; return x; } +static uint64_t node_seed(int i){ return 0x9E3779B97F4A7C15ULL ^ ((uint64_t)(i+1)*0xD1B54A32D192ED03ULL); } +static uint64_t edge_seed(int i){ return 0xC2B2AE3D27D4EB4FULL ^ ((uint64_t)(i+1)*0x165667B19E3779F9ULL); } +static char* rnd_str(uint64_t* st, size_t len){ + char* s = (char*)malloc(len + 1); + for (size_t i=0;iid = strdup(id); + size_t clen = (i % 500 == 0) ? (size_t)(17000 + (xs(&st) % 6000)) : (size_t)(xs(&st) % 300); + n->content = rnd_str(&st, clen); + n->node_type = rnd_str(&st, 4 + (xs(&st) % 8)); + n->label = (i % 2) ? rnd_str(&st, 3 + (xs(&st) % 10)) : NULL; + n->tier = rnd_str(&st, 4 + (xs(&st) % 6)); + n->tags = rnd_str(&st, xs(&st) % 40); + n->metadata = (i % 3) ? rnd_str(&st, xs(&st) % 60) : NULL; + n->salience = (double)(xs(&st) % 1000000) / 997.0; + n->importance = (double)(xs(&st) % 1000000) / 131.0; + n->confidence = (double)(xs(&st) % 1000000) / 733.0; + n->temporal_decay_rate = (double)(xs(&st) % 1000000) / 101.0; + n->activation_count = (int64_t)(xs(&st) % 100000); + n->last_activated = (int64_t)xs(&st); + n->created_at = (int64_t)(1600000000000LL + i); + n->updated_at = (int64_t)xs(&st); + n->background_activation = (double)(xs(&st) % 1000000) / 17.0; + n->working_memory_weight = (double)(xs(&st) % 1000000) / 29.0; + n->suppression_count = (int32_t)(xs(&st) % 50); + n->layer_id = (uint32_t)(xs(&st) % 5); + for (int k=0;kaccess_ts[k] = (int64_t)xs(&st); + n->access_head = (int32_t)(xs(&st) % STORE_BLL_K); + n->access_filled = (int32_t)(xs(&st) % (STORE_BLL_K + 1)); + n->wm_anchor = (double)(xs(&st) % 1000000) / 3.0; + n->emb = (float*)malloc(EMB_DIM * sizeof(float)); + for (int k=0;kemb[k], &u, 4); } + n->emb_dim = EMB_DIM; +} +static void gen_edge(int i, StoreEdge* e){ + memset(e, 0, sizeof *e); + uint64_t st = edge_seed(i); + char id[32], from[32], to[32]; + snprintf(id, sizeof id, "edge-%d", i); + snprintf(from, sizeof from, "node-%d", (int)(xs(&st) % NODE_COUNT)); + snprintf(to, sizeof to, "node-%d", (int)(xs(&st) % NODE_COUNT)); + e->id = strdup(id); e->from_id = strdup(from); e->to_id = strdup(to); + e->relation = rnd_str(&st, 3 + (xs(&st) % 12)); + e->metadata = (i % 4) ? rnd_str(&st, xs(&st) % 40) : NULL; + e->weight = (double)(xs(&st) % 1000000) / 111.0; + e->hebb = (double)(xs(&st) % 1000000) / 1000000.0; + e->confidence = (double)(xs(&st) % 1000000) / 777.0; + e->created_at = (int64_t)(1600000000000LL + i); + e->updated_at = (int64_t)xs(&st); + e->last_fired = (int64_t)xs(&st); + e->inhibitory = (int32_t)(xs(&st) % 2); + e->layer_id = (uint32_t)(xs(&st) % 5); +} +static int streq(const char* a, const char* b){ + if (!a && !b) return 1; + if (!a || !b) return 0; + return strcmp(a,b)==0; +} +static int cmp_node(const StoreNode* a, const StoreNode* b){ + if (!streq(a->id,b->id) || !streq(a->content,b->content) || + !streq(a->node_type,b->node_type) || !streq(a->label,b->label) || + !streq(a->tier,b->tier) || !streq(a->tags,b->tags) || + !streq(a->metadata,b->metadata)) return 0; + if (a->salience!=b->salience || a->importance!=b->importance || + a->confidence!=b->confidence || a->temporal_decay_rate!=b->temporal_decay_rate || + a->activation_count!=b->activation_count || a->last_activated!=b->last_activated || + a->created_at!=b->created_at || a->updated_at!=b->updated_at || + a->background_activation!=b->background_activation || + a->working_memory_weight!=b->working_memory_weight || + a->suppression_count!=b->suppression_count || a->layer_id!=b->layer_id || + a->access_head!=b->access_head || a->access_filled!=b->access_filled || + a->wm_anchor!=b->wm_anchor || a->emb_dim!=b->emb_dim) return 0; + for (int k=0;kaccess_ts[k]!=b->access_ts[k]) return 0; + if ((a->emb==NULL) != (b->emb==NULL)) return 0; + if (a->emb && memcmp(a->emb, b->emb, (size_t)a->emb_dim*4)!=0) return 0; + return 1; +} +static int cmp_edge(const StoreEdge* a, const StoreEdge* b){ + if (!streq(a->id,b->id) || !streq(a->from_id,b->from_id) || !streq(a->to_id,b->to_id) || + !streq(a->relation,b->relation) || !streq(a->metadata,b->metadata)) return 0; + if (a->weight!=b->weight || a->hebb!=b->hebb || a->confidence!=b->confidence || + a->created_at!=b->created_at || a->updated_at!=b->updated_at || + a->last_fired!=b->last_fired || a->inhibitory!=b->inhibitory || + a->layer_id!=b->layer_id) return 0; + return 1; +} +static void free_node_fields(StoreNode* n){ + free(n->id); free(n->content); free(n->node_type); free(n->label); + free(n->tier); free(n->tags); free(n->metadata); free(n->emb); free(n->unknown); +} +static void free_edge_fields(StoreEdge* e){ + free(e->id); free(e->from_id); free(e->to_id); free(e->relation); free(e->metadata); free(e->unknown); +} + +/* ════════════════════════════════════════════════════════════════════════════ + * TEST 1 — SMALL-POOL CORRECTNESS: full M1 workload (5k nodes / 20k edges) with + * a frame budget FAR smaller than the store → constant eviction + re-fault, yet + * every read is bit-exact and the pool stays bounded. + * ════════════════════════════════════════════════════════════════════════════ */ +static void test_small_pool_roundtrip(void){ + printf("\n== 1) small-pool correctness: %d nodes + %d edges, cap=%d frames ==\n", + NODE_COUNT, EDGE_COUNT, 32); + char path[600]; path_in(path, sizeof path, "small.store"); + unlink(path); + EngramPagedStore* s = store_create(path); + ok("store_create", s != NULL); + if (!s) return; + store__set_pool_frames(s, 32); /* pool << store */ + + for (int i=0;i 0); + ok("pool stayed bounded (resident <= cap)", st.resident <= st.cap); + ok("no dirty frames after checkpoint", st.dirty == 0); + + /* read back EVERY node bit-exact despite constant eviction/re-fault */ + int bad = 0; + for (int i=0;i=N) cold=200; + store_pool_stats(s,&a); + StoreNode g; if (store_get_node(s,id,&g)==1) store_node_free(&g); + store_pool_stats(s,&b); + cold_faults += (b.misses - a.misses); + } + } + printf(" hot re-get faults (post-warm)=%llu cold stream faults=%llu\n", + (unsigned long long)hot_faults, (unsigned long long)cold_faults); + ok("HOT pages stay resident (0 faults on re-access)", hot_faults == 0); + ok("COLD pages get evicted + re-faulted", cold_faults > 0); + store_pool_stats(s,&b); + double hr = (double)b.hits / (double)(b.hits + b.misses); + printf(" overall hit-rate = %.3f (hits=%llu misses=%llu)\n", + hr, (unsigned long long)b.hits, (unsigned long long)b.misses); + ok("hit-rate is sane (> 0.5)", hr > 0.5); + store_close(s); + unlink(path); + + /* ---- part B: no-steal (dirty pages never evicted before checkpoint) ---- */ + EngramPagedStore* s2 = store_create(path); + if (!s2){ ok("store_create(2)", 0); return; } + store__set_pool_frames(s2, 8); /* tiny budget */ + for (int i=0;i<1200;i++){ StoreNode n; gen_node(i,&n); store_put_node(s2,&n); free_node_fields(&n); } + /* NO sync: every mutated page is dirty and, by no-steal, unevictable */ + StorePoolStats d; store_pool_stats(s2,&d); + printf(" tiny cap=%zu, unsynced burst: resident=%zu dirty=%zu evictions=%llu\n", + d.cap, d.resident, d.dirty, (unsigned long long)d.evictions); + ok("dirty pages pinned in RAM beyond budget (no-steal)", d.dirty > d.cap && d.resident > d.cap); + /* a just-written node is served correctly from its dirty in-RAM page */ + { StoreNode want; gen_node(777,&want); StoreNode got; int hit=store_get_node(s2,want.id,&got); + ok("read served correctly from dirty (un-flushed) page", hit==1 && cmp_node(&want,&got)); + if (hit==1) store_node_free(&got); free_node_fields(&want); } + store_sync(s2); /* checkpoint → dirty become clean/evictable */ + store_pool_stats(s2,&d); + ok("checkpoint cleared all dirty frames", d.dirty == 0); + /* durability across reopen after the no-steal burst */ + store_close(s2); + EngramPagedStore* s3 = store_open(path); + store__set_pool_frames(s3, 8); + int miss=0; for (int i=0;i<1200;i++){ StoreNode want; gen_node(i,&want); + StoreNode got; int hit=store_get_node(s3,want.id,&got); + if (hit!=1 || !cmp_node(&want,&got)) miss++; + if (hit==1) store_node_free(&got); free_node_fields(&want); } + ok("all 1200 survive reopen, bit-exact, tiny pool", miss==0); + store_close(s3); + unlink(path); +} + +/* ════════════════════════════════════════════════════════════════════════════ + * TEST 3 — PINNED RESIDENCY: superblocks + index roots never evicted under heavy + * thrash; an explicitly pinned page stays until unpinned; a pinned hot layer's + * pages stay resident and are released on unpin. + * ════════════════════════════════════════════════════════════════════════════ */ +static void test_pinning(void){ + printf("\n== 3) pinned residency: superblocks / index roots / page / layer ==\n"); + char path[600]; path_in(path, sizeof path, "pin.store"); + unlink(path); + EngramPagedStore* s = store_create(path); + if (!s){ ok("store_create", 0); return; } + const int N = 1500; + for (int i=0;i=4: 2 SB + 2 roots)", st.pinned >= 4); + + /* unpin the page → it becomes evictable and is dropped under further thrash */ + store_unpin_page(s, P); + for (int i=0;i 0); + store_pool_stats(s,&st); + size_t pinned_with_layer = st.pinned; + for (int pass=0; pass<3; pass++) + for (int i=0;i= pinned_with_layer); + ok("layer pin holds >= npin extra frames", st.pinned >= (size_t)npin + 4); + + store_unpin_layer(s, 3); + store_pool_stats(s,&st); + size_t after_unpin_max = st.pinned; + for (int i=0;i 0); + unlink(path); +} + +/* ════════════════════════════════════════════════════════════════════════════ + * TEST 5 — CRASH SAFETY UNDER PAGING: WAL replay and checkpoint-crash recovery + * with a tiny pool (pages evict + re-fault during replay). + * ════════════════════════════════════════════════════════════════════════════ */ +static void test_crash_under_paging(void){ + printf("\n== 5) crash safety under a tiny pool (ENGRAM_POOL_FRAMES=16) ==\n"); + setenv("ENGRAM_POOL_FRAMES", "16", 1); /* every engram_open() below is paged */ + setenv("ENGRAM_WAL_SYNC", "always", 1); + + /* ---- 5a: power-loss → WAL replay ---- */ + char dir[600]; path_in(dir, sizeof dir, "crash_wal"); mkdir(dir, 0700); + EngramPagedStore* s = engram_open(dir); + if (!s){ ok("engram_open", 0); return; } + const int M = 400; + for (int i=0;i= (size_t)(1u<<20)); + ok("no eviction ever fired at default budget", st.evictions == 0); + ok("whole store resident (every page cached)", st.resident == store_page_count(s)); + store_close(s); + unlink(path); +} + +int main(void){ + mk_dir(); + printf("engram M4 buffer-pool gate — dir=%s\n", g_dir); + test_small_pool_roundtrip(); + test_eviction_policy(); + test_pinning(); + test_prefetch(); + test_crash_under_paging(); + test_default_is_phase1(); + printf("\n================ %d passed, %d failed ================\n", g_pass, g_fail); + return g_fail ? 1 : 0; +} diff --git a/lang/runtime/engram_store.c b/lang/runtime/engram_store.c index 678c1af..9cce290 100644 --- a/lang/runtime/engram_store.c +++ b/lang/runtime/engram_store.c @@ -143,13 +143,42 @@ struct EngramPagedStore { uint64_t ckpt_threshold; /* auto-checkpoint after this many ops (0 = never) */ }; -/* M2 buffer-pool hooks (defined in the M2 section at the bottom of this file). */ -typedef struct PgEnt { uint64_t id; uint8_t* buf; uint64_t lsn; int dirty; struct PgEnt* next; } PgEnt; +/* M2/M4 buffer-pool hooks (defined in the pool section at the bottom of this file). + * M2 shipped a write-back, no-steal cache (dirty→disk only at checkpoint). M4 + * turns it into a bounded, demand-paged buffer pool: a fixed frame budget, LRU + * eviction of CLEAN unpinned frames (no-steal preserved — dirty frames are never + * stolen), pinning of hot/structural pages, and bounded read-ahead. `lru_*` + * thread every resident frame onto an MRU→LRU list; `pin` is an explicit pin + * count (0 = unpinned). */ +typedef struct PgEnt { + uint64_t id; uint8_t* buf; uint64_t lsn; int dirty; struct PgEnt* next; + int pin; /* explicit pin count (0 = unpinned) */ + struct PgEnt* lru_prev; /* MRU→LRU doubly-linked list */ + struct PgEnt* lru_next; +} PgEnt; static PgCache* pc_new(void); static void pc_free(PgCache* c); static PgEnt* pc_get(EngramPagedStore* s, uint64_t id); static int pc_put(EngramPagedStore* s, uint64_t id, const uint8_t* buf, int dirty); static int pc_flush(EngramPagedStore* s); /* pwrite all dirty → clean */ +static void pc_prefetch(EngramPagedStore* s, uint64_t from_id, unsigned window); +static void store__autopin(EngramPagedStore* s); /* pin superblocks + index roots */ + +/* Per-layer pin record: the set of pages pinned on behalf of a hot layer, kept + * so store_unpin_layer can release exactly what store_pin_layer pinned. */ +typedef struct { uint32_t layer; uint64_t* pages; size_t n; } LayerPin; + +/* The bounded, demand-paged frame table (M4). Defined here (not in the pool + * section) so page_read / the scan loops can read its stats + prefetch window. */ +struct PgCache { + PgEnt** buckets; size_t nbuckets; size_t count; + size_t cap; /* max resident frames; 0 = unlimited */ + PgEnt* mru; PgEnt* lru; /* MRU (front) → LRU (back) recency list */ + unsigned prefetch; /* read-ahead window (pages); 0 = off */ + LayerPin* lp; size_t lp_n, lp_cap; /* hot-layer pin bookkeeping */ + /* stats (introspection only — never affect semantics) */ + uint64_t hits, misses, evictions, prefetch_reads; +}; /* ── little-endian scalar codecs ──────────────────────────────────────────── */ static void put_u16(uint8_t* p, uint16_t v){ p[0]=(uint8_t)v; p[1]=(uint8_t)(v>>8); } @@ -197,12 +226,12 @@ static uint64_t id_hash(const char* s){ static int page_read(EngramPagedStore* s, uint64_t id, uint8_t* buf){ if (s->cache){ PgEnt* e = pc_get(s, id); - if (e){ memcpy(buf, e->buf, STORE_PAGE_SIZE); return 0; } + if (e){ memcpy(buf, e->buf, STORE_PAGE_SIZE); s->cache->hits++; return 0; } } - off_t off = (off_t)id * STORE_PAGE_SIZE; + off_t off = (off_t)id * STORE_PAGE_SIZE; /* demand fault: not resident */ ssize_t r = pread(s->fd, buf, STORE_PAGE_SIZE, off); if (r != (ssize_t)STORE_PAGE_SIZE) return -1; - if (s->cache) pc_put(s, id, buf, 0); /* cache clean */ + if (s->cache){ s->cache->misses++; pc_put(s, id, buf, 0); } /* cache clean */ return 0; } static int page_write_raw(EngramPagedStore* s, uint64_t id, const uint8_t* buf){ @@ -834,6 +863,7 @@ EngramPagedStore* store_create(const char* path){ close(s->fd); free(s); return NULL; } if (store_sync(s)!=0){ close(s->fd); free(s); return NULL; } + store__autopin(s); /* keep superblocks + index roots resident */ return s; } @@ -861,6 +891,7 @@ EngramPagedStore* store_open(const char* path){ s->next_lsn = (s->last_checkpoint_lsn > s->sb_seq) ? s->last_checkpoint_lsn : s->sb_seq; s->cur_node_page = 0; s->cur_edge_page = 0; + store__autopin(s); /* keep superblocks + index roots resident */ return s; } @@ -1169,6 +1200,7 @@ int store_scan_nodes(EngramPagedStore* s, StoreNodeScanCb cb, void* ctx){ int count = 0; for (uint64_t pg = 2; pg < s->page_count; pg++){ if (page_read(s, pg, buf) != 0) continue; + if (s->cache) pc_prefetch(s, pg, s->cache->prefetch); /* sequential read-ahead */ if (buf[8] != STORE_PT_NODE) continue; int ns = slp_count(buf); for (int i = 0; i < ns; i++){ @@ -1198,6 +1230,7 @@ int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx){ int count = 0; for (uint64_t pg = 2; pg < s->page_count; pg++){ if (page_read(s, pg, buf) != 0) continue; + if (s->cache) pc_prefetch(s, pg, s->cache->prefetch); /* sequential read-ahead */ if (buf[8] != STORE_PT_EDGE) continue; int ns = slp_count(buf); for (int i = 0; i < ns; i++){ @@ -1247,8 +1280,36 @@ int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx){ #include -/* ── write-back buffer pool ────────────────────────────────────────────────── */ -struct PgCache { PgEnt** buckets; size_t nbuckets; size_t count; }; +/* ══════════════════════════════════════════════════════════════════════════════ + * M4 — demand-paging BUFFER POOL (bounded, LRU, pinned, read-ahead) + * + * A frame table (id→frame hash) capped at `cap` resident frames. On a page + * access that is not resident, page_read faults it in from neuron.egm; if the + * pool is full, the LRU eviction path reclaims a CLEAN, unpinned frame. This is + * purely additive residency — the on-disk format is unchanged, and with the + * DEFAULT cap (large) no eviction ever fires, so behaviour is byte-for-byte the + * Phase-1 resident store. + * + * Invariants preserved from M2 (write-back, NO-STEAL): + * • A DIRTY frame is NEVER evicted (never stolen) — its only durable copy is + * the fsync'd WAL, and the store page reaches disk solely at a checkpoint. + * pc_flush (checkpoint) is what turns dirty→clean and thus evictable. + * • A PINNED frame is never evicted. Structural pages are auto-pinned: the two + * superblocks (pages 0,1) and every index ROOT/INTERIOR page (type INDEX, + * leaf-flag 0). Leaves are pageable. Explicit pins (pin count) cover hot + * layers and any caller-designated page. + * Correctness under a pool SMALLER than the store rests on: every caller copies + * page bytes into a local stack buffer (memcpy in page_read / out in page_write) + * and never retains a frame pointer across another page access, so a frame may + * be evicted and later re-faulted with no aliasing hazard. A clean frame always + * matches disk, so a re-fault reproduces identical bytes. + * ════════════════════════════════════════════════════════════════════════════ */ + +/* default frame budget: large enough that today's whole store stays resident + * (== Phase 1). Override with env ENGRAM_POOL_FRAMES (0 = unlimited). */ +#ifndef ENGRAM_POOL_FRAMES_DEFAULT +#define ENGRAM_POOL_FRAMES_DEFAULT (1u<<20) /* ~1M frames × 16KiB = 16 GiB */ +#endif static PgCache* pc_new(void){ PgCache* c = (PgCache*)calloc(1, sizeof *c); @@ -1256,6 +1317,12 @@ static PgCache* pc_new(void){ c->nbuckets = 1024; c->buckets = (PgEnt**)calloc(c->nbuckets, sizeof(PgEnt*)); if (!c->buckets){ free(c); return NULL; } + c->cap = ENGRAM_POOL_FRAMES_DEFAULT; + c->prefetch = 8; + const char* pf = getenv("ENGRAM_POOL_FRAMES"); + if (pf && *pf){ char* end=NULL; unsigned long long v = strtoull(pf,&end,10); c->cap = (size_t)v; } + const char* pw = getenv("ENGRAM_PREFETCH"); + if (pw && *pw){ char* end=NULL; unsigned long v = strtoul(pw,&end,10); c->prefetch = (unsigned)v; } return c; } static void pc_free(PgCache* c){ @@ -1264,14 +1331,27 @@ static void pc_free(PgCache* c){ PgEnt* e = c->buckets[i]; while (e){ PgEnt* n=e->next; free(e->buf); free(e); e=n; } } + for (size_t i=0;ilp_n;i++) free(c->lp[i].pages); + free(c->lp); free(c->buckets); free(c); } -static PgEnt* pc_get(EngramPagedStore* s, uint64_t id){ - PgCache* c = s->cache; - PgEnt* e = c->buckets[id % c->nbuckets]; - while (e){ if (e->id==id) return e; e=e->next; } - return NULL; + +/* ── LRU recency list (front = MRU, back = LRU) ─────────────────────────────── */ +static void lru_unlink(PgCache* c, PgEnt* e){ + if (e->lru_prev) e->lru_prev->lru_next = e->lru_next; else c->mru = e->lru_next; + if (e->lru_next) e->lru_next->lru_prev = e->lru_prev; else c->lru = e->lru_prev; + e->lru_prev = e->lru_next = NULL; } +static void lru_push_front(PgCache* c, PgEnt* e){ + e->lru_prev = NULL; e->lru_next = c->mru; + if (c->mru) c->mru->lru_prev = e; c->mru = e; + if (!c->lru) c->lru = e; +} +static void lru_touch(PgCache* c, PgEnt* e){ + if (c->mru == e) return; + lru_unlink(c, e); lru_push_front(c, e); +} + static void pc_maybe_grow(PgCache* c){ if (c->count <= c->nbuckets*4) return; size_t nn = c->nbuckets*2; @@ -1283,9 +1363,54 @@ static void pc_maybe_grow(PgCache* c){ } free(c->buckets); c->buckets=nb; c->nbuckets=nn; } + +/* A frame is EVICTABLE iff it is clean, unpinned, not a superblock, and not an + * index root/interior page. This is the sole place the no-steal + structural-pin + * policy is enforced. */ +static int pc_evictable(const PgEnt* e){ + if (e->dirty) return 0; /* no-steal: dirty pages are pinned to RAM */ + if (e->pin > 0) return 0; /* explicit / hot-layer pin */ + if (e->id == 0 || e->id == 1) return 0; /* superblock + mirror */ + if (e->buf[8] == STORE_PT_INDEX && e->buf[IDX_LEAF_OFF] == 0) return 0; /* root/interior */ + return 1; +} +/* Detach `e` from both the hash chain and the recency list, and free it. */ +static void pc_remove(PgCache* c, PgEnt* e){ + size_t b = e->id % c->nbuckets; + PgEnt** pp = &c->buckets[b]; + while (*pp && *pp != e) pp = &(*pp)->next; + if (*pp == e) *pp = e->next; + lru_unlink(c, e); + free(e->buf); free(e); + c->count--; +} +/* Reclaim clean unpinned frames from the LRU end until under budget, or until no + * evictable frame remains (a dirty/pinned-heavy pool may transiently exceed cap — + * that is the no-steal guarantee, not a bug: the next checkpoint frees them). */ +static void pc_evict_to_budget(PgCache* c){ + if (!c->cap) return; /* unlimited */ + while (c->count > c->cap){ + PgEnt* e = c->lru; int freed = 0; + while (e){ + PgEnt* prev = e->lru_prev; /* walk LRU→MRU */ + if (pc_evictable(e)){ pc_remove(c, e); c->evictions++; freed = 1; break; } + e = prev; + } + if (!freed) break; /* nothing evictable — allowed to exceed cap */ + } +} + +static PgEnt* pc_get(EngramPagedStore* s, uint64_t id){ + PgCache* c = s->cache; + PgEnt* e = c->buckets[id % c->nbuckets]; + while (e){ if (e->id==id){ lru_touch(c, e); return e; } e=e->next; } + return NULL; +} +/* Insert-or-update a frame. New frames go to MRU; then evict down to budget. + * The just-touched frame is at MRU and can never be the eviction victim. */ static int pc_put(EngramPagedStore* s, uint64_t id, const uint8_t* buf, int dirty){ PgCache* c = s->cache; - PgEnt* e = pc_get(s, id); + PgEnt* e = pc_get(s, id); /* pc_get also bumps it to MRU on a hit */ if (!e){ e = (PgEnt*)calloc(1, sizeof *e); if (!e) return -1; @@ -1294,11 +1419,13 @@ static int pc_put(EngramPagedStore* s, uint64_t id, const uint8_t* buf, int dirt e->id = id; size_t b = id % c->nbuckets; e->next = c->buckets[b]; c->buckets[b] = e; c->count++; + lru_push_front(c, e); pc_maybe_grow(c); } memcpy(e->buf, buf, STORE_PAGE_SIZE); e->lsn = get_u64(buf + 16); if (dirty) e->dirty = 1; + pc_evict_to_budget(c); return 0; } static int pc_flush(EngramPagedStore* s){ @@ -1307,8 +1434,159 @@ static int pc_flush(EngramPagedStore* s){ for (size_t i=0;inbuckets;i++) for (PgEnt* e=c->buckets[i]; e; e=e->next) if (e->dirty){ if (page_write_raw(s, e->id, e->buf)!=0) return -1; e->dirty=0; } + /* Post-checkpoint the just-cleaned frames are now evictable; trim the pool + * back to budget so a dirty-heavy burst that transiently overshot cap does + * not leave the pool oversized. No-op at the default (unlimited-ish) cap. */ + pc_evict_to_budget(c); return 0; } +/* Bounded sequential read-ahead: fault the next `window` pages after `from_id` + * into any spare capacity, so a forward scan/leaf-walk hits them instead of + * faulting one-by-one. Never forces an eviction (fills slack only), never + * re-reads a resident page. Prefetch reads are counted separately from demand + * faults so a scan's fault count reflects on-demand misses only. */ +static void pc_prefetch(EngramPagedStore* s, uint64_t from_id, unsigned window){ + PgCache* c = s->cache; + if (!c || !window) return; + for (unsigned k=1; k<=window; k++){ + uint64_t id = from_id + k; + if (id >= s->page_count) break; + if (c->cap && c->count + 1 > c->cap) break; /* no eviction for read-ahead */ + if (c->buckets[id % c->nbuckets]){ + PgEnt* e = c->buckets[id % c->nbuckets]; + int resident = 0; while (e){ if (e->id==id){ resident=1; break; } e=e->next; } + if (resident) continue; + } + uint8_t buf[STORE_PAGE_SIZE]; + off_t off = (off_t)id * STORE_PAGE_SIZE; + if (pread(s->fd, buf, STORE_PAGE_SIZE, off) != (ssize_t)STORE_PAGE_SIZE) break; + pc_put(s, id, buf, 0); + c->prefetch_reads++; + } +} + +/* Non-LRU-touching frame lookup (for pin bookkeeping that must not reorder). */ +static PgEnt* pc_find(PgCache* c, uint64_t id){ + PgEnt* e = c->buckets[id % c->nbuckets]; + while (e){ if (e->id==id) return e; e=e->next; } + return NULL; +} + +/* ── public pin / prefetch / stats API (M4) ─────────────────────────────────── */ +int store_pin_page(EngramPagedStore* s, uint64_t page_id){ + if (!s || !s->cache) return -1; + uint8_t buf[STORE_PAGE_SIZE]; + if (page_read(s, page_id, buf) != 0) return -1; /* fault in + make resident */ + PgEnt* e = pc_find(s->cache, page_id); + if (!e) return -1; + e->pin++; + return 0; +} +int store_unpin_page(EngramPagedStore* s, uint64_t page_id){ + if (!s || !s->cache) return -1; + PgEnt* e = pc_find(s->cache, page_id); + if (e && e->pin > 0) e->pin--; + return 0; +} + +/* Pin every page currently holding a live record of `layer` (hot-layer residency). + * Records the pinned pages so store_unpin_layer releases exactly this set. Pages + * are pinned BEFORE their bodies are read so a small pool cannot evict them mid-scan. */ +int store_pin_layer(EngramPagedStore* s, uint32_t layer){ + if (!s || !s->cache) return -1; + uint64_t* pages = NULL; size_t np = 0, cap = 0; + uint8_t buf[STORE_PAGE_SIZE]; + for (uint64_t pg = 2; pg < s->page_count; pg++){ + if (page_read(s, pg, buf) != 0) continue; + int t = buf[8]; + if (t != STORE_PT_NODE && t != STORE_PT_EDGE) continue; + PgEnt* pe = pc_find(s->cache, pg); + if (!pe) continue; + pe->pin++; /* provisional pin: keeps pg resident */ + int ns = slp_count(buf), match = 0; + for (int i = 0; i < ns && !match; i++){ + uint16_t off, len, fl; slp_slot(buf, i, &off, &len, &fl); + if (fl != SLOT_LIVE) continue; + uint8_t* body; size_t blen; int live; + if (read_body(s, pg, (uint16_t)i, &body, &blen, &live) != 0) continue; + uint32_t lid = 0; + if (t == STORE_PT_NODE){ StoreNode c; node_parse(body, blen, &c); lid = c.layer_id; store_node_free(&c); } + else { StoreEdge c; edge_parse(body, blen, &c); lid = c.layer_id; store_edge_free(&c); } + free(body); + if (lid == layer) match = 1; + } + if (match){ + if (np == cap){ cap = cap ? cap*2 : 16; uint64_t* np2 = (uint64_t*)realloc(pages, cap*sizeof *pages); if (!np2){ free(pages); return -1; } pages = np2; } + pages[np++] = pg; /* keep the pin */ + } else { + pe->pin--; /* no match on this page: drop provisional pin */ + } + } + PgCache* c = s->cache; + if (c->lp_n == c->lp_cap){ c->lp_cap = c->lp_cap ? c->lp_cap*2 : 8; c->lp = (LayerPin*)realloc(c->lp, c->lp_cap*sizeof *c->lp); } + c->lp[c->lp_n].layer = layer; c->lp[c->lp_n].pages = pages; c->lp[c->lp_n].n = np; c->lp_n++; + return (int)np; +} +int store_unpin_layer(EngramPagedStore* s, uint32_t layer){ + if (!s || !s->cache) return -1; + PgCache* c = s->cache; + for (size_t i = 0; i < c->lp_n; i++){ + if (c->lp[i].layer != layer) continue; + for (size_t j = 0; j < c->lp[i].n; j++){ + PgEnt* e = pc_find(c, c->lp[i].pages[j]); + if (e && e->pin > 0) e->pin--; + } + free(c->lp[i].pages); + c->lp[i] = c->lp[--c->lp_n]; /* swap-remove */ + return 0; + } + return 0; +} + +/* Auto-pin the structural pages: both superblocks and the two index roots (plus + * the layer registry). A SHALLOW index root is a LEAF, so it is not covered by + * the "index interior" eviction rule — pinning it explicitly guarantees the root + * is never evicted even for a tiny tree. Deeper roots/interiors are additionally + * covered by pc_evictable's INDEX-non-leaf rule. Best-effort (ignores errors on + * a not-yet-built store). */ +static void store__autopin(EngramPagedStore* s){ + if (!s || !s->cache) return; + store_pin_page(s, 0); + store_pin_page(s, 1); + if (s->root_index_page) store_pin_page(s, s->root_index_page); + if (s->adj_index_page) store_pin_page(s, s->adj_index_page); + if (s->layer_registry_page) store_pin_page(s, s->layer_registry_page); +} + +/* Introspection + test hooks. */ +void store_pool_stats(const EngramPagedStore* s, StorePoolStats* out){ + if (!out) return; + memset(out, 0, sizeof *out); + if (!s || !s->cache) return; + const PgCache* c = s->cache; + out->cap = c->cap; out->resident = c->count; out->prefetch = c->prefetch; + out->hits = c->hits; out->misses = c->misses; + out->evictions = c->evictions; out->prefetch_reads = c->prefetch_reads; + size_t pinned = 0, dirty = 0; + for (size_t i=0;inbuckets;i++) + for (PgEnt* e=c->buckets[i]; e; e=e->next){ + if (!pc_evictable(e)) pinned++; + if (e->dirty) dirty++; + } + out->pinned = pinned; out->dirty = dirty; +} +int store_pool_resident(const EngramPagedStore* s, uint64_t page_id){ + if (!s || !s->cache) return -1; + return pc_find(s->cache, page_id) ? 1 : 0; +} +void store__set_pool_frames(EngramPagedStore* s, size_t frames){ + if (!s || !s->cache) return; + s->cache->cap = frames; + pc_evict_to_budget(s->cache); /* apply the new budget now */ +} +void store__set_prefetch(EngramPagedStore* s, unsigned window){ + if (s && s->cache) s->cache->prefetch = window; +} /* ── WAL log ───────────────────────────────────────────────────────────────── */ enum { OP_NODE_PUT=1, OP_EDGE_PUT, OP_TOMBSTONE, OP_SUPERSEDE, OP_LAYER_PUT, OP_LAYER_DEL, OP_FORGET, OP_HEBB_BATCH, OP_CHECKPOINT }; diff --git a/lang/runtime/engram_store.h b/lang/runtime/engram_store.h index d9491bb..c75592d 100644 --- a/lang/runtime/engram_store.h +++ b/lang/runtime/engram_store.h @@ -209,6 +209,41 @@ int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx); uint64_t engram_wal_next_lsn(const EngramPagedStore* s); uint64_t engram_last_checkpoint_lsn(const EngramPagedStore* s); +/* ── M4: demand-paging buffer pool (additive residency; on-disk format UNCHANGED) ── + * + * The write-back, no-steal cache of M2 becomes a bounded, demand-paged buffer + * pool. A fixed frame budget (env ENGRAM_POOL_FRAMES; 0 = unlimited; default + * large ⇒ whole store resident ⇒ identical to Phase 1) keeps only hot pages in + * RAM; a page access that is not resident faults in from neuron.egm, and under + * pressure a CLEAN, unpinned frame is evicted (LRU). Dirty frames are never + * stolen (M2 no-steal / WAL durability), and superblocks + index root/interior + * pages are auto-pinned. Prefetch (env ENGRAM_PREFETCH) reads ahead on scans. */ + +/* Pin / unpin an individual page (faults it in and keeps it resident until + * unpinned). Pin a hot layer's pages (WM/core) as a set. Idempotent counts. */ +int store_pin_page(EngramPagedStore* s, uint64_t page_id); +int store_unpin_page(EngramPagedStore* s, uint64_t page_id); +int store_pin_layer(EngramPagedStore* s, uint32_t layer); /* returns #pages pinned */ +int store_unpin_layer(EngramPagedStore* s, uint32_t layer); + +/* Buffer-pool introspection. */ +typedef struct StorePoolStats { + size_t cap; /* frame budget (0 = unlimited) */ + size_t resident; /* frames currently resident */ + size_t pinned; /* frames that cannot be evicted (dirty/pinned/structural) */ + size_t dirty; /* dirty (un-checkpointed) frames */ + unsigned prefetch; /* read-ahead window */ + uint64_t hits, misses; /* page_read cache hits / demand faults */ + uint64_t evictions; /* clean frames reclaimed */ + uint64_t prefetch_reads; /* pages brought in by read-ahead */ +} StorePoolStats; +void store_pool_stats(const EngramPagedStore* s, StorePoolStats* out); +int store_pool_resident(const EngramPagedStore* s, uint64_t page_id); + +/* Test hooks: set the frame budget / prefetch window at runtime (NOT format). */ +void store__set_pool_frames(EngramPagedStore* s, size_t frames); +void store__set_prefetch(EngramPagedStore* s, unsigned window); + /* Crash-test hooks (writes only under a throwaway dir). * store__crash — abandon all RAM state without flush/fsync (power loss). * store__flush_pages — pwrite dirty pages to disk WITHOUT a checkpoint (steal).