engram tiered storage M4: demand-paging buffer pool (Phase 2, additive)
Turn M2's write-back/no-steal cache into a bounded, demand-paged buffer pool so
the paged store can exceed RAM while keeping only hot pages resident. On-disk
format UNCHANGED (additive residency only; no migration). Default budget is large
enough that today's store stays fully resident, so default behaviour == Phase 1.
- Frame table capped at `cap` frames (env ENGRAM_POOL_FRAMES; 0 = unlimited;
default 1<<20). Not-resident access faults in from neuron.egm.
- LRU eviction of CLEAN, unpinned frames only. Dirty frames are never stolen
(M2 no-steal / WAL durability preserved) — turned evictable by a checkpoint's
pc_flush, which then trims the pool back to budget.
- Pinning: superblocks (0,1) + index root/interior pages auto-pinned; explicit
store_pin_page/unpin and store_pin_layer/unpin (hot WM/core layers).
- Bounded sequential read-ahead on scans (env ENGRAM_PREFETCH, default 8).
- Correctness rests on callers copying page bytes into local buffers and never
retaining a frame pointer across another access, so evict+re-fault is safe.
Gates (plain gcc, ASan/UBSan clean):
M4 run_bufpool_tests.sh ...... 37 passed, 0 failed (+ ASan/UBSan: 37/0)
small-pool round-trip (cap=32 vs 1599 pages, 2708 evictions): 5000 nodes +
4000 sampled edges bit-exact, crc clean, pool bounded to cap.
eviction: hot set 0 re-faults, cold evicted, hit-rate 0.989; no-steal burst
(cap=8) holds 309 dirty frames > cap, reads correct from dirty pages.
pinning: superblocks/roots/explicit page/hot-layer(19 pages) stay resident;
unpin makes them evictable.
prefetch: sequential scan 511 demand-faults OFF -> 4 ON.
crash-under-paging (ENGRAM_POOL_FRAMES=16): WAL replay + checkpoint-crash
phases 0-4 all recover bit-exact.
default pool: 0 evictions, whole store resident (== Phase 1).
No regression: M1 33/0, M2 36/0, M3 parity PASS, M3.5 PASS.
This commit is contained in:
+291
-13
@@ -143,13 +143,42 @@ struct EngramPagedStore {
|
||||
uint64_t ckpt_threshold; /* auto-checkpoint after this many ops (0 = never) */
|
||||
};
|
||||
|
||||
/* M2 buffer-pool hooks (defined in the M2 section at the bottom of this file). */
|
||||
typedef struct PgEnt { uint64_t id; uint8_t* buf; uint64_t lsn; int dirty; struct PgEnt* next; } PgEnt;
|
||||
/* M2/M4 buffer-pool hooks (defined in the pool section at the bottom of this file).
|
||||
* M2 shipped a write-back, no-steal cache (dirty→disk only at checkpoint). M4
|
||||
* turns it into a bounded, demand-paged buffer pool: a fixed frame budget, LRU
|
||||
* eviction of CLEAN unpinned frames (no-steal preserved — dirty frames are never
|
||||
* stolen), pinning of hot/structural pages, and bounded read-ahead. `lru_*`
|
||||
* thread every resident frame onto an MRU→LRU list; `pin` is an explicit pin
|
||||
* count (0 = unpinned). */
|
||||
typedef struct PgEnt {
|
||||
uint64_t id; uint8_t* buf; uint64_t lsn; int dirty; struct PgEnt* next;
|
||||
int pin; /* explicit pin count (0 = unpinned) */
|
||||
struct PgEnt* lru_prev; /* MRU→LRU doubly-linked list */
|
||||
struct PgEnt* lru_next;
|
||||
} PgEnt;
|
||||
static PgCache* pc_new(void);
|
||||
static void pc_free(PgCache* c);
|
||||
static PgEnt* pc_get(EngramPagedStore* s, uint64_t id);
|
||||
static int pc_put(EngramPagedStore* s, uint64_t id, const uint8_t* buf, int dirty);
|
||||
static int pc_flush(EngramPagedStore* s); /* pwrite all dirty → clean */
|
||||
static void pc_prefetch(EngramPagedStore* s, uint64_t from_id, unsigned window);
|
||||
static void store__autopin(EngramPagedStore* s); /* pin superblocks + index roots */
|
||||
|
||||
/* Per-layer pin record: the set of pages pinned on behalf of a hot layer, kept
|
||||
* so store_unpin_layer can release exactly what store_pin_layer pinned. */
|
||||
typedef struct { uint32_t layer; uint64_t* pages; size_t n; } LayerPin;
|
||||
|
||||
/* The bounded, demand-paged frame table (M4). Defined here (not in the pool
|
||||
* section) so page_read / the scan loops can read its stats + prefetch window. */
|
||||
struct PgCache {
|
||||
PgEnt** buckets; size_t nbuckets; size_t count;
|
||||
size_t cap; /* max resident frames; 0 = unlimited */
|
||||
PgEnt* mru; PgEnt* lru; /* MRU (front) → LRU (back) recency list */
|
||||
unsigned prefetch; /* read-ahead window (pages); 0 = off */
|
||||
LayerPin* lp; size_t lp_n, lp_cap; /* hot-layer pin bookkeeping */
|
||||
/* stats (introspection only — never affect semantics) */
|
||||
uint64_t hits, misses, evictions, prefetch_reads;
|
||||
};
|
||||
|
||||
/* ── little-endian scalar codecs ──────────────────────────────────────────── */
|
||||
static void put_u16(uint8_t* p, uint16_t v){ p[0]=(uint8_t)v; p[1]=(uint8_t)(v>>8); }
|
||||
@@ -197,12 +226,12 @@ static uint64_t id_hash(const char* s){
|
||||
static int page_read(EngramPagedStore* s, uint64_t id, uint8_t* buf){
|
||||
if (s->cache){
|
||||
PgEnt* e = pc_get(s, id);
|
||||
if (e){ memcpy(buf, e->buf, STORE_PAGE_SIZE); return 0; }
|
||||
if (e){ memcpy(buf, e->buf, STORE_PAGE_SIZE); s->cache->hits++; return 0; }
|
||||
}
|
||||
off_t off = (off_t)id * STORE_PAGE_SIZE;
|
||||
off_t off = (off_t)id * STORE_PAGE_SIZE; /* demand fault: not resident */
|
||||
ssize_t r = pread(s->fd, buf, STORE_PAGE_SIZE, off);
|
||||
if (r != (ssize_t)STORE_PAGE_SIZE) return -1;
|
||||
if (s->cache) pc_put(s, id, buf, 0); /* cache clean */
|
||||
if (s->cache){ s->cache->misses++; pc_put(s, id, buf, 0); } /* cache clean */
|
||||
return 0;
|
||||
}
|
||||
static int page_write_raw(EngramPagedStore* s, uint64_t id, const uint8_t* buf){
|
||||
@@ -834,6 +863,7 @@ EngramPagedStore* store_create(const char* path){
|
||||
close(s->fd); free(s); return NULL;
|
||||
}
|
||||
if (store_sync(s)!=0){ close(s->fd); free(s); return NULL; }
|
||||
store__autopin(s); /* keep superblocks + index roots resident */
|
||||
return s;
|
||||
}
|
||||
|
||||
@@ -861,6 +891,7 @@ EngramPagedStore* store_open(const char* path){
|
||||
s->next_lsn = (s->last_checkpoint_lsn > s->sb_seq) ? s->last_checkpoint_lsn : s->sb_seq;
|
||||
s->cur_node_page = 0;
|
||||
s->cur_edge_page = 0;
|
||||
store__autopin(s); /* keep superblocks + index roots resident */
|
||||
return s;
|
||||
}
|
||||
|
||||
@@ -1169,6 +1200,7 @@ int store_scan_nodes(EngramPagedStore* s, StoreNodeScanCb cb, void* ctx){
|
||||
int count = 0;
|
||||
for (uint64_t pg = 2; pg < s->page_count; pg++){
|
||||
if (page_read(s, pg, buf) != 0) continue;
|
||||
if (s->cache) pc_prefetch(s, pg, s->cache->prefetch); /* sequential read-ahead */
|
||||
if (buf[8] != STORE_PT_NODE) continue;
|
||||
int ns = slp_count(buf);
|
||||
for (int i = 0; i < ns; i++){
|
||||
@@ -1198,6 +1230,7 @@ int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx){
|
||||
int count = 0;
|
||||
for (uint64_t pg = 2; pg < s->page_count; pg++){
|
||||
if (page_read(s, pg, buf) != 0) continue;
|
||||
if (s->cache) pc_prefetch(s, pg, s->cache->prefetch); /* sequential read-ahead */
|
||||
if (buf[8] != STORE_PT_EDGE) continue;
|
||||
int ns = slp_count(buf);
|
||||
for (int i = 0; i < ns; i++){
|
||||
@@ -1247,8 +1280,36 @@ int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx){
|
||||
|
||||
#include <sys/time.h>
|
||||
|
||||
/* ── write-back buffer pool ────────────────────────────────────────────────── */
|
||||
struct PgCache { PgEnt** buckets; size_t nbuckets; size_t count; };
|
||||
/* ══════════════════════════════════════════════════════════════════════════════
|
||||
* M4 — demand-paging BUFFER POOL (bounded, LRU, pinned, read-ahead)
|
||||
*
|
||||
* A frame table (id→frame hash) capped at `cap` resident frames. On a page
|
||||
* access that is not resident, page_read faults it in from neuron.egm; if the
|
||||
* pool is full, the LRU eviction path reclaims a CLEAN, unpinned frame. This is
|
||||
* purely additive residency — the on-disk format is unchanged, and with the
|
||||
* DEFAULT cap (large) no eviction ever fires, so behaviour is byte-for-byte the
|
||||
* Phase-1 resident store.
|
||||
*
|
||||
* Invariants preserved from M2 (write-back, NO-STEAL):
|
||||
* • A DIRTY frame is NEVER evicted (never stolen) — its only durable copy is
|
||||
* the fsync'd WAL, and the store page reaches disk solely at a checkpoint.
|
||||
* pc_flush (checkpoint) is what turns dirty→clean and thus evictable.
|
||||
* • A PINNED frame is never evicted. Structural pages are auto-pinned: the two
|
||||
* superblocks (pages 0,1) and every index ROOT/INTERIOR page (type INDEX,
|
||||
* leaf-flag 0). Leaves are pageable. Explicit pins (pin count) cover hot
|
||||
* layers and any caller-designated page.
|
||||
* Correctness under a pool SMALLER than the store rests on: every caller copies
|
||||
* page bytes into a local stack buffer (memcpy in page_read / out in page_write)
|
||||
* and never retains a frame pointer across another page access, so a frame may
|
||||
* be evicted and later re-faulted with no aliasing hazard. A clean frame always
|
||||
* matches disk, so a re-fault reproduces identical bytes.
|
||||
* ════════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
/* default frame budget: large enough that today's whole store stays resident
|
||||
* (== Phase 1). Override with env ENGRAM_POOL_FRAMES (0 = unlimited). */
|
||||
#ifndef ENGRAM_POOL_FRAMES_DEFAULT
|
||||
#define ENGRAM_POOL_FRAMES_DEFAULT (1u<<20) /* ~1M frames × 16KiB = 16 GiB */
|
||||
#endif
|
||||
|
||||
static PgCache* pc_new(void){
|
||||
PgCache* c = (PgCache*)calloc(1, sizeof *c);
|
||||
@@ -1256,6 +1317,12 @@ static PgCache* pc_new(void){
|
||||
c->nbuckets = 1024;
|
||||
c->buckets = (PgEnt**)calloc(c->nbuckets, sizeof(PgEnt*));
|
||||
if (!c->buckets){ free(c); return NULL; }
|
||||
c->cap = ENGRAM_POOL_FRAMES_DEFAULT;
|
||||
c->prefetch = 8;
|
||||
const char* pf = getenv("ENGRAM_POOL_FRAMES");
|
||||
if (pf && *pf){ char* end=NULL; unsigned long long v = strtoull(pf,&end,10); c->cap = (size_t)v; }
|
||||
const char* pw = getenv("ENGRAM_PREFETCH");
|
||||
if (pw && *pw){ char* end=NULL; unsigned long v = strtoul(pw,&end,10); c->prefetch = (unsigned)v; }
|
||||
return c;
|
||||
}
|
||||
static void pc_free(PgCache* c){
|
||||
@@ -1264,14 +1331,27 @@ static void pc_free(PgCache* c){
|
||||
PgEnt* e = c->buckets[i];
|
||||
while (e){ PgEnt* n=e->next; free(e->buf); free(e); e=n; }
|
||||
}
|
||||
for (size_t i=0;i<c->lp_n;i++) free(c->lp[i].pages);
|
||||
free(c->lp);
|
||||
free(c->buckets); free(c);
|
||||
}
|
||||
static PgEnt* pc_get(EngramPagedStore* s, uint64_t id){
|
||||
PgCache* c = s->cache;
|
||||
PgEnt* e = c->buckets[id % c->nbuckets];
|
||||
while (e){ if (e->id==id) return e; e=e->next; }
|
||||
return NULL;
|
||||
|
||||
/* ── LRU recency list (front = MRU, back = LRU) ─────────────────────────────── */
|
||||
static void lru_unlink(PgCache* c, PgEnt* e){
|
||||
if (e->lru_prev) e->lru_prev->lru_next = e->lru_next; else c->mru = e->lru_next;
|
||||
if (e->lru_next) e->lru_next->lru_prev = e->lru_prev; else c->lru = e->lru_prev;
|
||||
e->lru_prev = e->lru_next = NULL;
|
||||
}
|
||||
static void lru_push_front(PgCache* c, PgEnt* e){
|
||||
e->lru_prev = NULL; e->lru_next = c->mru;
|
||||
if (c->mru) c->mru->lru_prev = e; c->mru = e;
|
||||
if (!c->lru) c->lru = e;
|
||||
}
|
||||
static void lru_touch(PgCache* c, PgEnt* e){
|
||||
if (c->mru == e) return;
|
||||
lru_unlink(c, e); lru_push_front(c, e);
|
||||
}
|
||||
|
||||
static void pc_maybe_grow(PgCache* c){
|
||||
if (c->count <= c->nbuckets*4) return;
|
||||
size_t nn = c->nbuckets*2;
|
||||
@@ -1283,9 +1363,54 @@ static void pc_maybe_grow(PgCache* c){
|
||||
}
|
||||
free(c->buckets); c->buckets=nb; c->nbuckets=nn;
|
||||
}
|
||||
|
||||
/* A frame is EVICTABLE iff it is clean, unpinned, not a superblock, and not an
|
||||
* index root/interior page. This is the sole place the no-steal + structural-pin
|
||||
* policy is enforced. */
|
||||
static int pc_evictable(const PgEnt* e){
|
||||
if (e->dirty) return 0; /* no-steal: dirty pages are pinned to RAM */
|
||||
if (e->pin > 0) return 0; /* explicit / hot-layer pin */
|
||||
if (e->id == 0 || e->id == 1) return 0; /* superblock + mirror */
|
||||
if (e->buf[8] == STORE_PT_INDEX && e->buf[IDX_LEAF_OFF] == 0) return 0; /* root/interior */
|
||||
return 1;
|
||||
}
|
||||
/* Detach `e` from both the hash chain and the recency list, and free it. */
|
||||
static void pc_remove(PgCache* c, PgEnt* e){
|
||||
size_t b = e->id % c->nbuckets;
|
||||
PgEnt** pp = &c->buckets[b];
|
||||
while (*pp && *pp != e) pp = &(*pp)->next;
|
||||
if (*pp == e) *pp = e->next;
|
||||
lru_unlink(c, e);
|
||||
free(e->buf); free(e);
|
||||
c->count--;
|
||||
}
|
||||
/* Reclaim clean unpinned frames from the LRU end until under budget, or until no
|
||||
* evictable frame remains (a dirty/pinned-heavy pool may transiently exceed cap —
|
||||
* that is the no-steal guarantee, not a bug: the next checkpoint frees them). */
|
||||
static void pc_evict_to_budget(PgCache* c){
|
||||
if (!c->cap) return; /* unlimited */
|
||||
while (c->count > c->cap){
|
||||
PgEnt* e = c->lru; int freed = 0;
|
||||
while (e){
|
||||
PgEnt* prev = e->lru_prev; /* walk LRU→MRU */
|
||||
if (pc_evictable(e)){ pc_remove(c, e); c->evictions++; freed = 1; break; }
|
||||
e = prev;
|
||||
}
|
||||
if (!freed) break; /* nothing evictable — allowed to exceed cap */
|
||||
}
|
||||
}
|
||||
|
||||
static PgEnt* pc_get(EngramPagedStore* s, uint64_t id){
|
||||
PgCache* c = s->cache;
|
||||
PgEnt* e = c->buckets[id % c->nbuckets];
|
||||
while (e){ if (e->id==id){ lru_touch(c, e); return e; } e=e->next; }
|
||||
return NULL;
|
||||
}
|
||||
/* Insert-or-update a frame. New frames go to MRU; then evict down to budget.
|
||||
* The just-touched frame is at MRU and can never be the eviction victim. */
|
||||
static int pc_put(EngramPagedStore* s, uint64_t id, const uint8_t* buf, int dirty){
|
||||
PgCache* c = s->cache;
|
||||
PgEnt* e = pc_get(s, id);
|
||||
PgEnt* e = pc_get(s, id); /* pc_get also bumps it to MRU on a hit */
|
||||
if (!e){
|
||||
e = (PgEnt*)calloc(1, sizeof *e);
|
||||
if (!e) return -1;
|
||||
@@ -1294,11 +1419,13 @@ static int pc_put(EngramPagedStore* s, uint64_t id, const uint8_t* buf, int dirt
|
||||
e->id = id;
|
||||
size_t b = id % c->nbuckets;
|
||||
e->next = c->buckets[b]; c->buckets[b] = e; c->count++;
|
||||
lru_push_front(c, e);
|
||||
pc_maybe_grow(c);
|
||||
}
|
||||
memcpy(e->buf, buf, STORE_PAGE_SIZE);
|
||||
e->lsn = get_u64(buf + 16);
|
||||
if (dirty) e->dirty = 1;
|
||||
pc_evict_to_budget(c);
|
||||
return 0;
|
||||
}
|
||||
static int pc_flush(EngramPagedStore* s){
|
||||
@@ -1307,8 +1434,159 @@ static int pc_flush(EngramPagedStore* s){
|
||||
for (size_t i=0;i<c->nbuckets;i++)
|
||||
for (PgEnt* e=c->buckets[i]; e; e=e->next)
|
||||
if (e->dirty){ if (page_write_raw(s, e->id, e->buf)!=0) return -1; e->dirty=0; }
|
||||
/* Post-checkpoint the just-cleaned frames are now evictable; trim the pool
|
||||
* back to budget so a dirty-heavy burst that transiently overshot cap does
|
||||
* not leave the pool oversized. No-op at the default (unlimited-ish) cap. */
|
||||
pc_evict_to_budget(c);
|
||||
return 0;
|
||||
}
|
||||
/* Bounded sequential read-ahead: fault the next `window` pages after `from_id`
|
||||
* into any spare capacity, so a forward scan/leaf-walk hits them instead of
|
||||
* faulting one-by-one. Never forces an eviction (fills slack only), never
|
||||
* re-reads a resident page. Prefetch reads are counted separately from demand
|
||||
* faults so a scan's fault count reflects on-demand misses only. */
|
||||
static void pc_prefetch(EngramPagedStore* s, uint64_t from_id, unsigned window){
|
||||
PgCache* c = s->cache;
|
||||
if (!c || !window) return;
|
||||
for (unsigned k=1; k<=window; k++){
|
||||
uint64_t id = from_id + k;
|
||||
if (id >= s->page_count) break;
|
||||
if (c->cap && c->count + 1 > c->cap) break; /* no eviction for read-ahead */
|
||||
if (c->buckets[id % c->nbuckets]){
|
||||
PgEnt* e = c->buckets[id % c->nbuckets];
|
||||
int resident = 0; while (e){ if (e->id==id){ resident=1; break; } e=e->next; }
|
||||
if (resident) continue;
|
||||
}
|
||||
uint8_t buf[STORE_PAGE_SIZE];
|
||||
off_t off = (off_t)id * STORE_PAGE_SIZE;
|
||||
if (pread(s->fd, buf, STORE_PAGE_SIZE, off) != (ssize_t)STORE_PAGE_SIZE) break;
|
||||
pc_put(s, id, buf, 0);
|
||||
c->prefetch_reads++;
|
||||
}
|
||||
}
|
||||
|
||||
/* Non-LRU-touching frame lookup (for pin bookkeeping that must not reorder). */
|
||||
static PgEnt* pc_find(PgCache* c, uint64_t id){
|
||||
PgEnt* e = c->buckets[id % c->nbuckets];
|
||||
while (e){ if (e->id==id) return e; e=e->next; }
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* ── public pin / prefetch / stats API (M4) ─────────────────────────────────── */
|
||||
int store_pin_page(EngramPagedStore* s, uint64_t page_id){
|
||||
if (!s || !s->cache) return -1;
|
||||
uint8_t buf[STORE_PAGE_SIZE];
|
||||
if (page_read(s, page_id, buf) != 0) return -1; /* fault in + make resident */
|
||||
PgEnt* e = pc_find(s->cache, page_id);
|
||||
if (!e) return -1;
|
||||
e->pin++;
|
||||
return 0;
|
||||
}
|
||||
int store_unpin_page(EngramPagedStore* s, uint64_t page_id){
|
||||
if (!s || !s->cache) return -1;
|
||||
PgEnt* e = pc_find(s->cache, page_id);
|
||||
if (e && e->pin > 0) e->pin--;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Pin every page currently holding a live record of `layer` (hot-layer residency).
|
||||
* Records the pinned pages so store_unpin_layer releases exactly this set. Pages
|
||||
* are pinned BEFORE their bodies are read so a small pool cannot evict them mid-scan. */
|
||||
int store_pin_layer(EngramPagedStore* s, uint32_t layer){
|
||||
if (!s || !s->cache) return -1;
|
||||
uint64_t* pages = NULL; size_t np = 0, cap = 0;
|
||||
uint8_t buf[STORE_PAGE_SIZE];
|
||||
for (uint64_t pg = 2; pg < s->page_count; pg++){
|
||||
if (page_read(s, pg, buf) != 0) continue;
|
||||
int t = buf[8];
|
||||
if (t != STORE_PT_NODE && t != STORE_PT_EDGE) continue;
|
||||
PgEnt* pe = pc_find(s->cache, pg);
|
||||
if (!pe) continue;
|
||||
pe->pin++; /* provisional pin: keeps pg resident */
|
||||
int ns = slp_count(buf), match = 0;
|
||||
for (int i = 0; i < ns && !match; i++){
|
||||
uint16_t off, len, fl; slp_slot(buf, i, &off, &len, &fl);
|
||||
if (fl != SLOT_LIVE) continue;
|
||||
uint8_t* body; size_t blen; int live;
|
||||
if (read_body(s, pg, (uint16_t)i, &body, &blen, &live) != 0) continue;
|
||||
uint32_t lid = 0;
|
||||
if (t == STORE_PT_NODE){ StoreNode c; node_parse(body, blen, &c); lid = c.layer_id; store_node_free(&c); }
|
||||
else { StoreEdge c; edge_parse(body, blen, &c); lid = c.layer_id; store_edge_free(&c); }
|
||||
free(body);
|
||||
if (lid == layer) match = 1;
|
||||
}
|
||||
if (match){
|
||||
if (np == cap){ cap = cap ? cap*2 : 16; uint64_t* np2 = (uint64_t*)realloc(pages, cap*sizeof *pages); if (!np2){ free(pages); return -1; } pages = np2; }
|
||||
pages[np++] = pg; /* keep the pin */
|
||||
} else {
|
||||
pe->pin--; /* no match on this page: drop provisional pin */
|
||||
}
|
||||
}
|
||||
PgCache* c = s->cache;
|
||||
if (c->lp_n == c->lp_cap){ c->lp_cap = c->lp_cap ? c->lp_cap*2 : 8; c->lp = (LayerPin*)realloc(c->lp, c->lp_cap*sizeof *c->lp); }
|
||||
c->lp[c->lp_n].layer = layer; c->lp[c->lp_n].pages = pages; c->lp[c->lp_n].n = np; c->lp_n++;
|
||||
return (int)np;
|
||||
}
|
||||
int store_unpin_layer(EngramPagedStore* s, uint32_t layer){
|
||||
if (!s || !s->cache) return -1;
|
||||
PgCache* c = s->cache;
|
||||
for (size_t i = 0; i < c->lp_n; i++){
|
||||
if (c->lp[i].layer != layer) continue;
|
||||
for (size_t j = 0; j < c->lp[i].n; j++){
|
||||
PgEnt* e = pc_find(c, c->lp[i].pages[j]);
|
||||
if (e && e->pin > 0) e->pin--;
|
||||
}
|
||||
free(c->lp[i].pages);
|
||||
c->lp[i] = c->lp[--c->lp_n]; /* swap-remove */
|
||||
return 0;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Auto-pin the structural pages: both superblocks and the two index roots (plus
|
||||
* the layer registry). A SHALLOW index root is a LEAF, so it is not covered by
|
||||
* the "index interior" eviction rule — pinning it explicitly guarantees the root
|
||||
* is never evicted even for a tiny tree. Deeper roots/interiors are additionally
|
||||
* covered by pc_evictable's INDEX-non-leaf rule. Best-effort (ignores errors on
|
||||
* a not-yet-built store). */
|
||||
static void store__autopin(EngramPagedStore* s){
|
||||
if (!s || !s->cache) return;
|
||||
store_pin_page(s, 0);
|
||||
store_pin_page(s, 1);
|
||||
if (s->root_index_page) store_pin_page(s, s->root_index_page);
|
||||
if (s->adj_index_page) store_pin_page(s, s->adj_index_page);
|
||||
if (s->layer_registry_page) store_pin_page(s, s->layer_registry_page);
|
||||
}
|
||||
|
||||
/* Introspection + test hooks. */
|
||||
void store_pool_stats(const EngramPagedStore* s, StorePoolStats* out){
|
||||
if (!out) return;
|
||||
memset(out, 0, sizeof *out);
|
||||
if (!s || !s->cache) return;
|
||||
const PgCache* c = s->cache;
|
||||
out->cap = c->cap; out->resident = c->count; out->prefetch = c->prefetch;
|
||||
out->hits = c->hits; out->misses = c->misses;
|
||||
out->evictions = c->evictions; out->prefetch_reads = c->prefetch_reads;
|
||||
size_t pinned = 0, dirty = 0;
|
||||
for (size_t i=0;i<c->nbuckets;i++)
|
||||
for (PgEnt* e=c->buckets[i]; e; e=e->next){
|
||||
if (!pc_evictable(e)) pinned++;
|
||||
if (e->dirty) dirty++;
|
||||
}
|
||||
out->pinned = pinned; out->dirty = dirty;
|
||||
}
|
||||
int store_pool_resident(const EngramPagedStore* s, uint64_t page_id){
|
||||
if (!s || !s->cache) return -1;
|
||||
return pc_find(s->cache, page_id) ? 1 : 0;
|
||||
}
|
||||
void store__set_pool_frames(EngramPagedStore* s, size_t frames){
|
||||
if (!s || !s->cache) return;
|
||||
s->cache->cap = frames;
|
||||
pc_evict_to_budget(s->cache); /* apply the new budget now */
|
||||
}
|
||||
void store__set_prefetch(EngramPagedStore* s, unsigned window){
|
||||
if (s && s->cache) s->cache->prefetch = window;
|
||||
}
|
||||
/* ── WAL log ───────────────────────────────────────────────────────────────── */
|
||||
enum { OP_NODE_PUT=1, OP_EDGE_PUT, OP_TOMBSTONE, OP_SUPERSEDE,
|
||||
OP_LAYER_PUT, OP_LAYER_DEL, OP_FORGET, OP_HEBB_BATCH, OP_CHECKPOINT };
|
||||
|
||||
@@ -209,6 +209,41 @@ int store_scan_edges(EngramPagedStore* s, StoreEdgeScanCb cb, void* ctx);
|
||||
uint64_t engram_wal_next_lsn(const EngramPagedStore* s);
|
||||
uint64_t engram_last_checkpoint_lsn(const EngramPagedStore* s);
|
||||
|
||||
/* ── M4: demand-paging buffer pool (additive residency; on-disk format UNCHANGED) ──
|
||||
*
|
||||
* The write-back, no-steal cache of M2 becomes a bounded, demand-paged buffer
|
||||
* pool. A fixed frame budget (env ENGRAM_POOL_FRAMES; 0 = unlimited; default
|
||||
* large ⇒ whole store resident ⇒ identical to Phase 1) keeps only hot pages in
|
||||
* RAM; a page access that is not resident faults in from neuron.egm, and under
|
||||
* pressure a CLEAN, unpinned frame is evicted (LRU). Dirty frames are never
|
||||
* stolen (M2 no-steal / WAL durability), and superblocks + index root/interior
|
||||
* pages are auto-pinned. Prefetch (env ENGRAM_PREFETCH) reads ahead on scans. */
|
||||
|
||||
/* Pin / unpin an individual page (faults it in and keeps it resident until
|
||||
* unpinned). Pin a hot layer's pages (WM/core) as a set. Idempotent counts. */
|
||||
int store_pin_page(EngramPagedStore* s, uint64_t page_id);
|
||||
int store_unpin_page(EngramPagedStore* s, uint64_t page_id);
|
||||
int store_pin_layer(EngramPagedStore* s, uint32_t layer); /* returns #pages pinned */
|
||||
int store_unpin_layer(EngramPagedStore* s, uint32_t layer);
|
||||
|
||||
/* Buffer-pool introspection. */
|
||||
typedef struct StorePoolStats {
|
||||
size_t cap; /* frame budget (0 = unlimited) */
|
||||
size_t resident; /* frames currently resident */
|
||||
size_t pinned; /* frames that cannot be evicted (dirty/pinned/structural) */
|
||||
size_t dirty; /* dirty (un-checkpointed) frames */
|
||||
unsigned prefetch; /* read-ahead window */
|
||||
uint64_t hits, misses; /* page_read cache hits / demand faults */
|
||||
uint64_t evictions; /* clean frames reclaimed */
|
||||
uint64_t prefetch_reads; /* pages brought in by read-ahead */
|
||||
} StorePoolStats;
|
||||
void store_pool_stats(const EngramPagedStore* s, StorePoolStats* out);
|
||||
int store_pool_resident(const EngramPagedStore* s, uint64_t page_id);
|
||||
|
||||
/* Test hooks: set the frame budget / prefetch window at runtime (NOT format). */
|
||||
void store__set_pool_frames(EngramPagedStore* s, size_t frames);
|
||||
void store__set_prefetch(EngramPagedStore* s, unsigned window);
|
||||
|
||||
/* Crash-test hooks (writes only under a throwaway dir).
|
||||
* store__crash — abandon all RAM state without flush/fsync (power loss).
|
||||
* store__flush_pages — pwrite dirty pages to disk WITHOUT a checkpoint (steal).
|
||||
|
||||
Reference in New Issue
Block a user