/* test_bufpool.c — M4 gate for the demand-paging BUFFER POOL (engram_store.{c,h}). * * Pure C. Build: gcc -O2 test_bufpool.c ../../lang/runtime/engram_store.c -o t * Writes ONLY under a throwaway /tmp dir. Never touches ~/.neuron or live ports. * * Proves the M4 pool preserves every M1/M2 invariant when the pool is SMALLER * than the store (pages evict + re-fault): small-pool round-trip correctness, * LRU eviction policy (hot resident / cold evicted / no dirty stolen), pinned * residency (superblocks, index roots, explicit page + hot-layer pins), bounded * read-ahead, and crash safety (WAL replay + checkpoint-crash) under paging. */ #include "../../lang/runtime/engram_store.h" #include #include #include #include #include #include #include static int g_pass = 0, g_fail = 0; static void ok(const char* name, int cond){ printf(" [%s] %s\n", cond ? "PASS" : "FAIL", name); if (cond) g_pass++; else g_fail++; } static char g_dir[512]; static void mk_dir(void){ snprintf(g_dir, sizeof g_dir, "/tmp/engram-bufpool-test-%d", (int)getpid()); mkdir(g_dir, 0700); } static void path_in(char* out, size_t cap, const char* name){ snprintf(out, cap, "%s/%s", g_dir, name); } /* ── deterministic generators (bit-exact regeneration for oracles) ─────────── */ static uint64_t xs(uint64_t* s){ uint64_t x=*s; x^=x<<13; x^=x>>7; x^=x<<17; *s=x; return x; } static uint64_t node_seed(int i){ return 0x9E3779B97F4A7C15ULL ^ ((uint64_t)(i+1)*0xD1B54A32D192ED03ULL); } static uint64_t edge_seed(int i){ return 0xC2B2AE3D27D4EB4FULL ^ ((uint64_t)(i+1)*0x165667B19E3779F9ULL); } static char* rnd_str(uint64_t* st, size_t len){ char* s = (char*)malloc(len + 1); for (size_t i=0;iid = strdup(id); size_t clen = (i % 500 == 0) ? (size_t)(17000 + (xs(&st) % 6000)) : (size_t)(xs(&st) % 300); n->content = rnd_str(&st, clen); n->node_type = rnd_str(&st, 4 + (xs(&st) % 8)); n->label = (i % 2) ? rnd_str(&st, 3 + (xs(&st) % 10)) : NULL; n->tier = rnd_str(&st, 4 + (xs(&st) % 6)); n->tags = rnd_str(&st, xs(&st) % 40); n->metadata = (i % 3) ? rnd_str(&st, xs(&st) % 60) : NULL; n->salience = (double)(xs(&st) % 1000000) / 997.0; n->importance = (double)(xs(&st) % 1000000) / 131.0; n->confidence = (double)(xs(&st) % 1000000) / 733.0; n->temporal_decay_rate = (double)(xs(&st) % 1000000) / 101.0; n->activation_count = (int64_t)(xs(&st) % 100000); n->last_activated = (int64_t)xs(&st); n->created_at = (int64_t)(1600000000000LL + i); n->updated_at = (int64_t)xs(&st); n->background_activation = (double)(xs(&st) % 1000000) / 17.0; n->working_memory_weight = (double)(xs(&st) % 1000000) / 29.0; n->suppression_count = (int32_t)(xs(&st) % 50); n->layer_id = (uint32_t)(xs(&st) % 5); for (int k=0;kaccess_ts[k] = (int64_t)xs(&st); n->access_head = (int32_t)(xs(&st) % STORE_BLL_K); n->access_filled = (int32_t)(xs(&st) % (STORE_BLL_K + 1)); n->wm_anchor = (double)(xs(&st) % 1000000) / 3.0; n->emb = (float*)malloc(EMB_DIM * sizeof(float)); for (int k=0;kemb[k], &u, 4); } n->emb_dim = EMB_DIM; } static void gen_edge(int i, StoreEdge* e){ memset(e, 0, sizeof *e); uint64_t st = edge_seed(i); char id[32], from[32], to[32]; snprintf(id, sizeof id, "edge-%d", i); snprintf(from, sizeof from, "node-%d", (int)(xs(&st) % NODE_COUNT)); snprintf(to, sizeof to, "node-%d", (int)(xs(&st) % NODE_COUNT)); e->id = strdup(id); e->from_id = strdup(from); e->to_id = strdup(to); e->relation = rnd_str(&st, 3 + (xs(&st) % 12)); e->metadata = (i % 4) ? rnd_str(&st, xs(&st) % 40) : NULL; e->weight = (double)(xs(&st) % 1000000) / 111.0; e->hebb = (double)(xs(&st) % 1000000) / 1000000.0; e->confidence = (double)(xs(&st) % 1000000) / 777.0; e->created_at = (int64_t)(1600000000000LL + i); e->updated_at = (int64_t)xs(&st); e->last_fired = (int64_t)xs(&st); e->inhibitory = (int32_t)(xs(&st) % 2); e->layer_id = (uint32_t)(xs(&st) % 5); } static int streq(const char* a, const char* b){ if (!a && !b) return 1; if (!a || !b) return 0; return strcmp(a,b)==0; } static int cmp_node(const StoreNode* a, const StoreNode* b){ if (!streq(a->id,b->id) || !streq(a->content,b->content) || !streq(a->node_type,b->node_type) || !streq(a->label,b->label) || !streq(a->tier,b->tier) || !streq(a->tags,b->tags) || !streq(a->metadata,b->metadata)) return 0; if (a->salience!=b->salience || a->importance!=b->importance || a->confidence!=b->confidence || a->temporal_decay_rate!=b->temporal_decay_rate || a->activation_count!=b->activation_count || a->last_activated!=b->last_activated || a->created_at!=b->created_at || a->updated_at!=b->updated_at || a->background_activation!=b->background_activation || a->working_memory_weight!=b->working_memory_weight || a->suppression_count!=b->suppression_count || a->layer_id!=b->layer_id || a->access_head!=b->access_head || a->access_filled!=b->access_filled || a->wm_anchor!=b->wm_anchor || a->emb_dim!=b->emb_dim) return 0; for (int k=0;kaccess_ts[k]!=b->access_ts[k]) return 0; if ((a->emb==NULL) != (b->emb==NULL)) return 0; if (a->emb && memcmp(a->emb, b->emb, (size_t)a->emb_dim*4)!=0) return 0; return 1; } static int cmp_edge(const StoreEdge* a, const StoreEdge* b){ if (!streq(a->id,b->id) || !streq(a->from_id,b->from_id) || !streq(a->to_id,b->to_id) || !streq(a->relation,b->relation) || !streq(a->metadata,b->metadata)) return 0; if (a->weight!=b->weight || a->hebb!=b->hebb || a->confidence!=b->confidence || a->created_at!=b->created_at || a->updated_at!=b->updated_at || a->last_fired!=b->last_fired || a->inhibitory!=b->inhibitory || a->layer_id!=b->layer_id) return 0; return 1; } static void free_node_fields(StoreNode* n){ free(n->id); free(n->content); free(n->node_type); free(n->label); free(n->tier); free(n->tags); free(n->metadata); free(n->emb); free(n->unknown); } static void free_edge_fields(StoreEdge* e){ free(e->id); free(e->from_id); free(e->to_id); free(e->relation); free(e->metadata); free(e->unknown); } /* ════════════════════════════════════════════════════════════════════════════ * TEST 1 — SMALL-POOL CORRECTNESS: full M1 workload (5k nodes / 20k edges) with * a frame budget FAR smaller than the store → constant eviction + re-fault, yet * every read is bit-exact and the pool stays bounded. * ════════════════════════════════════════════════════════════════════════════ */ static void test_small_pool_roundtrip(void){ printf("\n== 1) small-pool correctness: %d nodes + %d edges, cap=%d frames ==\n", NODE_COUNT, EDGE_COUNT, 32); char path[600]; path_in(path, sizeof path, "small.store"); unlink(path); EngramPagedStore* s = store_create(path); ok("store_create", s != NULL); if (!s) return; store__set_pool_frames(s, 32); /* pool << store */ for (int i=0;i 0); ok("pool stayed bounded (resident <= cap)", st.resident <= st.cap); ok("no dirty frames after checkpoint", st.dirty == 0); /* read back EVERY node bit-exact despite constant eviction/re-fault */ int bad = 0; for (int i=0;i=N) cold=200; store_pool_stats(s,&a); StoreNode g; if (store_get_node(s,id,&g)==1) store_node_free(&g); store_pool_stats(s,&b); cold_faults += (b.misses - a.misses); } } printf(" hot re-get faults (post-warm)=%llu cold stream faults=%llu\n", (unsigned long long)hot_faults, (unsigned long long)cold_faults); ok("HOT pages stay resident (0 faults on re-access)", hot_faults == 0); ok("COLD pages get evicted + re-faulted", cold_faults > 0); store_pool_stats(s,&b); double hr = (double)b.hits / (double)(b.hits + b.misses); printf(" overall hit-rate = %.3f (hits=%llu misses=%llu)\n", hr, (unsigned long long)b.hits, (unsigned long long)b.misses); ok("hit-rate is sane (> 0.5)", hr > 0.5); store_close(s); unlink(path); /* ---- part B: no-steal (dirty pages never evicted before checkpoint) ---- */ EngramPagedStore* s2 = store_create(path); if (!s2){ ok("store_create(2)", 0); return; } store__set_pool_frames(s2, 8); /* tiny budget */ for (int i=0;i<1200;i++){ StoreNode n; gen_node(i,&n); store_put_node(s2,&n); free_node_fields(&n); } /* NO sync: every mutated page is dirty and, by no-steal, unevictable */ StorePoolStats d; store_pool_stats(s2,&d); printf(" tiny cap=%zu, unsynced burst: resident=%zu dirty=%zu evictions=%llu\n", d.cap, d.resident, d.dirty, (unsigned long long)d.evictions); ok("dirty pages pinned in RAM beyond budget (no-steal)", d.dirty > d.cap && d.resident > d.cap); /* a just-written node is served correctly from its dirty in-RAM page */ { StoreNode want; gen_node(777,&want); StoreNode got; int hit=store_get_node(s2,want.id,&got); ok("read served correctly from dirty (un-flushed) page", hit==1 && cmp_node(&want,&got)); if (hit==1) store_node_free(&got); free_node_fields(&want); } store_sync(s2); /* checkpoint → dirty become clean/evictable */ store_pool_stats(s2,&d); ok("checkpoint cleared all dirty frames", d.dirty == 0); /* durability across reopen after the no-steal burst */ store_close(s2); EngramPagedStore* s3 = store_open(path); store__set_pool_frames(s3, 8); int miss=0; for (int i=0;i<1200;i++){ StoreNode want; gen_node(i,&want); StoreNode got; int hit=store_get_node(s3,want.id,&got); if (hit!=1 || !cmp_node(&want,&got)) miss++; if (hit==1) store_node_free(&got); free_node_fields(&want); } ok("all 1200 survive reopen, bit-exact, tiny pool", miss==0); store_close(s3); unlink(path); } /* ════════════════════════════════════════════════════════════════════════════ * TEST 3 — PINNED RESIDENCY: superblocks + index roots never evicted under heavy * thrash; an explicitly pinned page stays until unpinned; a pinned hot layer's * pages stay resident and are released on unpin. * ════════════════════════════════════════════════════════════════════════════ */ static void test_pinning(void){ printf("\n== 3) pinned residency: superblocks / index roots / page / layer ==\n"); char path[600]; path_in(path, sizeof path, "pin.store"); unlink(path); EngramPagedStore* s = store_create(path); if (!s){ ok("store_create", 0); return; } const int N = 1500; for (int i=0;i=4: 2 SB + 2 roots)", st.pinned >= 4); /* unpin the page → it becomes evictable and is dropped under further thrash */ store_unpin_page(s, P); for (int i=0;i 0); store_pool_stats(s,&st); size_t pinned_with_layer = st.pinned; for (int pass=0; pass<3; pass++) for (int i=0;i= pinned_with_layer); ok("layer pin holds >= npin extra frames", st.pinned >= (size_t)npin + 4); store_unpin_layer(s, 3); store_pool_stats(s,&st); size_t after_unpin_max = st.pinned; for (int i=0;i 0); unlink(path); } /* ════════════════════════════════════════════════════════════════════════════ * TEST 5 — CRASH SAFETY UNDER PAGING: WAL replay and checkpoint-crash recovery * with a tiny pool (pages evict + re-fault during replay). * ════════════════════════════════════════════════════════════════════════════ */ static void test_crash_under_paging(void){ printf("\n== 5) crash safety under a tiny pool (ENGRAM_POOL_FRAMES=16) ==\n"); setenv("ENGRAM_POOL_FRAMES", "16", 1); /* every engram_open() below is paged */ setenv("ENGRAM_WAL_SYNC", "always", 1); /* ---- 5a: power-loss → WAL replay ---- */ char dir[600]; path_in(dir, sizeof dir, "crash_wal"); mkdir(dir, 0700); EngramPagedStore* s = engram_open(dir); if (!s){ ok("engram_open", 0); return; } const int M = 400; for (int i=0;i= (size_t)(1u<<20)); ok("no eviction ever fired at default budget", st.evictions == 0); ok("whole store resident (every page cached)", st.resident == store_page_count(s)); store_close(s); unlink(path); } int main(void){ mk_dir(); printf("engram M4 buffer-pool gate — dir=%s\n", g_dir); test_small_pool_roundtrip(); test_eviction_policy(); test_pinning(); test_prefetch(); test_crash_under_paging(); test_default_is_phase1(); printf("\n================ %d passed, %d failed ================\n", g_pass, g_fail); return g_fail ? 1 : 0; }