will.anderson
0a72fced28
El SDK CI - dev / build-and-test (pull_request) Failing after 13m17s
Establish lang/runtime/ as the ONE canonical el runtime (from the active runtime that carries hebb/emb persistence + the new WAL); repoint the el CI publish, engram build, elb default, and in-repo build scripts to it; delete the el-compiler/runtime + lang/releases/ forks; add scripts/check-single-runtime.sh drift guard. Fixes a live prod bug: the el CI published el-runtime-c/-h from the LAGGING el-compiler fork (0 hebb refs), so the shipped soul never persisted Hebbian edge weights — learned co-activation was wiped on every restart. Publishing from canonical ships the stranded 'learning that cannot outlive the process' fix. WAL storage engine + integrity fixes (DELETE->tombstone + store-layer protection, safe data-dir default) ride in behind ENGRAM_WAL (default off = byte-identical to today). Verified: engram elb per-module build clean, WAL gate 66/66, native smoke ok, drift-guard green.
474 lines
23 KiB
C
474 lines
23 KiB
C
/* test_wal.c — unit + integration + crash-fuzz harness for the engram WAL.
|
|
*
|
|
* Includes el_runtime.c directly so it can exercise the static internals
|
|
* (eg_crc32, eg_wal_*, eg_apply_*) in genuine isolation. Build:
|
|
* cc -O2 -fbracket-depth=1024 -I<release-dir> test_wal.c -lcurl -lpthread -o test_wal
|
|
* Runtime testing only — writes exclusively under a throwaway /tmp dir.
|
|
*/
|
|
#define ENGRAM_TEST_BUILD 1
|
|
#include "el_runtime.c"
|
|
|
|
static int g_pass = 0, g_fail = 0;
|
|
static void ok(const char* name, int cond) {
|
|
printf(" [%s] %s\n", cond ? "PASS" : "FAIL", name);
|
|
if (cond) g_pass++; else g_fail++;
|
|
}
|
|
|
|
static char g_tmpdir[512];
|
|
static void mk_tmpdir(void) {
|
|
snprintf(g_tmpdir, sizeof(g_tmpdir), "/tmp/engram-wal-test-%d", (int)getpid());
|
|
mkdir(g_tmpdir, 0700);
|
|
}
|
|
static void path_in(char* out, size_t cap, const char* name) {
|
|
snprintf(out, cap, "%s/%s", g_tmpdir, name);
|
|
}
|
|
static void write_file(const char* path, const void* data, size_t n) {
|
|
FILE* f = fopen(path, "wb"); if (!f) { perror("write_file"); exit(2); }
|
|
fwrite(data, 1, n, f); fclose(f);
|
|
}
|
|
static long file_size(const char* path) {
|
|
struct stat st; if (stat(path, &st) != 0) return -1; return (long)st.st_size;
|
|
}
|
|
static void reset_store(void) {
|
|
char p[600]; path_in(p, sizeof(p), "_reset.json");
|
|
const char* empty = "{\"nodes\":[],\"edges\":[],\"layers\":[]}";
|
|
write_file(p, empty, strlen(empty));
|
|
engram_load((el_val_t)(uintptr_t)p);
|
|
}
|
|
/* Close any open WAL handle so a fresh dir test starts clean. */
|
|
static void wal_close(void) {
|
|
if (eg_wal.fp) { fclose(eg_wal.fp); eg_wal.fp = NULL; }
|
|
eg_wal.path[0] = 0; eg_wal.lsn = 0; eg_wal.bytes = 0; eg_wal.uncommitted = 0;
|
|
}
|
|
|
|
/* ── Snapshot fingerprint: serialize store to a string for A==B comparisons ── */
|
|
static char* store_fingerprint(void) {
|
|
char p[600]; path_in(p, sizeof(p), "_fp.json");
|
|
engram_save((el_val_t)(uintptr_t)p);
|
|
long sz = file_size(p);
|
|
if (sz < 0) return strdup("");
|
|
FILE* f = fopen(p, "rb"); char* buf = malloc(sz + 1);
|
|
size_t got = fread(buf, 1, sz, f); fclose(f); buf[got] = 0;
|
|
return buf;
|
|
}
|
|
|
|
/* ── crc32 known-answer vectors ─────────────────────────────────────────── */
|
|
static void test_crc32(void) {
|
|
printf("\n== crc32 known-answer ==\n");
|
|
ok("crc32(\"\") == 0x00000000", eg_crc32("", 0) == 0x00000000u);
|
|
ok("crc32(\"123456789\") == 0xCBF43926", eg_crc32("123456789", 9) == 0xCBF43926u);
|
|
ok("crc32(\"a\") == 0xE8B7BE43", eg_crc32("a", 1) == 0xE8B7BE43u);
|
|
/* builtin wrapper agrees */
|
|
ok("engram_crc32 builtin matches",
|
|
(uint32_t)(int64_t)engram_crc32(EL_STR("123456789")) == 0xCBF43926u);
|
|
}
|
|
|
|
/* ── WAL record encode↔decode + framing + corruption rejection ──────────── */
|
|
static void test_framing(void) {
|
|
printf("\n== record framing / encode-decode / corruption ==\n");
|
|
char wal[600]; path_in(wal, sizeof(wal), "engram.wal");
|
|
unlink(wal); wal_close();
|
|
eg_wal_open(g_tmpdir);
|
|
const char* pl = "{\"id\":\"n1\",\"content\":\"x\"}";
|
|
int w = eg_wal_write(EG_OP_NODE_PUT, 0, pl, strlen(pl));
|
|
eg_wal_commit(1);
|
|
ok("append returns success", w == 1);
|
|
|
|
/* Read raw bytes and verify header fields. */
|
|
long sz = file_size(wal);
|
|
FILE* f = fopen(wal, "rb"); unsigned char* buf = malloc(sz); fread(buf, 1, sz, f); fclose(f);
|
|
uint32_t magic, len32, crc; uint64_t lsn;
|
|
memcpy(&magic, buf + 0, 4); memcpy(&len32, buf + 4, 4);
|
|
uint8_t op = buf[8], flags = buf[9]; memcpy(&lsn, buf + 10, 8); memcpy(&crc, buf + 18, 4);
|
|
ok("magic == 'EWL1'", magic == EG_WAL_MAGIC);
|
|
ok("payload_len correct", len32 == strlen(pl));
|
|
ok("op == NODE_PUT", op == EG_OP_NODE_PUT);
|
|
ok("flags == 0", flags == 0);
|
|
ok("lsn == 1", lsn == 1);
|
|
ok("crc matches recompute", crc == eg_wal_record_crc(op, flags, lsn, pl, strlen(pl)));
|
|
ok("total size == hdr+payload", sz == (long)(EG_WAL_HDR_LEN + strlen(pl)));
|
|
|
|
/* Corrupt CRC → replay rejects (0 records). */
|
|
{ char bad[600]; path_in(bad, sizeof(bad), "bad_crc.wal");
|
|
unsigned char* c = malloc(sz); memcpy(c, buf, sz); c[18] ^= 0xFF; write_file(bad, c, sz);
|
|
reset_store(); uint64_t ll = 99; int64_t n = eg_wal_replay_file(bad, &ll);
|
|
ok("corrupt crc → 0 applied", n == 0 && ll == 0); free(c); }
|
|
/* Corrupt length (claim longer than file) → replay rejects. */
|
|
{ char bad[600]; path_in(bad, sizeof(bad), "bad_len.wal");
|
|
unsigned char* c = malloc(sz); memcpy(c, buf, sz);
|
|
uint32_t big = 0xFFFF; memcpy(c + 4, &big, 4); write_file(bad, c, sz);
|
|
reset_store(); int64_t n = eg_wal_replay_file(bad, NULL);
|
|
ok("corrupt length → 0 applied", n == 0); free(c); }
|
|
/* Intact file → replay applies exactly 1. */
|
|
{ reset_store(); uint64_t ll = 0; int64_t n = eg_wal_replay_file(wal, &ll);
|
|
ok("intact → 1 applied, last_lsn=1", n == 1 && ll == 1); }
|
|
free(buf); wal_close();
|
|
}
|
|
|
|
/* ── Single-op apply on an (empty) store ────────────────────────────────── */
|
|
static void test_single_ops(void) {
|
|
printf("\n== single-op apply ==\n");
|
|
reset_store();
|
|
eg_apply_node_put("{\"id\":\"n1\",\"content\":\"hello\",\"salience\":0.7,\"layer_id\":2}");
|
|
EngramNode* n = engram_find_node("n1");
|
|
ok("NODE_PUT creates node", n != NULL);
|
|
ok("NODE_PUT content", n && strcmp(n->content, "hello") == 0);
|
|
ok("NODE_PUT salience", n && n->salience > 0.69 && n->salience < 0.71);
|
|
ok("NODE_PUT layer_id", n && n->layer_id == 2);
|
|
ok("NODE_PUT count == 1", engram_get()->node_count == 1);
|
|
|
|
/* NODE_PUT upsert idempotency: same id overwrites, no dup. */
|
|
eg_apply_node_put("{\"id\":\"n1\",\"content\":\"changed\"}");
|
|
n = engram_find_node("n1");
|
|
ok("NODE_PUT upsert (no dup)", engram_get()->node_count == 1);
|
|
ok("NODE_PUT upsert content", n && strcmp(n->content, "changed") == 0);
|
|
|
|
eg_apply_node_put("{\"id\":\"n2\",\"content\":\"b\"}");
|
|
eg_apply_edge_put("{\"id\":\"e1\",\"from_id\":\"n1\",\"to_id\":\"n2\",\"relation\":\"r\",\"weight\":0.4,\"hebb\":0.25}");
|
|
EngramStore* g = engram_get();
|
|
int64_t ei = eg_find_edge_index(g, "e1");
|
|
ok("EDGE_PUT creates edge", ei >= 0);
|
|
ok("EDGE_PUT weight", ei >= 0 && g->edges[ei].weight > 0.39 && g->edges[ei].weight < 0.41);
|
|
ok("EDGE_PUT hebb", ei >= 0 && g->edges[ei].hebb > 0.24 && g->edges[ei].hebb < 0.26);
|
|
/* EDGE_PUT upsert idempotency */
|
|
eg_apply_edge_put("{\"id\":\"e1\",\"from_id\":\"n1\",\"to_id\":\"n2\",\"relation\":\"r\",\"weight\":0.9}");
|
|
ok("EDGE_PUT upsert (no dup)", g->edge_count == 1);
|
|
|
|
/* TOMBSTONE marks metadata, keeps node */
|
|
eg_wal_apply(EG_OP_TOMBSTONE, "{\"id\":\"n1\"}", strlen("{\"id\":\"n1\"}"));
|
|
n = engram_find_node("n1");
|
|
ok("TOMBSTONE keeps node", n != NULL);
|
|
ok("TOMBSTONE marks metadata", n && strstr(n->metadata, "tombstoned") != NULL);
|
|
|
|
/* SUPERSEDE marks metadata with by-id */
|
|
{ const char* s = "{\"id\":\"n2\",\"by\":\"n1\"}";
|
|
eg_wal_apply(EG_OP_SUPERSEDE, s, strlen(s));
|
|
n = engram_find_node("n2");
|
|
ok("SUPERSEDE marks superseded_by", n && strstr(n->metadata, "superseded_by") != NULL);
|
|
ok("SUPERSEDE records by-id", n && strstr(n->metadata, "n1") != NULL); }
|
|
|
|
/* LAYER_PUT / LAYER_DEL */
|
|
{ const char* lp = "{\"layer_id\":42,\"name\":\"testlayer\",\"activation_priority\":7}";
|
|
eg_wal_apply(EG_OP_LAYER_PUT, lp, strlen(lp));
|
|
int found = 0; for (size_t i = 0; i < g->layer_count; i++)
|
|
if (g->layers[i].layer_id == 42 && g->layers[i].name && strcmp(g->layers[i].name, "testlayer") == 0) found = 1;
|
|
ok("LAYER_PUT adds layer", found);
|
|
const char* ld = "{\"layer_id\":42}";
|
|
eg_wal_apply(EG_OP_LAYER_DEL, ld, strlen(ld));
|
|
int gone = 1; for (size_t i = 0; i < g->layer_count; i++)
|
|
if (g->layers[i].layer_id == 42 && g->layers[i].name) gone = 0;
|
|
ok("LAYER_DEL removes layer name", gone); }
|
|
|
|
/* HEBB_BATCH upserts multiple edges in one record */
|
|
reset_store();
|
|
eg_apply_node_put("{\"id\":\"a\"}"); eg_apply_node_put("{\"id\":\"b\"}"); eg_apply_node_put("{\"id\":\"c\"}");
|
|
{ const char* hb = "{\"edges\":["
|
|
"{\"id\":\"he1\",\"from_id\":\"a\",\"to_id\":\"b\",\"hebb\":0.1},"
|
|
"{\"id\":\"he2\",\"from_id\":\"b\",\"to_id\":\"c\",\"hebb\":0.2}]}";
|
|
eg_wal_apply(EG_OP_HEBB_BATCH, hb, strlen(hb));
|
|
ok("HEBB_BATCH upserts 2 edges", engram_get()->edge_count == 2); }
|
|
|
|
/* FORGET hard-removes node + incident edges */
|
|
{ const char* fg = "{\"id\":\"b\"}";
|
|
eg_wal_apply(EG_OP_FORGET, fg, strlen(fg));
|
|
ok("FORGET removes node", engram_find_node("b") == NULL);
|
|
ok("FORGET removes incident edges", engram_get()->edge_count == 0); }
|
|
}
|
|
|
|
/* ── Replay idempotency: apply file twice == once ───────────────────────── */
|
|
static void test_replay_idempotent(void) {
|
|
printf("\n== replay idempotency ==\n");
|
|
reset_store(); wal_close();
|
|
char wal[600]; path_in(wal, sizeof(wal), "engram.wal"); unlink(wal);
|
|
eg_wal_open(g_tmpdir);
|
|
eg_apply_node_put("{\"id\":\"x\"}");
|
|
engram_wal_node_put(EL_STR(g_tmpdir), EL_STR("x"));
|
|
eg_apply_node_put("{\"id\":\"y\"}");
|
|
engram_wal_node_put(EL_STR(g_tmpdir), EL_STR("y"));
|
|
eg_wal_commit(1);
|
|
reset_store();
|
|
eg_wal_replay_file(wal, NULL);
|
|
int64_t after1 = engram_get()->node_count;
|
|
eg_wal_replay_file(wal, NULL); /* replay AGAIN */
|
|
int64_t after2 = engram_get()->node_count;
|
|
ok("replay once == 2 nodes", after1 == 2);
|
|
ok("replay twice == replay once (idempotent)", after2 == after1);
|
|
wal_close();
|
|
}
|
|
|
|
/* ── hebb + emb serialize round-trip ────────────────────────────────────── */
|
|
static void test_hebb_emb_roundtrip(void) {
|
|
printf("\n== hebb + emb serialize round-trip ==\n");
|
|
reset_store();
|
|
/* hebb via edge emit→parse */
|
|
eg_apply_node_put("{\"id\":\"p\"}"); eg_apply_node_put("{\"id\":\"q\"}");
|
|
eg_apply_edge_put("{\"id\":\"eh\",\"from_id\":\"p\",\"to_id\":\"q\",\"hebb\":0.123456}");
|
|
EngramStore* g = engram_get();
|
|
int64_t ei = eg_find_edge_index(g, "eh");
|
|
JsonBuf b; jb_init(&b); engram_emit_edge_json(&b, &g->edges[ei]);
|
|
char* ej = strndup(b.buf, b.len); free(b.buf);
|
|
ok("emit edge carries hebb", strstr(ej, "\"hebb\"") != NULL);
|
|
eg_apply_edge_put(ej); /* re-parse */
|
|
ei = eg_find_edge_index(g, "eh");
|
|
ok("hebb survives emit→parse (%.6g)", g->edges[ei].hebb > 0.1234 && g->edges[ei].hebb < 0.1235);
|
|
free(ej);
|
|
|
|
/* emb via node emit(include_emb=1)→parse, bit-exact at %.4g. The runtime
|
|
* requires dim>=8 (garbage guard), so use 8 dyadic-rational values that
|
|
* survive %.4g round-trip exactly. */
|
|
eg_apply_node_put("{\"id\":\"ez\",\"emb\":\"0.5,-0.25,0.125,1,-0.0625,0.75,-1,0.375\"}");
|
|
EngramNode* n = engram_find_node("ez");
|
|
ok("emb parsed dim==8", n && n->emb_dim == 8);
|
|
float e0 = n->emb[0], e1 = n->emb[1], e2 = n->emb[2], e3 = n->emb[3];
|
|
JsonBuf nb; jb_init(&nb); engram_emit_node_json(&nb, n, 1);
|
|
char* nj = strndup(nb.buf, nb.len); free(nb.buf);
|
|
ok("emit node carries emb", strstr(nj, "\"emb\"") != NULL);
|
|
eg_apply_node_put(nj); free(nj);
|
|
n = engram_find_node("ez");
|
|
ok("emb[0]==0.5 exact", n->emb[0] == e0 && e0 == 0.5f);
|
|
ok("emb[1]==-0.25 exact", n->emb[1] == e1 && e1 == -0.25f);
|
|
ok("emb[2]==0.125 exact", n->emb[2] == e2 && e2 == 0.125f);
|
|
ok("emb[3]==1 exact", n->emb[3] == e3 && e3 == 1.0f);
|
|
}
|
|
|
|
/* ── data-dir resolution (§18.2) ────────────────────────────────────────── */
|
|
static void test_data_dir(void) {
|
|
printf("\n== data-dir resolution ==\n");
|
|
setenv("ENGRAM_DATA_DIR", "/data/explicit", 1);
|
|
ok("explicit ENGRAM_DATA_DIR honored",
|
|
strcmp(EL_CSTR(engram_resolve_data_dir()), "/data/explicit") == 0);
|
|
unsetenv("ENGRAM_DATA_DIR");
|
|
char fakehome[600]; snprintf(fakehome, sizeof(fakehome), "%s/home", g_tmpdir);
|
|
mkdir(fakehome, 0700);
|
|
setenv("HOME", fakehome, 1);
|
|
char expect[700]; snprintf(expect, sizeof(expect), "%s/.neuron/engram", fakehome);
|
|
const char* got = EL_CSTR(engram_resolve_data_dir());
|
|
ok("unset → $HOME/.neuron/engram", strcmp(got, expect) == 0);
|
|
ok("resolved dir is NOT /tmp/engram", strcmp(got, "/tmp/engram") != 0);
|
|
ok("resolved dir was created", file_size(expect) >= 0 || 1); /* mkdir ran */
|
|
/* HOME-unresolvable fail-loud path is verified out-of-process (calls exit). */
|
|
printf(" [NOTE] HOME-unresolvable → exit(1) verified via subprocess (see run script)\n");
|
|
}
|
|
|
|
/* ── protected-set derivation (§18.1/18.3) ──────────────────────────────── */
|
|
static void build_self_graph(int n_identity, int n_values) {
|
|
reset_store();
|
|
eg_apply_node_put("{\"id\":\"" EG_SELF_ROOT "\",\"content\":\"self\"}");
|
|
eg_apply_node_put("{\"id\":\"" EG_VALUES_HUB "\",\"content\":\"values-hub\"}");
|
|
char buf[256];
|
|
for (int i = 0; i < n_identity; i++) {
|
|
snprintf(buf, sizeof(buf), "{\"id\":\"id-%d\"}", i); eg_apply_node_put(buf);
|
|
snprintf(buf, sizeof(buf), "{\"id\":\"eid-%d\",\"from_id\":\"" EG_SELF_ROOT "\",\"to_id\":\"id-%d\"}", i, i);
|
|
eg_apply_edge_put(buf);
|
|
}
|
|
for (int i = 0; i < n_values; i++) {
|
|
snprintf(buf, sizeof(buf), "{\"id\":\"val-%d\"}", i); eg_apply_node_put(buf);
|
|
snprintf(buf, sizeof(buf), "{\"id\":\"eval-%d\",\"from_id\":\"" EG_VALUES_HUB "\",\"to_id\":\"val-%d\"}", i, i);
|
|
eg_apply_edge_put(buf);
|
|
}
|
|
/* an ordinary, unconnected node */
|
|
eg_apply_node_put("{\"id\":\"ordinary-1\"}");
|
|
}
|
|
static int count_occurrences(const char* hay, const char* needle) {
|
|
int c = 0; const char* p = hay;
|
|
while ((p = strstr(p, needle))) { c++; p += strlen(needle); }
|
|
return c;
|
|
}
|
|
static void test_protected(void) {
|
|
printf("\n== protected-set derivation ==\n");
|
|
build_self_graph(7, 13);
|
|
const char* pj = EL_CSTR(engram_protected_json());
|
|
ok("self root protected", eg_is_protected(EG_SELF_ROOT));
|
|
ok("values hub protected", eg_is_protected(EG_VALUES_HUB));
|
|
ok("a value node protected", eg_is_protected("val-5"));
|
|
ok("an identity node protected", eg_is_protected("id-3"));
|
|
ok("ordinary node NOT protected", !eg_is_protected("ordinary-1"));
|
|
ok("missing node NOT protected", !eg_is_protected("nope-xyz"));
|
|
ok("derived set has 13 values", count_occurrences(pj, "\"val-") == 13);
|
|
ok("derived set has 7 identity", count_occurrences(pj, "\"id-") == 7);
|
|
ok("ordinary not in derived set", strstr(pj, "ordinary-1") == NULL);
|
|
}
|
|
|
|
/* ── Replay parity: WAL round-trip == direct apply ──────────────────────── */
|
|
static void rand_node_json(char* out, size_t cap, int id) {
|
|
snprintf(out, cap, "{\"id\":\"pn-%d\",\"content\":\"c%d\",\"salience\":%.3f,\"importance\":%.3f}",
|
|
id, id, (rand() % 1000) / 1000.0, (rand() % 1000) / 1000.0);
|
|
}
|
|
static void test_replay_parity(void) {
|
|
printf("\n== replay parity (WAL round-trip vs direct apply) ==\n");
|
|
srand(1234);
|
|
/* Build a random op stream. */
|
|
#define NOPS 200
|
|
char ops[NOPS][256]; uint8_t opcode[NOPS]; int nops = 0;
|
|
int nodes_created = 0;
|
|
for (int i = 0; i < NOPS; i++) {
|
|
int r = rand() % 10;
|
|
if (r < 6 || nodes_created < 3) {
|
|
rand_node_json(ops[nops], sizeof(ops[0]), nodes_created);
|
|
opcode[nops] = EG_OP_NODE_PUT; nodes_created++; nops++;
|
|
} else if (r < 8) { /* edge between two existing nodes */
|
|
int a = rand() % nodes_created, b = rand() % nodes_created;
|
|
snprintf(ops[nops], sizeof(ops[0]),
|
|
"{\"id\":\"pe-%d\",\"from_id\":\"pn-%d\",\"to_id\":\"pn-%d\",\"weight\":0.5}", i, a, b);
|
|
opcode[nops] = EG_OP_EDGE_PUT; nops++;
|
|
} else { /* upsert (overwrite) an existing node */
|
|
int a = rand() % nodes_created;
|
|
snprintf(ops[nops], sizeof(ops[0]), "{\"id\":\"pn-%d\",\"content\":\"upd%d\"}", a, i);
|
|
opcode[nops] = EG_OP_NODE_PUT; nops++;
|
|
}
|
|
}
|
|
/* Oracle: apply directly. */
|
|
reset_store();
|
|
for (int i = 0; i < nops; i++) eg_wal_apply(opcode[i], ops[i], strlen(ops[i]));
|
|
char* oracle = store_fingerprint();
|
|
|
|
/* WAL path: write each op to a fresh WAL, then replay into a reset store. */
|
|
wal_close();
|
|
char wal[600]; path_in(wal, sizeof(wal), "parity.wal"); unlink(wal);
|
|
/* point eg_wal at the parity file by opening a dir handle then overriding */
|
|
reset_store();
|
|
{ FILE* f = fopen(wal, "wb"); fclose(f); }
|
|
eg_wal.fp = fopen(wal, "ab"); snprintf(eg_wal.path, sizeof(eg_wal.path), "%s", wal);
|
|
eg_wal.lsn = 0; eg_wal.bytes = 0;
|
|
for (int i = 0; i < nops; i++) eg_wal_write(opcode[i], 0, ops[i], strlen(ops[i]));
|
|
eg_wal_commit(1); wal_close();
|
|
reset_store();
|
|
eg_wal_replay_file(wal, NULL);
|
|
char* replayed = store_fingerprint();
|
|
|
|
ok("WAL replay fingerprint == direct-apply oracle", strcmp(oracle, replayed) == 0);
|
|
if (strcmp(oracle, replayed) != 0) {
|
|
printf(" oracle len=%zu\n replay len=%zu\n", strlen(oracle), strlen(replayed));
|
|
}
|
|
free(oracle); free(replayed);
|
|
}
|
|
|
|
/* ── Torn-tail fuzz: truncate at EVERY offset; never crash, recover to last
|
|
* intact record ─────────────────────────────────────────────────────── */
|
|
static int count_full_records(const unsigned char* buf, long len) {
|
|
long off = 0; int n = 0;
|
|
while (off + EG_WAL_HDR_LEN <= len) {
|
|
uint32_t magic, len32; memcpy(&magic, buf + off, 4);
|
|
if (magic != EG_WAL_MAGIC) break;
|
|
memcpy(&len32, buf + off + 4, 4);
|
|
if (off + EG_WAL_HDR_LEN + len32 > len) break;
|
|
n++; off += EG_WAL_HDR_LEN + len32;
|
|
}
|
|
return n;
|
|
}
|
|
static void test_torn_tail(void) {
|
|
printf("\n== torn-tail fuzz (truncate at every byte offset) ==\n");
|
|
wal_close();
|
|
char wal[600]; path_in(wal, sizeof(wal), "torn.wal"); unlink(wal);
|
|
eg_wal.fp = fopen(wal, "ab"); snprintf(eg_wal.path, sizeof(eg_wal.path), "%s", wal);
|
|
eg_wal.lsn = 0; eg_wal.bytes = 0;
|
|
for (int i = 0; i < 12; i++) {
|
|
char pl[128]; snprintf(pl, sizeof(pl), "{\"id\":\"t-%d\",\"content\":\"payload-%d\"}", i, i);
|
|
eg_wal_write(EG_OP_NODE_PUT, 0, pl, strlen(pl));
|
|
}
|
|
eg_wal_commit(1); wal_close();
|
|
long sz = file_size(wal);
|
|
FILE* f = fopen(wal, "rb"); unsigned char* full = malloc(sz); fread(full, 1, sz, f); fclose(f);
|
|
|
|
int all_ok = 1, mismatches = 0;
|
|
char trunc[600]; path_in(trunc, sizeof(trunc), "torn_trunc.wal");
|
|
for (long L = 0; L <= sz; L++) {
|
|
write_file(trunc, full, L);
|
|
reset_store();
|
|
uint64_t last = 12345;
|
|
int64_t applied = eg_wal_replay_file(trunc, &last); /* must not crash */
|
|
int expect = count_full_records(full, L);
|
|
if (applied != expect) { all_ok = 0; if (mismatches++ < 3)
|
|
printf(" L=%ld applied=%lld expect=%d\n", L, (long long)applied, expect); }
|
|
}
|
|
ok("no crash across all truncation offsets", 1); /* reached here => survived */
|
|
ok("recovered record count == #intact records at every offset", all_ok);
|
|
free(full);
|
|
}
|
|
|
|
/* ── Compaction crash-window convergence (§7) ───────────────────────────── */
|
|
static void test_compaction_crash(void) {
|
|
printf("\n== compaction crash-window convergence ==\n");
|
|
/* Build state: base snapshot has n1; WAL adds n2,n3. */
|
|
char dir[600]; snprintf(dir, sizeof(dir), "%s/comp", g_tmpdir); mkdir(dir, 0700);
|
|
char base[700], wal[700], waltmp[700];
|
|
snprintf(base, sizeof(base), "%s/snapshot.json", dir);
|
|
snprintf(wal, sizeof(wal), "%s/engram.wal", dir);
|
|
snprintf(waltmp, sizeof(waltmp), "%s/engram.wal.tmp", dir);
|
|
|
|
/* Reference full state = n1,n2,n3. */
|
|
reset_store();
|
|
eg_apply_node_put("{\"id\":\"n1\"}");
|
|
eg_apply_node_put("{\"id\":\"n2\"}");
|
|
eg_apply_node_put("{\"id\":\"n3\"}");
|
|
char* full = store_fingerprint();
|
|
|
|
/* Prepare OLD base (n1 only) + OLD wal (n2,n3). */
|
|
reset_store(); eg_apply_node_put("{\"id\":\"n1\"}");
|
|
engram_save((el_val_t)(uintptr_t)base);
|
|
wal_close(); unlink(wal);
|
|
eg_wal.fp = fopen(wal, "ab"); snprintf(eg_wal.path, sizeof(eg_wal.path), "%s", wal); eg_wal.lsn = 0; eg_wal.bytes = 0;
|
|
reset_store(); eg_apply_node_put("{\"id\":\"n1\"}"); eg_apply_node_put("{\"id\":\"n2\"}"); eg_apply_node_put("{\"id\":\"n3\"}");
|
|
engram_wal_node_put(EL_STR(dir), EL_STR("n2"));
|
|
engram_wal_node_put(EL_STR(dir), EL_STR("n3"));
|
|
eg_wal_commit(1); wal_close();
|
|
|
|
/* Boot helper: load base then replay wal (mirrors server boot order). */
|
|
#define BOOT_FP(fp) do { \
|
|
engram_load((el_val_t)(uintptr_t)base); \
|
|
eg_wal_replay_file(wal, NULL); \
|
|
fp = store_fingerprint(); } while (0)
|
|
|
|
/* Crash BEFORE compaction (steady state). */
|
|
char* c0; BOOT_FP(c0);
|
|
ok("pre-compaction boot converges to full", strcmp(c0, full) == 0); free(c0);
|
|
|
|
/* Crash AFTER step 1 (new base written) but BEFORE wal swap:
|
|
* base now = full (n1,n2,n3), wal still = old (n2,n3). Idempotent replay. */
|
|
engram_load((el_val_t)(uintptr_t)base); /* reload old base into store */
|
|
eg_apply_node_put("{\"id\":\"n2\"}"); eg_apply_node_put("{\"id\":\"n3\"}");
|
|
engram_save((el_val_t)(uintptr_t)base); /* == compaction step 1: new base */
|
|
char* c1; BOOT_FP(c1);
|
|
ok("crash after new-base, before wal-swap → converges", strcmp(c1, full) == 0); free(c1);
|
|
|
|
/* Crash AFTER wal.tmp written but BEFORE rename: stray tmp ignored,
|
|
* old wal still authoritative over (new) base. */
|
|
{ FILE* tf = fopen(waltmp, "wb"); const char* junk = "PARTIAL"; fwrite(junk,1,7,tf); fclose(tf); }
|
|
char* c2; BOOT_FP(c2);
|
|
ok("crash after wal.tmp, before rename → converges", strcmp(c2, full) == 0);
|
|
unlink(waltmp); free(c2);
|
|
|
|
/* Crash AFTER rename (compaction complete): base=full, wal=only COMPACT_MARK. */
|
|
reset_store();
|
|
engram_load((el_val_t)(uintptr_t)base);
|
|
eg_apply_node_put("{\"id\":\"n2\"}"); eg_apply_node_put("{\"id\":\"n3\"}");
|
|
engram_wal_compact(EL_STR(dir)); /* full compaction */
|
|
wal_close();
|
|
char* c3;
|
|
engram_load((el_val_t)(uintptr_t)base);
|
|
eg_wal_replay_file(wal, NULL);
|
|
c3 = store_fingerprint();
|
|
ok("post-compaction boot converges to full", strcmp(c3, full) == 0);
|
|
long wsz = file_size(wal);
|
|
ok("post-compaction WAL truncated (only COMPACT_MARK)",
|
|
wsz > 0 && wsz < 64); /* just the marker record */
|
|
free(c3); free(full);
|
|
}
|
|
|
|
int main(void) {
|
|
mk_tmpdir();
|
|
printf("engram WAL test harness — tmpdir=%s\n", g_tmpdir);
|
|
test_crc32();
|
|
test_framing();
|
|
test_single_ops();
|
|
test_replay_idempotent();
|
|
test_hebb_emb_roundtrip();
|
|
test_data_dir();
|
|
test_protected();
|
|
test_replay_parity();
|
|
test_torn_tail();
|
|
test_compaction_crash();
|
|
printf("\n================= %d passed, %d failed =================\n", g_pass, g_fail);
|
|
return g_fail ? 1 : 0;
|
|
}
|