el/engram/test/test_scan_collision.c

/* test_scan_collision.c — regression gate for the "saved but not findable" bug.
 *
 * ROOT CAUSE UNDER TEST: store_scan_nodes / store_scan_edges (the boot-load
 * path that populates the resident in-RAM graph — engram_store_boot ->
 * eg_load_node_cb) deduplicated emitted records by their 64-bit id_hash
 * (FNV-1a-64), NOT by the full id string. Two DISTINCT ids that collide under
 * id_hash therefore emitted only the FIRST: the second node/edge was durably
 * present in neuron.egm (store_get_node finds it), physically on a live page,
 * yet was SILENTLY DROPPED from the resident load. After any store reopen it
 * was unretrievable by id, absent from lexical search, and missing from the
 * recent list — exactly the reported symptom.
 *
 * The two ids below are real FNV-1a-64 collisions (found offline via Brent's
 * cycle detection over fnv1a(hex16(x))); both hash to 0x15141fdadfa24abe.
 *
 * Pure C. Writes ONLY under a throwaway /tmp dir. Never touches ~/.neuron.
 */
#include "../../lang/runtime/engram_store.h"

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <unistd.h>
#include <sys/stat.h>

static int g_pass = 0, g_fail = 0;
static void ok(const char* name, int cond){
    printf("  [%s] %s\n", cond ? "PASS" : "FAIL", name);
    if (cond) g_pass++; else g_fail++;
}

/* Confirmed FNV-1a-64 collision (distinct strings, equal id_hash). */
#define ID_A "d2c61ec7d015dc98"
#define ID_B "bf85e965a2aefbdd"

static uint64_t fnv1a(const char* s){
    uint64_t h = 1469598103934665603ULL;
    for (; *s; ++s){ h ^= (uint8_t)*s; h *= 1099511628211ULL; }
    return h;
}

static char g_dir[512];
static void mk_dir(void){
    snprintf(g_dir, sizeof g_dir, "/tmp/engram-scancol-%d", (int)getpid());
    mkdir(g_dir, 0700);
}

/* ── scan collectors: record which ids the boot-load scan actually emits ── */
typedef struct { const char* want[8]; int seen[8]; int nwant; int total; } Collect;
static void node_cb(const StoreNode* n, void* ctx){
    Collect* c = ctx; c->total++;
    for (int i=0;i<c->nwant;i++) if (n->id && strcmp(n->id, c->want[i])==0) c->seen[i]=1;
}
static void edge_cb(const StoreEdge* e, void* ctx){
    Collect* c = ctx; c->total++;
    for (int i=0;i<c->nwant;i++) if (e->id && strcmp(e->id, c->want[i])==0) c->seen[i]=1;
}

static void mk_node(StoreNode* n, const char* id, const char* content){
    memset(n, 0, sizeof *n);
    n->id = strdup(id);
    n->content = strdup(content);
    n->node_type = strdup("Memory");
    n->label = strdup(content);
    n->tier = strdup("Working");
    n->tags = strdup("");
    n->metadata = strdup("{}");
    n->salience = 0.5; n->importance = 0.5; n->confidence = 1.0;
    n->created_at = 1700000000000LL; n->updated_at = 1700000000000LL;
    n->last_activated = 1700000000000LL;
}
static void mk_edge(StoreEdge* e, const char* id, const char* from, const char* to){
    memset(e, 0, sizeof *e);
    e->id = strdup(id); e->from_id = strdup(from); e->to_id = strdup(to);
    e->relation = strdup("assoc"); e->metadata = strdup("{}");
    e->weight = 1.0; e->confidence = 1.0;
    e->created_at = 1700000000000LL; e->updated_at = 1700000000000LL;
}

int main(void){
    mk_dir();
    printf("== scan-collision regression (saved-but-not-findable) ==\n");
    printf("  id_hash(%s) = %016llx\n", ID_A, (unsigned long long)fnv1a(ID_A));
    printf("  id_hash(%s) = %016llx\n", ID_B, (unsigned long long)fnv1a(ID_B));
    ok("precondition: the two ids genuinely collide under id_hash",
       fnv1a(ID_A) == fnv1a(ID_B) && strcmp(ID_A, ID_B) != 0);

    /* ---- Control: a single node survives a full store round-trip. ---- */
    {
        EngramPagedStore* s = engram_open(g_dir);
        StoreNode n; mk_node(&n, ID_A, "alpha distinctiveword");
        store_put_node(s, &n);
        engram_close(s);                 /* checkpoint + close */

        EngramPagedStore* r = engram_open(g_dir);
        StoreNode got;
        ok("control: single node found by id after reopen", store_get_node(r, ID_A, &got)==1);
        if (0) {} else store_node_free(&got);
        Collect c = {{ID_A}, {0}, 1, 0};
        store_scan_nodes(r, node_cb, &c);
        ok("control: single node emitted by boot-load scan", c.seen[0]==1);
        engram_close(r);
        store_node_free(&n);
    }

    /* ---- Bug: two id-hash-colliding NODES, both durable, both must load. ---- */
    {
        char dir2[600]; snprintf(dir2, sizeof dir2, "%s/nodes", g_dir); mkdir(dir2, 0700);
        EngramPagedStore* s = engram_open(dir2);
        StoreNode a, b;
        mk_node(&a, ID_A, "alpha distinctiveword-A");
        mk_node(&b, ID_B, "beta distinctiveword-B");
        store_put_node(s, &a);
        store_put_node(s, &b);
        engram_close(s);
        store_node_free(&a); store_node_free(&b);

        EngramPagedStore* r = engram_open(dir2);
        /* Both are individually durable (store_get_node disambiguates by strcmp). */
        StoreNode ga, gb;
        int hit_a = store_get_node(r, ID_A, &ga); if (hit_a==1) store_node_free(&ga);
        int hit_b = store_get_node(r, ID_B, &gb); if (hit_b==1) store_node_free(&gb);
        ok("both colliding nodes are durably present (store_get_node)", hit_a==1 && hit_b==1);

        /* THE REGRESSION: the boot-load scan must emit BOTH, not silently drop one. */
        Collect c = {{ID_A, ID_B}, {0,0}, 2, 0};
        store_scan_nodes(r, node_cb, &c);
        printf("    scan emitted A=%d B=%d (total=%d)\n", c.seen[0], c.seen[1], c.total);
        ok("boot-load scan emits node A (would be resident)", c.seen[0]==1);
        ok("boot-load scan emits node B (the dropped/unretrievable one)", c.seen[1]==1);
        engram_close(r);
    }

    /* ---- Bug: two id-hash-colliding EDGES, both must load. ---- */
    {
        char dir3[600]; snprintf(dir3, sizeof dir3, "%s/edges", g_dir); mkdir(dir3, 0700);
        EngramPagedStore* s = engram_open(dir3);
        StoreNode na, nb; mk_node(&na, "src", "s"); mk_node(&nb, "dst", "d");
        store_put_node(s, &na); store_put_node(s, &nb);
        StoreEdge ea, eb;
        mk_edge(&ea, ID_A, "src", "dst");
        mk_edge(&eb, ID_B, "src", "dst");
        store_put_edge(s, &ea);
        store_put_edge(s, &eb);
        engram_close(s);
        store_node_free(&na); store_node_free(&nb);
        store_edge_free(&ea); store_edge_free(&eb);

        EngramPagedStore* r = engram_open(dir3);
        Collect c = {{ID_A, ID_B}, {0,0}, 2, 0};
        store_scan_edges(r, edge_cb, &c);
        printf("    scan emitted edgeA=%d edgeB=%d\n", c.seen[0], c.seen[1]);
        ok("boot-load scan emits edge A", c.seen[0]==1);
        ok("boot-load scan emits edge B (the dropped one)", c.seen[1]==1);
        engram_close(r);
    }

    printf("\n  %d passed, %d failed\n", g_pass, g_fail);
    /* cleanup */
    char cmd[600]; snprintf(cmd, sizeof cmd, "rm -rf %s", g_dir); if (system(cmd)){}
    return g_fail ? 1 : 0;
}