el/engram/test/test_m7_traversal.c

/* test_m7_traversal.c — M7 index-driven activation traversal.
 *
 * Milestone 7 replaces the O(E) full adjacency rebuild that spreading activation
 * paid before every BFS with an incrementally-maintained per-node index, behind
 * the ENGRAM_STORE flag (flag-off = unchanged behavior). This harness links the
 * REAL el_runtime.c engram builtins (+ engram_store.c) and drives activation
 * directly — no EL interpreter, no store boot (the index optimization is a pure
 * in-RAM concern; the flag is read from the environment).
 *
 * Modes (argv[1]):
 *   parity-off <dir>  — ENGRAM_STORE unset: build a fixed graph, run a scripted
 *                       sequence of activations WITH mid-sequence edge/node
 *                       inserts, dump each activation's JSON to <dir>/off_actN.json.
 *   parity-on  <dir>  — ENGRAM_STORE=1: identical graph + identical sequence,
 *                       dump to <dir>/on_actN.json. The runner asserts the off/on
 *                       files are BYTE-IDENTICAL (same activated set, weights,
 *                       ordering, hops, WM promotion).
 *   perf <off|on> <dir> <nodes> <edges> <iters>
 *                     — build a large graph, then loop `iters` times doing
 *                       (add 1 edge + activate). Prints wall-time and the M7
 *                       instrumentation counters (rebuild calls / rebuild
 *                       edge-work / incremental appends).
 *
 * Writes ONLY under the caller-provided throwaway dir.
 */
#include "el_runtime.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

/* M7 instrumentation getters (test-only; defined in el_runtime.c). */
extern int64_t engram_adj_rebuild_calls(void);
extern int64_t engram_adj_rebuild_edge_work(void);
extern int64_t engram_adj_incr_appends(void);
extern double  engram_adj_maint_seconds(void);
extern void    engram_adj_test_force_dirty(void);
extern int     engram_store_enabled(void);

static el_val_t S(const char* s){ return EL_STR(s); }
static el_val_t F(double d){ return el_from_float(d); }

/* Deterministic LCG so off/on processes build byte-identical graphs. */
static uint64_t g_rng = 0x9E3779B97F4A7C15ULL;
static void rng_seed(uint64_t s){ g_rng = s ? s : 1; }
static uint64_t rng_next(void){ g_rng = g_rng * 6364136223846793005ULL + 1442695040888963407ULL; return g_rng >> 17; }

static el_val_t* g_handles = NULL;   /* node id handles from engram_node_full */
static int64_t   g_nnodes  = 0;

static void write_file(const char* path, const char* content){
    FILE* f = fopen(path, "wb");
    if (!f){ fprintf(stderr, "cannot open %s\n", path); exit(2); }
    if (content) fwrite(content, 1, strlen(content), f);
    fclose(f);
}

/* Build `n` nodes whose content carries query-matchable tokens, then `m`
 * deterministic edges among them. Handles are retained for later connect. */
static void build_graph(int64_t n, int64_t m){
    g_handles = malloc((size_t)n * sizeof(el_val_t));
    g_nnodes  = n;
    static const char* topics[] = {
        "storage engine durable log", "spreading activation graph traversal",
        "hebbian potentiation memory", "buffer pool paging checkpoint",
        "adjacency index edge lookup", "working memory promotion",
        "b-tree primary index", "embeddings nearest neighbour" };
    for (int64_t i = 0; i < n; i++){
        char content[256];
        snprintf(content, sizeof content,
                 "node %lld about %s and storage engine activation index",
                 (long long)i, topics[(size_t)(i % 8)]);
        char label[32]; snprintf(label, sizeof label, "n%lld", (long long)i);
        g_handles[i] = engram_node_full(S(content), S("Concept"), S(label),
                         F(0.7), F(0.6), F(1.0), S("Semantic"), S("storage,graph,index"));
    }
    for (int64_t k = 0; k < m; k++){
        int64_t a = (int64_t)(rng_next() % (uint64_t)n);
        int64_t b = (int64_t)(rng_next() % (uint64_t)n);
        if (a == b) b = (b + 1) % n;
        engram_connect(g_handles[a], g_handles[b], F(0.6), S("associate"));
    }
}

static const char* Q1 = "storage engine activation and the durable log";
static const char* Q2 = "adjacency index graph traversal";

/* One scripted activation with an optional forced full-rebuild first. */
static el_val_t act(const char* q, int depth, int force_rebuild){
    if (force_rebuild) engram_adj_test_force_dirty();
    return engram_activate_json(S(q), (el_val_t)depth);
}

/* Run the scripted parity sequence and dump each activation JSON. `tag` names
 * the output set. When force_rebuild is set, every activation first forces the
 * O(E) full-rebuild path (the pre-M7 "scan" behavior); otherwise the M7
 * incremental index is used. The graph build + query sequence are byte-for-byte
 * deterministic, so any difference between two runs is attributable solely to
 * the difference in adjacency maintenance (and/or the ENGRAM_STORE flag). */
static int run_parity(const char* dir, const char* tag, int force_rebuild){
    char p[1024];
    rng_seed(0xC0FFEE123ULL);
    build_graph(60, 140);

    el_val_t a1 = act(Q1, 3, force_rebuild);
    snprintf(p, sizeof p, "%s/%s_act1.json", dir, tag); write_file(p, EL_CSTR(a1));

    /* Mutate the graph BETWEEN activations: this is exactly where the M7 path
     * appends incrementally while the rebuild path marks dirty + fully rebuilds.
     * Parity must hold across this divergence in HOW the index is maintained. */
    engram_connect(g_handles[0], g_handles[7],  F(0.8), S("depends-on"));
    engram_connect(g_handles[7], g_handles[23], F(0.7), S("enables"));
    engram_connect(g_handles[23], g_handles[41],F(0.5), S("uses"));
    el_val_t hnew = engram_node_full(S("freshly minted storage index node about activation"),
                       S("Concept"), S("nnew"), F(0.8), F(0.7), F(1.0), S("Semantic"), S("storage,index"));
    engram_connect(g_handles[0], hnew, F(0.9), S("about"));

    el_val_t a2 = act(Q1, 3, force_rebuild);
    snprintf(p, sizeof p, "%s/%s_act2.json", dir, tag); write_file(p, EL_CSTR(a2));
    el_val_t a3 = act(Q2, 2, force_rebuild);
    snprintf(p, sizeof p, "%s/%s_act3.json", dir, tag); write_file(p, EL_CSTR(a3));
    el_val_t a4 = act(Q1, 3, force_rebuild);
    snprintf(p, sizeof p, "%s/%s_act4.json", dir, tag); write_file(p, EL_CSTR(a4));

    printf("[parity-%s] enabled=%d force_rebuild=%d nodes=%lld edges=%lld "
           "rebuilds=%lld rebuild_edge_work=%lld incr_appends=%lld\n",
           tag, engram_store_enabled(), force_rebuild,
           (long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count(),
           (long long)engram_adj_rebuild_calls(), (long long)engram_adj_rebuild_edge_work(),
           (long long)engram_adj_incr_appends());
    return 0;
}

static double now_sec(void){
    struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts);
    return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9;
}

static int run_perf(const char* dir, const char* tag, int64_t n, int64_t m, int64_t iters){
    (void)dir;
    rng_seed(0xBEEF7777ULL);
    double t_build0 = now_sec();
    build_graph(n, m);
    double t_build = now_sec() - t_build0;

    int64_t rb0 = engram_adj_rebuild_calls();
    int64_t rw0 = engram_adj_rebuild_edge_work();
    int64_t ap0 = engram_adj_incr_appends();
    double  mt0 = engram_adj_maint_seconds();

    double t0 = now_sec();
    for (int64_t it = 0; it < iters; it++){
        /* One structural mutation per query — the curiosity-loop cadence that
         * makes the OLD path rebuild the whole adjacency before every BFS. */
        int64_t a = (int64_t)(rng_next() % (uint64_t)n);
        int64_t b = (int64_t)(rng_next() % (uint64_t)n);
        if (a == b) b = (b + 1) % n;
        engram_connect(g_handles[a], g_handles[b], F(0.6), S("associate"));
        el_val_t r = engram_activate_json(S(Q1), (el_val_t)2);
        (void)r;
    }
    double elapsed = now_sec() - t0;

    double maint = engram_adj_maint_seconds() - mt0;
    printf("[perf-%s] flag=%d nodes=%lld edges=%lld iters=%lld build=%.3fs "
           "loop=%.3fs per_query=%.3fms adj_maint=%.4fs adj_maint_per_query=%.4fms | "
           "rebuilds=%lld rebuild_edge_work=%lld incr_appends=%lld\n",
           tag, engram_store_enabled(),
           (long long)(int64_t)engram_node_count(), (long long)(int64_t)engram_edge_count(),
           (long long)iters, t_build, elapsed, (elapsed / (double)iters) * 1e3,
           maint, (maint / (double)iters) * 1e3,
           (long long)(engram_adj_rebuild_calls() - rb0),
           (long long)(engram_adj_rebuild_edge_work() - rw0),
           (long long)(engram_adj_incr_appends() - ap0));
    return 0;
}

int main(int argc, char** argv){
    if (argc < 3){ fprintf(stderr, "usage: %s <parity-off|parity-on|perf> ...\n", argv[0]); return 2; }
    const char* mode = argv[1];

    if (!strcmp(mode, "parity-off")){
        /* flag-off, rebuild path = today's scan behavior (the baseline). */
        if (engram_store_enabled()){ fprintf(stderr, "parity-off requires ENGRAM_STORE unset\n"); return 2; }
        return run_parity(argv[2], "off", 0);
    }
    if (!strcmp(mode, "parity-on-rebuild")){
        /* flag-on, but force the O(E) rebuild before each activation. */
        if (!engram_store_enabled()){ fprintf(stderr, "parity-on-rebuild requires ENGRAM_STORE=1\n"); return 2; }
        return run_parity(argv[2], "onrb", 1);
    }
    if (!strcmp(mode, "parity-on-incr")){
        /* flag-on, M7 incremental index (the code path under test). */
        if (!engram_store_enabled()){ fprintf(stderr, "parity-on-incr requires ENGRAM_STORE=1\n"); return 2; }
        return run_parity(argv[2], "onincr", 0);
    }
    if (!strcmp(mode, "perf")){
        /* perf <off|on> <dir> <nodes> <edges> <iters> */
        if (argc < 7){ fprintf(stderr, "usage: %s perf <off|on> <dir> <nodes> <edges> <iters>\n", argv[0]); return 2; }
        const char* tag = argv[2];
        int64_t n = strtoll(argv[4], NULL, 10);
        int64_t m = strtoll(argv[5], NULL, 10);
        int64_t iters = strtoll(argv[6], NULL, 10);
        return run_perf(argv[3], tag, n, m, iters);
    }
    fprintf(stderr, "unknown mode %s\n", mode);
    return 2;
}