From f6a0777f9019a9b4e62dab179449840cd39a73e6 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Wed, 12 Aug 2026 23:06:49 -0500 Subject: [PATCH] M10: reify dense neighborhoods into first-class persisted records; geometry-priming reads them (default OFF) Reification, not a cache. Densely co-wired relational neighborhoods are crystallized into DURABLE first-class store records that survive restart, load on boot, and evolve via supersede+provenance -- so the geometry-priming hot path READS persisted structure instead of computing a per-query descriptor (the M9 3.2x/13x latency blocker). engram_geometry.{h,c}: - engram_geo_reify_store(): detect hub-anchored neighborhoods on the hebb-weighted graph (greedy non-redundant cover), compute each centered descriptor ONCE against the true store-wide mean, persist as node_type="Neighborhood" (raw centroid in emb, membership+scalars+axis-extents in a compact GEO1 metadata schema) + member edges, superseding any prior same-hub record. The mean is persisted once as "GeoMeanFrame". - resident loaded form (index_new/add/finalize/lookup): parses the durable records at boot (never recomputes geometry); O(seeds) membership lookup, miss -> centroid-nearest. - descriptor: skip the Jacobi eigensolve when top_axes==0; reject structural records as members (id-convention + node_type guards) so re-reify/ad-hoc stay clean. el_runtime.c: - boot routes Neighborhood/GeoMeanFrame records OUT of the activation graph into the reify index, and skips member edges from adjacency -> ENGRAM_GEOMETRY_PRIMING OFF is byte-identical to M8/M9 (verified across 15 queries). - priming hot path reads the persisted membership; ENGRAM_GEO_PRIMING_NOCACHE=1 keeps the M9 per-query descriptor for ad-hoc geometries / A/B control. A/B on a COPY (128 neighborhoods): priming ON is now FLAT latency (1.06x median / 1.03x p90 vs OFF) where the M9 per-query path is 3.30x/10.5x. Reified records provably inert when OFF. Restart survival + supersede verified. Build 0 warnings (my code); ASan/UBSan clean on module and full server hot path. Recall quality re-eval against the TRUE mean still shows no reliable gain (mean coherence -0.017), so geometry-priming STAYS default-OFF -- but the latency blocker is removed and the durable structure now exists. See docs/architecture/design/engram-m10-reification.md. --- lang/runtime/el_runtime.c | 230 ++++++++++------- lang/runtime/engram_geometry.c | 450 ++++++++++++++++++++++++++++++++- lang/runtime/engram_geometry.h | 107 ++++++++ 3 files changed, 700 insertions(+), 87 deletions(-) diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index ec0db33..0ef6bc3 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -7438,6 +7438,13 @@ static char* engram_first_n_chars(const char* s, size_t n) { #include "engram_vindex.h" /* M8: ANN (HNSW) index for activation seed selection */ #include "engram_geometry.h" /* M9: centered relational-neighborhood geometry (priming) */ +/* M10 REIFICATION: resident loaded form of the first-class persisted neighborhood + * records (Neighborhood + GeoMeanFrame). Built once at boot from the durable store + * (it PARSES persisted structure, never recomputes geometry). The geometry-priming + * hot path reads THIS instead of computing a per-query descriptor. NULL until boot; + * empty (count 0) on a store that has not been reified — priming then no-ops. */ +static GeoReifyIndex* _eg_reify = NULL; + static EngramPagedStore* g_engram_store = NULL; int engram_store_enabled(void) { @@ -7597,6 +7604,16 @@ static void eg_store_put_edge(const EngramEdge* e) { * so the store-on boot behaves byte-identically to the JSON path (M3.5 parity). */ static void eg_load_node_cb(const StoreNode* sn, void* ctx) { EngramStore* g = (EngramStore*)ctx; + /* M10: first-class reified records (Neighborhood / GeoMeanFrame) are DURABLE + * STRUCTURE, not corpus content. Absorb them into the reify index and keep them + * OUT of the resident activation graph, so seed selection / vindex / results / + * embedding backfill are byte-identical to a store that was never reified. */ + if (sn->node_type && + (strcmp(sn->node_type, ENGRAM_GEO_NBHD_TYPE) == 0 || + strcmp(sn->node_type, ENGRAM_GEO_MEANFRAME_TYPE) == 0)) { + if (_eg_reify) engram_geo_reify_index_add(_eg_reify, sn); + return; + } engram_grow_nodes(); EngramNode* n = &g->nodes[g->node_count]; memset(n, 0, sizeof *n); @@ -7628,6 +7645,12 @@ static void eg_load_node_cb(const StoreNode* sn, void* ctx) { } static void eg_load_edge_cb(const StoreEdge* se, void* ctx) { EngramStore* g = (EngramStore*)ctx; + /* M10: skip the persisted member links (from a "nbhd-…" record). They are + * durable structure joining a neighborhood to its members, but inert to + * activation adjacency — dropping them here keeps spreading activation + * byte-identical to pre-reify. (Matched by id convention, so no real edge, + * whatever its relation string, is ever affected.) */ + if (se->from_id && strncmp(se->from_id, ENGRAM_GEO_NBHD_ID_PREFIX, 5) == 0) return; engram_grow_edges(); EngramEdge* e = &g->edges[g->edge_count]; memset(e, 0, sizeof *e); @@ -7697,8 +7720,12 @@ el_val_t engram_store_boot(el_val_t data_dir) { if (!g_engram_store) return (el_val_t)0; EngramStore* g = engram_get(); eg_reset_resident(g); + /* M10: build the resident reify index alongside the graph load — eg_load_node_cb + * feeds the first-class Neighborhood/GeoMeanFrame records into it. */ + if (!_eg_reify) _eg_reify = engram_geo_reify_index_new(); store_scan_nodes(g_engram_store, eg_load_node_cb, g); store_scan_edges(g_engram_store, eg_load_edge_cb, g); + if (_eg_reify) engram_geo_reify_index_finalize(_eg_reify); StoreLayer* ls = NULL; size_t ln = 0; if (store_list_layers(g_engram_store, &ls, &ln) == 0) { for (size_t i = 0; i < ln; i++) eg_load_layer_cb(g, &ls[i]); @@ -9369,97 +9396,130 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { } free(seed_dup); - /* ── M9 GEOMETRY PRIMING (ENGRAM_GEOMETRY_PRIMING, default OFF) ────── - * COMPOSES with the M8 seed set above: uses the CENTERED geometry of - * the seed neighborhood to (a) damp off-domain seeds by centered - * membership (disambiguation) and (b) prime nearby members sub- - * threshold (a warm floor). Flag OFF → this whole block is skipped and - * the seed set/activation are exactly what M8 produced. Read-only over - * the graph except for the bounded, sub-threshold seed additions here. */ - if (eg_geometry_priming_on() && g_engram_store && q_emb && q_dim > 0 && nsel > 0) { - const float* gmean = eg_geo_mean_sync(q_dim); - if (gmean) { - /* Seed ids = the M8-selected semantic seeds; vids maps the - * resident-array VIndex ordinals (== g->nodes[] index) to store - * ids so the descriptor's ANN expansion resolves to paged nodes. */ - const char** seed_ids = malloc((size_t)nsel * sizeof(char*)); - char** vids = malloc((size_t)g->node_count * sizeof(char*)); - if (seed_ids && vids) { - for (int s = 0; s < nsel; s++) seed_ids[s] = g->nodes[sel[s]].id; - for (int64_t i = 0; i < g->node_count; i++) vids[i] = g->nodes[i].id; - GeoDescriptor* geo = engram_geometry_descriptor( - g_engram_store, _eg_vindex, vids, (int)g->node_count, - seed_ids, (size_t)nsel, NULL, gmean); + /* ── GEOMETRY PRIMING (ENGRAM_GEOMETRY_PRIMING, default OFF) ───────── + * COMPOSES with the M8 seed set above: resolves the seed neighborhood's + * CENTERED geometry to (a) damp off-domain seeds by membership + * (disambiguation) and (b) prime nearby members sub-threshold (warm floor). + * Flag OFF → this whole block is skipped and the seed set/activation are + * exactly what M8 produced (byte-identical). Read-only over the graph + * except the bounded, sub-threshold seed additions here. + * + * M10: the neighborhood is READ from the PERSISTED first-class reify index + * (a durable Neighborhood record's membership map) — O(seeds) hash lookup, + * miss → centroid-nearest, NO geometry computed on the activation path. The + * membership is centered against the true store-wide mean persisted in the + * GeoMeanFrame record. Set ENGRAM_GEO_PRIMING_NOCACHE=1 to instead compute + * the descriptor fresh per query (the M9 on-the-fly path — kept for ad-hoc + * geometries and as the A/B latency control). */ + if (eg_geometry_priming_on() && nsel > 0) { + static int _nocache = -1; + if (_nocache < 0) { const char* s = getenv("ENGRAM_GEO_PRIMING_NOCACHE"); + _nocache = (s && s[0] && s[0] != '0') ? 1 : 0; } + + const char** seed_ids = malloc((size_t)nsel * sizeof(char*)); + if (seed_ids) { + for (int s = 0; s < nsel; s++) seed_ids[s] = g->nodes[sel[s]].id; + + char* const* mids = NULL; /* resolved neighborhood member ids */ + const double* mw = NULL; /* their centered membership in [0,1] */ + int mn = 0; + GeoDescriptor* geo = NULL; /* on-the-fly path only (freed below) */ + char** tmid = NULL; double* tmw = NULL; + + if (!_nocache && _eg_reify && engram_geo_reify_count(_eg_reify) > 0) { + /* HOT PATH — read persisted structure, no compute. */ + const GeoNeighborhood* nb = engram_geo_reify_lookup( + _eg_reify, seed_ids, (size_t)nsel, q_emb, q_dim); + if (nb && nb->n_members > 0) { + mids = nb->member_ids; mw = nb->member_w; mn = nb->n_members; + } + } else if (g_engram_store && q_emb && q_dim > 0) { + /* AD-HOC / NOCACHE control — compute the descriptor fresh. */ + const float* gmean = eg_geo_mean_sync(q_dim); + char** vids = malloc((size_t)g->node_count * sizeof(char*)); + if (gmean && vids) { + for (int64_t i = 0; i < g->node_count; i++) vids[i] = g->nodes[i].id; + geo = engram_geometry_descriptor( + g_engram_store, _eg_vindex, vids, (int)g->node_count, + seed_ids, (size_t)nsel, NULL, gmean); + } + free(vids); if (geo && geo->n_members > 0) { - const double lo = eg_geo_seed_lo(); - const double pscl = eg_geo_prime_scale(); - const int pmax = eg_geo_prime_max(); - /* Centered membership per resident idx (-1 = not in the - * geometry → left untouched by the reweight). */ - double* geo_m = malloc((size_t)g->node_count * sizeof(double)); - if (geo_m) { - for (int64_t i = 0; i < g->node_count; i++) geo_m[i] = -1.0; + tmid = malloc((size_t)geo->n_members * sizeof(char*)); + tmw = malloc((size_t)geo->n_members * sizeof(double)); + if (tmid && tmw) { for (int m = 0; m < geo->n_members; m++) { - int64_t ri = engram_find_node_index(geo->members[m].id); - if (ri >= 0 && ri < g->node_count) { - double mv = geo->members[m].membership; - if (mv < 0.0) mv = 0.0; else if (mv > 1.0) mv = 1.0; - geo_m[ri] = mv; - } + tmid[m] = geo->members[m].id; tmw[m] = geo->members[m].membership; } - /* (a) DAMP-ONLY seed reweight: factor = lo+(1-lo)*memb - * ∈ [lo,1]. Off-domain seeds (low centered membership) - * lose weight; the neighborhood anchor (memb→1) is - * unchanged. Never amplifies. Updates both the frontier - * act (drives propagation) and best_bg (drives this - * node's own WM weight). */ - for (int64_t s = 0; s < seed_count; s++) { - int64_t si = seeds[s].idx; - if (si < 0 || si >= g->node_count) continue; - double mv = geo_m[si]; - if (mv < 0.0) continue; /* not in geometry */ - double factor = lo + (1.0 - lo) * mv; - seeds[s].act *= factor; - best_bg[si] *= factor; - } - /* (b) PRIME sub-threshold: descriptor members not - * already reached get a warm floor act=memb*pscl - * (pscl < WM gate ⇒ cannot self-promote) and enter the - * frontier so a warm gradient spreads one hop then dies - * at the 0.02 BFS cutoff. Capped at pmax; ISE skipped. - * Safe: BFS keeps max, so this only RAISES a floor and - * never caps a stronger legitimate activation. */ - int primed = 0; - for (int m = 0; m < geo->n_members && primed < pmax; m++) { - int64_t ri = engram_find_node_index(geo->members[m].id); - if (ri < 0 || ri >= g->node_count) continue; - if (reached[ri]) continue; /* already a seed */ - EngramNode* pn = &g->nodes[ri]; - if (pn->node_type && - strcmp(pn->node_type, "InternalStateEvent") == 0) - continue; - double mv = geo->members[m].membership; - if (mv < 0.0) mv = 0.0; else if (mv > 1.0) mv = 1.0; - double pact = mv * pscl; - if (pact < 0.01) continue; /* too cold to matter */ - seeds[seed_count].idx = ri; - seeds[seed_count].act = pact; - seeds[seed_count].created_at = pn->created_at; - seed_count++; - best_bg[ri] = pact; - best_hops[ri] = 0; - reached[ri] = 1; - primed++; - } - _eg_act_geo_primed += primed; - free(geo_m); + mids = tmid; mw = tmw; mn = geo->n_members; } - engram_geo_free(geo); - } else if (geo) { - engram_geo_free(geo); } } - free(seed_ids); free(vids); + + if (mn > 0 && mids && mw) { + const double lo = eg_geo_seed_lo(); + const double pscl = eg_geo_prime_scale(); + const int pmax = eg_geo_prime_max(); + /* membership per resident idx (-1 = not in the neighborhood). */ + double* geo_m = malloc((size_t)g->node_count * sizeof(double)); + if (geo_m) { + for (int64_t i = 0; i < g->node_count; i++) geo_m[i] = -1.0; + for (int m = 0; m < mn; m++) { + int64_t ri = engram_find_node_index(mids[m]); + if (ri >= 0 && ri < g->node_count) { + double mv = mw[m]; + if (mv < 0.0) mv = 0.0; else if (mv > 1.0) mv = 1.0; + geo_m[ri] = mv; + } + } + /* (a) DAMP-ONLY seed reweight: factor = lo+(1-lo)*memb ∈ [lo,1]. + * Off-domain members lose weight; anchor (memb→1) unchanged; + * seeds outside the neighborhood are left untouched. Never + * amplifies. Updates frontier act + best_bg (WM weight). */ + for (int64_t s = 0; s < seed_count; s++) { + int64_t si = seeds[s].idx; + if (si < 0 || si >= g->node_count) continue; + double mv = geo_m[si]; + if (mv < 0.0) continue; /* not in neighborhood */ + double factor = lo + (1.0 - lo) * mv; + seeds[s].act *= factor; + best_bg[si] *= factor; + } + /* (b) PRIME sub-threshold: neighborhood members not already + * reached get a warm floor act=memb*pscl (pscl < WM gate ⇒ + * cannot self-promote) and enter the frontier so a warm + * gradient spreads one hop then dies at the 0.02 BFS cutoff. + * Capped at pmax; ISE skipped. Safe: BFS keeps max, so this + * only RAISES a floor, never caps a legit activation. */ + int primed = 0; + for (int m = 0; m < mn && primed < pmax; m++) { + int64_t ri = engram_find_node_index(mids[m]); + if (ri < 0 || ri >= g->node_count) continue; + if (reached[ri]) continue; /* already a seed */ + EngramNode* pn = &g->nodes[ri]; + if (pn->node_type && + strcmp(pn->node_type, "InternalStateEvent") == 0) + continue; + double mv = mw[m]; + if (mv < 0.0) mv = 0.0; else if (mv > 1.0) mv = 1.0; + double pact = mv * pscl; + if (pact < 0.01) continue; /* too cold to matter */ + seeds[seed_count].idx = ri; + seeds[seed_count].act = pact; + seeds[seed_count].created_at = pn->created_at; + seed_count++; + best_bg[ri] = pact; + best_hops[ri] = 0; + reached[ri] = 1; + primed++; + } + _eg_act_geo_primed += primed; + free(geo_m); + } + } + free(tmid); free(tmw); + if (geo) engram_geo_free(geo); + free(seed_ids); } } } diff --git a/lang/runtime/engram_geometry.c b/lang/runtime/engram_geometry.c index 0407a14..e033e4b 100644 --- a/lang/runtime/engram_geometry.c +++ b/lang/runtime/engram_geometry.c @@ -6,6 +6,9 @@ #include #include #include +#include +#include +#include /* Must match ENGRAM_HEBB_GAIN in el_runtime.c (eff = weight*(1+GAIN*hebb)). */ #define GEO_HEBB_GAIN 0.5 @@ -91,8 +94,26 @@ static double ccos_dir(const float* a, const float* gm, const float* dir, int di struct GeoMeanCache { float* mean; int dim; uint64_t n; }; typedef struct { double* sum; int dim; uint64_t n; int err; } GeoMeanAcc; +/* The reified records (Neighborhood / GeoMeanFrame) carry an emb (centroid / mean) + * but are STRUCTURE, not corpus content — they must never pollute the store-wide + * mean, the hub scan, or the descriptor. One predicate, used everywhere. */ +static int geo_is_reified_type(const char* nt){ + return nt && (strcmp(nt,ENGRAM_GEO_NBHD_TYPE)==0 || + strcmp(nt,ENGRAM_GEO_MEANFRAME_TYPE)==0); +} +/* Identify a structural record by its id convention (no store read needed), so the + * descriptor never admits a reified Neighborhood / GeoMeanFrame as a neighborhood + * MEMBER even when the ANN index or adjacency still references it (re-reify/refresh + * on a store that already holds reified records; ad-hoc descriptors alike). */ +static int geo_is_structural_id(const char* id){ + if(!id) return 0; + if(strcmp(id,ENGRAM_GEO_MEANFRAME_ID)==0) return 1; + size_t p=strlen(ENGRAM_GEO_NBHD_ID_PREFIX); + return strncmp(id,ENGRAM_GEO_NBHD_ID_PREFIX,p)==0; +} static void geo_mean_cb(const StoreNode* n, void* ctx){ GeoMeanAcc* a=ctx; if(a->err) return; + if(geo_is_reified_type(n->node_type)) return; /* skip structural records */ if(!(n->emb && n->emb_dim>0)) return; /* skip unembedded */ if(a->dim==0){ a->dim=n->emb_dim; @@ -255,6 +276,7 @@ GeoDescriptor* engram_geometry_descriptor( int got=vindex_search(vindex, prov, k, 0, rids, dd); for(int r=0;r=(uint64_t)n_vids) continue; + if(geo_is_structural_id(vids[rids[r]])) continue; /* never a member */ double memb = 1.0 - (double)dd[r]; /* cosine sim in [-1,1] */ if(memb<0) memb=0; int mi=ms_upsert(&ms, vids[rids[r]], memb*0.9); /* <1: not a seed */ @@ -272,6 +294,8 @@ GeoDescriptor* engram_geometry_descriptor( if(store_get_edges_from(store, ms.id[i], &es, &ne)==0 && es){ for(size_t e=0;e0?total_var:0); - /* ── principal axes via dual PCA (Jacobi on the m×m Gram of centered embs) ── */ + /* ── principal axes via dual PCA (Jacobi on the m×m Gram of centered embs) ── + * Skipped entirely when top_axes==0: the eigensolve is the dominant cost, and + * priming needs only members+membership, so reified records that don't want the + * ellipsoid pass top_axes=0 and pay nothing here (centroid+radius still filled). */ int n_axes=0; GeoAxis* axes=NULL; - if(nemb>=2 && nemb<=GEO_EIG_CAP){ + if(P.top_axes>0 && nemb>=2 && nemb<=GEO_EIG_CAP){ int m=nemb; /* centered, row-major m×dim */ float* Xc=malloc((size_t)m*dim*sizeof(float)); @@ -491,3 +520,420 @@ void engram_geo_free(GeoDescriptor* g){ free(g->members); free(g->edges); free(g); } + +/* ═══════════════════════════════════════════════════════════════════════════ + * M10 — REIFICATION: persist / load / lookup first-class neighborhood records. + * ═══════════════════════════════════════════════════════════════════════════ */ + +static int64_t geo_now_ms(void){ + struct timespec ts; + if(clock_gettime(CLOCK_REALTIME,&ts)==0) + return (int64_t)ts.tv_sec*1000 + ts.tv_nsec/1000000; + return (int64_t)time(NULL)*1000; +} + +void engram_geo_reify_default_params(GeoReifyParams* p){ + if(!p) return; + p->min_weighted_degree=0; + p->max_neighborhoods=128; + p->cover_membership=0.5; + p->persist_member_edges=1; + engram_geo_default_params(&p->descriptor); + p->descriptor.top_axes=4; /* keep a small ellipsoid summary; cheap */ + p->descriptor.max_members=256; /* reified neighborhoods stay compact */ +} + +/* ── tiny growable string builder ─────────────────────────────────────────── */ +typedef struct { char* s; size_t n, cap; } SB; +static int sb_reserve(SB* b, size_t add){ + if(b->n+add+1<=b->cap) return 0; + size_t nc=b->cap?b->cap:256; while(ncn+add+1) nc*=2; + char* t=realloc(b->s,nc); if(!t) return -1; b->s=t; b->cap=nc; return 0; +} +static int sb_puts(SB* b, const char* s){ + size_t l=strlen(s); if(sb_reserve(b,l)) return -1; + memcpy(b->s+b->n,s,l); b->n+=l; b->s[b->n]=0; return 0; +} +static int sb_fmt(SB* b, const char* fmt, ...){ + char tmp[512]; va_list ap; va_start(ap,fmt); + int k=vsnprintf(tmp,sizeof tmp,fmt,ap); va_end(ap); + if(k<0) return -1; if(k>=(int)sizeof tmp) k=sizeof tmp-1; + return sb_puts(b,tmp); +} + +/* Serialize a descriptor's DURABLE geometry into the GEO1 metadata schema. + * (The raw centroid is stored separately as the record's emb.) */ +static char* geo_nbhd_metadata(const GeoDescriptor* g, const char* hub, + const char* meanid){ + SB b={0}; + if(sb_puts(&b,"GEO1\n")) { free(b.s); return NULL; } + sb_fmt(&b,"hub %s\n", hub?hub:""); + sb_fmt(&b,"mean %s\n", meanid?meanid:""); + sb_fmt(&b,"s %.9g %.9g %d %.9g %d %d\n", + g->radius, g->total_variance, g->k_core, g->co_registration, + g->n_embedded, g->n_members); + sb_puts(&b,"e"); + for(int i=0;in_axes;i++) sb_fmt(&b," %.9g", g->axes[i].extent); + sb_puts(&b,"\n"); + for(int i=0;in_members;i++){ + sb_fmt(&b,"m %s %.9g %.9g %d\n", + g->members[i].id, g->members[i].membership, + g->members[i].centrality, g->members[i].core); + } + return b.s; /* caller frees */ +} + +/* ── string set (greedy-cover claimed ids) + string→id list (hub→old nbhd) ──── */ +static uint64_t geo_djb2(const char* s){ + uint64_t h=5381; for(;*s;s++) h=((h<<5)+h)^(unsigned char)*s; return h; +} +typedef struct SSNode { char* key; struct SSNode* next; } SSNode; +typedef struct { SSNode** b; size_t nb; } SSet; +static void ss_init(SSet* s, size_t nb){ s->nb=nb; s->b=calloc(nb,sizeof*s->b); } +static int ss_has(const SSet* s, const char* k){ + if(!s->b) return 0; SSNode* n=s->b[geo_djb2(k)%s->nb]; + for(;n;n=n->next) if(strcmp(n->key,k)==0) return 1; return 0; +} +static void ss_add(SSet* s, const char* k){ + if(!s->b||ss_has(s,k)) return; size_t i=geo_djb2(k)%s->nb; + SSNode* n=malloc(sizeof*n); if(!n) return; n->key=strdup(k); n->next=s->b[i]; s->b[i]=n; +} +static void ss_free(SSet* s){ + if(!s->b) return; + for(size_t i=0;inb;i++){ SSNode* n=s->b[i]; while(n){ SSNode* x=n->next; free(n->key); free(n); n=x; } } + free(s->b); s->b=NULL; +} + +typedef struct { char** id; int n, cap; } StrVec; +static void sv_push(StrVec* v, const char* s){ + if(v->n==v->cap){ v->cap=v->cap?v->cap*2:64; v->id=realloc(v->id,(size_t)v->cap*sizeof*v->id); } + v->id[v->n++]=strdup(s); +} +static void sv_free(StrVec* v){ for(int i=0;in;i++) free(v->id[i]); free(v->id); } + +/* pass 1 collector: all non-structural node ids; also record existing Neighborhood + * records as (hub -> old_id) so a re-reify supersedes the prior version. */ +typedef struct { + StrVec cand; /* candidate node ids (content nodes) */ + StrVec old_hub, old_id; /* parallel: existing nbhd hub + its id */ +} ReifyScan; +static void geo_reify_scan_cb(const StoreNode* n, void* ctx){ + ReifyScan* rs=ctx; if(!n->id||!n->node_type) { if(n->id) sv_push(&rs->cand,n->id); return; } + if(strcmp(n->node_type,ENGRAM_GEO_NBHD_TYPE)==0){ + /* parse hub from metadata GEO1 (line "hub ") for supersede lineage */ + const char* md=n->metadata?n->metadata:""; + const char* p=strstr(md,"hub "); + if(p && (p==md || p[-1]=='\n')){ + p+=4; const char* e=p; while(*e && *e!='\n') e++; + char* hub=strndup(p,(size_t)(e-p)); + sv_push(&rs->old_hub,hub); sv_push(&rs->old_id,n->id); free(hub); + } + return; /* structural: not a candidate */ + } + if(strcmp(n->node_type,ENGRAM_GEO_MEANFRAME_TYPE)==0) return; + sv_push(&rs->cand,n->id); +} + +/* weighted strong-edge degree of a node (from+to), matching eff_w/threshold. */ +static double geo_weighted_degree(EngramPagedStore* st, const char* id, double emin){ + double deg=0; StoreEdge* es=NULL; size_t ne=0; + if(store_get_edges_from(st,id,&es,&ne)==0 && es){ + for(size_t e=0;e=emin) deg+=w; } + } + store_edges_free(es,ne); es=NULL; ne=0; + if(store_get_edges_to(st,id,&es,&ne)==0 && es){ + for(size_t e=0;e=emin) deg+=w; } + } + store_edges_free(es,ne); + return deg; +} + +int engram_geo_reify_store(EngramPagedStore* store, VIndex* vindex, + char** vids, int n_vids, + const GeoReifyParams* params){ + if(!store) return -1; + GeoReifyParams P; if(params) P=*params; else engram_geo_reify_default_params(&P); + + /* 1. true store-wide mean → persist the GeoMeanFrame record (once). */ + GeoMeanCache* mc=engram_geo_mean_build(store); + if(!mc) return -2; + int dim=engram_geo_mean_dim(mc); + const float* mean=engram_geo_mean_vec(mc); + int64_t now=geo_now_ms(); + { StoreNode mf; memset(&mf,0,sizeof mf); + mf.id=(char*)ENGRAM_GEO_MEANFRAME_ID; mf.node_type=(char*)ENGRAM_GEO_MEANFRAME_TYPE; + mf.content=(char*)"geo-mean-frame"; mf.tier=(char*)"Semantic"; mf.metadata=(char*)"{}"; + mf.emb=(float*)mean; mf.emb_dim=dim; mf.created_at=now; mf.updated_at=now; + if(store_put_node(store,&mf)<0){ engram_geo_mean_free(mc); return -3; } + } + + /* 2. scan: candidate ids + existing (hub→old id) for supersede. */ + ReifyScan rs; memset(&rs,0,sizeof rs); + if(store_scan_nodes(store,geo_reify_scan_cb,&rs)<0){ + sv_free(&rs.cand); sv_free(&rs.old_hub); sv_free(&rs.old_id); + engram_geo_mean_free(mc); return -4; + } + + /* 3. weighted degree per candidate; sort desc. */ + int N=rs.cand.n; + double* deg=malloc((size_t)N*sizeof(double)); + int* ord=malloc((size_t)N*sizeof(int)); + for(int i=0;ideg[ord[best]]) best=b; + int t=ord[a]; ord[a]=ord[best]; ord[best]=t; } + + /* 4. greedy non-redundant cover: reify each qualifying hub once. */ + SSet claimed; ss_init(&claimed, (size_t)(N>16?N:16)); + int persisted=0; + for(int oi=0; oi0 && deg[i]<(double)P.min_weighted_degree) break; /* sorted: rest smaller */ + if(ss_has(&claimed,hub)) continue; + const char* seeds[1]={hub}; + GeoDescriptor* g=engram_geometry_descriptor(store,vindex,vids,n_vids, + seeds,1,&P.descriptor,mean); + if(!g || g->n_members<=0){ if(g) engram_geo_free(g); continue; } + /* claim members above cover threshold (incl. the hub itself) */ + for(int m=0;mn_members;m++) + if(g->members[m].membership>=P.cover_membership) ss_add(&claimed,g->members[m].id); + + /* build record: id = nbhd--, emb = RAW centroid = centered+mean */ + char nid[512]; snprintf(nid,sizeof nid,"%s%s-%lld",ENGRAM_GEO_NBHD_ID_PREFIX,hub,(long long)now); + float* raw=NULL; + if(g->n_embedded>0 && g->centroid && g->global_mean){ + raw=malloc((size_t)dim*sizeof(float)); + if(raw) for(int d=0;dcentroid[d]+g->global_mean[d]; + } + char* md=geo_nbhd_metadata(g,hub,ENGRAM_GEO_MEANFRAME_ID); + StoreNode nn; memset(&nn,0,sizeof nn); + nn.id=nid; nn.node_type=(char*)ENGRAM_GEO_NBHD_TYPE; + nn.content=(char*)"reified-neighborhood"; nn.tier=(char*)"Semantic"; + nn.metadata=md?md:(char*)"{}"; nn.emb=raw; nn.emb_dim=raw?dim:0; + nn.created_at=now; nn.updated_at=now; + int wrc=store_put_node(store,&nn); + free(raw); free(md); + if(wrc<0){ engram_geo_free(g); continue; } + + /* provenance: supersede any prior neighborhood for this hub. */ + for(int k=0;kn_members;m++){ + char eid[600]; snprintf(eid,sizeof eid,"%s->%s",nid,g->members[m].id); + StoreEdge se; memset(&se,0,sizeof se); + se.id=eid; se.from_id=nid; se.to_id=g->members[m].id; + se.relation=(char*)ENGRAM_GEO_MEMBER_RELATION; + se.metadata=(char*)"{}"; se.weight=g->members[m].membership; + se.confidence=1.0; se.created_at=now; se.updated_at=now; + store_put_edge(store,&se); + } + } + engram_geo_free(g); + persisted++; + } + + ss_free(&claimed); + free(deg); free(ord); + sv_free(&rs.cand); sv_free(&rs.old_hub); sv_free(&rs.old_id); + engram_geo_mean_free(mc); + return persisted; +} + +/* ═══════════════ resident loaded form + hot-path lookup ═════════════════════ */ + +typedef struct { + char* id; + char* hub_id; + int n_members; + char** member_ids; + double* member_w; + double radius, co_reg; + int k_core, n_embedded; + float* centroid_raw; /* dim floats or NULL */ + float* centroid_unit; /* centered+normalized (finalize) or NULL */ + int dim; + GeoNeighborhood view; +} RNbhd; + +typedef struct RE { char* id; int nbhd; double w; struct RE* next; } RE; + +struct GeoReifyIndex { + RNbhd* nb; int n, cap; + float* mean; int mean_dim; + RE** buckets; size_t nbuckets; + double* score; /* scratch[n], reused per lookup */ +}; + +GeoReifyIndex* engram_geo_reify_index_new(void){ + GeoReifyIndex* ix=calloc(1,sizeof*ix); return ix; +} + +/* parse a GEO1 metadata blob into an RNbhd (members + scalars). */ +static int geo_parse_nbhd(const char* md, RNbhd* r){ + if(!md) return -1; + if(strncmp(md,"GEO1",4)!=0) return -1; + /* count member lines to size arrays */ + int cap=0; for(const char* p=md; (p=strstr(p,"\nm ")); p+=3) cap++; + r->member_ids=cap?calloc((size_t)cap,sizeof(char*)):NULL; + r->member_w =cap?calloc((size_t)cap,sizeof(double)):NULL; + r->n_members=0; + const char* line=md; + while(line && *line){ + const char* nl=strchr(line,'\n'); + size_t len= nl? (size_t)(nl-line) : strlen(line); + char buf[600]; if(len>=sizeof buf) len=sizeof buf-1; + memcpy(buf,line,len); buf[len]=0; + if(buf[0]=='h'&&buf[1]=='u'&&buf[2]=='b'&&buf[3]==' '){ + free(r->hub_id); r->hub_id=strdup(buf+4); + } else if(buf[0]=='s'&&buf[1]==' '){ + int kc=0,ne=0,nm=0; double rad=0,tv=0,cr=0; + sscanf(buf+2,"%lf %lf %d %lf %d %d",&rad,&tv,&kc,&cr,&ne,&nm); + r->radius=rad; r->co_reg=cr; r->k_core=kc; r->n_embedded=ne; + } else if(buf[0]=='m'&&buf[1]==' '){ + char mid[512]; double w=0,c=0; int core=0; + if(sscanf(buf+2,"%511s %lf %lf %d",mid,&w,&c,&core)>=2 && r->member_ids){ + r->member_ids[r->n_members]=strdup(mid); + r->member_w[r->n_members]=w; + r->n_members++; + } + } + line = nl? nl+1 : NULL; + } + return 0; +} + +int engram_geo_reify_index_add(GeoReifyIndex* ix, const StoreNode* n){ + if(!ix||!n||!n->node_type) return 0; + if(strcmp(n->node_type,ENGRAM_GEO_MEANFRAME_TYPE)==0){ + if(n->emb && n->emb_dim>0){ + free(ix->mean); + ix->mean=malloc((size_t)n->emb_dim*sizeof(float)); + if(ix->mean){ memcpy(ix->mean,n->emb,(size_t)n->emb_dim*sizeof(float)); ix->mean_dim=n->emb_dim; } + } + return 0; + } + if(strcmp(n->node_type,ENGRAM_GEO_NBHD_TYPE)!=0) return 0; + if(ix->n==ix->cap){ ix->cap=ix->cap?ix->cap*2:16; + RNbhd* t=realloc(ix->nb,(size_t)ix->cap*sizeof*t); if(!t) return -1; ix->nb=t; } + RNbhd* r=&ix->nb[ix->n]; memset(r,0,sizeof*r); + r->id=strdup(n->id?n->id:""); + if(geo_parse_nbhd(n->metadata,r)!=0){ free(r->id); return 0; } /* skip malformed */ + if(n->emb && n->emb_dim>0){ + r->dim=n->emb_dim; + r->centroid_raw=malloc((size_t)n->emb_dim*sizeof(float)); + if(r->centroid_raw) memcpy(r->centroid_raw,n->emb,(size_t)n->emb_dim*sizeof(float)); + } + ix->n++; + return 0; +} + +int engram_geo_reify_index_finalize(GeoReifyIndex* ix){ + if(!ix) return -1; + /* member → neighborhood hash */ + size_t total=0; for(int i=0;in;i++) total+=(size_t)ix->nb[i].n_members; + ix->nbuckets = total? (total*2+1) : 1; + ix->buckets=calloc(ix->nbuckets,sizeof(RE*)); + if(!ix->buckets) return -1; + for(int i=0;in;i++){ + RNbhd* r=&ix->nb[i]; + for(int m=0;mn_members;m++){ + size_t b=geo_djb2(r->member_ids[m])%ix->nbuckets; + RE* e=malloc(sizeof*e); if(!e) continue; + e->id=r->member_ids[m]; e->nbhd=i; e->w=r->member_w[m]; e->next=ix->buckets[b]; ix->buckets[b]=e; + } + /* centered, normalized centroid for the nearest-fallback */ + if(r->centroid_raw && ix->mean && ix->mean_dim==r->dim){ + r->centroid_unit=malloc((size_t)r->dim*sizeof(float)); + if(r->centroid_unit){ + double nrm=0; for(int d=0;ddim;d++){ double v=(double)r->centroid_raw[d]-ix->mean[d]; r->centroid_unit[d]=(float)v; nrm+=v*v; } + nrm=sqrt(nrm); if(nrm>1e-12) for(int d=0;ddim;d++) r->centroid_unit[d]=(float)(r->centroid_unit[d]/nrm); + else { free(r->centroid_unit); r->centroid_unit=NULL; } + } + } + /* fill the borrowed view */ + r->view.id=r->id; r->view.hub_id=r->hub_id; r->view.n_members=r->n_members; + r->view.member_ids=r->member_ids; r->view.member_w=r->member_w; + r->view.radius=r->radius; r->view.co_registration=r->co_reg; + r->view.k_core=r->k_core; r->view.n_embedded=r->n_embedded; + } + ix->score=ix->n?calloc((size_t)ix->n,sizeof(double)):NULL; + return 0; +} + +static void geo__reify_load_cb(const StoreNode* n, void* ctx){ + engram_geo_reify_index_add((GeoReifyIndex*)ctx, n); +} +GeoReifyIndex* engram_geo_reify_load(EngramPagedStore* store){ + if(!store) return NULL; + GeoReifyIndex* ix=engram_geo_reify_index_new(); if(!ix) return NULL; + store_scan_nodes(store, geo__reify_load_cb, ix); + if(ix->n==0 && ix->mean==NULL){ engram_geo_reify_index_free(ix); return NULL; } + engram_geo_reify_index_finalize(ix); + return ix; +} + +const GeoNeighborhood* engram_geo_reify_lookup( + const GeoReifyIndex* ix, + const char* const* seed_ids, size_t n_seeds, + const float* q_emb, int q_dim){ + if(!ix||ix->n<=0) return NULL; + /* (a) membership route: score each neighborhood by summed seed membership. */ + if(ix->score && ix->buckets && seed_ids && n_seeds>0){ + for(int i=0;in;i++) ((GeoReifyIndex*)ix)->score[i]=0.0; + int any=0; + for(size_t s=0;sbuckets[geo_djb2(id)%ix->nbuckets]; e; e=e->next) + if(strcmp(e->id,id)==0){ ((GeoReifyIndex*)ix)->score[e->nbhd]+=e->w; any=1; } + } + if(any){ + int best=-1; double bv=-1; + for(int i=0;in;i++) if(ix->score[i]>bv){ bv=ix->score[i]; best=i; } + if(best>=0 && bv>0) return &ix->nb[best].view; + } + } + /* (b) centroid-nearest fallback (centered query vs centered centroids). */ + if(q_emb && q_dim>0 && ix->mean && ix->mean_dim==q_dim){ + double nq=0; float* cq=malloc((size_t)q_dim*sizeof(float)); + if(!cq) return NULL; + for(int d=0;dmean[d]; cq[d]=(float)v; nq+=v*v; } + nq=sqrt(nq); + if(nq>1e-12){ + int best=-1; double bc=-1e9; + for(int i=0;in;i++){ RNbhd* r=&ix->nb[i]; if(!r->centroid_unit) continue; + double s=0; for(int d=0;dcentroid_unit[d]; + s/=nq; if(s>bc){ bc=s; best=i; } } + free(cq); + if(best>=0) return &ix->nb[best].view; + } else free(cq); + } + return NULL; +} + +int engram_geo_reify_count(const GeoReifyIndex* ix){ return ix?ix->n:0; } +const float* engram_geo_reify_mean(const GeoReifyIndex* ix, int* dim){ + if(!ix||!ix->mean){ if(dim)*dim=0; return NULL; } + if(dim)*dim=ix->mean_dim; return ix->mean; +} + +void engram_geo_reify_index_free(GeoReifyIndex* ix){ + if(!ix) return; + if(ix->buckets){ + for(size_t b=0;bnbuckets;b++){ RE* e=ix->buckets[b]; while(e){ RE* x=e->next; free(e); e=x; } } + free(ix->buckets); + } + for(int i=0;in;i++){ RNbhd* r=&ix->nb[i]; + free(r->id); free(r->hub_id); + for(int m=0;mn_members;m++) free(r->member_ids[m]); + free(r->member_ids); free(r->member_w); + free(r->centroid_raw); free(r->centroid_unit); + } + free(ix->nb); free(ix->score); free(ix->mean); + free(ix); +} diff --git a/lang/runtime/engram_geometry.h b/lang/runtime/engram_geometry.h index 49afb27..0811b96 100644 --- a/lang/runtime/engram_geometry.h +++ b/lang/runtime/engram_geometry.h @@ -158,4 +158,111 @@ GeoDescriptor* engram_geometry_descriptor( void engram_geo_free(GeoDescriptor* g); +/* ═══════════════════════════════════════════════════════════════════════════ + * M10 — REIFICATION: densely co-wired relational neighborhoods crystallized into + * FIRST-CLASS, PERSISTED store records (design doc §2; memory 885f5945). This is + * NOT a cache — it is durable structure. A reified neighborhood is a real store + * NODE (node_type "Neighborhood") that survives restart, is loaded on boot, and + * EVOLVES via supersede+provenance when the pattern shifts. The geometry-priming + * HOT PATH reads these persisted records (never computes geometry on the + * activation path). Ad-hoc/transient geometries still use the on-the-fly + * engram_geometry_descriptor above. + * + * Two record types, both ordinary TLV store nodes (no new on-disk format): + * - "GeoMeanFrame" : the store-wide centering mean, persisted ONCE (emb = mean + * vector, id ENGRAM_GEO_MEANFRAME_ID). Referenced by every + * neighborhood so priming centers against the SAME true mean. + * - "Neighborhood" : one reified neighborhood. emb = the RAW centroid (prototype + * point, so it stays centroid-ANN-able; centered_centroid = + * emb - meanframe). metadata = the compact "GEO1" schema: + * hub id, meanframe ref, scalar shape (radius, total_variance, + * k_core, co_registration, n_embedded), axis EXTENTS (ellipsoid + * half-widths), and the MEMBER list {id -> membership, centrality, + * core}. Member links are also persisted as edges relation="member". + * + * v1 honest simplifications (documented; extensible without migration): axis + * DIRECTION vectors are not persisted (extents capture the ellipsoid scale; the + * directions are recomputable via the on-the-fly descriptor for viz/operators); + * with hebb potentiation ~0 on today's store the "hebb-weighted" degree reduces to + * AUTHORED edge weight, so detected neighborhoods currently reflect authored edges — + * the design is unchanged and self-correcting once hebb accrues. + * ═══════════════════════════════════════════════════════════════════════════ */ + +#define ENGRAM_GEO_NBHD_TYPE "Neighborhood" +#define ENGRAM_GEO_MEANFRAME_TYPE "GeoMeanFrame" +#define ENGRAM_GEO_MEANFRAME_ID "geo-meanframe" /* stable id of the singleton */ +#define ENGRAM_GEO_NBHD_ID_PREFIX "nbhd-" /* id = nbhd-- */ +#define ENGRAM_GEO_MEMBER_RELATION "member" + +typedef struct { + int min_weighted_degree; /* hub qualifies iff strong-edge weighted degree >= this + * (0 = no floor: just rank + take top max_neighborhoods) */ + int max_neighborhoods; /* homeostatic budget cap (default 128) */ + double cover_membership; /* skip a hub already a member (w>=this) of an accepted + * neighborhood — greedy non-redundant cover (default 0.5) */ + int persist_member_edges; /* 1 = also write relation="member" edges (default 1) */ + GeoParams descriptor; /* per-neighborhood params (top_axes may be 0 = skip eigensolve) */ +} GeoReifyParams; + +/* Defaults: min_weighted_degree=0, max_neighborhoods=128, cover_membership=0.5, + * persist_member_edges=1, descriptor = engram_geo_default_params but top_axes=4, + * max_members=256 (reified neighborhoods stay compact). */ +void engram_geo_reify_default_params(GeoReifyParams* p); + +/* WRITE PATH (offline / consolidation — NEVER the activation hot path). + * Detect dense hub neighborhoods on the hebb-weighted graph, compute each one's + * CENTERED descriptor ONCE against the true store-wide mean, and PERSIST them as + * first-class records: the GeoMeanFrame (once) + one Neighborhood node per detected + * neighborhood (+ member edges), superseding any prior same-hub record with + * provenance. Read-then-write over `store`. Returns #neighborhoods persisted, or <0. + * Skips existing Neighborhood/GeoMeanFrame nodes when detecting (idempotent re-reify). */ +int engram_geo_reify_store(EngramPagedStore* store, VIndex* vindex, + char** vids, int n_vids, + const GeoReifyParams* params); + +/* ── Resident loaded form of the persisted records (boot-time; READ-ONLY) ───── + * The durable Neighborhood/GeoMeanFrame records are the source of truth; this + * index is their LOADED form (like the resident node array is the loaded form of + * the node records, or adjacency the loaded form of edges). It never recomputes + * geometry — it parses. Build it by feeding the runtime's boot node scan, or in + * one pass with engram_geo_reify_load. */ +typedef struct GeoReifyIndex GeoReifyIndex; + +GeoReifyIndex* engram_geo_reify_index_new(void); +/* Feed one store node; if it is a Neighborhood or GeoMeanFrame record it is parsed + * and absorbed (else ignored). The node is BORROWED (copied as needed). 0/<0. */ +int engram_geo_reify_index_add(GeoReifyIndex* ix, const StoreNode* n); +/* Build the member->neighborhood hash after all adds. Call once. 0/<0. */ +int engram_geo_reify_index_finalize(GeoReifyIndex* ix); +/* One-pass convenience: scan the store and build the finalized index. NULL if the + * store holds no reified records. */ +GeoReifyIndex* engram_geo_reify_load(EngramPagedStore* store); + +/* A borrowed view of one persisted neighborhood (owned by the index). */ +typedef struct { + const char* id; + const char* hub_id; + int n_members; + char* const* member_ids; /* parallel arrays, length n_members */ + const double* member_w; /* membership in [0,1] */ + double radius; + double co_registration; + int k_core; + int n_embedded; +} GeoNeighborhood; + +/* HOT-PATH LOOKUP (no geometry compute): resolve the seed set to the best + * persisted neighborhood — the one with the greatest summed seed membership; on a + * miss (no seed is a member of any neighborhood) fall back to the centroid nearest + * the query embedding (centered by the loaded mean frame). q_emb may be NULL (then + * a miss returns NULL). Returns a BORROWED handle (do NOT free) or NULL. */ +const GeoNeighborhood* engram_geo_reify_lookup( + const GeoReifyIndex* ix, + const char* const* seed_ids, size_t n_seeds, + const float* q_emb, int q_dim); + +int engram_geo_reify_count(const GeoReifyIndex* ix); +const float* engram_geo_reify_mean(const GeoReifyIndex* ix, int* dim); /* loaded true mean or NULL */ +void engram_geo_reify_index_free(GeoReifyIndex* ix); + #endif /* ENGRAM_GEOMETRY_H */