M10: reify dense neighborhoods into first-class persisted records; geometry-priming reads them (default OFF)
Reification, not a cache. Densely co-wired relational neighborhoods are crystallized
into DURABLE first-class store records that survive restart, load on boot, and evolve
via supersede+provenance -- so the geometry-priming hot path READS persisted structure
instead of computing a per-query descriptor (the M9 3.2x/13x latency blocker).
engram_geometry.{h,c}:
- engram_geo_reify_store(): detect hub-anchored neighborhoods on the hebb-weighted
graph (greedy non-redundant cover), compute each centered descriptor ONCE against
the true store-wide mean, persist as node_type="Neighborhood" (raw centroid in emb,
membership+scalars+axis-extents in a compact GEO1 metadata schema) + member edges,
superseding any prior same-hub record. The mean is persisted once as "GeoMeanFrame".
- resident loaded form (index_new/add/finalize/lookup): parses the durable records at
boot (never recomputes geometry); O(seeds) membership lookup, miss -> centroid-nearest.
- descriptor: skip the Jacobi eigensolve when top_axes==0; reject structural records as
members (id-convention + node_type guards) so re-reify/ad-hoc stay clean.
el_runtime.c:
- boot routes Neighborhood/GeoMeanFrame records OUT of the activation graph into the
reify index, and skips member edges from adjacency -> ENGRAM_GEOMETRY_PRIMING OFF is
byte-identical to M8/M9 (verified across 15 queries).
- priming hot path reads the persisted membership; ENGRAM_GEO_PRIMING_NOCACHE=1 keeps
the M9 per-query descriptor for ad-hoc geometries / A/B control.
A/B on a COPY (128 neighborhoods): priming ON is now FLAT latency (1.06x median / 1.03x
p90 vs OFF) where the M9 per-query path is 3.30x/10.5x. Reified records provably inert
when OFF. Restart survival + supersede verified. Build 0 warnings (my code); ASan/UBSan
clean on module and full server hot path. Recall quality re-eval against the TRUE mean
still shows no reliable gain (mean coherence -0.017), so geometry-priming STAYS default-OFF
-- but the latency blocker is removed and the durable structure now exists. See
docs/architecture/design/engram-m10-reification.md.
This commit is contained in:
+145
-85
@@ -7438,6 +7438,13 @@ static char* engram_first_n_chars(const char* s, size_t n) {
|
||||
#include "engram_vindex.h" /* M8: ANN (HNSW) index for activation seed selection */
|
||||
#include "engram_geometry.h" /* M9: centered relational-neighborhood geometry (priming) */
|
||||
|
||||
/* M10 REIFICATION: resident loaded form of the first-class persisted neighborhood
|
||||
* records (Neighborhood + GeoMeanFrame). Built once at boot from the durable store
|
||||
* (it PARSES persisted structure, never recomputes geometry). The geometry-priming
|
||||
* hot path reads THIS instead of computing a per-query descriptor. NULL until boot;
|
||||
* empty (count 0) on a store that has not been reified — priming then no-ops. */
|
||||
static GeoReifyIndex* _eg_reify = NULL;
|
||||
|
||||
static EngramPagedStore* g_engram_store = NULL;
|
||||
|
||||
int engram_store_enabled(void) {
|
||||
@@ -7597,6 +7604,16 @@ static void eg_store_put_edge(const EngramEdge* e) {
|
||||
* so the store-on boot behaves byte-identically to the JSON path (M3.5 parity). */
|
||||
static void eg_load_node_cb(const StoreNode* sn, void* ctx) {
|
||||
EngramStore* g = (EngramStore*)ctx;
|
||||
/* M10: first-class reified records (Neighborhood / GeoMeanFrame) are DURABLE
|
||||
* STRUCTURE, not corpus content. Absorb them into the reify index and keep them
|
||||
* OUT of the resident activation graph, so seed selection / vindex / results /
|
||||
* embedding backfill are byte-identical to a store that was never reified. */
|
||||
if (sn->node_type &&
|
||||
(strcmp(sn->node_type, ENGRAM_GEO_NBHD_TYPE) == 0 ||
|
||||
strcmp(sn->node_type, ENGRAM_GEO_MEANFRAME_TYPE) == 0)) {
|
||||
if (_eg_reify) engram_geo_reify_index_add(_eg_reify, sn);
|
||||
return;
|
||||
}
|
||||
engram_grow_nodes();
|
||||
EngramNode* n = &g->nodes[g->node_count];
|
||||
memset(n, 0, sizeof *n);
|
||||
@@ -7628,6 +7645,12 @@ static void eg_load_node_cb(const StoreNode* sn, void* ctx) {
|
||||
}
|
||||
static void eg_load_edge_cb(const StoreEdge* se, void* ctx) {
|
||||
EngramStore* g = (EngramStore*)ctx;
|
||||
/* M10: skip the persisted member links (from a "nbhd-…" record). They are
|
||||
* durable structure joining a neighborhood to its members, but inert to
|
||||
* activation adjacency — dropping them here keeps spreading activation
|
||||
* byte-identical to pre-reify. (Matched by id convention, so no real edge,
|
||||
* whatever its relation string, is ever affected.) */
|
||||
if (se->from_id && strncmp(se->from_id, ENGRAM_GEO_NBHD_ID_PREFIX, 5) == 0) return;
|
||||
engram_grow_edges();
|
||||
EngramEdge* e = &g->edges[g->edge_count];
|
||||
memset(e, 0, sizeof *e);
|
||||
@@ -7697,8 +7720,12 @@ el_val_t engram_store_boot(el_val_t data_dir) {
|
||||
if (!g_engram_store) return (el_val_t)0;
|
||||
EngramStore* g = engram_get();
|
||||
eg_reset_resident(g);
|
||||
/* M10: build the resident reify index alongside the graph load — eg_load_node_cb
|
||||
* feeds the first-class Neighborhood/GeoMeanFrame records into it. */
|
||||
if (!_eg_reify) _eg_reify = engram_geo_reify_index_new();
|
||||
store_scan_nodes(g_engram_store, eg_load_node_cb, g);
|
||||
store_scan_edges(g_engram_store, eg_load_edge_cb, g);
|
||||
if (_eg_reify) engram_geo_reify_index_finalize(_eg_reify);
|
||||
StoreLayer* ls = NULL; size_t ln = 0;
|
||||
if (store_list_layers(g_engram_store, &ls, &ln) == 0) {
|
||||
for (size_t i = 0; i < ln; i++) eg_load_layer_cb(g, &ls[i]);
|
||||
@@ -9369,97 +9396,130 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
|
||||
}
|
||||
free(seed_dup);
|
||||
|
||||
/* ── M9 GEOMETRY PRIMING (ENGRAM_GEOMETRY_PRIMING, default OFF) ──────
|
||||
* COMPOSES with the M8 seed set above: uses the CENTERED geometry of
|
||||
* the seed neighborhood to (a) damp off-domain seeds by centered
|
||||
* membership (disambiguation) and (b) prime nearby members sub-
|
||||
* threshold (a warm floor). Flag OFF → this whole block is skipped and
|
||||
* the seed set/activation are exactly what M8 produced. Read-only over
|
||||
* the graph except for the bounded, sub-threshold seed additions here. */
|
||||
if (eg_geometry_priming_on() && g_engram_store && q_emb && q_dim > 0 && nsel > 0) {
|
||||
const float* gmean = eg_geo_mean_sync(q_dim);
|
||||
if (gmean) {
|
||||
/* Seed ids = the M8-selected semantic seeds; vids maps the
|
||||
* resident-array VIndex ordinals (== g->nodes[] index) to store
|
||||
* ids so the descriptor's ANN expansion resolves to paged nodes. */
|
||||
const char** seed_ids = malloc((size_t)nsel * sizeof(char*));
|
||||
char** vids = malloc((size_t)g->node_count * sizeof(char*));
|
||||
if (seed_ids && vids) {
|
||||
for (int s = 0; s < nsel; s++) seed_ids[s] = g->nodes[sel[s]].id;
|
||||
for (int64_t i = 0; i < g->node_count; i++) vids[i] = g->nodes[i].id;
|
||||
GeoDescriptor* geo = engram_geometry_descriptor(
|
||||
g_engram_store, _eg_vindex, vids, (int)g->node_count,
|
||||
seed_ids, (size_t)nsel, NULL, gmean);
|
||||
/* ── GEOMETRY PRIMING (ENGRAM_GEOMETRY_PRIMING, default OFF) ─────────
|
||||
* COMPOSES with the M8 seed set above: resolves the seed neighborhood's
|
||||
* CENTERED geometry to (a) damp off-domain seeds by membership
|
||||
* (disambiguation) and (b) prime nearby members sub-threshold (warm floor).
|
||||
* Flag OFF → this whole block is skipped and the seed set/activation are
|
||||
* exactly what M8 produced (byte-identical). Read-only over the graph
|
||||
* except the bounded, sub-threshold seed additions here.
|
||||
*
|
||||
* M10: the neighborhood is READ from the PERSISTED first-class reify index
|
||||
* (a durable Neighborhood record's membership map) — O(seeds) hash lookup,
|
||||
* miss → centroid-nearest, NO geometry computed on the activation path. The
|
||||
* membership is centered against the true store-wide mean persisted in the
|
||||
* GeoMeanFrame record. Set ENGRAM_GEO_PRIMING_NOCACHE=1 to instead compute
|
||||
* the descriptor fresh per query (the M9 on-the-fly path — kept for ad-hoc
|
||||
* geometries and as the A/B latency control). */
|
||||
if (eg_geometry_priming_on() && nsel > 0) {
|
||||
static int _nocache = -1;
|
||||
if (_nocache < 0) { const char* s = getenv("ENGRAM_GEO_PRIMING_NOCACHE");
|
||||
_nocache = (s && s[0] && s[0] != '0') ? 1 : 0; }
|
||||
|
||||
const char** seed_ids = malloc((size_t)nsel * sizeof(char*));
|
||||
if (seed_ids) {
|
||||
for (int s = 0; s < nsel; s++) seed_ids[s] = g->nodes[sel[s]].id;
|
||||
|
||||
char* const* mids = NULL; /* resolved neighborhood member ids */
|
||||
const double* mw = NULL; /* their centered membership in [0,1] */
|
||||
int mn = 0;
|
||||
GeoDescriptor* geo = NULL; /* on-the-fly path only (freed below) */
|
||||
char** tmid = NULL; double* tmw = NULL;
|
||||
|
||||
if (!_nocache && _eg_reify && engram_geo_reify_count(_eg_reify) > 0) {
|
||||
/* HOT PATH — read persisted structure, no compute. */
|
||||
const GeoNeighborhood* nb = engram_geo_reify_lookup(
|
||||
_eg_reify, seed_ids, (size_t)nsel, q_emb, q_dim);
|
||||
if (nb && nb->n_members > 0) {
|
||||
mids = nb->member_ids; mw = nb->member_w; mn = nb->n_members;
|
||||
}
|
||||
} else if (g_engram_store && q_emb && q_dim > 0) {
|
||||
/* AD-HOC / NOCACHE control — compute the descriptor fresh. */
|
||||
const float* gmean = eg_geo_mean_sync(q_dim);
|
||||
char** vids = malloc((size_t)g->node_count * sizeof(char*));
|
||||
if (gmean && vids) {
|
||||
for (int64_t i = 0; i < g->node_count; i++) vids[i] = g->nodes[i].id;
|
||||
geo = engram_geometry_descriptor(
|
||||
g_engram_store, _eg_vindex, vids, (int)g->node_count,
|
||||
seed_ids, (size_t)nsel, NULL, gmean);
|
||||
}
|
||||
free(vids);
|
||||
if (geo && geo->n_members > 0) {
|
||||
const double lo = eg_geo_seed_lo();
|
||||
const double pscl = eg_geo_prime_scale();
|
||||
const int pmax = eg_geo_prime_max();
|
||||
/* Centered membership per resident idx (-1 = not in the
|
||||
* geometry → left untouched by the reweight). */
|
||||
double* geo_m = malloc((size_t)g->node_count * sizeof(double));
|
||||
if (geo_m) {
|
||||
for (int64_t i = 0; i < g->node_count; i++) geo_m[i] = -1.0;
|
||||
tmid = malloc((size_t)geo->n_members * sizeof(char*));
|
||||
tmw = malloc((size_t)geo->n_members * sizeof(double));
|
||||
if (tmid && tmw) {
|
||||
for (int m = 0; m < geo->n_members; m++) {
|
||||
int64_t ri = engram_find_node_index(geo->members[m].id);
|
||||
if (ri >= 0 && ri < g->node_count) {
|
||||
double mv = geo->members[m].membership;
|
||||
if (mv < 0.0) mv = 0.0; else if (mv > 1.0) mv = 1.0;
|
||||
geo_m[ri] = mv;
|
||||
}
|
||||
tmid[m] = geo->members[m].id; tmw[m] = geo->members[m].membership;
|
||||
}
|
||||
/* (a) DAMP-ONLY seed reweight: factor = lo+(1-lo)*memb
|
||||
* ∈ [lo,1]. Off-domain seeds (low centered membership)
|
||||
* lose weight; the neighborhood anchor (memb→1) is
|
||||
* unchanged. Never amplifies. Updates both the frontier
|
||||
* act (drives propagation) and best_bg (drives this
|
||||
* node's own WM weight). */
|
||||
for (int64_t s = 0; s < seed_count; s++) {
|
||||
int64_t si = seeds[s].idx;
|
||||
if (si < 0 || si >= g->node_count) continue;
|
||||
double mv = geo_m[si];
|
||||
if (mv < 0.0) continue; /* not in geometry */
|
||||
double factor = lo + (1.0 - lo) * mv;
|
||||
seeds[s].act *= factor;
|
||||
best_bg[si] *= factor;
|
||||
}
|
||||
/* (b) PRIME sub-threshold: descriptor members not
|
||||
* already reached get a warm floor act=memb*pscl
|
||||
* (pscl < WM gate ⇒ cannot self-promote) and enter the
|
||||
* frontier so a warm gradient spreads one hop then dies
|
||||
* at the 0.02 BFS cutoff. Capped at pmax; ISE skipped.
|
||||
* Safe: BFS keeps max, so this only RAISES a floor and
|
||||
* never caps a stronger legitimate activation. */
|
||||
int primed = 0;
|
||||
for (int m = 0; m < geo->n_members && primed < pmax; m++) {
|
||||
int64_t ri = engram_find_node_index(geo->members[m].id);
|
||||
if (ri < 0 || ri >= g->node_count) continue;
|
||||
if (reached[ri]) continue; /* already a seed */
|
||||
EngramNode* pn = &g->nodes[ri];
|
||||
if (pn->node_type &&
|
||||
strcmp(pn->node_type, "InternalStateEvent") == 0)
|
||||
continue;
|
||||
double mv = geo->members[m].membership;
|
||||
if (mv < 0.0) mv = 0.0; else if (mv > 1.0) mv = 1.0;
|
||||
double pact = mv * pscl;
|
||||
if (pact < 0.01) continue; /* too cold to matter */
|
||||
seeds[seed_count].idx = ri;
|
||||
seeds[seed_count].act = pact;
|
||||
seeds[seed_count].created_at = pn->created_at;
|
||||
seed_count++;
|
||||
best_bg[ri] = pact;
|
||||
best_hops[ri] = 0;
|
||||
reached[ri] = 1;
|
||||
primed++;
|
||||
}
|
||||
_eg_act_geo_primed += primed;
|
||||
free(geo_m);
|
||||
mids = tmid; mw = tmw; mn = geo->n_members;
|
||||
}
|
||||
engram_geo_free(geo);
|
||||
} else if (geo) {
|
||||
engram_geo_free(geo);
|
||||
}
|
||||
}
|
||||
free(seed_ids); free(vids);
|
||||
|
||||
if (mn > 0 && mids && mw) {
|
||||
const double lo = eg_geo_seed_lo();
|
||||
const double pscl = eg_geo_prime_scale();
|
||||
const int pmax = eg_geo_prime_max();
|
||||
/* membership per resident idx (-1 = not in the neighborhood). */
|
||||
double* geo_m = malloc((size_t)g->node_count * sizeof(double));
|
||||
if (geo_m) {
|
||||
for (int64_t i = 0; i < g->node_count; i++) geo_m[i] = -1.0;
|
||||
for (int m = 0; m < mn; m++) {
|
||||
int64_t ri = engram_find_node_index(mids[m]);
|
||||
if (ri >= 0 && ri < g->node_count) {
|
||||
double mv = mw[m];
|
||||
if (mv < 0.0) mv = 0.0; else if (mv > 1.0) mv = 1.0;
|
||||
geo_m[ri] = mv;
|
||||
}
|
||||
}
|
||||
/* (a) DAMP-ONLY seed reweight: factor = lo+(1-lo)*memb ∈ [lo,1].
|
||||
* Off-domain members lose weight; anchor (memb→1) unchanged;
|
||||
* seeds outside the neighborhood are left untouched. Never
|
||||
* amplifies. Updates frontier act + best_bg (WM weight). */
|
||||
for (int64_t s = 0; s < seed_count; s++) {
|
||||
int64_t si = seeds[s].idx;
|
||||
if (si < 0 || si >= g->node_count) continue;
|
||||
double mv = geo_m[si];
|
||||
if (mv < 0.0) continue; /* not in neighborhood */
|
||||
double factor = lo + (1.0 - lo) * mv;
|
||||
seeds[s].act *= factor;
|
||||
best_bg[si] *= factor;
|
||||
}
|
||||
/* (b) PRIME sub-threshold: neighborhood members not already
|
||||
* reached get a warm floor act=memb*pscl (pscl < WM gate ⇒
|
||||
* cannot self-promote) and enter the frontier so a warm
|
||||
* gradient spreads one hop then dies at the 0.02 BFS cutoff.
|
||||
* Capped at pmax; ISE skipped. Safe: BFS keeps max, so this
|
||||
* only RAISES a floor, never caps a legit activation. */
|
||||
int primed = 0;
|
||||
for (int m = 0; m < mn && primed < pmax; m++) {
|
||||
int64_t ri = engram_find_node_index(mids[m]);
|
||||
if (ri < 0 || ri >= g->node_count) continue;
|
||||
if (reached[ri]) continue; /* already a seed */
|
||||
EngramNode* pn = &g->nodes[ri];
|
||||
if (pn->node_type &&
|
||||
strcmp(pn->node_type, "InternalStateEvent") == 0)
|
||||
continue;
|
||||
double mv = mw[m];
|
||||
if (mv < 0.0) mv = 0.0; else if (mv > 1.0) mv = 1.0;
|
||||
double pact = mv * pscl;
|
||||
if (pact < 0.01) continue; /* too cold to matter */
|
||||
seeds[seed_count].idx = ri;
|
||||
seeds[seed_count].act = pact;
|
||||
seeds[seed_count].created_at = pn->created_at;
|
||||
seed_count++;
|
||||
best_bg[ri] = pact;
|
||||
best_hops[ri] = 0;
|
||||
reached[ri] = 1;
|
||||
primed++;
|
||||
}
|
||||
_eg_act_geo_primed += primed;
|
||||
free(geo_m);
|
||||
}
|
||||
}
|
||||
free(tmid); free(tmw);
|
||||
if (geo) engram_geo_free(geo);
|
||||
free(seed_ids);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,9 @@
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <stdarg.h>
|
||||
#include <time.h>
|
||||
|
||||
/* Must match ENGRAM_HEBB_GAIN in el_runtime.c (eff = weight*(1+GAIN*hebb)). */
|
||||
#define GEO_HEBB_GAIN 0.5
|
||||
@@ -91,8 +94,26 @@ static double ccos_dir(const float* a, const float* gm, const float* dir, int di
|
||||
struct GeoMeanCache { float* mean; int dim; uint64_t n; };
|
||||
|
||||
typedef struct { double* sum; int dim; uint64_t n; int err; } GeoMeanAcc;
|
||||
/* The reified records (Neighborhood / GeoMeanFrame) carry an emb (centroid / mean)
|
||||
* but are STRUCTURE, not corpus content — they must never pollute the store-wide
|
||||
* mean, the hub scan, or the descriptor. One predicate, used everywhere. */
|
||||
static int geo_is_reified_type(const char* nt){
|
||||
return nt && (strcmp(nt,ENGRAM_GEO_NBHD_TYPE)==0 ||
|
||||
strcmp(nt,ENGRAM_GEO_MEANFRAME_TYPE)==0);
|
||||
}
|
||||
/* Identify a structural record by its id convention (no store read needed), so the
|
||||
* descriptor never admits a reified Neighborhood / GeoMeanFrame as a neighborhood
|
||||
* MEMBER even when the ANN index or adjacency still references it (re-reify/refresh
|
||||
* on a store that already holds reified records; ad-hoc descriptors alike). */
|
||||
static int geo_is_structural_id(const char* id){
|
||||
if(!id) return 0;
|
||||
if(strcmp(id,ENGRAM_GEO_MEANFRAME_ID)==0) return 1;
|
||||
size_t p=strlen(ENGRAM_GEO_NBHD_ID_PREFIX);
|
||||
return strncmp(id,ENGRAM_GEO_NBHD_ID_PREFIX,p)==0;
|
||||
}
|
||||
static void geo_mean_cb(const StoreNode* n, void* ctx){
|
||||
GeoMeanAcc* a=ctx; if(a->err) return;
|
||||
if(geo_is_reified_type(n->node_type)) return; /* skip structural records */
|
||||
if(!(n->emb && n->emb_dim>0)) return; /* skip unembedded */
|
||||
if(a->dim==0){
|
||||
a->dim=n->emb_dim;
|
||||
@@ -255,6 +276,7 @@ GeoDescriptor* engram_geometry_descriptor(
|
||||
int got=vindex_search(vindex, prov, k, 0, rids, dd);
|
||||
for(int r=0;r<got;r++){
|
||||
if(rids[r]>=(uint64_t)n_vids) continue;
|
||||
if(geo_is_structural_id(vids[rids[r]])) continue; /* never a member */
|
||||
double memb = 1.0 - (double)dd[r]; /* cosine sim in [-1,1] */
|
||||
if(memb<0) memb=0;
|
||||
int mi=ms_upsert(&ms, vids[rids[r]], memb*0.9); /* <1: not a seed */
|
||||
@@ -272,6 +294,8 @@ GeoDescriptor* engram_geometry_descriptor(
|
||||
if(store_get_edges_from(store, ms.id[i], &es, &ne)==0 && es){
|
||||
for(size_t e=0;e<ne;e++){
|
||||
if(es[e].tombstoned || es[e].inhibitory) continue;
|
||||
if(es[e].relation && strcmp(es[e].relation,ENGRAM_GEO_MEMBER_RELATION)==0) continue;
|
||||
if(geo_is_structural_id(es[e].to_id)) continue;
|
||||
double w=eff_w(es[e].weight, es[e].hebb);
|
||||
if(w < P.edge_min_weight) continue;
|
||||
int mi=ms_upsert(&ms, es[e].to_id, w);
|
||||
@@ -283,6 +307,8 @@ GeoDescriptor* engram_geometry_descriptor(
|
||||
if(store_get_edges_to(store, ms.id[i], &es, &ne)==0 && es){
|
||||
for(size_t e=0;e<ne;e++){
|
||||
if(es[e].tombstoned || es[e].inhibitory) continue;
|
||||
if(es[e].relation && strcmp(es[e].relation,ENGRAM_GEO_MEMBER_RELATION)==0) continue;
|
||||
if(geo_is_structural_id(es[e].from_id)) continue;
|
||||
double w=eff_w(es[e].weight, es[e].hebb);
|
||||
if(w < P.edge_min_weight) continue;
|
||||
int mi=ms_upsert(&ms, es[e].from_id, w);
|
||||
@@ -338,9 +364,12 @@ GeoDescriptor* engram_geometry_descriptor(
|
||||
if(nemb) total_var/=nemb;
|
||||
double radius=sqrt(total_var>0?total_var:0);
|
||||
|
||||
/* ── principal axes via dual PCA (Jacobi on the m×m Gram of centered embs) ── */
|
||||
/* ── principal axes via dual PCA (Jacobi on the m×m Gram of centered embs) ──
|
||||
* Skipped entirely when top_axes==0: the eigensolve is the dominant cost, and
|
||||
* priming needs only members+membership, so reified records that don't want the
|
||||
* ellipsoid pass top_axes=0 and pay nothing here (centroid+radius still filled). */
|
||||
int n_axes=0; GeoAxis* axes=NULL;
|
||||
if(nemb>=2 && nemb<=GEO_EIG_CAP){
|
||||
if(P.top_axes>0 && nemb>=2 && nemb<=GEO_EIG_CAP){
|
||||
int m=nemb;
|
||||
/* centered, row-major m×dim */
|
||||
float* Xc=malloc((size_t)m*dim*sizeof(float));
|
||||
@@ -491,3 +520,420 @@ void engram_geo_free(GeoDescriptor* g){
|
||||
free(g->members); free(g->edges);
|
||||
free(g);
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════════
|
||||
* M10 — REIFICATION: persist / load / lookup first-class neighborhood records.
|
||||
* ═══════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
static int64_t geo_now_ms(void){
|
||||
struct timespec ts;
|
||||
if(clock_gettime(CLOCK_REALTIME,&ts)==0)
|
||||
return (int64_t)ts.tv_sec*1000 + ts.tv_nsec/1000000;
|
||||
return (int64_t)time(NULL)*1000;
|
||||
}
|
||||
|
||||
void engram_geo_reify_default_params(GeoReifyParams* p){
|
||||
if(!p) return;
|
||||
p->min_weighted_degree=0;
|
||||
p->max_neighborhoods=128;
|
||||
p->cover_membership=0.5;
|
||||
p->persist_member_edges=1;
|
||||
engram_geo_default_params(&p->descriptor);
|
||||
p->descriptor.top_axes=4; /* keep a small ellipsoid summary; cheap */
|
||||
p->descriptor.max_members=256; /* reified neighborhoods stay compact */
|
||||
}
|
||||
|
||||
/* ── tiny growable string builder ─────────────────────────────────────────── */
|
||||
typedef struct { char* s; size_t n, cap; } SB;
|
||||
static int sb_reserve(SB* b, size_t add){
|
||||
if(b->n+add+1<=b->cap) return 0;
|
||||
size_t nc=b->cap?b->cap:256; while(nc<b->n+add+1) nc*=2;
|
||||
char* t=realloc(b->s,nc); if(!t) return -1; b->s=t; b->cap=nc; return 0;
|
||||
}
|
||||
static int sb_puts(SB* b, const char* s){
|
||||
size_t l=strlen(s); if(sb_reserve(b,l)) return -1;
|
||||
memcpy(b->s+b->n,s,l); b->n+=l; b->s[b->n]=0; return 0;
|
||||
}
|
||||
static int sb_fmt(SB* b, const char* fmt, ...){
|
||||
char tmp[512]; va_list ap; va_start(ap,fmt);
|
||||
int k=vsnprintf(tmp,sizeof tmp,fmt,ap); va_end(ap);
|
||||
if(k<0) return -1; if(k>=(int)sizeof tmp) k=sizeof tmp-1;
|
||||
return sb_puts(b,tmp);
|
||||
}
|
||||
|
||||
/* Serialize a descriptor's DURABLE geometry into the GEO1 metadata schema.
|
||||
* (The raw centroid is stored separately as the record's emb.) */
|
||||
static char* geo_nbhd_metadata(const GeoDescriptor* g, const char* hub,
|
||||
const char* meanid){
|
||||
SB b={0};
|
||||
if(sb_puts(&b,"GEO1\n")) { free(b.s); return NULL; }
|
||||
sb_fmt(&b,"hub %s\n", hub?hub:"");
|
||||
sb_fmt(&b,"mean %s\n", meanid?meanid:"");
|
||||
sb_fmt(&b,"s %.9g %.9g %d %.9g %d %d\n",
|
||||
g->radius, g->total_variance, g->k_core, g->co_registration,
|
||||
g->n_embedded, g->n_members);
|
||||
sb_puts(&b,"e");
|
||||
for(int i=0;i<g->n_axes;i++) sb_fmt(&b," %.9g", g->axes[i].extent);
|
||||
sb_puts(&b,"\n");
|
||||
for(int i=0;i<g->n_members;i++){
|
||||
sb_fmt(&b,"m %s %.9g %.9g %d\n",
|
||||
g->members[i].id, g->members[i].membership,
|
||||
g->members[i].centrality, g->members[i].core);
|
||||
}
|
||||
return b.s; /* caller frees */
|
||||
}
|
||||
|
||||
/* ── string set (greedy-cover claimed ids) + string→id list (hub→old nbhd) ──── */
|
||||
static uint64_t geo_djb2(const char* s){
|
||||
uint64_t h=5381; for(;*s;s++) h=((h<<5)+h)^(unsigned char)*s; return h;
|
||||
}
|
||||
typedef struct SSNode { char* key; struct SSNode* next; } SSNode;
|
||||
typedef struct { SSNode** b; size_t nb; } SSet;
|
||||
static void ss_init(SSet* s, size_t nb){ s->nb=nb; s->b=calloc(nb,sizeof*s->b); }
|
||||
static int ss_has(const SSet* s, const char* k){
|
||||
if(!s->b) return 0; SSNode* n=s->b[geo_djb2(k)%s->nb];
|
||||
for(;n;n=n->next) if(strcmp(n->key,k)==0) return 1; return 0;
|
||||
}
|
||||
static void ss_add(SSet* s, const char* k){
|
||||
if(!s->b||ss_has(s,k)) return; size_t i=geo_djb2(k)%s->nb;
|
||||
SSNode* n=malloc(sizeof*n); if(!n) return; n->key=strdup(k); n->next=s->b[i]; s->b[i]=n;
|
||||
}
|
||||
static void ss_free(SSet* s){
|
||||
if(!s->b) return;
|
||||
for(size_t i=0;i<s->nb;i++){ SSNode* n=s->b[i]; while(n){ SSNode* x=n->next; free(n->key); free(n); n=x; } }
|
||||
free(s->b); s->b=NULL;
|
||||
}
|
||||
|
||||
typedef struct { char** id; int n, cap; } StrVec;
|
||||
static void sv_push(StrVec* v, const char* s){
|
||||
if(v->n==v->cap){ v->cap=v->cap?v->cap*2:64; v->id=realloc(v->id,(size_t)v->cap*sizeof*v->id); }
|
||||
v->id[v->n++]=strdup(s);
|
||||
}
|
||||
static void sv_free(StrVec* v){ for(int i=0;i<v->n;i++) free(v->id[i]); free(v->id); }
|
||||
|
||||
/* pass 1 collector: all non-structural node ids; also record existing Neighborhood
|
||||
* records as (hub -> old_id) so a re-reify supersedes the prior version. */
|
||||
typedef struct {
|
||||
StrVec cand; /* candidate node ids (content nodes) */
|
||||
StrVec old_hub, old_id; /* parallel: existing nbhd hub + its id */
|
||||
} ReifyScan;
|
||||
static void geo_reify_scan_cb(const StoreNode* n, void* ctx){
|
||||
ReifyScan* rs=ctx; if(!n->id||!n->node_type) { if(n->id) sv_push(&rs->cand,n->id); return; }
|
||||
if(strcmp(n->node_type,ENGRAM_GEO_NBHD_TYPE)==0){
|
||||
/* parse hub from metadata GEO1 (line "hub <id>") for supersede lineage */
|
||||
const char* md=n->metadata?n->metadata:"";
|
||||
const char* p=strstr(md,"hub ");
|
||||
if(p && (p==md || p[-1]=='\n')){
|
||||
p+=4; const char* e=p; while(*e && *e!='\n') e++;
|
||||
char* hub=strndup(p,(size_t)(e-p));
|
||||
sv_push(&rs->old_hub,hub); sv_push(&rs->old_id,n->id); free(hub);
|
||||
}
|
||||
return; /* structural: not a candidate */
|
||||
}
|
||||
if(strcmp(n->node_type,ENGRAM_GEO_MEANFRAME_TYPE)==0) return;
|
||||
sv_push(&rs->cand,n->id);
|
||||
}
|
||||
|
||||
/* weighted strong-edge degree of a node (from+to), matching eff_w/threshold. */
|
||||
static double geo_weighted_degree(EngramPagedStore* st, const char* id, double emin){
|
||||
double deg=0; StoreEdge* es=NULL; size_t ne=0;
|
||||
if(store_get_edges_from(st,id,&es,&ne)==0 && es){
|
||||
for(size_t e=0;e<ne;e++){ if(es[e].tombstoned||es[e].inhibitory) continue;
|
||||
double w=eff_w(es[e].weight,es[e].hebb); if(w>=emin) deg+=w; }
|
||||
}
|
||||
store_edges_free(es,ne); es=NULL; ne=0;
|
||||
if(store_get_edges_to(st,id,&es,&ne)==0 && es){
|
||||
for(size_t e=0;e<ne;e++){ if(es[e].tombstoned||es[e].inhibitory) continue;
|
||||
double w=eff_w(es[e].weight,es[e].hebb); if(w>=emin) deg+=w; }
|
||||
}
|
||||
store_edges_free(es,ne);
|
||||
return deg;
|
||||
}
|
||||
|
||||
int engram_geo_reify_store(EngramPagedStore* store, VIndex* vindex,
|
||||
char** vids, int n_vids,
|
||||
const GeoReifyParams* params){
|
||||
if(!store) return -1;
|
||||
GeoReifyParams P; if(params) P=*params; else engram_geo_reify_default_params(&P);
|
||||
|
||||
/* 1. true store-wide mean → persist the GeoMeanFrame record (once). */
|
||||
GeoMeanCache* mc=engram_geo_mean_build(store);
|
||||
if(!mc) return -2;
|
||||
int dim=engram_geo_mean_dim(mc);
|
||||
const float* mean=engram_geo_mean_vec(mc);
|
||||
int64_t now=geo_now_ms();
|
||||
{ StoreNode mf; memset(&mf,0,sizeof mf);
|
||||
mf.id=(char*)ENGRAM_GEO_MEANFRAME_ID; mf.node_type=(char*)ENGRAM_GEO_MEANFRAME_TYPE;
|
||||
mf.content=(char*)"geo-mean-frame"; mf.tier=(char*)"Semantic"; mf.metadata=(char*)"{}";
|
||||
mf.emb=(float*)mean; mf.emb_dim=dim; mf.created_at=now; mf.updated_at=now;
|
||||
if(store_put_node(store,&mf)<0){ engram_geo_mean_free(mc); return -3; }
|
||||
}
|
||||
|
||||
/* 2. scan: candidate ids + existing (hub→old id) for supersede. */
|
||||
ReifyScan rs; memset(&rs,0,sizeof rs);
|
||||
if(store_scan_nodes(store,geo_reify_scan_cb,&rs)<0){
|
||||
sv_free(&rs.cand); sv_free(&rs.old_hub); sv_free(&rs.old_id);
|
||||
engram_geo_mean_free(mc); return -4;
|
||||
}
|
||||
|
||||
/* 3. weighted degree per candidate; sort desc. */
|
||||
int N=rs.cand.n;
|
||||
double* deg=malloc((size_t)N*sizeof(double));
|
||||
int* ord=malloc((size_t)N*sizeof(int));
|
||||
for(int i=0;i<N;i++){ deg[i]=geo_weighted_degree(store,rs.cand.id[i],P.descriptor.edge_min_weight); ord[i]=i; }
|
||||
/* simple insertion-ish selection sort by degree desc (N a few thousand, one-time) */
|
||||
for(int a=0;a<N;a++){ int best=a; for(int b=a+1;b<N;b++) if(deg[ord[b]]>deg[ord[best]]) best=b;
|
||||
int t=ord[a]; ord[a]=ord[best]; ord[best]=t; }
|
||||
|
||||
/* 4. greedy non-redundant cover: reify each qualifying hub once. */
|
||||
SSet claimed; ss_init(&claimed, (size_t)(N>16?N:16));
|
||||
int persisted=0;
|
||||
for(int oi=0; oi<N && persisted<P.max_neighborhoods; oi++){
|
||||
int i=ord[oi]; const char* hub=rs.cand.id[i];
|
||||
if(P.min_weighted_degree>0 && deg[i]<(double)P.min_weighted_degree) break; /* sorted: rest smaller */
|
||||
if(ss_has(&claimed,hub)) continue;
|
||||
const char* seeds[1]={hub};
|
||||
GeoDescriptor* g=engram_geometry_descriptor(store,vindex,vids,n_vids,
|
||||
seeds,1,&P.descriptor,mean);
|
||||
if(!g || g->n_members<=0){ if(g) engram_geo_free(g); continue; }
|
||||
/* claim members above cover threshold (incl. the hub itself) */
|
||||
for(int m=0;m<g->n_members;m++)
|
||||
if(g->members[m].membership>=P.cover_membership) ss_add(&claimed,g->members[m].id);
|
||||
|
||||
/* build record: id = nbhd-<hub>-<now>, emb = RAW centroid = centered+mean */
|
||||
char nid[512]; snprintf(nid,sizeof nid,"%s%s-%lld",ENGRAM_GEO_NBHD_ID_PREFIX,hub,(long long)now);
|
||||
float* raw=NULL;
|
||||
if(g->n_embedded>0 && g->centroid && g->global_mean){
|
||||
raw=malloc((size_t)dim*sizeof(float));
|
||||
if(raw) for(int d=0;d<dim;d++) raw[d]=g->centroid[d]+g->global_mean[d];
|
||||
}
|
||||
char* md=geo_nbhd_metadata(g,hub,ENGRAM_GEO_MEANFRAME_ID);
|
||||
StoreNode nn; memset(&nn,0,sizeof nn);
|
||||
nn.id=nid; nn.node_type=(char*)ENGRAM_GEO_NBHD_TYPE;
|
||||
nn.content=(char*)"reified-neighborhood"; nn.tier=(char*)"Semantic";
|
||||
nn.metadata=md?md:(char*)"{}"; nn.emb=raw; nn.emb_dim=raw?dim:0;
|
||||
nn.created_at=now; nn.updated_at=now;
|
||||
int wrc=store_put_node(store,&nn);
|
||||
free(raw); free(md);
|
||||
if(wrc<0){ engram_geo_free(g); continue; }
|
||||
|
||||
/* provenance: supersede any prior neighborhood for this hub. */
|
||||
for(int k=0;k<rs.old_hub.n;k++) if(strcmp(rs.old_hub.id[k],hub)==0){
|
||||
store_supersede(store, rs.old_id.id[k], nid);
|
||||
}
|
||||
|
||||
/* member links (durable, but inert to activation — runtime skips them). */
|
||||
if(P.persist_member_edges){
|
||||
for(int m=0;m<g->n_members;m++){
|
||||
char eid[600]; snprintf(eid,sizeof eid,"%s->%s",nid,g->members[m].id);
|
||||
StoreEdge se; memset(&se,0,sizeof se);
|
||||
se.id=eid; se.from_id=nid; se.to_id=g->members[m].id;
|
||||
se.relation=(char*)ENGRAM_GEO_MEMBER_RELATION;
|
||||
se.metadata=(char*)"{}"; se.weight=g->members[m].membership;
|
||||
se.confidence=1.0; se.created_at=now; se.updated_at=now;
|
||||
store_put_edge(store,&se);
|
||||
}
|
||||
}
|
||||
engram_geo_free(g);
|
||||
persisted++;
|
||||
}
|
||||
|
||||
ss_free(&claimed);
|
||||
free(deg); free(ord);
|
||||
sv_free(&rs.cand); sv_free(&rs.old_hub); sv_free(&rs.old_id);
|
||||
engram_geo_mean_free(mc);
|
||||
return persisted;
|
||||
}
|
||||
|
||||
/* ═══════════════ resident loaded form + hot-path lookup ═════════════════════ */
|
||||
|
||||
typedef struct {
|
||||
char* id;
|
||||
char* hub_id;
|
||||
int n_members;
|
||||
char** member_ids;
|
||||
double* member_w;
|
||||
double radius, co_reg;
|
||||
int k_core, n_embedded;
|
||||
float* centroid_raw; /* dim floats or NULL */
|
||||
float* centroid_unit; /* centered+normalized (finalize) or NULL */
|
||||
int dim;
|
||||
GeoNeighborhood view;
|
||||
} RNbhd;
|
||||
|
||||
typedef struct RE { char* id; int nbhd; double w; struct RE* next; } RE;
|
||||
|
||||
struct GeoReifyIndex {
|
||||
RNbhd* nb; int n, cap;
|
||||
float* mean; int mean_dim;
|
||||
RE** buckets; size_t nbuckets;
|
||||
double* score; /* scratch[n], reused per lookup */
|
||||
};
|
||||
|
||||
GeoReifyIndex* engram_geo_reify_index_new(void){
|
||||
GeoReifyIndex* ix=calloc(1,sizeof*ix); return ix;
|
||||
}
|
||||
|
||||
/* parse a GEO1 metadata blob into an RNbhd (members + scalars). */
|
||||
static int geo_parse_nbhd(const char* md, RNbhd* r){
|
||||
if(!md) return -1;
|
||||
if(strncmp(md,"GEO1",4)!=0) return -1;
|
||||
/* count member lines to size arrays */
|
||||
int cap=0; for(const char* p=md; (p=strstr(p,"\nm ")); p+=3) cap++;
|
||||
r->member_ids=cap?calloc((size_t)cap,sizeof(char*)):NULL;
|
||||
r->member_w =cap?calloc((size_t)cap,sizeof(double)):NULL;
|
||||
r->n_members=0;
|
||||
const char* line=md;
|
||||
while(line && *line){
|
||||
const char* nl=strchr(line,'\n');
|
||||
size_t len= nl? (size_t)(nl-line) : strlen(line);
|
||||
char buf[600]; if(len>=sizeof buf) len=sizeof buf-1;
|
||||
memcpy(buf,line,len); buf[len]=0;
|
||||
if(buf[0]=='h'&&buf[1]=='u'&&buf[2]=='b'&&buf[3]==' '){
|
||||
free(r->hub_id); r->hub_id=strdup(buf+4);
|
||||
} else if(buf[0]=='s'&&buf[1]==' '){
|
||||
int kc=0,ne=0,nm=0; double rad=0,tv=0,cr=0;
|
||||
sscanf(buf+2,"%lf %lf %d %lf %d %d",&rad,&tv,&kc,&cr,&ne,&nm);
|
||||
r->radius=rad; r->co_reg=cr; r->k_core=kc; r->n_embedded=ne;
|
||||
} else if(buf[0]=='m'&&buf[1]==' '){
|
||||
char mid[512]; double w=0,c=0; int core=0;
|
||||
if(sscanf(buf+2,"%511s %lf %lf %d",mid,&w,&c,&core)>=2 && r->member_ids){
|
||||
r->member_ids[r->n_members]=strdup(mid);
|
||||
r->member_w[r->n_members]=w;
|
||||
r->n_members++;
|
||||
}
|
||||
}
|
||||
line = nl? nl+1 : NULL;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int engram_geo_reify_index_add(GeoReifyIndex* ix, const StoreNode* n){
|
||||
if(!ix||!n||!n->node_type) return 0;
|
||||
if(strcmp(n->node_type,ENGRAM_GEO_MEANFRAME_TYPE)==0){
|
||||
if(n->emb && n->emb_dim>0){
|
||||
free(ix->mean);
|
||||
ix->mean=malloc((size_t)n->emb_dim*sizeof(float));
|
||||
if(ix->mean){ memcpy(ix->mean,n->emb,(size_t)n->emb_dim*sizeof(float)); ix->mean_dim=n->emb_dim; }
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
if(strcmp(n->node_type,ENGRAM_GEO_NBHD_TYPE)!=0) return 0;
|
||||
if(ix->n==ix->cap){ ix->cap=ix->cap?ix->cap*2:16;
|
||||
RNbhd* t=realloc(ix->nb,(size_t)ix->cap*sizeof*t); if(!t) return -1; ix->nb=t; }
|
||||
RNbhd* r=&ix->nb[ix->n]; memset(r,0,sizeof*r);
|
||||
r->id=strdup(n->id?n->id:"");
|
||||
if(geo_parse_nbhd(n->metadata,r)!=0){ free(r->id); return 0; } /* skip malformed */
|
||||
if(n->emb && n->emb_dim>0){
|
||||
r->dim=n->emb_dim;
|
||||
r->centroid_raw=malloc((size_t)n->emb_dim*sizeof(float));
|
||||
if(r->centroid_raw) memcpy(r->centroid_raw,n->emb,(size_t)n->emb_dim*sizeof(float));
|
||||
}
|
||||
ix->n++;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int engram_geo_reify_index_finalize(GeoReifyIndex* ix){
|
||||
if(!ix) return -1;
|
||||
/* member → neighborhood hash */
|
||||
size_t total=0; for(int i=0;i<ix->n;i++) total+=(size_t)ix->nb[i].n_members;
|
||||
ix->nbuckets = total? (total*2+1) : 1;
|
||||
ix->buckets=calloc(ix->nbuckets,sizeof(RE*));
|
||||
if(!ix->buckets) return -1;
|
||||
for(int i=0;i<ix->n;i++){
|
||||
RNbhd* r=&ix->nb[i];
|
||||
for(int m=0;m<r->n_members;m++){
|
||||
size_t b=geo_djb2(r->member_ids[m])%ix->nbuckets;
|
||||
RE* e=malloc(sizeof*e); if(!e) continue;
|
||||
e->id=r->member_ids[m]; e->nbhd=i; e->w=r->member_w[m]; e->next=ix->buckets[b]; ix->buckets[b]=e;
|
||||
}
|
||||
/* centered, normalized centroid for the nearest-fallback */
|
||||
if(r->centroid_raw && ix->mean && ix->mean_dim==r->dim){
|
||||
r->centroid_unit=malloc((size_t)r->dim*sizeof(float));
|
||||
if(r->centroid_unit){
|
||||
double nrm=0; for(int d=0;d<r->dim;d++){ double v=(double)r->centroid_raw[d]-ix->mean[d]; r->centroid_unit[d]=(float)v; nrm+=v*v; }
|
||||
nrm=sqrt(nrm); if(nrm>1e-12) for(int d=0;d<r->dim;d++) r->centroid_unit[d]=(float)(r->centroid_unit[d]/nrm);
|
||||
else { free(r->centroid_unit); r->centroid_unit=NULL; }
|
||||
}
|
||||
}
|
||||
/* fill the borrowed view */
|
||||
r->view.id=r->id; r->view.hub_id=r->hub_id; r->view.n_members=r->n_members;
|
||||
r->view.member_ids=r->member_ids; r->view.member_w=r->member_w;
|
||||
r->view.radius=r->radius; r->view.co_registration=r->co_reg;
|
||||
r->view.k_core=r->k_core; r->view.n_embedded=r->n_embedded;
|
||||
}
|
||||
ix->score=ix->n?calloc((size_t)ix->n,sizeof(double)):NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void geo__reify_load_cb(const StoreNode* n, void* ctx){
|
||||
engram_geo_reify_index_add((GeoReifyIndex*)ctx, n);
|
||||
}
|
||||
GeoReifyIndex* engram_geo_reify_load(EngramPagedStore* store){
|
||||
if(!store) return NULL;
|
||||
GeoReifyIndex* ix=engram_geo_reify_index_new(); if(!ix) return NULL;
|
||||
store_scan_nodes(store, geo__reify_load_cb, ix);
|
||||
if(ix->n==0 && ix->mean==NULL){ engram_geo_reify_index_free(ix); return NULL; }
|
||||
engram_geo_reify_index_finalize(ix);
|
||||
return ix;
|
||||
}
|
||||
|
||||
const GeoNeighborhood* engram_geo_reify_lookup(
|
||||
const GeoReifyIndex* ix,
|
||||
const char* const* seed_ids, size_t n_seeds,
|
||||
const float* q_emb, int q_dim){
|
||||
if(!ix||ix->n<=0) return NULL;
|
||||
/* (a) membership route: score each neighborhood by summed seed membership. */
|
||||
if(ix->score && ix->buckets && seed_ids && n_seeds>0){
|
||||
for(int i=0;i<ix->n;i++) ((GeoReifyIndex*)ix)->score[i]=0.0;
|
||||
int any=0;
|
||||
for(size_t s=0;s<n_seeds;s++){
|
||||
const char* id=seed_ids[s]; if(!id) continue;
|
||||
for(RE* e=ix->buckets[geo_djb2(id)%ix->nbuckets]; e; e=e->next)
|
||||
if(strcmp(e->id,id)==0){ ((GeoReifyIndex*)ix)->score[e->nbhd]+=e->w; any=1; }
|
||||
}
|
||||
if(any){
|
||||
int best=-1; double bv=-1;
|
||||
for(int i=0;i<ix->n;i++) if(ix->score[i]>bv){ bv=ix->score[i]; best=i; }
|
||||
if(best>=0 && bv>0) return &ix->nb[best].view;
|
||||
}
|
||||
}
|
||||
/* (b) centroid-nearest fallback (centered query vs centered centroids). */
|
||||
if(q_emb && q_dim>0 && ix->mean && ix->mean_dim==q_dim){
|
||||
double nq=0; float* cq=malloc((size_t)q_dim*sizeof(float));
|
||||
if(!cq) return NULL;
|
||||
for(int d=0;d<q_dim;d++){ double v=(double)q_emb[d]-ix->mean[d]; cq[d]=(float)v; nq+=v*v; }
|
||||
nq=sqrt(nq);
|
||||
if(nq>1e-12){
|
||||
int best=-1; double bc=-1e9;
|
||||
for(int i=0;i<ix->n;i++){ RNbhd* r=&ix->nb[i]; if(!r->centroid_unit) continue;
|
||||
double s=0; for(int d=0;d<q_dim;d++) s+=(double)cq[d]*r->centroid_unit[d];
|
||||
s/=nq; if(s>bc){ bc=s; best=i; } }
|
||||
free(cq);
|
||||
if(best>=0) return &ix->nb[best].view;
|
||||
} else free(cq);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int engram_geo_reify_count(const GeoReifyIndex* ix){ return ix?ix->n:0; }
|
||||
const float* engram_geo_reify_mean(const GeoReifyIndex* ix, int* dim){
|
||||
if(!ix||!ix->mean){ if(dim)*dim=0; return NULL; }
|
||||
if(dim)*dim=ix->mean_dim; return ix->mean;
|
||||
}
|
||||
|
||||
void engram_geo_reify_index_free(GeoReifyIndex* ix){
|
||||
if(!ix) return;
|
||||
if(ix->buckets){
|
||||
for(size_t b=0;b<ix->nbuckets;b++){ RE* e=ix->buckets[b]; while(e){ RE* x=e->next; free(e); e=x; } }
|
||||
free(ix->buckets);
|
||||
}
|
||||
for(int i=0;i<ix->n;i++){ RNbhd* r=&ix->nb[i];
|
||||
free(r->id); free(r->hub_id);
|
||||
for(int m=0;m<r->n_members;m++) free(r->member_ids[m]);
|
||||
free(r->member_ids); free(r->member_w);
|
||||
free(r->centroid_raw); free(r->centroid_unit);
|
||||
}
|
||||
free(ix->nb); free(ix->score); free(ix->mean);
|
||||
free(ix);
|
||||
}
|
||||
|
||||
@@ -158,4 +158,111 @@ GeoDescriptor* engram_geometry_descriptor(
|
||||
|
||||
void engram_geo_free(GeoDescriptor* g);
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════════════════════
|
||||
* M10 — REIFICATION: densely co-wired relational neighborhoods crystallized into
|
||||
* FIRST-CLASS, PERSISTED store records (design doc §2; memory 885f5945). This is
|
||||
* NOT a cache — it is durable structure. A reified neighborhood is a real store
|
||||
* NODE (node_type "Neighborhood") that survives restart, is loaded on boot, and
|
||||
* EVOLVES via supersede+provenance when the pattern shifts. The geometry-priming
|
||||
* HOT PATH reads these persisted records (never computes geometry on the
|
||||
* activation path). Ad-hoc/transient geometries still use the on-the-fly
|
||||
* engram_geometry_descriptor above.
|
||||
*
|
||||
* Two record types, both ordinary TLV store nodes (no new on-disk format):
|
||||
* - "GeoMeanFrame" : the store-wide centering mean, persisted ONCE (emb = mean
|
||||
* vector, id ENGRAM_GEO_MEANFRAME_ID). Referenced by every
|
||||
* neighborhood so priming centers against the SAME true mean.
|
||||
* - "Neighborhood" : one reified neighborhood. emb = the RAW centroid (prototype
|
||||
* point, so it stays centroid-ANN-able; centered_centroid =
|
||||
* emb - meanframe). metadata = the compact "GEO1" schema:
|
||||
* hub id, meanframe ref, scalar shape (radius, total_variance,
|
||||
* k_core, co_registration, n_embedded), axis EXTENTS (ellipsoid
|
||||
* half-widths), and the MEMBER list {id -> membership, centrality,
|
||||
* core}. Member links are also persisted as edges relation="member".
|
||||
*
|
||||
* v1 honest simplifications (documented; extensible without migration): axis
|
||||
* DIRECTION vectors are not persisted (extents capture the ellipsoid scale; the
|
||||
* directions are recomputable via the on-the-fly descriptor for viz/operators);
|
||||
* with hebb potentiation ~0 on today's store the "hebb-weighted" degree reduces to
|
||||
* AUTHORED edge weight, so detected neighborhoods currently reflect authored edges —
|
||||
* the design is unchanged and self-correcting once hebb accrues.
|
||||
* ═══════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
#define ENGRAM_GEO_NBHD_TYPE "Neighborhood"
|
||||
#define ENGRAM_GEO_MEANFRAME_TYPE "GeoMeanFrame"
|
||||
#define ENGRAM_GEO_MEANFRAME_ID "geo-meanframe" /* stable id of the singleton */
|
||||
#define ENGRAM_GEO_NBHD_ID_PREFIX "nbhd-" /* id = nbhd-<hub>-<built_at> */
|
||||
#define ENGRAM_GEO_MEMBER_RELATION "member"
|
||||
|
||||
typedef struct {
|
||||
int min_weighted_degree; /* hub qualifies iff strong-edge weighted degree >= this
|
||||
* (0 = no floor: just rank + take top max_neighborhoods) */
|
||||
int max_neighborhoods; /* homeostatic budget cap (default 128) */
|
||||
double cover_membership; /* skip a hub already a member (w>=this) of an accepted
|
||||
* neighborhood — greedy non-redundant cover (default 0.5) */
|
||||
int persist_member_edges; /* 1 = also write relation="member" edges (default 1) */
|
||||
GeoParams descriptor; /* per-neighborhood params (top_axes may be 0 = skip eigensolve) */
|
||||
} GeoReifyParams;
|
||||
|
||||
/* Defaults: min_weighted_degree=0, max_neighborhoods=128, cover_membership=0.5,
|
||||
* persist_member_edges=1, descriptor = engram_geo_default_params but top_axes=4,
|
||||
* max_members=256 (reified neighborhoods stay compact). */
|
||||
void engram_geo_reify_default_params(GeoReifyParams* p);
|
||||
|
||||
/* WRITE PATH (offline / consolidation — NEVER the activation hot path).
|
||||
* Detect dense hub neighborhoods on the hebb-weighted graph, compute each one's
|
||||
* CENTERED descriptor ONCE against the true store-wide mean, and PERSIST them as
|
||||
* first-class records: the GeoMeanFrame (once) + one Neighborhood node per detected
|
||||
* neighborhood (+ member edges), superseding any prior same-hub record with
|
||||
* provenance. Read-then-write over `store`. Returns #neighborhoods persisted, or <0.
|
||||
* Skips existing Neighborhood/GeoMeanFrame nodes when detecting (idempotent re-reify). */
|
||||
int engram_geo_reify_store(EngramPagedStore* store, VIndex* vindex,
|
||||
char** vids, int n_vids,
|
||||
const GeoReifyParams* params);
|
||||
|
||||
/* ── Resident loaded form of the persisted records (boot-time; READ-ONLY) ─────
|
||||
* The durable Neighborhood/GeoMeanFrame records are the source of truth; this
|
||||
* index is their LOADED form (like the resident node array is the loaded form of
|
||||
* the node records, or adjacency the loaded form of edges). It never recomputes
|
||||
* geometry — it parses. Build it by feeding the runtime's boot node scan, or in
|
||||
* one pass with engram_geo_reify_load. */
|
||||
typedef struct GeoReifyIndex GeoReifyIndex;
|
||||
|
||||
GeoReifyIndex* engram_geo_reify_index_new(void);
|
||||
/* Feed one store node; if it is a Neighborhood or GeoMeanFrame record it is parsed
|
||||
* and absorbed (else ignored). The node is BORROWED (copied as needed). 0/<0. */
|
||||
int engram_geo_reify_index_add(GeoReifyIndex* ix, const StoreNode* n);
|
||||
/* Build the member->neighborhood hash after all adds. Call once. 0/<0. */
|
||||
int engram_geo_reify_index_finalize(GeoReifyIndex* ix);
|
||||
/* One-pass convenience: scan the store and build the finalized index. NULL if the
|
||||
* store holds no reified records. */
|
||||
GeoReifyIndex* engram_geo_reify_load(EngramPagedStore* store);
|
||||
|
||||
/* A borrowed view of one persisted neighborhood (owned by the index). */
|
||||
typedef struct {
|
||||
const char* id;
|
||||
const char* hub_id;
|
||||
int n_members;
|
||||
char* const* member_ids; /* parallel arrays, length n_members */
|
||||
const double* member_w; /* membership in [0,1] */
|
||||
double radius;
|
||||
double co_registration;
|
||||
int k_core;
|
||||
int n_embedded;
|
||||
} GeoNeighborhood;
|
||||
|
||||
/* HOT-PATH LOOKUP (no geometry compute): resolve the seed set to the best
|
||||
* persisted neighborhood — the one with the greatest summed seed membership; on a
|
||||
* miss (no seed is a member of any neighborhood) fall back to the centroid nearest
|
||||
* the query embedding (centered by the loaded mean frame). q_emb may be NULL (then
|
||||
* a miss returns NULL). Returns a BORROWED handle (do NOT free) or NULL. */
|
||||
const GeoNeighborhood* engram_geo_reify_lookup(
|
||||
const GeoReifyIndex* ix,
|
||||
const char* const* seed_ids, size_t n_seeds,
|
||||
const float* q_emb, int q_dim);
|
||||
|
||||
int engram_geo_reify_count(const GeoReifyIndex* ix);
|
||||
const float* engram_geo_reify_mean(const GeoReifyIndex* ix, int* dim); /* loaded true mean or NULL */
|
||||
void engram_geo_reify_index_free(GeoReifyIndex* ix);
|
||||
|
||||
#endif /* ENGRAM_GEOMETRY_H */
|
||||
|
||||
Reference in New Issue
Block a user