From 8cae0f94eba604e5b0eba733fa49c71fd1790330 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Wed, 12 Aug 2026 19:54:41 -0500 Subject: [PATCH] M9 refinement: operate geometry descriptor in mean-centered (isotropic) embedding space The nomic-embed-text space over the corpus is strongly anisotropic (mean pairwise cosine ~0.55), which compresses cosine-based domain separation almost to nothing so the design-doc s5 operators (distance/overlap/Wasserstein) cannot discriminate. Subtracting the global mean of the normalized embeddings restores isotropy (mean pairwise cosine ~0) and sharpens the operators. - add GeoMeanCache (engram_geo_mean_build / _maybe_refresh / _vec / _free): a store-derived centering offset over the embed-eligible set, cached and refreshed on significant drift; lives in geometry.c, not the store. - engram_geometry_descriptor gains an optional global_mean: when supplied the centroid, per-member cosine distance, and co-registration run in centered space (GM=zeros reproduces the legacy raw path exactly). - co-registration choice (b): the ANN query stays in raw unit space (index unchanged) since centering is a rigid translation that ~preserves neighborhood membership; only the descriptor statistics move to the centered frame. Covariance/axes/radius are translation-invariant and therefore unchanged. - test: synthetic ground-truth suite stays green (PERF + ASan/UBSan), plus new centered/raw/mean-cache assertions. - add bench_discrimination.c (env-gated, read-only, skips in CI): on a copy of the real store the two-domain overlap operator drops 1.13 -> 0.008 and cross-centroid cosine 0.899 -> 0.003 after centering, Euclid distance unchanged (translation-invariant control). No change to activation/retrieval behavior; wiring geometry into retrieval is a separate, behavior-changing cutover. --- engram/test/bench_discrimination.c | 149 +++++++++++++++++++++++++++++ engram/test/run_geometry_tests.sh | 9 ++ engram/test/test_geometry.c | 34 +++++-- lang/runtime/engram_geometry.c | 126 +++++++++++++++++++++--- lang/runtime/engram_geometry.h | 60 +++++++++++- 5 files changed, 355 insertions(+), 23 deletions(-) create mode 100644 engram/test/bench_discrimination.c diff --git a/engram/test/bench_discrimination.c b/engram/test/bench_discrimination.c new file mode 100644 index 0000000..fba8cb4 --- /dev/null +++ b/engram/test/bench_discrimination.c @@ -0,0 +1,149 @@ +/* bench_discrimination.c — M9 REFINEMENT bench: measures whether mean-centering + * the anisotropic nomic-embed-text space sharpens the §5 geometry operators on + * REAL data. Read-only over a COPY of the live store (never the live file). + * + * usage: bench_discrimination [store.egm] + * (or set ENGRAM_BENCH_STORE). If no store is given/openable it prints + * SKIP and exits 0 — so it is safe in CI without live data. + * + * It picks two semantically distinct cohorts by keyword (domain A vs domain B), + * computes the global mean over the embed-eligible set (via engram_geo_mean_build + * — the same offset the descriptor uses), then reports BEFORE (raw unit space) + * vs AFTER (mean-centered space): + * - cross-centroid cosine (lower = better separated) + * - cross-centroid Euclid dist (translation-invariant: a control) + * - intra-cohesion per domain (member cos to own centroid) + * - overlap operator (cross_cos / sqrt(intraA*intraB): ~1 = domains + * indistinguishable, ~0 = cleanly separated) + * - angular separation ratio z (centroid angle / summed angular spread) + * - mean pairwise cosine sample (the anisotropy headline; ~0.55 raw -> ~0 ctr) + * + * Pure C11; links engram_store.c + engram_geometry.c; -lm. + */ +#include "engram_store.h" +#include "engram_geometry.h" +#include +#include +#include +#include +#include + +#define CAP_DOMAIN 400 +#define CAP_SAMPLE 800 + +typedef struct { float** v; int n, cap, dim; } VecSet; +static void vs_init(VecSet* s){ s->v=NULL; s->n=0; s->cap=0; s->dim=0; } +static void vs_push(VecSet* s, const float* e, int dim, int cap){ + if(s->n>=cap) return; + if(s->dim==0) s->dim=dim; + if(s->n==s->cap){ int nc=s->cap?s->cap*2:64; s->v=realloc(s->v,(size_t)nc*sizeof*s->v); s->cap=nc; } + float* c=malloc((size_t)dim*sizeof(float)); + double nn=0; for(int d=0;dv[s->n++]=c; +} +static void vs_free(VecSet* s){ for(int i=0;in;i++) free(s->v[i]); free(s->v); } + +typedef struct { VecSet A, B, S; long idx; } Coh; +static int has(const char* h, const char* n){ return h && strcasestr(h,n)!=NULL; } +static void cb(const StoreNode* n, void* ctx){ + Coh* c=ctx; + if(!(n->emb && n->emb_dim>0)) return; + /* every 5th embedded node -> isotropy sample */ + if((c->idx++ % 5)==0) vs_push(&c->S, n->emb, n->emb_dim, CAP_SAMPLE); + const char* t=n->content; const char* g=n->tags; + int A = has(t,"quantiz")||has(g,"quantiz")||has(t,"lorablation")||has(t,"70B")||has(t,"LoRA merge"); + int B = has(t,"kubernetes")||has(t,"terraform")||has(t,"argo")||has(g,"infrastructure")||has(t,"vault")||has(t,"cloudflare"); + if(A && !B) vs_push(&c->A, n->emb, n->emb_dim, CAP_DOMAIN); + else if(B && !A) vs_push(&c->B, n->emb, n->emb_dim, CAP_DOMAIN); +} + +/* mean of a VecSet into out (dim doubles). */ +static void mean_of(const VecSet* s, const float* gm, double* out){ + int dim=s->dim; for(int d=0;dn;i++) for(int d=0;dv[i][d]-(gm?gm[d]:0.0); + if(s->n) for(int d=0;dn; +} +static double dnorm(const double* a, int dim){ double s=0; for(int d=0;d1)c=1; if(c<-1)c=-1; return c; +} +static double deuclid(const double* a, const double* b, int dim){ + double s=0; for(int d=0;ddim; double nc=dnorm(c,dim); if(nc<1e-12||s->n==0) return 0; + double acc=0; for(int i=0;in;i++){ + double dot=0, nv=0; + for(int d=0;dv[i][d]-(gm?gm[d]:0.0); dot+=v*c[d]; nv+=v*v; } + nv=sqrt(nv); if(nv<1e-12) continue; double cc=dot/(nv*nc); + if(cc>1)cc=1; if(cc<-1)cc=-1; acc+=cc; + } + return acc/s->n; +} +/* mean pairwise cosine over a sample (isotropy metric). */ +static double mean_pairwise_cos(const VecSet* s, const float* gm){ + int dim=s->dim; if(s->n<2) return 0; double acc=0; long np=0; + for(int i=0;in;i++) for(int j=i+1;jn;j++){ + double dot=0, na=0, nb=0; + for(int d=0;dv[i][d]-(gm?gm[d]:0.0), b=(double)s->v[j][d]-(gm?gm[d]:0.0); + dot+=a*b; na+=a*a; nb+=b*b; } + na=sqrt(na); nb=sqrt(nb); if(na<1e-12||nb<1e-12) continue; + double c=dot/(na*nb); if(c>1)c=1; if(c<-1)c=-1; acc+=c; np++; + } + return np? acc/np : 0; +} + +static void report(const char* label, Coh* c, const float* gm){ + int dim=c->A.dim; double* ca=malloc((size_t)dim*sizeof(double)); double* cb=malloc((size_t)dim*sizeof(double)); + mean_of(&c->A, gm, ca); mean_of(&c->B, gm, cb); + double xcos=dcos(ca,cb,dim), xeuc=deuclid(ca,cb,dim); + double cohA=cohesion(&c->A,gm,ca), cohB=cohesion(&c->B,gm,cb); + double overlap = (cohA>0&&cohB>0)? xcos/sqrt(cohA*cohB) : xcos; + double theta = acos(xcos<-1?-1:(xcos>1?1:xcos)); + double sig = acos(cohA<-1?-1:(cohA>1?1:cohA)) + acos(cohB<-1?-1:(cohB>1?1:cohB)); + double z = (sig>1e-9)? theta/sig : 0; + double mpc = mean_pairwise_cos(&c->S, gm); + printf(" [%s]\n", label); + printf(" cross-centroid cosine = %+.4f (lower = better separated)\n", xcos); + printf(" cross-centroid Euclid = %.4f (translation-invariant control)\n", xeuc); + printf(" intra-cohesion A / B = %.4f / %.4f\n", cohA, cohB); + printf(" OVERLAP operator = %.4f (~1 = indistinguishable, ~0 = clean)\n", overlap); + printf(" angular separation z = %.3f (centroid-angle / summed spread; >1 = separated)\n", z); + printf(" mean pairwise cosine = %+.4f (isotropy: ~0.55 anisotropic -> ~0 isotropic)\n", mpc); + free(ca); free(cb); +} + +int main(int argc, char** argv){ + const char* path = (argc>1)? argv[1] : getenv("ENGRAM_BENCH_STORE"); + if(!path){ printf("SKIP: no store path (arg or ENGRAM_BENCH_STORE)\n"); return 0; } + EngramPagedStore* st=store_open(path); + if(!st){ printf("SKIP: could not open %s\n", path); return 0; } + + Coh c; vs_init(&c.A); vs_init(&c.B); vs_init(&c.S); c.idx=0; + store_scan_nodes(st, cb, &c); + printf("=== two-domain discrimination bench (real store copy) ===\n"); + printf("domain A (quantization) n=%d ; domain B (infrastructure) n=%d ; sample n=%d ; dim=%d\n", + c.A.n, c.B.n, c.S.n, c.A.dim); + if(c.A.n<3 || c.B.n<3){ printf("SKIP: a cohort is too small to be meaningful\n"); + vs_free(&c.A); vs_free(&c.B); vs_free(&c.S); store_close(st); return 0; } + + GeoMeanCache* mc=engram_geo_mean_build(st); + const float* gm=engram_geo_mean_vec(mc); + printf("global-mean cache: dim=%d over %llu embedded nodes\n\n", + engram_geo_mean_dim(mc), (unsigned long long)engram_geo_mean_count(mc)); + + printf("BEFORE (raw anisotropic unit space):\n"); + report("RAW", &c, NULL); + printf("\nAFTER (mean-centered isotropic space):\n"); + report("CENTERED", &c, gm); + + engram_geo_mean_free(mc); + vs_free(&c.A); vs_free(&c.B); vs_free(&c.S); + store_close(st); + return 0; +} diff --git a/engram/test/run_geometry_tests.sh b/engram/test/run_geometry_tests.sh index ee37b82..424e8f6 100755 --- a/engram/test/run_geometry_tests.sh +++ b/engram/test/run_geometry_tests.sh @@ -19,3 +19,12 @@ echo echo "### PASS 2: SAFETY (ASan/UBSan)" $CC $WARN -O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer -I"$RT" $SRC -lm -o "$TMP/safe" ASAN_OPTIONS=${ASAN_OPTIONS:-detect_leaks=0} UBSAN_OPTIONS=halt_on_error=1 "$TMP/safe" + +# PASS 3 (OPTIONAL): mean-centering discrimination bench on a COPY of a real +# store. Skips cleanly unless ENGRAM_BENCH_STORE points at a store .egm — never +# touches the live store. Read-only; not part of the pass/fail gate. +echo +echo "### PASS 3: DISCRIMINATION BENCH (optional; set ENGRAM_BENCH_STORE)" +BSRC="$HERE/bench_discrimination.c $RT/engram_geometry.c $RT/engram_store.c $RT/engram_vindex.c" +$CC $WARN -O2 -I"$RT" $BSRC -lm -o "$TMP/bench" +"$TMP/bench" "${ENGRAM_BENCH_STORE:-}" diff --git a/engram/test/test_geometry.c b/engram/test/test_geometry.c index 6f7a3d7..22fe357 100644 --- a/engram/test/test_geometry.c +++ b/engram/test/test_geometry.c @@ -78,18 +78,31 @@ int main(void){ /* seed inside cluster A -> expect an A-dominated neighborhood */ st=store_open(path); + /* global-mean cache over the embedded set: the centering offset */ + GeoMeanCache* mc=engram_geo_mean_build(st); + const float* gm=engram_geo_mean_vec(mc); + CHECK(mc!=NULL && engram_geo_mean_dim(mc)==DIM, "global-mean cache built over embedded set"); + CHECK(engram_geo_mean_count(mc)==(uint64_t)(NA+NB), "global mean averaged all embedded nodes"); const char* seeds[1]={aids[0]}; - GeoDescriptor* g=engram_geometry_descriptor(st, ix, ids, nids, seeds, 1, &P); + /* CENTERED descriptor: pass the global mean so geometry runs in isotropic space */ + GeoDescriptor* g=engram_geometry_descriptor(st, ix, ids, nids, seeds, 1, &P, gm); CHECK(g!=NULL, "descriptor computed"); if(g){ printf(" members=%d embedded=%d edges=%d k_core=%d radius=%.4f co_reg=%.3f n_axes=%d\n", g->n_members,g->n_embedded,g->n_edges,g->k_core,g->radius,g->co_registration,g->n_axes); - /* centroid near cluster-A center (+e0): centroid[0] should dominate */ - int argmax=0; for(int d=1;ddim;d++) if(fabsf(g->centroid[d])>fabsf(g->centroid[argmax])) argmax=d; - printf(" centroid dominant axis = %d (expect 0), centroid[0]=%.3f centroid[1]=%.3f\n", + /* geometry ran in CENTERED space: g->centroid is the centered centroid, + * g->global_mean the applied offset. Reconstruct the raw prototype + * (centroid + global_mean) and check it sits on cluster-A's axis. */ + CHECK(g->global_mean!=NULL, "descriptor recorded the centering offset (centered mode)"); + int argmax=0; float best=-1.f; + for(int d=0;ddim;d++){ float raw=g->centroid[d]+(g->global_mean?g->global_mean[d]:0.f); + if(fabsf(raw)>best){ best=fabsf(raw); argmax=d; } } + printf(" raw-prototype dominant axis = %d (expect 0); centered c[0]=%.3f c[1]=%.3f\n", argmax, g->centroid[0], g->centroid[1]); - CHECK(argmax==0, "centroid sits on cluster-A's axis (near members)"); + CHECK(argmax==0, "raw prototype sits on cluster-A's axis (near members)"); + /* centering pushes A off cluster-B's axis: centered c[0] > c[1] */ + CHECK(g->centroid[0] > g->centroid[1], "centered centroid leans off B's axis (isotropy)"); /* hub should be A0 (the intra-A hub with NA-1 strong edges) */ CHECK(g->hub_id && strcmp(g->hub_id,"A0")==0, "hub = the relational center A0"); @@ -128,12 +141,17 @@ int main(void){ engram_geo_free(g); /* edge cases: NULL store, no seeds, relational-only (NULL vindex) */ - CHECK(engram_geometry_descriptor(NULL,ix,ids,nids,seeds,1,&P)==NULL, "NULL store -> NULL"); - CHECK(engram_geometry_descriptor(st,ix,ids,nids,seeds,0,&P)==NULL, "zero seeds -> NULL"); - GeoDescriptor* g2=engram_geometry_descriptor(st, NULL, NULL, 0, seeds, 1, &P); + CHECK(engram_geometry_descriptor(NULL,ix,ids,nids,seeds,1,&P,gm)==NULL, "NULL store -> NULL"); + CHECK(engram_geometry_descriptor(st,ix,ids,nids,seeds,0,&P,gm)==NULL, "zero seeds -> NULL"); + GeoDescriptor* g2=engram_geometry_descriptor(st, NULL, NULL, 0, seeds, 1, &P, gm); CHECK(g2!=NULL && g2->n_members>=NA-1, "relational-only path (no vindex) works"); engram_geo_free(g2); + /* raw (uncentered) mode still supported: global_mean=NULL -> no offset recorded */ + GeoDescriptor* g3=engram_geometry_descriptor(st, ix, ids, nids, seeds, 1, &P, NULL); + CHECK(g3!=NULL && g3->global_mean==NULL, "raw mode (global_mean=NULL) leaves offset unset"); + engram_geo_free(g3); + engram_geo_mean_free(mc); for(int i=0;i1)c=1; if(c<-1)c=-1; return c; +} +/* cosine of (a-gm) against a pre-centered UNIT direction `dir`. */ +static double ccos_dir(const float* a, const float* gm, const float* dir, int dim){ + double na=sqrt(cnorm2(a,gm,dim)); if(na<1e-12) return 0.0; + double s=0; for(int d=0;d1)c=1; if(c<-1)c=-1; return c; +} + +/* ───────────────────────── global-mean cache ─────────────────────────────── + * Store-derived centering offset: the mean of the L2-normalized embeddings over + * the embed-eligible set. See engram_geometry.h for the anisotropy rationale. */ +struct GeoMeanCache { float* mean; int dim; uint64_t n; }; + +typedef struct { double* sum; int dim; uint64_t n; int err; } GeoMeanAcc; +static void geo_mean_cb(const StoreNode* n, void* ctx){ + GeoMeanAcc* a=ctx; if(a->err) return; + if(!(n->emb && n->emb_dim>0)) return; /* skip unembedded */ + if(a->dim==0){ + a->dim=n->emb_dim; + a->sum=calloc((size_t)a->dim,sizeof(double)); + if(!a->sum){ a->err=1; return; } + } + if(n->emb_dim!=a->dim) return; /* skip off-dim */ + double s=0; for(int d=0;ddim;d++) s+=(double)n->emb[d]*n->emb[d]; + double nn=sqrt(s); if(nn<1e-12) return; /* skip ~zero */ + for(int d=0;ddim;d++) a->sum[d]+=(double)n->emb[d]/nn; + a->n++; +} +typedef struct { uint64_t n; } GeoCntAcc; +static void geo_cnt_cb(const StoreNode* n, void* ctx){ + if(n->emb && n->emb_dim>0) ((GeoCntAcc*)ctx)->n++; +} + +GeoMeanCache* engram_geo_mean_build(EngramPagedStore* store){ + if(!store) return NULL; + GeoMeanAcc a; memset(&a,0,sizeof a); + if(store_scan_nodes(store,geo_mean_cb,&a)<0){ free(a.sum); return NULL; } + if(a.err || a.n==0 || !a.sum){ free(a.sum); return NULL; } + GeoMeanCache* c=calloc(1,sizeof*c); + if(!c){ free(a.sum); return NULL; } + c->mean=malloc((size_t)a.dim*sizeof(float)); + if(!c->mean){ free(a.sum); free(c); return NULL; } + for(int d=0;dmean[d]=(float)(a.sum[d]/(double)a.n); + c->dim=a.dim; c->n=a.n; free(a.sum); + return c; +} +const float* engram_geo_mean_vec(const GeoMeanCache* c){ return c?c->mean:NULL; } +int engram_geo_mean_dim(const GeoMeanCache* c){ return c?c->dim:0; } +uint64_t engram_geo_mean_count(const GeoMeanCache* c){ return c?c->n:0; } + +int engram_geo_mean_maybe_refresh(GeoMeanCache* c, EngramPagedStore* store, double frac){ + if(!c||!store) return -1; + GeoCntAcc cn={0}; + if(store_scan_nodes(store,geo_cnt_cb,&cn)<0) return -1; + double base=(double)(c->n?c->n:1); + double drift=fabs((double)cn.n-(double)c->n)/base; + if(drift<=frac) return 0; /* no significant change */ + GeoMeanAcc a; memset(&a,0,sizeof a); + if(store_scan_nodes(store,geo_mean_cb,&a)<0){ free(a.sum); return -1; } + if(a.err || a.n==0 || !a.sum){ free(a.sum); return -1; } + float* nm=malloc((size_t)a.dim*sizeof(float)); + if(!nm){ free(a.sum); return -1; } + for(int d=0;dmean); + c->mean=nm; c->dim=a.dim; c->n=a.n; + return 1; +} +void engram_geo_mean_free(GeoMeanCache* c){ if(c){ free(c->mean); free(c); } } /* Attach a member's emb (normalized) + salience by point-reading the store. */ static void ms_load_node(MemSet* s, int i, EngramPagedStore* st){ @@ -123,7 +204,8 @@ GeoDescriptor* engram_geometry_descriptor( EngramPagedStore* store, VIndex* vindex, char** vids, int n_vids, const char* const* seed_ids, size_t n_seeds, - const GeoParams* params) + const GeoParams* params, + const float* global_mean) { if(!store || !seed_ids || n_seeds==0) return NULL; GeoParams P; if(params) P=*params; else engram_geo_default_params(&P); @@ -139,7 +221,17 @@ GeoDescriptor* engram_geometry_descriptor( } if(dim==0) dim = 768; /* no embedded seed: still build the relational side */ - MemSet ms; if(ms_init(&ms,dim)!=0){ ms_free(&ms); return NULL; } + /* Centering offset. When a global_mean is supplied the semantic cosine math + * runs in mean-centered (isotropic) space; otherwise GM is an all-zeros + * vector so the identical code path reproduces raw unit-space cosines. */ + int centered = (global_mean != NULL); + float* zeros = NULL; + const float* GM; + if(centered) GM = global_mean; + else { zeros = calloc((size_t)dim,sizeof(float)); + if(!zeros) return NULL; GM = zeros; } + + MemSet ms; if(ms_init(&ms,dim)!=0){ ms_free(&ms); free(zeros); return NULL; } /* 1. seeds (membership 1.0) */ for(size_t i=0;i0 && normcopy(centroid,dim,cdir)==0); + int have_cdir = (nemb>0 && normcopy(ccen,dim,cdir)==0); for(int i=0;i1)cs=1; if(cs<-1)cs=-1; + double cs=ccos_dir(ms.emb[i],GM,cdir,dim); distc[i]=1.0-cs; } else distc[i]=-1.0; /* unknown */ } @@ -304,9 +398,10 @@ GeoDescriptor* engram_geometry_descriptor( edges[n_edges].eff_weight=w; edges[n_edges].hebb=es[e].hebb; n_edges++; centrality[i]+=w; centrality[j]+=w; degree[i]++; degree[j]++; - /* co-registration: relational strength vs semantic proximity */ + /* co-registration: relational strength vs semantic proximity + * (semantic proximity measured in the CENTERED frame). */ if(ms.emb[i] && ms.emb[j]){ - double cs=dotf(ms.emb[i],ms.emb[j],dim); if(cs>1)cs=1; if(cs<-1)cs=-1; + double cs=ccos(ms.emb[i],ms.emb[j],GM,dim); double x=w, y=cs; cr_n++; cr_sx+=x; cr_sy+=y; cr_sxx+=x*x; cr_syy+=y*y; cr_sxy+=x*y; } @@ -358,7 +453,12 @@ GeoDescriptor* engram_geometry_descriptor( GeoDescriptor* g=calloc(1,sizeof(GeoDescriptor)); g->dim=dim; g->hub_id = strdup(ms.id[hub]); - g->centroid = centroid; /* transfer ownership */ + g->centroid = ccen; /* CENTERED centroid; transfer ownership */ + free(centroid); + if(centered){ + g->global_mean = malloc((size_t)dim*sizeof(float)); + if(g->global_mean) memcpy(g->global_mean, GM, (size_t)dim*sizeof(float)); + } else g->global_mean = NULL; g->n_axes=n_axes; g->axes=axes; g->total_variance=total_var; g->radius=radius; g->n_members=M; g->n_embedded=nemb; @@ -377,14 +477,14 @@ GeoDescriptor* engram_geometry_descriptor( g->co_registration=co_reg; free(centrality); free(degree); free(core); free(distc); free(eidx); - free(cdir); + free(cdir); free(zeros); ms_free(&ms); return g; } void engram_geo_free(GeoDescriptor* g){ if(!g) return; - free(g->hub_id); free(g->centroid); + free(g->hub_id); free(g->centroid); free(g->global_mean); for(int i=0;in_axes;i++) free(g->axes[i].axis); free(g->axes); for(int i=0;in_members;i++) free(g->members[i].id); diff --git a/lang/runtime/engram_geometry.h b/lang/runtime/engram_geometry.h index 84fcffc..49afb27 100644 --- a/lang/runtime/engram_geometry.h +++ b/lang/runtime/engram_geometry.h @@ -50,7 +50,18 @@ typedef struct { int dim; /* ── anchor ── */ char* hub_id; /* highest-centrality member: the relational hub */ - float* centroid; /* v̄ ∈ R^dim: mean of L2-normalized member embs */ + float* centroid; /* v̄ ∈ R^dim: mean of the member embeddings in the + * frame the descriptor operated in. When centered + * (global_mean != NULL) this is the CENTERED + * centroid (mean of L2-normalized embs minus the + * global mean): the neighborhood's location in the + * isotropic/whitened frame. Add global_mean back to + * recover the raw prototype point. When uncentered + * it is the raw mean of L2-normalized member embs. */ + float* global_mean; /* the centering offset actually applied (dim floats), + * or NULL if the descriptor ran in raw space. The §5 + * operators (distance/overlap/Wasserstein) are only + * discriminative in the centered frame — see notes. */ /* ── shape (compact covariance): top principal axes + extents ── */ int n_axes; GeoAxis* axes; /* orientation + extents of the ellipsoid */ @@ -84,6 +95,39 @@ typedef struct { * kcore_k=0 (auto), top_axes=8, max_members=400. */ void engram_geo_default_params(GeoParams* p); +/* ── Global-mean cache (mean-centering / whitening the anisotropic emb space) ── + * The nomic-embed-text space over the engram corpus is strongly ANISOTROPIC: + * every embedding sits in a narrow cone (mean pairwise cosine ~0.55), which + * compresses cosine-based domain separation almost to nothing. Subtracting the + * GLOBAL MEAN of the (L2-normalized) embeddings recenters the cloud on the + * origin (mean pairwise cosine -> ~0), restoring isotropy so the §5 operators + * discriminate. The mean is a store-level derived quantity, like the ANN index: + * built once from the paged store, cached, and refreshed when the embedded set + * drifts. It lives here (not in the store) so this stays a contained, read-only + * addition; a runtime owns one GeoMeanCache per open store alongside its VIndex. */ +typedef struct GeoMeanCache GeoMeanCache; + +/* Scan every live node in `store` and compute the mean of the L2-normalized + * embeddings over the embed-eligible set (nodes carrying an emb vector; the + * unembedded telemetry/system nodes are skipped). Returns a malloc'd cache, or + * NULL on error / no embedded nodes. The offset vector is NOT renormalized — it + * is a translation, applied by subtraction. */ +GeoMeanCache* engram_geo_mean_build(EngramPagedStore* store); + +/* The cached offset (dim floats) — pass to engram_geometry_descriptor as + * global_mean. Valid until the cache is freed/refreshed. */ +const float* engram_geo_mean_vec(const GeoMeanCache* c); +int engram_geo_mean_dim(const GeoMeanCache* c); +uint64_t engram_geo_mean_count(const GeoMeanCache* c); /* #embedded nodes used */ + +/* Recompute the mean IN PLACE iff the embedded-node count has drifted by more + * than `frac` (e.g. 0.10 = 10%) since the cache was built — "recompute on + * significant change". Returns 1 if it rebuilt, 0 if unchanged, <0 on error. */ +int engram_geo_mean_maybe_refresh(GeoMeanCache* c, EngramPagedStore* store, + double frac); + +void engram_geo_mean_free(GeoMeanCache* c); + /* Compute the geometry descriptor of the neighborhood grown from seed_ids. * READ-ONLY over store + vindex. * store — an opened store (borrowed; not modified). @@ -92,13 +136,25 @@ void engram_geo_default_params(GeoParams* p); * (vids[node_id] == store id). Required iff vindex != NULL. * n_vids — length of vids. * params — NULL to use engram_geo_default_params. + * global_mean — optional centering offset (dim floats, from engram_geo_mean_*). + * When non-NULL the SEMANTIC geometry is computed in mean-centered + * (isotropic) space: every normalized member emb has global_mean + * subtracted before the centroid / cosine-distance / co-registration + * math, so those operators discriminate. NULL = raw space (legacy). + * NOTE: the ANN neighbor query still runs in RAW unit-vector space — + * centering is a rigid translation that ~preserves neighborhood + * MEMBERSHIP, so the index needs no rebuild; only the descriptor + * STATISTICS move to the centered frame (co-registration choice (b)). + * The eigen/covariance shape (axes, radius) is translation-invariant + * and therefore identical in either frame. * Returns a malloc'd descriptor (free with engram_geo_free), or NULL on error * (no seeds resolvable, OOM). */ GeoDescriptor* engram_geometry_descriptor( EngramPagedStore* store, VIndex* vindex, char** vids, int n_vids, const char* const* seed_ids, size_t n_seeds, - const GeoParams* params); + const GeoParams* params, + const float* global_mean); void engram_geo_free(GeoDescriptor* g);