M9 refinement: operate geometry descriptor in mean-centered (isotropic) embedding space
The nomic-embed-text space over the corpus is strongly anisotropic (mean pairwise cosine ~0.55), which compresses cosine-based domain separation almost to nothing so the design-doc s5 operators (distance/overlap/Wasserstein) cannot discriminate. Subtracting the global mean of the normalized embeddings restores isotropy (mean pairwise cosine ~0) and sharpens the operators. - add GeoMeanCache (engram_geo_mean_build / _maybe_refresh / _vec / _free): a store-derived centering offset over the embed-eligible set, cached and refreshed on significant drift; lives in geometry.c, not the store. - engram_geometry_descriptor gains an optional global_mean: when supplied the centroid, per-member cosine distance, and co-registration run in centered space (GM=zeros reproduces the legacy raw path exactly). - co-registration choice (b): the ANN query stays in raw unit space (index unchanged) since centering is a rigid translation that ~preserves neighborhood membership; only the descriptor statistics move to the centered frame. Covariance/axes/radius are translation-invariant and therefore unchanged. - test: synthetic ground-truth suite stays green (PERF + ASan/UBSan), plus new centered/raw/mean-cache assertions. - add bench_discrimination.c (env-gated, read-only, skips in CI): on a copy of the real store the two-domain overlap operator drops 1.13 -> 0.008 and cross-centroid cosine 0.899 -> 0.003 after centering, Euclid distance unchanged (translation-invariant control). No change to activation/retrieval behavior; wiring geometry into retrieval is a separate, behavior-changing cutover.
This commit is contained in:
+113
-13
@@ -61,9 +61,90 @@ static int normcopy(const float* v, int dim, float* out){
|
||||
for(int i=0;i<dim;i++) out[i]=(float)(v[i]/n);
|
||||
return 0;
|
||||
}
|
||||
static double dotf(const float* a, const float* b, int dim){
|
||||
double s=0; for(int i=0;i<dim;i++) s+=(double)a[i]*b[i]; return s;
|
||||
/* ── mean-centered cosine geometry (whitening the anisotropic emb space) ──────
|
||||
* All three operate on RAW (unnormalized-here, but member embs are already unit)
|
||||
* vectors, subtracting the global-mean offset `gm` on the fly. With gm all-zeros
|
||||
* they reduce EXACTLY to the raw unit-space cosine — so the same code path serves
|
||||
* both the centered and legacy-raw modes. */
|
||||
static double cnorm2(const float* a, const float* gm, int dim){
|
||||
double s=0; for(int d=0;d<dim;d++){ double v=(double)a[d]-gm[d]; s+=v*v; } return s;
|
||||
}
|
||||
static double cdot(const float* a, const float* b, const float* gm, int dim){
|
||||
double s=0; for(int d=0;d<dim;d++){ double av=(double)a[d]-gm[d], bv=(double)b[d]-gm[d]; s+=av*bv; }
|
||||
return s;
|
||||
}
|
||||
static double ccos(const float* a, const float* b, const float* gm, int dim){
|
||||
double na=sqrt(cnorm2(a,gm,dim)), nb=sqrt(cnorm2(b,gm,dim));
|
||||
if(na<1e-12||nb<1e-12) return 0.0;
|
||||
double c=cdot(a,b,gm,dim)/(na*nb); if(c>1)c=1; if(c<-1)c=-1; return c;
|
||||
}
|
||||
/* cosine of (a-gm) against a pre-centered UNIT direction `dir`. */
|
||||
static double ccos_dir(const float* a, const float* gm, const float* dir, int dim){
|
||||
double na=sqrt(cnorm2(a,gm,dim)); if(na<1e-12) return 0.0;
|
||||
double s=0; for(int d=0;d<dim;d++) s+=((double)a[d]-gm[d])*(double)dir[d];
|
||||
double c=s/na; if(c>1)c=1; if(c<-1)c=-1; return c;
|
||||
}
|
||||
|
||||
/* ───────────────────────── global-mean cache ───────────────────────────────
|
||||
* Store-derived centering offset: the mean of the L2-normalized embeddings over
|
||||
* the embed-eligible set. See engram_geometry.h for the anisotropy rationale. */
|
||||
struct GeoMeanCache { float* mean; int dim; uint64_t n; };
|
||||
|
||||
typedef struct { double* sum; int dim; uint64_t n; int err; } GeoMeanAcc;
|
||||
static void geo_mean_cb(const StoreNode* n, void* ctx){
|
||||
GeoMeanAcc* a=ctx; if(a->err) return;
|
||||
if(!(n->emb && n->emb_dim>0)) return; /* skip unembedded */
|
||||
if(a->dim==0){
|
||||
a->dim=n->emb_dim;
|
||||
a->sum=calloc((size_t)a->dim,sizeof(double));
|
||||
if(!a->sum){ a->err=1; return; }
|
||||
}
|
||||
if(n->emb_dim!=a->dim) return; /* skip off-dim */
|
||||
double s=0; for(int d=0;d<a->dim;d++) s+=(double)n->emb[d]*n->emb[d];
|
||||
double nn=sqrt(s); if(nn<1e-12) return; /* skip ~zero */
|
||||
for(int d=0;d<a->dim;d++) a->sum[d]+=(double)n->emb[d]/nn;
|
||||
a->n++;
|
||||
}
|
||||
typedef struct { uint64_t n; } GeoCntAcc;
|
||||
static void geo_cnt_cb(const StoreNode* n, void* ctx){
|
||||
if(n->emb && n->emb_dim>0) ((GeoCntAcc*)ctx)->n++;
|
||||
}
|
||||
|
||||
GeoMeanCache* engram_geo_mean_build(EngramPagedStore* store){
|
||||
if(!store) return NULL;
|
||||
GeoMeanAcc a; memset(&a,0,sizeof a);
|
||||
if(store_scan_nodes(store,geo_mean_cb,&a)<0){ free(a.sum); return NULL; }
|
||||
if(a.err || a.n==0 || !a.sum){ free(a.sum); return NULL; }
|
||||
GeoMeanCache* c=calloc(1,sizeof*c);
|
||||
if(!c){ free(a.sum); return NULL; }
|
||||
c->mean=malloc((size_t)a.dim*sizeof(float));
|
||||
if(!c->mean){ free(a.sum); free(c); return NULL; }
|
||||
for(int d=0;d<a.dim;d++) c->mean[d]=(float)(a.sum[d]/(double)a.n);
|
||||
c->dim=a.dim; c->n=a.n; free(a.sum);
|
||||
return c;
|
||||
}
|
||||
const float* engram_geo_mean_vec(const GeoMeanCache* c){ return c?c->mean:NULL; }
|
||||
int engram_geo_mean_dim(const GeoMeanCache* c){ return c?c->dim:0; }
|
||||
uint64_t engram_geo_mean_count(const GeoMeanCache* c){ return c?c->n:0; }
|
||||
|
||||
int engram_geo_mean_maybe_refresh(GeoMeanCache* c, EngramPagedStore* store, double frac){
|
||||
if(!c||!store) return -1;
|
||||
GeoCntAcc cn={0};
|
||||
if(store_scan_nodes(store,geo_cnt_cb,&cn)<0) return -1;
|
||||
double base=(double)(c->n?c->n:1);
|
||||
double drift=fabs((double)cn.n-(double)c->n)/base;
|
||||
if(drift<=frac) return 0; /* no significant change */
|
||||
GeoMeanAcc a; memset(&a,0,sizeof a);
|
||||
if(store_scan_nodes(store,geo_mean_cb,&a)<0){ free(a.sum); return -1; }
|
||||
if(a.err || a.n==0 || !a.sum){ free(a.sum); return -1; }
|
||||
float* nm=malloc((size_t)a.dim*sizeof(float));
|
||||
if(!nm){ free(a.sum); return -1; }
|
||||
for(int d=0;d<a.dim;d++) nm[d]=(float)(a.sum[d]/(double)a.n);
|
||||
free(a.sum); free(c->mean);
|
||||
c->mean=nm; c->dim=a.dim; c->n=a.n;
|
||||
return 1;
|
||||
}
|
||||
void engram_geo_mean_free(GeoMeanCache* c){ if(c){ free(c->mean); free(c); } }
|
||||
|
||||
/* Attach a member's emb (normalized) + salience by point-reading the store. */
|
||||
static void ms_load_node(MemSet* s, int i, EngramPagedStore* st){
|
||||
@@ -123,7 +204,8 @@ GeoDescriptor* engram_geometry_descriptor(
|
||||
EngramPagedStore* store, VIndex* vindex,
|
||||
char** vids, int n_vids,
|
||||
const char* const* seed_ids, size_t n_seeds,
|
||||
const GeoParams* params)
|
||||
const GeoParams* params,
|
||||
const float* global_mean)
|
||||
{
|
||||
if(!store || !seed_ids || n_seeds==0) return NULL;
|
||||
GeoParams P; if(params) P=*params; else engram_geo_default_params(&P);
|
||||
@@ -139,7 +221,17 @@ GeoDescriptor* engram_geometry_descriptor(
|
||||
}
|
||||
if(dim==0) dim = 768; /* no embedded seed: still build the relational side */
|
||||
|
||||
MemSet ms; if(ms_init(&ms,dim)!=0){ ms_free(&ms); return NULL; }
|
||||
/* Centering offset. When a global_mean is supplied the semantic cosine math
|
||||
* runs in mean-centered (isotropic) space; otherwise GM is an all-zeros
|
||||
* vector so the identical code path reproduces raw unit-space cosines. */
|
||||
int centered = (global_mean != NULL);
|
||||
float* zeros = NULL;
|
||||
const float* GM;
|
||||
if(centered) GM = global_mean;
|
||||
else { zeros = calloc((size_t)dim,sizeof(float));
|
||||
if(!zeros) return NULL; GM = zeros; }
|
||||
|
||||
MemSet ms; if(ms_init(&ms,dim)!=0){ ms_free(&ms); free(zeros); return NULL; }
|
||||
|
||||
/* 1. seeds (membership 1.0) */
|
||||
for(size_t i=0;i<n_seeds;i++) ms_upsert(&ms, seed_ids[i], 1.0);
|
||||
@@ -223,15 +315,17 @@ GeoDescriptor* engram_geometry_descriptor(
|
||||
for(int d=0;d<dim;d++) centroid[d]+=ms.emb[i][d]; }
|
||||
if(nemb){ for(int d=0;d<dim;d++) centroid[d]/=(float)nemb; }
|
||||
|
||||
/* ── radius + per-member cosine distance to centroid ── */
|
||||
/* ── radius + per-member cosine distance to centroid (CENTERED frame) ── */
|
||||
double total_var=0;
|
||||
double* distc=calloc((size_t)M,sizeof(double));
|
||||
/* normalize centroid direction for cosine distances */
|
||||
/* centered centroid (= raw centroid - global mean) and its unit direction */
|
||||
float* ccen=malloc((size_t)dim*sizeof(float));
|
||||
for(int d=0;d<dim;d++) ccen[d]=centroid[d]-GM[d];
|
||||
float* cdir=malloc((size_t)dim*sizeof(float));
|
||||
int have_cdir = (nemb>0 && normcopy(centroid,dim,cdir)==0);
|
||||
int have_cdir = (nemb>0 && normcopy(ccen,dim,cdir)==0);
|
||||
for(int i=0;i<M;i++){
|
||||
if(ms.emb[i] && have_cdir){
|
||||
double cs=dotf(ms.emb[i],cdir,dim); if(cs>1)cs=1; if(cs<-1)cs=-1;
|
||||
double cs=ccos_dir(ms.emb[i],GM,cdir,dim);
|
||||
distc[i]=1.0-cs;
|
||||
} else distc[i]=-1.0; /* unknown */
|
||||
}
|
||||
@@ -304,9 +398,10 @@ GeoDescriptor* engram_geometry_descriptor(
|
||||
edges[n_edges].eff_weight=w; edges[n_edges].hebb=es[e].hebb;
|
||||
n_edges++;
|
||||
centrality[i]+=w; centrality[j]+=w; degree[i]++; degree[j]++;
|
||||
/* co-registration: relational strength vs semantic proximity */
|
||||
/* co-registration: relational strength vs semantic proximity
|
||||
* (semantic proximity measured in the CENTERED frame). */
|
||||
if(ms.emb[i] && ms.emb[j]){
|
||||
double cs=dotf(ms.emb[i],ms.emb[j],dim); if(cs>1)cs=1; if(cs<-1)cs=-1;
|
||||
double cs=ccos(ms.emb[i],ms.emb[j],GM,dim);
|
||||
double x=w, y=cs;
|
||||
cr_n++; cr_sx+=x; cr_sy+=y; cr_sxx+=x*x; cr_syy+=y*y; cr_sxy+=x*y;
|
||||
}
|
||||
@@ -358,7 +453,12 @@ GeoDescriptor* engram_geometry_descriptor(
|
||||
GeoDescriptor* g=calloc(1,sizeof(GeoDescriptor));
|
||||
g->dim=dim;
|
||||
g->hub_id = strdup(ms.id[hub]);
|
||||
g->centroid = centroid; /* transfer ownership */
|
||||
g->centroid = ccen; /* CENTERED centroid; transfer ownership */
|
||||
free(centroid);
|
||||
if(centered){
|
||||
g->global_mean = malloc((size_t)dim*sizeof(float));
|
||||
if(g->global_mean) memcpy(g->global_mean, GM, (size_t)dim*sizeof(float));
|
||||
} else g->global_mean = NULL;
|
||||
g->n_axes=n_axes; g->axes=axes;
|
||||
g->total_variance=total_var; g->radius=radius;
|
||||
g->n_members=M; g->n_embedded=nemb;
|
||||
@@ -377,14 +477,14 @@ GeoDescriptor* engram_geometry_descriptor(
|
||||
g->co_registration=co_reg;
|
||||
|
||||
free(centrality); free(degree); free(core); free(distc); free(eidx);
|
||||
free(cdir);
|
||||
free(cdir); free(zeros);
|
||||
ms_free(&ms);
|
||||
return g;
|
||||
}
|
||||
|
||||
void engram_geo_free(GeoDescriptor* g){
|
||||
if(!g) return;
|
||||
free(g->hub_id); free(g->centroid);
|
||||
free(g->hub_id); free(g->centroid); free(g->global_mean);
|
||||
for(int i=0;i<g->n_axes;i++) free(g->axes[i].axis);
|
||||
free(g->axes);
|
||||
for(int i=0;i<g->n_members;i++) free(g->members[i].id);
|
||||
|
||||
@@ -50,7 +50,18 @@ typedef struct {
|
||||
int dim;
|
||||
/* ── anchor ── */
|
||||
char* hub_id; /* highest-centrality member: the relational hub */
|
||||
float* centroid; /* v̄ ∈ R^dim: mean of L2-normalized member embs */
|
||||
float* centroid; /* v̄ ∈ R^dim: mean of the member embeddings in the
|
||||
* frame the descriptor operated in. When centered
|
||||
* (global_mean != NULL) this is the CENTERED
|
||||
* centroid (mean of L2-normalized embs minus the
|
||||
* global mean): the neighborhood's location in the
|
||||
* isotropic/whitened frame. Add global_mean back to
|
||||
* recover the raw prototype point. When uncentered
|
||||
* it is the raw mean of L2-normalized member embs. */
|
||||
float* global_mean; /* the centering offset actually applied (dim floats),
|
||||
* or NULL if the descriptor ran in raw space. The §5
|
||||
* operators (distance/overlap/Wasserstein) are only
|
||||
* discriminative in the centered frame — see notes. */
|
||||
/* ── shape (compact covariance): top principal axes + extents ── */
|
||||
int n_axes;
|
||||
GeoAxis* axes; /* orientation + extents of the ellipsoid */
|
||||
@@ -84,6 +95,39 @@ typedef struct {
|
||||
* kcore_k=0 (auto), top_axes=8, max_members=400. */
|
||||
void engram_geo_default_params(GeoParams* p);
|
||||
|
||||
/* ── Global-mean cache (mean-centering / whitening the anisotropic emb space) ──
|
||||
* The nomic-embed-text space over the engram corpus is strongly ANISOTROPIC:
|
||||
* every embedding sits in a narrow cone (mean pairwise cosine ~0.55), which
|
||||
* compresses cosine-based domain separation almost to nothing. Subtracting the
|
||||
* GLOBAL MEAN of the (L2-normalized) embeddings recenters the cloud on the
|
||||
* origin (mean pairwise cosine -> ~0), restoring isotropy so the §5 operators
|
||||
* discriminate. The mean is a store-level derived quantity, like the ANN index:
|
||||
* built once from the paged store, cached, and refreshed when the embedded set
|
||||
* drifts. It lives here (not in the store) so this stays a contained, read-only
|
||||
* addition; a runtime owns one GeoMeanCache per open store alongside its VIndex. */
|
||||
typedef struct GeoMeanCache GeoMeanCache;
|
||||
|
||||
/* Scan every live node in `store` and compute the mean of the L2-normalized
|
||||
* embeddings over the embed-eligible set (nodes carrying an emb vector; the
|
||||
* unembedded telemetry/system nodes are skipped). Returns a malloc'd cache, or
|
||||
* NULL on error / no embedded nodes. The offset vector is NOT renormalized — it
|
||||
* is a translation, applied by subtraction. */
|
||||
GeoMeanCache* engram_geo_mean_build(EngramPagedStore* store);
|
||||
|
||||
/* The cached offset (dim floats) — pass to engram_geometry_descriptor as
|
||||
* global_mean. Valid until the cache is freed/refreshed. */
|
||||
const float* engram_geo_mean_vec(const GeoMeanCache* c);
|
||||
int engram_geo_mean_dim(const GeoMeanCache* c);
|
||||
uint64_t engram_geo_mean_count(const GeoMeanCache* c); /* #embedded nodes used */
|
||||
|
||||
/* Recompute the mean IN PLACE iff the embedded-node count has drifted by more
|
||||
* than `frac` (e.g. 0.10 = 10%) since the cache was built — "recompute on
|
||||
* significant change". Returns 1 if it rebuilt, 0 if unchanged, <0 on error. */
|
||||
int engram_geo_mean_maybe_refresh(GeoMeanCache* c, EngramPagedStore* store,
|
||||
double frac);
|
||||
|
||||
void engram_geo_mean_free(GeoMeanCache* c);
|
||||
|
||||
/* Compute the geometry descriptor of the neighborhood grown from seed_ids.
|
||||
* READ-ONLY over store + vindex.
|
||||
* store — an opened store (borrowed; not modified).
|
||||
@@ -92,13 +136,25 @@ void engram_geo_default_params(GeoParams* p);
|
||||
* (vids[node_id] == store id). Required iff vindex != NULL.
|
||||
* n_vids — length of vids.
|
||||
* params — NULL to use engram_geo_default_params.
|
||||
* global_mean — optional centering offset (dim floats, from engram_geo_mean_*).
|
||||
* When non-NULL the SEMANTIC geometry is computed in mean-centered
|
||||
* (isotropic) space: every normalized member emb has global_mean
|
||||
* subtracted before the centroid / cosine-distance / co-registration
|
||||
* math, so those operators discriminate. NULL = raw space (legacy).
|
||||
* NOTE: the ANN neighbor query still runs in RAW unit-vector space —
|
||||
* centering is a rigid translation that ~preserves neighborhood
|
||||
* MEMBERSHIP, so the index needs no rebuild; only the descriptor
|
||||
* STATISTICS move to the centered frame (co-registration choice (b)).
|
||||
* The eigen/covariance shape (axes, radius) is translation-invariant
|
||||
* and therefore identical in either frame.
|
||||
* Returns a malloc'd descriptor (free with engram_geo_free), or NULL on error
|
||||
* (no seeds resolvable, OOM). */
|
||||
GeoDescriptor* engram_geometry_descriptor(
|
||||
EngramPagedStore* store, VIndex* vindex,
|
||||
char** vids, int n_vids,
|
||||
const char* const* seed_ids, size_t n_seeds,
|
||||
const GeoParams* params);
|
||||
const GeoParams* params,
|
||||
const float* global_mean);
|
||||
|
||||
void engram_geo_free(GeoDescriptor* g);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user