will.anderson
816b258255
Adds engram_geo_displacement(A, B, core_frac) — a read-only interoceptive primitive that measures how far a neighborhood descriptor B has drifted from a baseline A and decomposes it into GROWTH (periphery extends, core fixed) vs CORRUPTION (the invariant core displaces). The core is the top core_frac of A's members by centrality; per shared member (matched by id) the displacement is the change in radial position (dist_centroid). Centroid separation (L2 + cosine) and radius delta give the aggregate move. Pure function, no store mutation, no flag. HONESTLY PARTIAL: a live self-drift reading needs a persisted SelfAnchor baseline to compare "now" against, and no persisted self node / anchored self-neighborhood exists in this store yet. The primitive takes an EXPLICIT baseline so it is real and testable today; capturing a durable SelfAnchor and wiring the ENGRAM_DRIFT_SENSOR live reading is a flagged follow-up. We do not fabricate a self silently. MEASURED on synthetic descriptors: - GROWTH (periphery 0.50->0.90, core fixed): core_disp=0.000, periph_disp=0.400, centroid_sep=0.000, radius_delta=0.400. - CORRUPTION (core 0.10->0.60, periphery fixed): core_disp=0.500, periph_disp=0.000, centroid_sep=0.566. - Identity A vs A: zero drift. The sensor discriminates cleanly (corruption core_disp >> growth core_disp). ASan+UBSan clean.
285 lines
17 KiB
C
285 lines
17 KiB
C
/* engram_geometry.h — M9 FOUNDATION: the relational-neighborhood GEOMETRY
|
|
* DESCRIPTOR (design doc §3, §5; memory node e94371bd).
|
|
*
|
|
* Computes, for a relational neighborhood grown from a seed set, the compact
|
|
* (KB-not-MB) joint geometry Will specified: the SEMANTIC geometry (centroid,
|
|
* covariance / principal axes, radius) braided with the RELATIONAL geometry
|
|
* (k-core skeleton, hub->periphery centrality gradient), plus soft membership.
|
|
*
|
|
* Two coordinate systems, one shape — "a constellation: bright prototype at the
|
|
* center, a cloud of members at varying distance, the strongest edges as a
|
|
* backbone, fading at the edges."
|
|
*
|
|
* Built ON the two standalone M-era modules only:
|
|
* - engram_vindex : semantic neighbors (the cloud) via ANN.
|
|
* - engram_store : node embeddings + hebb adjacency (the skeleton), read-only.
|
|
* It does NOT link or touch el_runtime.c, and it is a pure READ over the graph:
|
|
* it never modifies nodes, edges, activation, the index, or any retrieval path.
|
|
*
|
|
* Pure C11, stdlib + libm only. The descriptor is a foundation object; it is NOT
|
|
* wired into retrieval/priming yet (that is the next M9 step).
|
|
*/
|
|
#ifndef ENGRAM_GEOMETRY_H
|
|
#define ENGRAM_GEOMETRY_H
|
|
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
#include "engram_store.h"
|
|
#include "engram_vindex.h"
|
|
|
|
/* One member of the neighborhood + its place in the gradient. */
|
|
typedef struct {
|
|
char* id;
|
|
double membership; /* soft membership in [0,1] (semantic+relational blend) */
|
|
double centrality; /* skeleton weighted-degree — relational salience */
|
|
double salience; /* the node's own stored salience */
|
|
int core; /* k-core number (0 = fringe / not in any core) */
|
|
double dist_centroid; /* cosine distance of member emb to centroid (semantic)*/
|
|
int embedded; /* 1 if the member carried an emb vector */
|
|
} GeoMember;
|
|
|
|
/* One skeleton edge (indices into members[]). eff_weight = weight*(1+0.5*hebb),
|
|
* clamped to 1.0 — the effective propagation strength eg_edge_eff_weight uses. */
|
|
typedef struct { uint32_t a, b; double eff_weight; double hebb; } GeoEdge;
|
|
|
|
/* A compact principal axis of the ellipsoid: unit direction in R^dim + extent
|
|
* (sqrt of the covariance eigenvalue = the ellipsoid's half-width along it). */
|
|
typedef struct { float* axis; double extent; } GeoAxis;
|
|
|
|
typedef struct {
|
|
int dim;
|
|
/* ── anchor ── */
|
|
char* hub_id; /* highest-centrality member: the relational hub */
|
|
float* centroid; /* v̄ ∈ R^dim: mean of the member embeddings in the
|
|
* frame the descriptor operated in. When centered
|
|
* (global_mean != NULL) this is the CENTERED
|
|
* centroid (mean of L2-normalized embs minus the
|
|
* global mean): the neighborhood's location in the
|
|
* isotropic/whitened frame. Add global_mean back to
|
|
* recover the raw prototype point. When uncentered
|
|
* it is the raw mean of L2-normalized member embs. */
|
|
float* global_mean; /* the centering offset actually applied (dim floats),
|
|
* or NULL if the descriptor ran in raw space. The §5
|
|
* operators (distance/overlap/Wasserstein) are only
|
|
* discriminative in the centered frame — see notes. */
|
|
/* ── shape (compact covariance): top principal axes + extents ── */
|
|
int n_axes;
|
|
GeoAxis* axes; /* orientation + extents of the ellipsoid */
|
|
double total_variance; /* trace(Σ) = mean squared member dist to centroid*/
|
|
/* ── scale ── */
|
|
double radius; /* sqrt(total_variance) — the neighborhood breadth*/
|
|
/* ── members + gradient ── */
|
|
int n_members;
|
|
GeoMember* members; /* soft membership {id->weight} + centrality/salience */
|
|
/* ── skeleton ── */
|
|
int n_edges;
|
|
GeoEdge* edges; /* strong internal hebb edges = the backbone */
|
|
int k_core; /* the maximum core number present in the skeleton*/
|
|
/* ── diagnostics ── */
|
|
double co_registration;/* corr(hebb strength, semantic proximity) over */
|
|
/* internal edges: >0 = geometries agree (reify); */
|
|
/* <0 = disagree (surprising links / dream cands). */
|
|
int n_embedded; /* members that carried an emb vector */
|
|
} GeoDescriptor;
|
|
|
|
typedef struct {
|
|
int ann_k; /* semantic expansion: ANN neighbors per seed (0=off) */
|
|
int hop_relational; /* 1 = include seeds' hebb neighbors as members */
|
|
double edge_min_weight; /* skeleton: ignore internal edges below this eff wt */
|
|
int kcore_k; /* target k for the reported k-core (0 = auto/max) */
|
|
int top_axes; /* principal axes to retain (default 8) */
|
|
int max_members; /* cap neighborhood size (guards the eigensolve cost) */
|
|
} GeoParams;
|
|
|
|
/* Fill p with sane defaults: ann_k=24, hop_relational=1, edge_min_weight=0.05,
|
|
* kcore_k=0 (auto), top_axes=8, max_members=400. */
|
|
void engram_geo_default_params(GeoParams* p);
|
|
|
|
/* ── Global-mean cache (mean-centering / whitening the anisotropic emb space) ──
|
|
* The nomic-embed-text space over the engram corpus is strongly ANISOTROPIC:
|
|
* every embedding sits in a narrow cone (mean pairwise cosine ~0.55), which
|
|
* compresses cosine-based domain separation almost to nothing. Subtracting the
|
|
* GLOBAL MEAN of the (L2-normalized) embeddings recenters the cloud on the
|
|
* origin (mean pairwise cosine -> ~0), restoring isotropy so the §5 operators
|
|
* discriminate. The mean is a store-level derived quantity, like the ANN index:
|
|
* built once from the paged store, cached, and refreshed when the embedded set
|
|
* drifts. It lives here (not in the store) so this stays a contained, read-only
|
|
* addition; a runtime owns one GeoMeanCache per open store alongside its VIndex. */
|
|
typedef struct GeoMeanCache GeoMeanCache;
|
|
|
|
/* Scan every live node in `store` and compute the mean of the L2-normalized
|
|
* embeddings over the embed-eligible set (nodes carrying an emb vector; the
|
|
* unembedded telemetry/system nodes are skipped). Returns a malloc'd cache, or
|
|
* NULL on error / no embedded nodes. The offset vector is NOT renormalized — it
|
|
* is a translation, applied by subtraction. */
|
|
GeoMeanCache* engram_geo_mean_build(EngramPagedStore* store);
|
|
|
|
/* The cached offset (dim floats) — pass to engram_geometry_descriptor as
|
|
* global_mean. Valid until the cache is freed/refreshed. */
|
|
const float* engram_geo_mean_vec(const GeoMeanCache* c);
|
|
int engram_geo_mean_dim(const GeoMeanCache* c);
|
|
uint64_t engram_geo_mean_count(const GeoMeanCache* c); /* #embedded nodes used */
|
|
|
|
/* Recompute the mean IN PLACE iff the embedded-node count has drifted by more
|
|
* than `frac` (e.g. 0.10 = 10%) since the cache was built — "recompute on
|
|
* significant change". Returns 1 if it rebuilt, 0 if unchanged, <0 on error. */
|
|
int engram_geo_mean_maybe_refresh(GeoMeanCache* c, EngramPagedStore* store,
|
|
double frac);
|
|
|
|
void engram_geo_mean_free(GeoMeanCache* c);
|
|
|
|
/* Compute the geometry descriptor of the neighborhood grown from seed_ids.
|
|
* READ-ONLY over store + vindex.
|
|
* store — an opened store (borrowed; not modified).
|
|
* vindex — optional ANN index for semantic expansion; NULL disables it.
|
|
* vids — the ordinal->store-id map returned by vindex_build_from_store
|
|
* (vids[node_id] == store id). Required iff vindex != NULL.
|
|
* n_vids — length of vids.
|
|
* params — NULL to use engram_geo_default_params.
|
|
* global_mean — optional centering offset (dim floats, from engram_geo_mean_*).
|
|
* When non-NULL the SEMANTIC geometry is computed in mean-centered
|
|
* (isotropic) space: every normalized member emb has global_mean
|
|
* subtracted before the centroid / cosine-distance / co-registration
|
|
* math, so those operators discriminate. NULL = raw space (legacy).
|
|
* NOTE: the ANN neighbor query still runs in RAW unit-vector space —
|
|
* centering is a rigid translation that ~preserves neighborhood
|
|
* MEMBERSHIP, so the index needs no rebuild; only the descriptor
|
|
* STATISTICS move to the centered frame (co-registration choice (b)).
|
|
* The eigen/covariance shape (axes, radius) is translation-invariant
|
|
* and therefore identical in either frame.
|
|
* Returns a malloc'd descriptor (free with engram_geo_free), or NULL on error
|
|
* (no seeds resolvable, OOM). */
|
|
GeoDescriptor* engram_geometry_descriptor(
|
|
EngramPagedStore* store, VIndex* vindex,
|
|
char** vids, int n_vids,
|
|
const char* const* seed_ids, size_t n_seeds,
|
|
const GeoParams* params,
|
|
const float* global_mean);
|
|
|
|
void engram_geo_free(GeoDescriptor* g);
|
|
|
|
/* ── M-INTEROCEPTION P3: drift-sensor primitive (descriptor displacement) ────
|
|
* Read-only. GROWTH vs CORRUPTION split of how far B drifted from baseline A.
|
|
* See engram_geometry.c for the honesty note on the missing SelfAnchor. */
|
|
typedef struct {
|
|
double centroid_sep; /* L2 distance between centroids (same frame) */
|
|
double centroid_cos; /* 1 - cosine(centroidA, centroidB) */
|
|
double radius_delta; /* |radiusA - radiusB| — neighborhood scale change */
|
|
double core_disp; /* mean radial displacement of the invariant core */
|
|
double periph_disp; /* mean radial displacement of the periphery */
|
|
int core_matched; /* # core members matched by id across A,B */
|
|
int periph_matched; /* # periphery members matched by id across A,B */
|
|
} GeoDisplacement;
|
|
|
|
void engram_geo_displacement(const GeoDescriptor* a, const GeoDescriptor* b,
|
|
double core_frac, GeoDisplacement* out);
|
|
|
|
/* ═══════════════════════════════════════════════════════════════════════════
|
|
* M10 — REIFICATION: densely co-wired relational neighborhoods crystallized into
|
|
* FIRST-CLASS, PERSISTED store records (design doc §2; memory 885f5945). This is
|
|
* NOT a cache — it is durable structure. A reified neighborhood is a real store
|
|
* NODE (node_type "Neighborhood") that survives restart, is loaded on boot, and
|
|
* EVOLVES via supersede+provenance when the pattern shifts. The geometry-priming
|
|
* HOT PATH reads these persisted records (never computes geometry on the
|
|
* activation path). Ad-hoc/transient geometries still use the on-the-fly
|
|
* engram_geometry_descriptor above.
|
|
*
|
|
* Two record types, both ordinary TLV store nodes (no new on-disk format):
|
|
* - "GeoMeanFrame" : the store-wide centering mean, persisted ONCE (emb = mean
|
|
* vector, id ENGRAM_GEO_MEANFRAME_ID). Referenced by every
|
|
* neighborhood so priming centers against the SAME true mean.
|
|
* - "Neighborhood" : one reified neighborhood. emb = the RAW centroid (prototype
|
|
* point, so it stays centroid-ANN-able; centered_centroid =
|
|
* emb - meanframe). metadata = the compact "GEO1" schema:
|
|
* hub id, meanframe ref, scalar shape (radius, total_variance,
|
|
* k_core, co_registration, n_embedded), axis EXTENTS (ellipsoid
|
|
* half-widths), and the MEMBER list {id -> membership, centrality,
|
|
* core}. Member links are also persisted as edges relation="member".
|
|
*
|
|
* v1 honest simplifications (documented; extensible without migration): axis
|
|
* DIRECTION vectors are not persisted (extents capture the ellipsoid scale; the
|
|
* directions are recomputable via the on-the-fly descriptor for viz/operators);
|
|
* with hebb potentiation ~0 on today's store the "hebb-weighted" degree reduces to
|
|
* AUTHORED edge weight, so detected neighborhoods currently reflect authored edges —
|
|
* the design is unchanged and self-correcting once hebb accrues.
|
|
* ═══════════════════════════════════════════════════════════════════════════ */
|
|
|
|
#define ENGRAM_GEO_NBHD_TYPE "Neighborhood"
|
|
#define ENGRAM_GEO_MEANFRAME_TYPE "GeoMeanFrame"
|
|
#define ENGRAM_GEO_MEANFRAME_ID "geo-meanframe" /* stable id of the singleton */
|
|
#define ENGRAM_GEO_NBHD_ID_PREFIX "nbhd-" /* id = nbhd-<hub>-<built_at> */
|
|
#define ENGRAM_GEO_MEMBER_RELATION "member"
|
|
|
|
typedef struct {
|
|
int min_weighted_degree; /* hub qualifies iff strong-edge weighted degree >= this
|
|
* (0 = no floor: just rank + take top max_neighborhoods) */
|
|
int max_neighborhoods; /* homeostatic budget cap (default 128) */
|
|
double cover_membership; /* skip a hub already a member (w>=this) of an accepted
|
|
* neighborhood — greedy non-redundant cover (default 0.5) */
|
|
int persist_member_edges; /* 1 = also write relation="member" edges (default 1) */
|
|
GeoParams descriptor; /* per-neighborhood params (top_axes may be 0 = skip eigensolve) */
|
|
} GeoReifyParams;
|
|
|
|
/* Defaults: min_weighted_degree=0, max_neighborhoods=128, cover_membership=0.5,
|
|
* persist_member_edges=1, descriptor = engram_geo_default_params but top_axes=4,
|
|
* max_members=256 (reified neighborhoods stay compact). */
|
|
void engram_geo_reify_default_params(GeoReifyParams* p);
|
|
|
|
/* WRITE PATH (offline / consolidation — NEVER the activation hot path).
|
|
* Detect dense hub neighborhoods on the hebb-weighted graph, compute each one's
|
|
* CENTERED descriptor ONCE against the true store-wide mean, and PERSIST them as
|
|
* first-class records: the GeoMeanFrame (once) + one Neighborhood node per detected
|
|
* neighborhood (+ member edges), superseding any prior same-hub record with
|
|
* provenance. Read-then-write over `store`. Returns #neighborhoods persisted, or <0.
|
|
* Skips existing Neighborhood/GeoMeanFrame nodes when detecting (idempotent re-reify). */
|
|
int engram_geo_reify_store(EngramPagedStore* store, VIndex* vindex,
|
|
char** vids, int n_vids,
|
|
const GeoReifyParams* params);
|
|
|
|
/* ── Resident loaded form of the persisted records (boot-time; READ-ONLY) ─────
|
|
* The durable Neighborhood/GeoMeanFrame records are the source of truth; this
|
|
* index is their LOADED form (like the resident node array is the loaded form of
|
|
* the node records, or adjacency the loaded form of edges). It never recomputes
|
|
* geometry — it parses. Build it by feeding the runtime's boot node scan, or in
|
|
* one pass with engram_geo_reify_load. */
|
|
typedef struct GeoReifyIndex GeoReifyIndex;
|
|
|
|
GeoReifyIndex* engram_geo_reify_index_new(void);
|
|
/* Feed one store node; if it is a Neighborhood or GeoMeanFrame record it is parsed
|
|
* and absorbed (else ignored). The node is BORROWED (copied as needed). 0/<0. */
|
|
int engram_geo_reify_index_add(GeoReifyIndex* ix, const StoreNode* n);
|
|
/* Build the member->neighborhood hash after all adds. Call once. 0/<0. */
|
|
int engram_geo_reify_index_finalize(GeoReifyIndex* ix);
|
|
/* One-pass convenience: scan the store and build the finalized index. NULL if the
|
|
* store holds no reified records. */
|
|
GeoReifyIndex* engram_geo_reify_load(EngramPagedStore* store);
|
|
|
|
/* A borrowed view of one persisted neighborhood (owned by the index). */
|
|
typedef struct {
|
|
const char* id;
|
|
const char* hub_id;
|
|
int n_members;
|
|
char* const* member_ids; /* parallel arrays, length n_members */
|
|
const double* member_w; /* membership in [0,1] */
|
|
double radius;
|
|
double co_registration;
|
|
int k_core;
|
|
int n_embedded;
|
|
} GeoNeighborhood;
|
|
|
|
/* HOT-PATH LOOKUP (no geometry compute): resolve the seed set to the best
|
|
* persisted neighborhood — the one with the greatest summed seed membership; on a
|
|
* miss (no seed is a member of any neighborhood) fall back to the centroid nearest
|
|
* the query embedding (centered by the loaded mean frame). q_emb may be NULL (then
|
|
* a miss returns NULL). Returns a BORROWED handle (do NOT free) or NULL. */
|
|
const GeoNeighborhood* engram_geo_reify_lookup(
|
|
const GeoReifyIndex* ix,
|
|
const char* const* seed_ids, size_t n_seeds,
|
|
const float* q_emb, int q_dim);
|
|
|
|
int engram_geo_reify_count(const GeoReifyIndex* ix);
|
|
const float* engram_geo_reify_mean(const GeoReifyIndex* ix, int* dim); /* loaded true mean or NULL */
|
|
void engram_geo_reify_index_free(GeoReifyIndex* ix);
|
|
|
|
#endif /* ENGRAM_GEOMETRY_H */
|