neuron-technologies/el
2
0
Fork 0
Code Issues 1 Pull Requests 6 Actions Packages Projects Releases 5 Wiki Activity

self-review 2026-05-27: semantic auto-linking on embed

Browse Source 
Add engram_auto_link_semantic(): when a node is embedded, scan all
embedded nodes for cosine sim >= 0.72 and create bidirectional
"semantic-similar" edges to the top-3 matches. Runs once at write
time rather than at every query, converting the expensive O(N) scan
from live activation into durable graph structure.

Fixes the core connectivity problem: 63 edges / 5364 nodes (0.012
edges/node). Verified: new Knowledge nodes now auto-link at sim 0.77–
0.95 with correct threshold discrimination. One checkpoint per insert
(not one per edge) bounds the persistence overhead.

Excludes InternalStateEvents and inbox/outbox transient nodes to keep
semantic graph clean.
This commit is contained in:
Will Anderson
2026-05-27 08:45:44 -05:00
parent ef1db34846
commit 8f922e68b3
2 changed files with 132 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engram/dist/engram
Vendored
BIN
View File
Binary file not shown.
lang/releases/v1.0.0-20260501/el_runtime.c
+132
View File
@@ -5987,6 +5987,7 @@ static int engram_load_binary(const char* path);
static void engram_embed_node(EngramNode* n);
static float* engram_embed_query(const char* text, uint32_t* dim_out);
static float engram_cosine_sim(const float* a, const float* b, uint32_t dim);
static void engram_auto_link_semantic(EngramNode* n);
static void engram_checkpoint(void);
static void engram_emit_ise_internal(const char* content, const char* label);
@@ -7585,6 +7586,8 @@ static void engram_embed_node(EngramNode* n) {
free(n->embedding);
n->embedding = vec;
n->embedding_dim = dim;
/* Auto-link: wire this node into the semantic graph. */
engram_auto_link_semantic(n);
}
/* ── Engram: cosine similarity ───────────────────────────────────────────── */
@@ -7601,6 +7604,135 @@ static float engram_cosine_sim(const float* a, const float* b, uint32_t dim) {
return (float)(denom > 1e-9 ? dot / denom : 0.0);
}
/* ── Engram: semantic auto-linking ─────────────────────────────────────────
* When a node is embedded, find the top-K most similar existing nodes above
* a cosine similarity threshold and create bidirectional "semantic-similar"
* edges. This builds graph connectivity organically: every new Knowledge,
* Memory, Belief, or Lesson node becomes reachable from its nearest semantic
* neighbors via BFS spreading activation, not just via the expensive O(N)
* cosine scan that previously ran at query time only.
*
* Design choices:
* K = 3 — enough connectivity without flooding the edge table
* threshold = 0.72 — empirically: below ~0.70 admits off-topic pairs
* bidirectional — spreading activation needs both directions to propagate
* skip ISEs — ephemeral; linking them pollutes semantic graph with noise
* one checkpoint after all edges — avoids N binary writes per insert
* guard against self-link and duplicate edges
*/
#define AUTOLINK_K 3
#define AUTOLINK_THRESHOLD 0.72f
static int engram_is_linkable_type(const EngramNode* n) {
if (!n->node_type) return 1; /* unknown type: allow */
if (strcmp(n->node_type, "InternalStateEvent") == 0) return 0;
/* Skip inbox/outbox/loop transient nodes by tag */
if (n->tags) {
if (strstr(n->tags, "soul-inbox")) return 0;
if (strstr(n->tags, "soul-outbox")) return 0;
if (strstr(n->tags, "loop-outcome")) return 0;
if (strstr(n->tags, "internal-state")) return 0;
}
return 1;
}
/* O(E) raw edge check — avoids EL value boxing. */
static int engram_has_edge_raw(const char* from, const char* to) {
EngramStore* g = engram_get();
for (int64_t i = 0; i < g->edge_count; i++) {
EngramEdge* e = &g->edges[i];
if (e->from_id && e->to_id &&
strcmp(e->from_id, from) == 0 &&
strcmp(e->to_id, to) == 0) return 1;
}
return 0;
}
static void engram_auto_link_semantic(EngramNode* n) {
if (!n || !n->id || !n->embedding || n->embedding_dim == 0) return;
if (!engram_is_linkable_type(n)) return;
EngramStore* g = engram_get();
if (!g || g->node_count < 10) return; /* too few nodes to link meaningfully */
typedef struct { int64_t idx; float sim; } SimEntry;
SimEntry top[AUTOLINK_K];
int top_count = 0;
for (int64_t i = 0; i < g->node_count; i++) {
EngramNode* other = &g->nodes[i];
if (other == n) continue; /* skip self */
if (!engram_is_linkable_type(other)) continue;
if (!other->embedding || other->embedding_dim != n->embedding_dim) continue;
float sim = engram_cosine_sim(n->embedding, other->embedding, n->embedding_dim);
if (sim < AUTOLINK_THRESHOLD) continue;
if (top_count < AUTOLINK_K) {
top[top_count++] = (SimEntry){i, sim};
} else {
/* Replace the weakest entry if this one is stronger. */
int min_j = 0;
for (int j = 1; j < AUTOLINK_K; j++)
if (top[j].sim < top[min_j].sim) min_j = j;
if (sim > top[min_j].sim)
top[min_j] = (SimEntry){i, sim};
}
}
if (top_count == 0) return;
int64_t now = engram_now_ms();
int added = 0;
for (int j = 0; j < top_count; j++) {
EngramNode* other = &g->nodes[top[j].idx];
if (!other->id || !*other->id) continue;
/* Forward: n → other */
if (!engram_has_edge_raw(n->id, other->id)) {
engram_grow_edges();
EngramEdge* e = &g->edges[g->edge_count];
memset(e, 0, sizeof(*e));
e->id = engram_new_id();
e->from_id = strdup(n->id);
e->to_id = strdup(other->id);
e->relation = strdup("semantic-similar");
e->metadata = strdup("{}");
e->weight = (double)top[j].sim;
e->confidence = 1.0;
e->created_at = now; e->updated_at = now;
e->layer_id = ENGRAM_LAYER_DEFAULT;
g->edge_count++;
added++;
}
/* Reverse: other → n */
if (!engram_has_edge_raw(other->id, n->id)) {
engram_grow_edges();
EngramEdge* e = &g->edges[g->edge_count];
memset(e, 0, sizeof(*e));
e->id = engram_new_id();
e->from_id = strdup(other->id);
e->to_id = strdup(n->id);
e->relation = strdup("semantic-similar");
e->metadata = strdup("{}");
e->weight = (double)top[j].sim;
e->confidence = 1.0;
e->created_at = now; e->updated_at = now;
e->layer_id = ENGRAM_LAYER_DEFAULT;
g->edge_count++;
added++;
}
}
/* One checkpoint covers all edges added in this call. */
if (added > 0) {
engram_checkpoint();
}
}
/* ── Engram: ML-KEM-1024 key management ─────────────────────────────────────*/
static int engram_keys_init(void) {