diff --git a/engram/dist/engram b/engram/dist/engram index e775904..01ea9d0 100755 Binary files a/engram/dist/engram and b/engram/dist/engram differ diff --git a/lang/releases/v1.0.0-20260501/el_runtime.c b/lang/releases/v1.0.0-20260501/el_runtime.c index f7f78f6..f4f605f 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.c +++ b/lang/releases/v1.0.0-20260501/el_runtime.c @@ -5454,8 +5454,12 @@ void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal, #define ENGRAM_DECAY_LAMBDA 0.693147 /* Two-layer activation constants. - * ENGRAM_WM_THRESHOLD: minimum background_activation for a node to be - * considered for working-memory promotion (layer 2 candidate gate). + * ENGRAM_WM_THRESHOLD: SUPERSEDED — defined here for legacy reference only. + * The actual per-call threshold is computed by engram_type_threshold() which + * returns per-node-type values (0.05 Safety/DharmaSelf, 0.15 Canonical, + * 0.25 Lesson, 0.30 Belief/Entity, 0.40 Note/Memory/Working). This constant + * is NOT used in engram_activate(); it matches the Canonical tier value only + * by coincidence. (2026-07-01 self-review: clarified stale doc) * ENGRAM_WM_DECAY: per-turn decay applied to working_memory_weight for * nodes NOT re-activated in the current turn (conversational thread * continuity: a node promoted in turn N persists with reduced weight @@ -5653,6 +5657,20 @@ typedef struct EngramLayer { int injectable; /* can be added/removed at runtime? */ } EngramLayer; +/* ID → index hash map. Open-addressing with linear probing. + * Slots hold a strdup'd key and the array index of that node. + * Tombstones (deleted entries) use key=ENGRAM_IDMAP_TOMB and idx=-1. + * Rebuild required after engram_forget (shift-delete changes all indices + * above the deleted position). */ +#define ENGRAM_IDMAP_TOMB ((char*)1) /* sentinel pointer, never dereferenced */ +#define ENGRAM_IDMAP_LOAD_NUM 3 /* grow when count*3 >= capacity*2 */ +#define ENGRAM_IDMAP_LOAD_DEN 2 + +typedef struct { + char* key; /* NULL = empty, ENGRAM_IDMAP_TOMB = deleted, else strdup'd */ + int64_t idx; +} EngramIdSlot; + typedef struct EngramStore { EngramNode* nodes; int64_t node_count; @@ -5668,6 +5686,22 @@ typedef struct EngramStore { EngramLayer* layers; size_t layer_count; size_t layer_capacity; + /* O(1) node-id lookup: open-addressing hash map over node IDs. + * Maintained in sync with the nodes array. Null until first use. */ + EngramIdSlot* id_map; + size_t id_map_cap; /* power-of-2 slot count */ + size_t id_map_used; /* live entries (excluding tombstones) */ + /* Per-node adjacency index: for each node i, adj_from[i] lists edges + * where nodes[i] is the 'from' end; adj_to[i] lists edges where it is + * the 'to' end. Both store edge indices into g->edges[]. + * Rebuilt lazily via engram_adj_rebuild() before any BFS call. Set + * adj_dirty=1 whenever an edge is added, deleted, or nodes shift. */ + int** adj_from; /* adj_from[node_idx] → int* array of edge indices */ + int* adj_from_len; + int** adj_to; + int* adj_to_len; + int adj_dirty; /* 1 = rebuild needed before next BFS */ + int64_t adj_node_count; /* node_count at time of last adj_rebuild */ } EngramStore; static EngramStore* engram_global = NULL; @@ -5800,18 +5834,245 @@ static int64_t engram_now_ms(void) { return (int64_t)tv.tv_sec * 1000LL + (int64_t)tv.tv_usec / 1000LL; } +/* Forward declaration: engram_find_node_index is defined after the id_map + * helpers but called here. Without this, C99 -Wimplicit-function-declaration + * treats the call as an implicit non-static declaration, then conflicts with + * the later `static` definition. (2026-07-01 self-review: pre-existing) */ +static int64_t engram_find_node_index(const char* id); + static EngramNode* engram_find_node(const char* id) { if (!id) return NULL; EngramStore* g = engram_get(); - for (int64_t i = 0; i < g->node_count; i++) { - if (g->nodes[i].id && strcmp(g->nodes[i].id, id) == 0) return &g->nodes[i]; - } + int64_t idx = engram_find_node_index(id); + if (idx >= 0) return &g->nodes[idx]; return NULL; } +/* ── ID hash map helpers ───────────────────────────────────────────────────── + * Open-addressing, linear-probing hash map. Keys are node-id C strings. + * Values are int64_t indices into g->nodes[]. + * + * Rules: + * - id_map is NULL until the first insertion (lazy init). + * - Capacity is always a power of two. + * - Load factor kept below 2/3: when used*3 >= cap*2, rehash to 2*cap. + * - Deletion uses ENGRAM_IDMAP_TOMB sentinels (key == (char*)1). + * - After engram_forget (shift-delete) the whole map is rebuilt from + * scratch because all indices above the deleted position change. + */ + +static uint64_t engram_id_hash(const char* s) { + /* FNV-1a 64-bit */ + uint64_t h = 14695981039346656037ULL; + while (*s) { h ^= (unsigned char)*s++; h *= 1099511628211ULL; } + return h; +} + +/* Allocate a zeroed id_map of `cap` slots (cap must be power-of-two). */ +static EngramIdSlot* engram_idmap_alloc(size_t cap) { + return calloc(cap, sizeof(EngramIdSlot)); +} + +/* Low-level insert (no rehash check, no free of existing). Used during + * rehash and initial build where we know the load is controlled. */ +static void engram_idmap_put_raw(EngramIdSlot* map, size_t cap, + char* key, int64_t idx) { + size_t mask = cap - 1; + size_t slot = (size_t)engram_id_hash(key) & mask; + while (map[slot].key != NULL && map[slot].key != ENGRAM_IDMAP_TOMB) { + slot = (slot + 1) & mask; + } + map[slot].key = key; + map[slot].idx = idx; +} + +/* Insert or update id → idx into the store's id_map. Rehashes if needed. */ +static void engram_idmap_put(EngramStore* g, const char* id, int64_t idx) { + if (!id || !*id) return; + /* Lazy init */ + if (!g->id_map) { + g->id_map_cap = 64; + g->id_map_used = 0; + g->id_map = engram_idmap_alloc(g->id_map_cap); + if (!g->id_map) return; /* OOM: fall back to linear scan */ + } + /* Rehash if load factor would exceed 2/3 */ + if ((g->id_map_used + 1) * ENGRAM_IDMAP_LOAD_NUM >= g->id_map_cap * ENGRAM_IDMAP_LOAD_DEN) { + size_t new_cap = g->id_map_cap * 2; + EngramIdSlot* new_map = engram_idmap_alloc(new_cap); + if (!new_map) return; /* OOM: keep old map, insert below */ + for (size_t s = 0; s < g->id_map_cap; s++) { + if (g->id_map[s].key && g->id_map[s].key != ENGRAM_IDMAP_TOMB) { + engram_idmap_put_raw(new_map, new_cap, + g->id_map[s].key, g->id_map[s].idx); + } + } + free(g->id_map); + g->id_map = new_map; + g->id_map_cap = new_cap; + } + /* Probe for existing key or empty/tomb slot */ + size_t mask = g->id_map_cap - 1; + size_t slot = (size_t)engram_id_hash(id) & mask; + size_t tomb_slot = SIZE_MAX; + while (g->id_map[slot].key != NULL) { + if (g->id_map[slot].key == ENGRAM_IDMAP_TOMB) { + if (tomb_slot == SIZE_MAX) tomb_slot = slot; + } else if (strcmp(g->id_map[slot].key, id) == 0) { + g->id_map[slot].idx = idx; /* update */ + return; + } + slot = (slot + 1) & mask; + } + /* Use tombstone slot if found (avoids growing used count unnecessarily) */ + if (tomb_slot != SIZE_MAX) slot = tomb_slot; + g->id_map[slot].key = el_strdup(id); + g->id_map[slot].idx = idx; + g->id_map_used++; +} + +/* Look up id in the store's id_map. Returns index or -1 if not found. */ +static int64_t engram_idmap_get(const EngramStore* g, const char* id) { + if (!g->id_map || !id || !*id) return -1; + size_t mask = g->id_map_cap - 1; + size_t slot = (size_t)engram_id_hash(id) & mask; + while (g->id_map[slot].key != NULL) { + if (g->id_map[slot].key != ENGRAM_IDMAP_TOMB && + strcmp(g->id_map[slot].key, id) == 0) { + return g->id_map[slot].idx; + } + slot = (slot + 1) & mask; + } + return -1; +} + +/* Free and null-out the id_map (called on full reset). */ +static void engram_idmap_free(EngramStore* g) { + if (!g->id_map) return; + for (size_t s = 0; s < g->id_map_cap; s++) { + if (g->id_map[s].key && g->id_map[s].key != ENGRAM_IDMAP_TOMB) + free(g->id_map[s].key); + } + free(g->id_map); + g->id_map = NULL; + g->id_map_cap = 0; + g->id_map_used = 0; +} + +/* Rebuild id_map from scratch after a structural change (e.g. shift-delete). + * Frees old map and constructs a fresh one. */ +static void engram_idmap_rebuild(EngramStore* g) { + engram_idmap_free(g); + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].id && *g->nodes[i].id) + engram_idmap_put(g, g->nodes[i].id, i); + } +} + +/* ── Adjacency index helpers ───────────────────────────────────────────────── + * Per-node adjacency lists: adj_from[i] holds edge indices where + * g->edges[ei].from_id == g->nodes[i].id, adj_to[i] for the 'to' side. + * BFS uses these instead of scanning all edges on every hop. + * Called once per activation call when adj_dirty != 0. + */ +static void engram_adj_free(EngramStore* g) { + int64_t old_nc = g->adj_node_count; + if (g->adj_from) { + for (int64_t i = 0; i < old_nc; i++) free(g->adj_from[i]); + free(g->adj_from); g->adj_from = NULL; + free(g->adj_from_len); g->adj_from_len = NULL; + } + if (g->adj_to) { + for (int64_t i = 0; i < old_nc; i++) free(g->adj_to[i]); + free(g->adj_to); g->adj_to = NULL; + free(g->adj_to_len); g->adj_to_len = NULL; + } + g->adj_node_count = 0; + g->adj_dirty = 1; +} + +static void engram_adj_rebuild(EngramStore* g) { + /* Free old adjacency arrays */ + if (g->adj_from) { + /* Use adj_node_count (count at build time) not current node_count — + * nodes may have been added since the last rebuild, and adj arrays + * only have adj_node_count entries. */ + int64_t old_nc = g->adj_node_count; + for (int64_t i = 0; i < old_nc; i++) { + free(g->adj_from[i]); free(g->adj_to[i]); + } + free(g->adj_from); free(g->adj_from_len); + free(g->adj_to); free(g->adj_to_len); + } + g->adj_from = NULL; g->adj_from_len = NULL; + g->adj_to = NULL; g->adj_to_len = NULL; + g->adj_node_count = 0; + if (g->node_count == 0) { g->adj_dirty = 0; return; } + + /* Count degree per node */ + int* from_cnt = calloc((size_t)g->node_count, sizeof(int)); + int* to_cnt = calloc((size_t)g->node_count, sizeof(int)); + if (!from_cnt || !to_cnt) { free(from_cnt); free(to_cnt); return; } + for (int64_t ei = 0; ei < g->edge_count; ei++) { + EngramEdge* e = &g->edges[ei]; + if (!e->from_id || !e->to_id) continue; + int64_t fi = engram_idmap_get(g, e->from_id); + int64_t ti = engram_idmap_get(g, e->to_id); + if (fi >= 0) from_cnt[fi]++; + if (ti >= 0) to_cnt[ti]++; + } + /* Allocate per-node arrays */ + g->adj_from = calloc((size_t)g->node_count, sizeof(int*)); + g->adj_from_len = calloc((size_t)g->node_count, sizeof(int)); + g->adj_to = calloc((size_t)g->node_count, sizeof(int*)); + g->adj_to_len = calloc((size_t)g->node_count, sizeof(int)); + if (!g->adj_from || !g->adj_from_len || !g->adj_to || !g->adj_to_len) { + free(from_cnt); free(to_cnt); + free(g->adj_from); g->adj_from = NULL; + free(g->adj_from_len); g->adj_from_len = NULL; + free(g->adj_to); g->adj_to = NULL; + free(g->adj_to_len); g->adj_to_len = NULL; + return; + } + for (int64_t i = 0; i < g->node_count; i++) { + if (from_cnt[i] > 0) + g->adj_from[i] = malloc((size_t)from_cnt[i] * sizeof(int)); + if (to_cnt[i] > 0) + g->adj_to[i] = malloc((size_t)to_cnt[i] * sizeof(int)); + } + /* Fill */ + int* from_pos = calloc((size_t)g->node_count, sizeof(int)); + int* to_pos = calloc((size_t)g->node_count, sizeof(int)); + if (!from_pos || !to_pos) { + free(from_cnt); free(to_cnt); free(from_pos); free(to_pos); return; + } + for (int64_t ei = 0; ei < g->edge_count; ei++) { + EngramEdge* e = &g->edges[ei]; + if (!e->from_id || !e->to_id) continue; + int64_t fi = engram_idmap_get(g, e->from_id); + int64_t ti = engram_idmap_get(g, e->to_id); + if (fi >= 0 && g->adj_from[fi]) + g->adj_from[fi][from_pos[fi]++] = (int)ei; + if (ti >= 0 && g->adj_to[ti]) + g->adj_to[ti][to_pos[ti]++] = (int)ei; + } + /* Copy counts */ + for (int64_t i = 0; i < g->node_count; i++) { + g->adj_from_len[i] = from_cnt[i]; + g->adj_to_len[i] = to_cnt[i]; + } + free(from_cnt); free(to_cnt); free(from_pos); free(to_pos); + g->adj_node_count = g->node_count; + g->adj_dirty = 0; +} + static int64_t engram_find_node_index(const char* id) { if (!id) return -1; EngramStore* g = engram_get(); + /* Fast O(1) path via id_map */ + int64_t fast = engram_idmap_get(g, id); + if (fast >= 0) return fast; + /* Fallback linear scan (id_map not yet built or OOM) */ for (int64_t i = 0; i < g->node_count; i++) { if (g->nodes[i].id && strcmp(g->nodes[i].id, id) == 0) return i; } @@ -5922,7 +6183,10 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) { n->created_at = now; n->updated_at = now; n->layer_id = ENGRAM_LAYER_DEFAULT; + int64_t new_idx = g->node_count; g->node_count++; + engram_idmap_put(g, n->id, new_idx); + g->adj_dirty = 1; return el_wrap_str(el_strdup(n->id)); } @@ -5958,7 +6222,10 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label, n->created_at = now; n->updated_at = now; n->layer_id = ENGRAM_LAYER_DEFAULT; + int64_t new_idx_full = g->node_count; g->node_count++; + engram_idmap_put(g, n->id, new_idx_full); + g->adj_dirty = 1; return el_wrap_str(el_strdup(n->id)); } @@ -6025,7 +6292,10 @@ el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t labe if (lid < 0) lid = (int64_t)ENGRAM_LAYER_DEFAULT; if (!engram_find_layer((uint32_t)lid)) lid = (int64_t)ENGRAM_LAYER_DEFAULT; n->layer_id = (uint32_t)lid; + int64_t new_idx_layered = g->node_count; g->node_count++; + engram_idmap_put(g, n->id, new_idx_layered); + g->adj_dirty = 1; return el_wrap_str(el_strdup(n->id)); } @@ -6188,6 +6458,10 @@ void engram_forget(el_val_t node_id) { } } g->edge_count = w; + /* Shift-delete changed all indices above the removed position. + * Rebuild id_map and mark adjacency index dirty. */ + engram_idmap_rebuild(g); + engram_adj_free(g); } el_val_t engram_node_count(void) { @@ -6291,6 +6565,7 @@ void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t e->last_fired = 0; e->layer_id = ENGRAM_LAYER_DEFAULT; g->edge_count++; + g->adj_dirty = 1; } el_val_t engram_edge_between(el_val_t from_id, el_val_t to_id) { @@ -6550,6 +6825,12 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { el_val_t out = el_list_empty(); if (!q || g->node_count == 0) return out; + /* Rebuild adjacency index if the edge/node topology changed since the + * last activation call. This is O(E) one-time cost vs O(E) per BFS step + * without the index. On a 40K-edge graph this drops BFS from O(frontier + * * E) to O(frontier * avg_degree). (2026-07-01 self-review) */ + if (g->adj_dirty || !g->adj_from) engram_adj_rebuild(g); + int64_t now_ms = engram_now_ms(); /* Per-node layer-1 tracking. */ @@ -6612,22 +6893,47 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { while (fhead < ftail) { Frontier f = fr[fhead++]; if (f.hops >= max_depth) continue; - const char* cur_id = g->nodes[f.idx].id; - for (int64_t ei = 0; ei < g->edge_count; ei++) { + int64_t cur = f.idx; + int64_t new_hops = f.hops + 1; + /* Use adjacency index: iterate only edges incident to `cur`. + * adj_from[cur] holds edge indices where cur is the 'from' node; + * adj_to[cur] holds edge indices where cur is the 'to' node. + * If adj index is unavailable (OOM during rebuild), fall back to + * full edge scan so activation is never silently wrong. */ + int use_adj = (g->adj_from != NULL && g->adj_to != NULL); + int from_len = use_adj ? g->adj_from_len[cur] : 0; + int to_len = use_adj ? g->adj_to_len[cur] : 0; + int edge_scan_count = use_adj ? (from_len + to_len) : (int)g->edge_count; + for (int scan_i = 0; scan_i < edge_scan_count; scan_i++) { + int64_t ei; + int64_t oi; + if (use_adj) { + ei = (scan_i < from_len) + ? g->adj_from[cur][scan_i] + : g->adj_to[cur][scan_i - from_len]; + EngramEdge* e = &g->edges[ei]; + oi = (scan_i < from_len) + ? engram_idmap_get(g, e->to_id) + : engram_idmap_get(g, e->from_id); + } else { + /* Fallback: linear scan */ + ei = scan_i; + EngramEdge* e = &g->edges[ei]; + const char* other = NULL; + const char* cur_id = g->nodes[cur].id; + if (e->from_id && strcmp(e->from_id, cur_id) == 0) other = e->to_id; + else if (e->to_id && strcmp(e->to_id, cur_id) == 0) other = e->from_id; + else continue; + oi = engram_find_node_index(other); + } + if (oi < 0 || oi >= g->node_count) continue; EngramEdge* e = &g->edges[ei]; - const char* other = NULL; - if (e->from_id && strcmp(e->from_id, cur_id) == 0) other = e->to_id; - else if (e->to_id && strcmp(e->to_id, cur_id) == 0) other = e->from_id; - else continue; - int64_t oi = engram_find_node_index(other); - if (oi < 0) continue; EngramNode* on = &g->nodes[oi]; double tbonus = engram_temporal_proximity_bonus(on->created_at, seed_epoch); double tdecay = engram_temporal_decay(on, now_ms); double dampen = engram_activation_dampen(on); double new_act = f.act * e->weight * SPREAD_DECAY * (1.0 + tbonus) * tdecay * dampen; - int64_t new_hops = f.hops + 1; if (!reached[oi] || new_act > best_bg[oi]) { best_bg[oi] = new_act; best_hops[oi] = new_hops; @@ -7080,6 +7386,8 @@ el_val_t engram_load(el_val_t path) { free(g->edges[i].relation); free(g->edges[i].metadata); } g->edge_count = 0; + engram_idmap_free(g); + engram_adj_free(g); /* Walk nodes array */ const char* nodes_p = json_find_key(data, "nodes"); @@ -7126,7 +7434,9 @@ el_val_t engram_load(el_val_t path) { } else { nn->layer_id = ENGRAM_LAYER_DEFAULT; } + int64_t load_idx = g->node_count; g->node_count++; + if (nn->id && *nn->id) engram_idmap_put(g, nn->id, load_idx); free(obj); nodes_p = end; nodes_p = eg_skip_ws(nodes_p); @@ -7134,6 +7444,7 @@ el_val_t engram_load(el_val_t path) { } } } + g->adj_dirty = 1; /* Walk edges array */ const char* edges_p = json_find_key(data, "edges"); if (edges_p) { @@ -7307,8 +7618,11 @@ el_val_t engram_load_merge(el_val_t path) { } else { nn->layer_id = ENGRAM_LAYER_DEFAULT; } + int64_t merge_idx = g->node_count; g->node_count++; added_nodes++; + if (nn->id && *nn->id) engram_idmap_put(g, nn->id, merge_idx); + g->adj_dirty = 1; } free(obj); nodes_p = end; @@ -7400,6 +7714,34 @@ el_val_t engram_get_node_json(el_val_t id) { return el_wrap_str(b.buf); } +/* engram_get_node_by_label — find the first node whose label field exactly + * matches the given string. Returns the node as a JSON object string, or "{}" + * if no match is found. + * + * Used by chat.el to retrieve well-known nodes (e.g. "conv:history", + * "session:summary") by their stable label rather than by ID, which is immune + * to vector index drift across restarts. + * + * Exact match (strcmp, not istr_contains) because labels like "conv:history" + * must not collide with nodes whose content happens to contain that substring. + * + * Added 2026-07-01 self-review: was called in chat.el but never defined, + * causing build failure since June 30. */ +el_val_t engram_get_node_by_label(el_val_t label) { + const char* lbl = EL_CSTR(label); + if (!lbl || !*lbl) return el_wrap_str(el_strdup("{}")); + EngramStore* g = engram_get(); + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + if (n->label && strcmp(n->label, lbl) == 0) { + JsonBuf b; jb_init(&b); + engram_emit_node_json(&b, n); + return el_wrap_str(b.buf); + } + } + return el_wrap_str(el_strdup("{}")); +} + el_val_t engram_search_json(el_val_t query, el_val_t limit) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query);