fix(release-snapshot): fflush stdout in println and update Knowledge threshold

lang/releases/v1.0.0-20260501/el_runtime.c

@@ -5454,8 +5454,12 @@ void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal,
 #define ENGRAM_DECAY_LAMBDA  0.693147
 
 /* Two-layer activation constants.
- * ENGRAM_WM_THRESHOLD: minimum background_activation for a node to be
- *   considered for working-memory promotion (layer 2 candidate gate).
+ * ENGRAM_WM_THRESHOLD: SUPERSEDED — defined here for legacy reference only.
+ *   The actual per-call threshold is computed by engram_type_threshold() which
+ *   returns per-node-type values (0.05 Safety/DharmaSelf, 0.15 Canonical,
+ *   0.25 Lesson, 0.30 Belief/Entity, 0.40 Note/Memory/Working). This constant
+ *   is NOT used in engram_activate(); it matches the Canonical tier value only
+ *   by coincidence. (2026-07-01 self-review: clarified stale doc)
  * ENGRAM_WM_DECAY: per-turn decay applied to working_memory_weight for
  *   nodes NOT re-activated in the current turn (conversational thread
  *   continuity: a node promoted in turn N persists with reduced weight
@@ -5653,6 +5657,20 @@ typedef struct EngramLayer {
     int      injectable;           /* can be added/removed at runtime? */
 } EngramLayer;
 
+/* ID → index hash map. Open-addressing with linear probing.
+ * Slots hold a strdup'd key and the array index of that node.
+ * Tombstones (deleted entries) use key=ENGRAM_IDMAP_TOMB and idx=-1.
+ * Rebuild required after engram_forget (shift-delete changes all indices
+ * above the deleted position). */
+#define ENGRAM_IDMAP_TOMB ((char*)1)   /* sentinel pointer, never dereferenced */
+#define ENGRAM_IDMAP_LOAD_NUM 3        /* grow when count*3 >= capacity*2     */
+#define ENGRAM_IDMAP_LOAD_DEN 2
+
+typedef struct {
+    char*   key;   /* NULL = empty, ENGRAM_IDMAP_TOMB = deleted, else strdup'd */
+    int64_t idx;
+} EngramIdSlot;
+
 typedef struct EngramStore {
     EngramNode*  nodes;
     int64_t      node_count;
@@ -5668,6 +5686,22 @@ typedef struct EngramStore {
     EngramLayer* layers;
     size_t       layer_count;
     size_t       layer_capacity;
+    /* O(1) node-id lookup: open-addressing hash map over node IDs.
+     * Maintained in sync with the nodes array. Null until first use. */
+    EngramIdSlot* id_map;
+    size_t        id_map_cap;   /* power-of-2 slot count */
+    size_t        id_map_used;  /* live entries (excluding tombstones) */
+    /* Per-node adjacency index: for each node i, adj_from[i] lists edges
+     * where nodes[i] is the 'from' end; adj_to[i] lists edges where it is
+     * the 'to' end. Both store edge indices into g->edges[].
+     * Rebuilt lazily via engram_adj_rebuild() before any BFS call. Set
+     * adj_dirty=1 whenever an edge is added, deleted, or nodes shift. */
+    int**    adj_from;      /* adj_from[node_idx] → int* array of edge indices */
+    int*     adj_from_len;
+    int**    adj_to;
+    int*     adj_to_len;
+    int      adj_dirty;     /* 1 = rebuild needed before next BFS */
+    int64_t  adj_node_count; /* node_count at time of last adj_rebuild */
 } EngramStore;
 
 static EngramStore* engram_global = NULL;
@@ -5800,18 +5834,245 @@ static int64_t engram_now_ms(void) {
     return (int64_t)tv.tv_sec * 1000LL + (int64_t)tv.tv_usec / 1000LL;
 }
 
+/* Forward declaration: engram_find_node_index is defined after the id_map
+ * helpers but called here. Without this, C99 -Wimplicit-function-declaration
+ * treats the call as an implicit non-static declaration, then conflicts with
+ * the later `static` definition. (2026-07-01 self-review: pre-existing) */
+static int64_t engram_find_node_index(const char* id);
+
 static EngramNode* engram_find_node(const char* id) {
     if (!id) return NULL;
     EngramStore* g = engram_get();
-    for (int64_t i = 0; i < g->node_count; i++) {
-        if (g->nodes[i].id && strcmp(g->nodes[i].id, id) == 0) return &g->nodes[i];
-    }
+    int64_t idx = engram_find_node_index(id);
+    if (idx >= 0) return &g->nodes[idx];
     return NULL;
 }
 
+/* ── ID hash map helpers ─────────────────────────────────────────────────────
+ * Open-addressing, linear-probing hash map.  Keys are node-id C strings.
+ * Values are int64_t indices into g->nodes[].
+ *
+ * Rules:
+ *   - id_map is NULL until the first insertion (lazy init).
+ *   - Capacity is always a power of two.
+ *   - Load factor kept below 2/3: when used*3 >= cap*2, rehash to 2*cap.
+ *   - Deletion uses ENGRAM_IDMAP_TOMB sentinels (key == (char*)1).
+ *   - After engram_forget (shift-delete) the whole map is rebuilt from
+ *     scratch because all indices above the deleted position change.
+ */
+
+static uint64_t engram_id_hash(const char* s) {
+    /* FNV-1a 64-bit */
+    uint64_t h = 14695981039346656037ULL;
+    while (*s) { h ^= (unsigned char)*s++; h *= 1099511628211ULL; }
+    return h;
+}
+
+/* Allocate a zeroed id_map of `cap` slots (cap must be power-of-two). */
+static EngramIdSlot* engram_idmap_alloc(size_t cap) {
+    return calloc(cap, sizeof(EngramIdSlot));
+}
+
+/* Low-level insert (no rehash check, no free of existing). Used during
+ * rehash and initial build where we know the load is controlled. */
+static void engram_idmap_put_raw(EngramIdSlot* map, size_t cap,
+                                 char* key, int64_t idx) {
+    size_t mask = cap - 1;
+    size_t slot = (size_t)engram_id_hash(key) & mask;
+    while (map[slot].key != NULL && map[slot].key != ENGRAM_IDMAP_TOMB) {
+        slot = (slot + 1) & mask;
+    }
+    map[slot].key = key;
+    map[slot].idx = idx;
+}
+
+/* Insert or update id → idx into the store's id_map. Rehashes if needed. */
+static void engram_idmap_put(EngramStore* g, const char* id, int64_t idx) {
+    if (!id || !*id) return;
+    /* Lazy init */
+    if (!g->id_map) {
+        g->id_map_cap  = 64;
+        g->id_map_used = 0;
+        g->id_map = engram_idmap_alloc(g->id_map_cap);
+        if (!g->id_map) return; /* OOM: fall back to linear scan */
+    }
+    /* Rehash if load factor would exceed 2/3 */
+    if ((g->id_map_used + 1) * ENGRAM_IDMAP_LOAD_NUM >= g->id_map_cap * ENGRAM_IDMAP_LOAD_DEN) {
+        size_t new_cap = g->id_map_cap * 2;
+        EngramIdSlot* new_map = engram_idmap_alloc(new_cap);
+        if (!new_map) return; /* OOM: keep old map, insert below */
+        for (size_t s = 0; s < g->id_map_cap; s++) {
+            if (g->id_map[s].key && g->id_map[s].key != ENGRAM_IDMAP_TOMB) {
+                engram_idmap_put_raw(new_map, new_cap,
+                                     g->id_map[s].key, g->id_map[s].idx);
+            }
+        }
+        free(g->id_map);
+        g->id_map     = new_map;
+        g->id_map_cap = new_cap;
+    }
+    /* Probe for existing key or empty/tomb slot */
+    size_t mask = g->id_map_cap - 1;
+    size_t slot = (size_t)engram_id_hash(id) & mask;
+    size_t tomb_slot = SIZE_MAX;
+    while (g->id_map[slot].key != NULL) {
+        if (g->id_map[slot].key == ENGRAM_IDMAP_TOMB) {
+            if (tomb_slot == SIZE_MAX) tomb_slot = slot;
+        } else if (strcmp(g->id_map[slot].key, id) == 0) {
+            g->id_map[slot].idx = idx; /* update */
+            return;
+        }
+        slot = (slot + 1) & mask;
+    }
+    /* Use tombstone slot if found (avoids growing used count unnecessarily) */
+    if (tomb_slot != SIZE_MAX) slot = tomb_slot;
+    g->id_map[slot].key = el_strdup(id);
+    g->id_map[slot].idx = idx;
+    g->id_map_used++;
+}
+
+/* Look up id in the store's id_map. Returns index or -1 if not found. */
+static int64_t engram_idmap_get(const EngramStore* g, const char* id) {
+    if (!g->id_map || !id || !*id) return -1;
+    size_t mask = g->id_map_cap - 1;
+    size_t slot = (size_t)engram_id_hash(id) & mask;
+    while (g->id_map[slot].key != NULL) {
+        if (g->id_map[slot].key != ENGRAM_IDMAP_TOMB &&
+            strcmp(g->id_map[slot].key, id) == 0) {
+            return g->id_map[slot].idx;
+        }
+        slot = (slot + 1) & mask;
+    }
+    return -1;
+}
+
+/* Free and null-out the id_map (called on full reset). */
+static void engram_idmap_free(EngramStore* g) {
+    if (!g->id_map) return;
+    for (size_t s = 0; s < g->id_map_cap; s++) {
+        if (g->id_map[s].key && g->id_map[s].key != ENGRAM_IDMAP_TOMB)
+            free(g->id_map[s].key);
+    }
+    free(g->id_map);
+    g->id_map     = NULL;
+    g->id_map_cap = 0;
+    g->id_map_used = 0;
+}
+
+/* Rebuild id_map from scratch after a structural change (e.g. shift-delete).
+ * Frees old map and constructs a fresh one. */
+static void engram_idmap_rebuild(EngramStore* g) {
+    engram_idmap_free(g);
+    for (int64_t i = 0; i < g->node_count; i++) {
+        if (g->nodes[i].id && *g->nodes[i].id)
+            engram_idmap_put(g, g->nodes[i].id, i);
+    }
+}
+
+/* ── Adjacency index helpers ─────────────────────────────────────────────────
+ * Per-node adjacency lists: adj_from[i] holds edge indices where
+ * g->edges[ei].from_id == g->nodes[i].id, adj_to[i] for the 'to' side.
+ * BFS uses these instead of scanning all edges on every hop.
+ * Called once per activation call when adj_dirty != 0.
+ */
+static void engram_adj_free(EngramStore* g) {
+    int64_t old_nc = g->adj_node_count;
+    if (g->adj_from) {
+        for (int64_t i = 0; i < old_nc; i++) free(g->adj_from[i]);
+        free(g->adj_from); g->adj_from = NULL;
+        free(g->adj_from_len); g->adj_from_len = NULL;
+    }
+    if (g->adj_to) {
+        for (int64_t i = 0; i < old_nc; i++) free(g->adj_to[i]);
+        free(g->adj_to); g->adj_to = NULL;
+        free(g->adj_to_len); g->adj_to_len = NULL;
+    }
+    g->adj_node_count = 0;
+    g->adj_dirty = 1;
+}
+
+static void engram_adj_rebuild(EngramStore* g) {
+    /* Free old adjacency arrays */
+    if (g->adj_from) {
+        /* Use adj_node_count (count at build time) not current node_count —
+         * nodes may have been added since the last rebuild, and adj arrays
+         * only have adj_node_count entries. */
+        int64_t old_nc = g->adj_node_count;
+        for (int64_t i = 0; i < old_nc; i++) {
+            free(g->adj_from[i]); free(g->adj_to[i]);
+        }
+        free(g->adj_from); free(g->adj_from_len);
+        free(g->adj_to);   free(g->adj_to_len);
+    }
+    g->adj_from     = NULL; g->adj_from_len = NULL;
+    g->adj_to       = NULL; g->adj_to_len   = NULL;
+    g->adj_node_count = 0;
+    if (g->node_count == 0) { g->adj_dirty = 0; return; }
+
+    /* Count degree per node */
+    int* from_cnt = calloc((size_t)g->node_count, sizeof(int));
+    int* to_cnt   = calloc((size_t)g->node_count, sizeof(int));
+    if (!from_cnt || !to_cnt) { free(from_cnt); free(to_cnt); return; }
+    for (int64_t ei = 0; ei < g->edge_count; ei++) {
+        EngramEdge* e = &g->edges[ei];
+        if (!e->from_id || !e->to_id) continue;
+        int64_t fi = engram_idmap_get(g, e->from_id);
+        int64_t ti = engram_idmap_get(g, e->to_id);
+        if (fi >= 0) from_cnt[fi]++;
+        if (ti >= 0) to_cnt[ti]++;
+    }
+    /* Allocate per-node arrays */
+    g->adj_from     = calloc((size_t)g->node_count, sizeof(int*));
+    g->adj_from_len = calloc((size_t)g->node_count, sizeof(int));
+    g->adj_to       = calloc((size_t)g->node_count, sizeof(int*));
+    g->adj_to_len   = calloc((size_t)g->node_count, sizeof(int));
+    if (!g->adj_from || !g->adj_from_len || !g->adj_to || !g->adj_to_len) {
+        free(from_cnt); free(to_cnt);
+        free(g->adj_from); g->adj_from = NULL;
+        free(g->adj_from_len); g->adj_from_len = NULL;
+        free(g->adj_to); g->adj_to = NULL;
+        free(g->adj_to_len); g->adj_to_len = NULL;
+        return;
+    }
+    for (int64_t i = 0; i < g->node_count; i++) {
+        if (from_cnt[i] > 0)
+            g->adj_from[i] = malloc((size_t)from_cnt[i] * sizeof(int));
+        if (to_cnt[i] > 0)
+            g->adj_to[i]   = malloc((size_t)to_cnt[i]   * sizeof(int));
+    }
+    /* Fill */
+    int* from_pos = calloc((size_t)g->node_count, sizeof(int));
+    int* to_pos   = calloc((size_t)g->node_count, sizeof(int));
+    if (!from_pos || !to_pos) {
+        free(from_cnt); free(to_cnt); free(from_pos); free(to_pos); return;
+    }
+    for (int64_t ei = 0; ei < g->edge_count; ei++) {
+        EngramEdge* e = &g->edges[ei];
+        if (!e->from_id || !e->to_id) continue;
+        int64_t fi = engram_idmap_get(g, e->from_id);
+        int64_t ti = engram_idmap_get(g, e->to_id);
+        if (fi >= 0 && g->adj_from[fi])
+            g->adj_from[fi][from_pos[fi]++] = (int)ei;
+        if (ti >= 0 && g->adj_to[ti])
+            g->adj_to[ti][to_pos[ti]++] = (int)ei;
+    }
+    /* Copy counts */
+    for (int64_t i = 0; i < g->node_count; i++) {
+        g->adj_from_len[i] = from_cnt[i];
+        g->adj_to_len[i]   = to_cnt[i];
+    }
+    free(from_cnt); free(to_cnt); free(from_pos); free(to_pos);
+    g->adj_node_count = g->node_count;
+    g->adj_dirty = 0;
+}
+
 static int64_t engram_find_node_index(const char* id) {
     if (!id) return -1;
     EngramStore* g = engram_get();
+    /* Fast O(1) path via id_map */
+    int64_t fast = engram_idmap_get(g, id);
+    if (fast >= 0) return fast;
+    /* Fallback linear scan (id_map not yet built or OOM) */
     for (int64_t i = 0; i < g->node_count; i++) {
         if (g->nodes[i].id && strcmp(g->nodes[i].id, id) == 0) return i;
     }
@@ -5922,7 +6183,10 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) {
     n->created_at = now;
     n->updated_at = now;
     n->layer_id   = ENGRAM_LAYER_DEFAULT;
+    int64_t new_idx = g->node_count;
     g->node_count++;
+    engram_idmap_put(g, n->id, new_idx);
+    g->adj_dirty = 1;
     return el_wrap_str(el_strdup(n->id));
 }
 
@@ -5958,7 +6222,10 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
     n->created_at = now;
     n->updated_at = now;
     n->layer_id   = ENGRAM_LAYER_DEFAULT;
+    int64_t new_idx_full = g->node_count;
     g->node_count++;
+    engram_idmap_put(g, n->id, new_idx_full);
+    g->adj_dirty = 1;
     return el_wrap_str(el_strdup(n->id));
 }
 
@@ -6025,7 +6292,10 @@ el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t labe
     if (lid < 0) lid = (int64_t)ENGRAM_LAYER_DEFAULT;
     if (!engram_find_layer((uint32_t)lid)) lid = (int64_t)ENGRAM_LAYER_DEFAULT;
     n->layer_id = (uint32_t)lid;
+    int64_t new_idx_layered = g->node_count;
     g->node_count++;
+    engram_idmap_put(g, n->id, new_idx_layered);
+    g->adj_dirty = 1;
     return el_wrap_str(el_strdup(n->id));
 }
 
@@ -6188,6 +6458,10 @@ void engram_forget(el_val_t node_id) {
         }
     }
     g->edge_count = w;
+    /* Shift-delete changed all indices above the removed position.
+     * Rebuild id_map and mark adjacency index dirty. */
+    engram_idmap_rebuild(g);
+    engram_adj_free(g);
 }
 
 el_val_t engram_node_count(void) {
@@ -6291,6 +6565,7 @@ void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t
     e->last_fired = 0;
     e->layer_id   = ENGRAM_LAYER_DEFAULT;
     g->edge_count++;
+    g->adj_dirty = 1;
 }
 
 el_val_t engram_edge_between(el_val_t from_id, el_val_t to_id) {
@@ -6550,6 +6825,12 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
     el_val_t out = el_list_empty();
     if (!q || g->node_count == 0) return out;
 
+    /* Rebuild adjacency index if the edge/node topology changed since the
+     * last activation call. This is O(E) one-time cost vs O(E) per BFS step
+     * without the index. On a 40K-edge graph this drops BFS from O(frontier
+     * * E) to O(frontier * avg_degree). (2026-07-01 self-review) */
+    if (g->adj_dirty || !g->adj_from) engram_adj_rebuild(g);
+
     int64_t now_ms = engram_now_ms();
 
     /* Per-node layer-1 tracking. */
@@ -6612,22 +6893,47 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
     while (fhead < ftail) {
         Frontier f = fr[fhead++];
         if (f.hops >= max_depth) continue;
-        const char* cur_id = g->nodes[f.idx].id;
-        for (int64_t ei = 0; ei < g->edge_count; ei++) {
+        int64_t cur = f.idx;
+        int64_t new_hops = f.hops + 1;
+        /* Use adjacency index: iterate only edges incident to `cur`.
+         * adj_from[cur] holds edge indices where cur is the 'from' node;
+         * adj_to[cur] holds edge indices where cur is the 'to' node.
+         * If adj index is unavailable (OOM during rebuild), fall back to
+         * full edge scan so activation is never silently wrong. */
+        int use_adj = (g->adj_from != NULL && g->adj_to != NULL);
+        int from_len = use_adj ? g->adj_from_len[cur] : 0;
+        int to_len   = use_adj ? g->adj_to_len[cur]   : 0;
+        int edge_scan_count = use_adj ? (from_len + to_len) : (int)g->edge_count;
+        for (int scan_i = 0; scan_i < edge_scan_count; scan_i++) {
+            int64_t ei;
+            int64_t oi;
+            if (use_adj) {
+                ei = (scan_i < from_len)
+                     ? g->adj_from[cur][scan_i]
+                     : g->adj_to[cur][scan_i - from_len];
+                EngramEdge* e = &g->edges[ei];
+                oi = (scan_i < from_len)
+                     ? engram_idmap_get(g, e->to_id)
+                     : engram_idmap_get(g, e->from_id);
+            } else {
+                /* Fallback: linear scan */
+                ei = scan_i;
+                EngramEdge* e = &g->edges[ei];
+                const char* other = NULL;
+                const char* cur_id = g->nodes[cur].id;
+                if (e->from_id && strcmp(e->from_id, cur_id) == 0) other = e->to_id;
+                else if (e->to_id && strcmp(e->to_id, cur_id) == 0) other = e->from_id;
+                else continue;
+                oi = engram_find_node_index(other);
+            }
+            if (oi < 0 || oi >= g->node_count) continue;
             EngramEdge* e = &g->edges[ei];
-            const char* other = NULL;
-            if (e->from_id && strcmp(e->from_id, cur_id) == 0) other = e->to_id;
-            else if (e->to_id && strcmp(e->to_id, cur_id) == 0) other = e->from_id;
-            else continue;
-            int64_t oi = engram_find_node_index(other);
-            if (oi < 0) continue;
             EngramNode* on = &g->nodes[oi];
             double tbonus  = engram_temporal_proximity_bonus(on->created_at, seed_epoch);
             double tdecay  = engram_temporal_decay(on, now_ms);
             double dampen  = engram_activation_dampen(on);
             double new_act = f.act * e->weight * SPREAD_DECAY * (1.0 + tbonus)
                            * tdecay * dampen;
-            int64_t new_hops = f.hops + 1;
             if (!reached[oi] || new_act > best_bg[oi]) {
                 best_bg[oi]   = new_act;
                 best_hops[oi] = new_hops;
@@ -7080,6 +7386,8 @@ el_val_t engram_load(el_val_t path) {
         free(g->edges[i].relation); free(g->edges[i].metadata);
     }
     g->edge_count = 0;
+    engram_idmap_free(g);
+    engram_adj_free(g);
 
     /* Walk nodes array */
     const char* nodes_p = json_find_key(data, "nodes");
@@ -7126,7 +7434,9 @@ el_val_t engram_load(el_val_t path) {
                 } else {
                     nn->layer_id = ENGRAM_LAYER_DEFAULT;
                 }
+                int64_t load_idx = g->node_count;
                 g->node_count++;
+                if (nn->id && *nn->id) engram_idmap_put(g, nn->id, load_idx);
                 free(obj);
                 nodes_p = end;
                 nodes_p = eg_skip_ws(nodes_p);
@@ -7134,6 +7444,7 @@ el_val_t engram_load(el_val_t path) {
             }
         }
     }
+    g->adj_dirty = 1;
     /* Walk edges array */
     const char* edges_p = json_find_key(data, "edges");
     if (edges_p) {
@@ -7307,8 +7618,11 @@ el_val_t engram_load_merge(el_val_t path) {
                     } else {
                         nn->layer_id = ENGRAM_LAYER_DEFAULT;
                     }
+                    int64_t merge_idx = g->node_count;
                     g->node_count++;
                     added_nodes++;
+                    if (nn->id && *nn->id) engram_idmap_put(g, nn->id, merge_idx);
+                    g->adj_dirty = 1;
                 }
                 free(obj);
                 nodes_p = end;
@@ -7400,6 +7714,34 @@ el_val_t engram_get_node_json(el_val_t id) {
     return el_wrap_str(b.buf);
 }
 
+/* engram_get_node_by_label — find the first node whose label field exactly
+ * matches the given string. Returns the node as a JSON object string, or "{}"
+ * if no match is found.
+ *
+ * Used by chat.el to retrieve well-known nodes (e.g. "conv:history",
+ * "session:summary") by their stable label rather than by ID, which is immune
+ * to vector index drift across restarts.
+ *
+ * Exact match (strcmp, not istr_contains) because labels like "conv:history"
+ * must not collide with nodes whose content happens to contain that substring.
+ *
+ * Added 2026-07-01 self-review: was called in chat.el but never defined,
+ * causing build failure since June 30. */
+el_val_t engram_get_node_by_label(el_val_t label) {
+    const char* lbl = EL_CSTR(label);
+    if (!lbl || !*lbl) return el_wrap_str(el_strdup("{}"));
+    EngramStore* g = engram_get();
+    for (int64_t i = 0; i < g->node_count; i++) {
+        EngramNode* n = &g->nodes[i];
+        if (n->label && strcmp(n->label, lbl) == 0) {
+            JsonBuf b; jb_init(&b);
+            engram_emit_node_json(&b, n);
+            return el_wrap_str(b.buf);
+        }
+    }
+    return el_wrap_str(el_strdup("{}"));
+}
+
 el_val_t engram_search_json(el_val_t query, el_val_t limit) {
     EngramStore* g = engram_get();
     const char* q = EL_CSTR(query);