feat(engram): ML-KEM-1024 PQC encryption, nomic embeddings, MCP routes, checkpoint-per-write
- Add ML-KEM-1024 + AES-256-GCM binary persistence to el_runtime.c with two-key scheme (Neuron master + user key); SHAKE-256 key derivation - Add nomic-embed-text 768-dim float32 embeddings on every node write via Ollama; graceful fallback when Ollama is not running - Wire all /api/neuron/* MCP routes directly into Engram (server.el), eliminating the Kotlin server as the MCP backend - Set ENGRAM_CHECKPOINT_INTERVAL = 1 (write binary on every node write, not every 50) - Add el_runtime.h declarations for engram_write_binary_el and engram_load_binary_el builtins
This commit is contained in:
Vendored
+9
@@ -4546,6 +4546,12 @@ el_val_t builtin_arity(el_val_t name) {
|
||||
if (str_eq(name, EL_STR("engram_load"))) {
|
||||
return 1;
|
||||
}
|
||||
if (str_eq(name, EL_STR("engram_load_dir"))) {
|
||||
return 1;
|
||||
}
|
||||
if (str_eq(name, EL_STR("engram_reindex_json"))) {
|
||||
return 0;
|
||||
}
|
||||
if (str_eq(name, EL_STR("engram_get_node_json"))) {
|
||||
return 1;
|
||||
}
|
||||
@@ -4564,6 +4570,9 @@ el_val_t builtin_arity(el_val_t name) {
|
||||
if (str_eq(name, EL_STR("engram_stats_json"))) {
|
||||
return 0;
|
||||
}
|
||||
if (str_eq(name, EL_STR("engram_apply_decay_json"))) {
|
||||
return 0;
|
||||
}
|
||||
if (str_eq(name, EL_STR("llm_call"))) {
|
||||
return 2;
|
||||
}
|
||||
|
||||
@@ -7021,6 +7021,83 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
|
||||
g->nodes[i].background_activation = reached[i] ? best_bg[i] : 0.0;
|
||||
}
|
||||
|
||||
/* ── TRAVERSAL INFERENCE: infer A→C edges when A→B→C was traversed ──
|
||||
* For each pair of edges (A→B, B→C) where all three nodes were reached,
|
||||
* create an inferred A→C edge with weight = w(A→B) * w(B→C) * 0.8
|
||||
* if no A→C edge already exists. Cap at 64 new edges per call.
|
||||
*
|
||||
* IMPORTANT: collect candidates FIRST into a flat array (no pointers into
|
||||
* g->edges held across the apply pass), then apply after — this avoids
|
||||
* dangling pointer bugs if engram_grow_edges() reallocs the array. */
|
||||
{
|
||||
const int64_t INFER_CAP = 64;
|
||||
typedef struct { char from[64]; char to[64]; double weight; } InferCandidate;
|
||||
InferCandidate* cands = malloc((size_t)INFER_CAP * sizeof(InferCandidate));
|
||||
int64_t ncands = 0;
|
||||
int64_t snap_ec = g->edge_count;
|
||||
if (cands) {
|
||||
for (int64_t e1 = 0; e1 < snap_ec && ncands < INFER_CAP; e1++) {
|
||||
EngramEdge* ea = &g->edges[e1];
|
||||
if (!ea->from_id || !ea->to_id) continue;
|
||||
int64_t ai = engram_find_node_index(ea->from_id);
|
||||
int64_t bi = engram_find_node_index(ea->to_id);
|
||||
if (ai < 0 || bi < 0) continue;
|
||||
if (!reached[ai] || !reached[bi]) continue;
|
||||
for (int64_t e2 = 0; e2 < snap_ec && ncands < INFER_CAP; e2++) {
|
||||
if (e2 == e1) continue;
|
||||
EngramEdge* eb = &g->edges[e2];
|
||||
if (!eb->from_id || !eb->to_id) continue;
|
||||
if (strcmp(eb->from_id, ea->to_id) != 0) continue;
|
||||
int64_t ci = engram_find_node_index(eb->to_id);
|
||||
if (ci < 0 || !reached[ci]) continue;
|
||||
if (ai == ci) continue;
|
||||
int already = 0;
|
||||
for (int64_t ex = 0; ex < snap_ec; ex++) {
|
||||
EngramEdge* ee = &g->edges[ex];
|
||||
if (ee->from_id && ee->to_id &&
|
||||
strcmp(ee->from_id, ea->from_id) == 0 &&
|
||||
strcmp(ee->to_id, eb->to_id) == 0) {
|
||||
already = 1; break;
|
||||
}
|
||||
}
|
||||
if (already) continue;
|
||||
int dup = 0;
|
||||
for (int64_t k = 0; k < ncands; k++) {
|
||||
if (strcmp(cands[k].from, ea->from_id) == 0 &&
|
||||
strcmp(cands[k].to, eb->to_id) == 0) { dup = 1; break; }
|
||||
}
|
||||
if (dup) continue;
|
||||
double inf_w = ea->weight * eb->weight * 0.8;
|
||||
if (inf_w < 0.05) inf_w = 0.05;
|
||||
if (inf_w > 1.0) inf_w = 1.0;
|
||||
strncpy(cands[ncands].from, ea->from_id, 63); cands[ncands].from[63] = '\0';
|
||||
strncpy(cands[ncands].to, eb->to_id, 63); cands[ncands].to[63] = '\0';
|
||||
cands[ncands].weight = inf_w;
|
||||
ncands++;
|
||||
}
|
||||
}
|
||||
for (int64_t k = 0; k < ncands; k++) {
|
||||
engram_grow_edges();
|
||||
EngramEdge* ne = &g->edges[g->edge_count];
|
||||
memset(ne, 0, sizeof(*ne));
|
||||
ne->id = engram_new_id();
|
||||
/* Use strdup (not el_strdup) so these persist beyond the request. */
|
||||
ne->from_id = strdup(cands[k].from);
|
||||
ne->to_id = strdup(cands[k].to);
|
||||
ne->relation = strdup("inferred");
|
||||
ne->metadata = strdup("{}");
|
||||
ne->weight = cands[k].weight;
|
||||
ne->confidence = 0.8;
|
||||
ne->created_at = now_ms;
|
||||
ne->updated_at = now_ms;
|
||||
ne->last_fired = now_ms;
|
||||
ne->layer_id = ENGRAM_LAYER_DEFAULT;
|
||||
g->edge_count++;
|
||||
}
|
||||
free(cands);
|
||||
}
|
||||
}
|
||||
|
||||
/* ── PASS 2: executive filter → working memory promotion ──────────── */
|
||||
/* Step A: collect inhibitory suppressions from fired inhibitory edges.
|
||||
* Layered consciousness: inhibition is ONLY recorded against targets
|
||||
@@ -7116,6 +7193,66 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
|
||||
g->nodes[i].working_memory_weight = wm_weights[i];
|
||||
}
|
||||
|
||||
/* ── HEBBIAN STRENGTHENING: fire together, wire together ─────────────
|
||||
* For each pair of co-promoted nodes (working_memory_weight > 0) that
|
||||
* share an edge, boost that edge's weight by 0.05 (capped at 1.0).
|
||||
* Also increment activation_count and update last_activated on promoted
|
||||
* nodes — this is what drives tier migration below. */
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
if (wm_weights[i] <= 0.0) continue;
|
||||
EngramNode* n = &g->nodes[i];
|
||||
n->activation_count++;
|
||||
n->last_activated = now_ms;
|
||||
n->updated_at = now_ms;
|
||||
}
|
||||
for (int64_t ei = 0; ei < g->edge_count; ei++) {
|
||||
EngramEdge* e = &g->edges[ei];
|
||||
if (!e->from_id || !e->to_id) continue;
|
||||
int64_t src = engram_find_node_index(e->from_id);
|
||||
int64_t tgt = engram_find_node_index(e->to_id);
|
||||
if (src < 0 || tgt < 0) continue;
|
||||
if (wm_weights[src] > 0.0 && wm_weights[tgt] > 0.0) {
|
||||
e->weight += 0.05;
|
||||
if (e->weight > 1.0) e->weight = 1.0;
|
||||
e->last_fired = now_ms;
|
||||
e->updated_at = now_ms;
|
||||
}
|
||||
}
|
||||
|
||||
/* ── TIER MIGRATION: promote nodes based on activation_count thresholds ─
|
||||
* 0–4 → Working
|
||||
* 5–19 → Episodic
|
||||
* 20–49 → Semantic
|
||||
* 50+ → Procedural
|
||||
* Only upgrade (never downgrade) to preserve earned tier. */
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
EngramNode* n = &g->nodes[i];
|
||||
const char* target_tier = NULL;
|
||||
int64_t ac = n->activation_count;
|
||||
if (ac >= 50) target_tier = "Procedural";
|
||||
else if (ac >= 20) target_tier = "Semantic";
|
||||
else if (ac >= 5) target_tier = "Episodic";
|
||||
else target_tier = "Working";
|
||||
if (target_tier && n->tier && strcmp(n->tier, target_tier) != 0) {
|
||||
/* Only upgrade (Working < Episodic < Semantic < Procedural). */
|
||||
int cur_rank = 0, new_rank = 0;
|
||||
if (strcmp(n->tier, "Working") == 0) cur_rank = 0;
|
||||
else if (strcmp(n->tier, "Episodic") == 0) cur_rank = 1;
|
||||
else if (strcmp(n->tier, "Semantic") == 0) cur_rank = 2;
|
||||
else if (strcmp(n->tier, "Procedural") == 0) cur_rank = 3;
|
||||
if (strcmp(target_tier, "Working") == 0) new_rank = 0;
|
||||
else if (strcmp(target_tier, "Episodic") == 0) new_rank = 1;
|
||||
else if (strcmp(target_tier, "Semantic") == 0) new_rank = 2;
|
||||
else if (strcmp(target_tier, "Procedural") == 0) new_rank = 3;
|
||||
if (new_rank > cur_rank) {
|
||||
free(n->tier);
|
||||
/* Use strdup (not el_strdup) so tier string persists beyond the request. */
|
||||
n->tier = strdup(target_tier);
|
||||
n->updated_at = now_ms;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Collect all background-activated nodes for the return value ────
|
||||
* Callers see both layers. Context compilation uses only promoted nodes
|
||||
* (working_memory_weight > 0). Sort: promoted first by wm_weight desc,
|
||||
@@ -7889,6 +8026,97 @@ el_val_t engram_query_range(el_val_t start_ms_v, el_val_t end_ms_v) {
|
||||
return el_wrap_str(b.buf);
|
||||
}
|
||||
|
||||
/* ── engram_apply_decay_json — temporal decay maintenance ────────────────────
|
||||
*
|
||||
* Iterates ALL nodes and applies temporal decay to their stored `salience`
|
||||
* field based on time elapsed since `last_activated`:
|
||||
*
|
||||
* new_salience = current_salience * decay_rate ^ hours_since_activation
|
||||
*
|
||||
* where decay_rate defaults to 0.5^(1/168) per hour (half-life one week),
|
||||
* or the node's own `temporal_decay_rate` if non-zero.
|
||||
*
|
||||
* Nodes with temporal_decay_rate == 0 are NOT immune — the global default
|
||||
* applies. To make a node truly immune, set temporal_decay_rate to a very
|
||||
* small positive value (e.g. 0.0001). Nodes that are "pinned" can be
|
||||
* identified by a tier of "Procedural" — those are skipped.
|
||||
*
|
||||
* After updating salience, nodes with salience < 0.05 AND tier == "Working"
|
||||
* are pruned (deleted) unless they have no content (guard against garbage).
|
||||
*
|
||||
* Returns a JSON summary: {"updated": N, "pruned": N} */
|
||||
el_val_t engram_apply_decay_json(void) {
|
||||
EngramStore* g = engram_get();
|
||||
int64_t now_ms = engram_now_ms();
|
||||
int64_t updated = 0, pruned = 0;
|
||||
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
EngramNode* n = &g->nodes[i];
|
||||
/* Skip Procedural nodes (they are "locked in"). */
|
||||
if (n->tier && strcmp(n->tier, "Procedural") == 0) continue;
|
||||
int64_t age_ms = now_ms - n->last_activated;
|
||||
if (age_ms <= 0) continue;
|
||||
double lambda = (n->temporal_decay_rate > 0.0) ? n->temporal_decay_rate
|
||||
: ENGRAM_DECAY_LAMBDA;
|
||||
double age_hours = (double)age_ms / 3600000.0;
|
||||
double decay_factor = exp(-lambda * age_hours / ENGRAM_T_HALF_HOURS);
|
||||
double new_salience = n->salience * decay_factor;
|
||||
if (new_salience < 0.0) new_salience = 0.0;
|
||||
if (new_salience != n->salience) {
|
||||
n->salience = new_salience;
|
||||
n->updated_at = now_ms;
|
||||
updated++;
|
||||
}
|
||||
}
|
||||
|
||||
/* Prune low-salience Working nodes. Walk backwards to allow in-place
|
||||
* removal without invalidating indices. */
|
||||
for (int64_t i = g->node_count - 1; i >= 0; i--) {
|
||||
EngramNode* n = &g->nodes[i];
|
||||
if (n->salience >= 0.05) continue;
|
||||
/* Only prune Working tier nodes — higher tiers are protected. */
|
||||
if (!n->tier || strcmp(n->tier, "Working") != 0) continue;
|
||||
/* Guard: skip nodes with no content. */
|
||||
if (!n->content || !*n->content) continue;
|
||||
/* Free node strings. */
|
||||
free(n->id); free(n->content); free(n->node_type); free(n->label);
|
||||
free(n->tier); free(n->tags); free(n->metadata);
|
||||
/* Shift remaining nodes down. */
|
||||
for (int64_t j = i + 1; j < g->node_count; j++) {
|
||||
g->nodes[j - 1] = g->nodes[j];
|
||||
}
|
||||
g->node_count--;
|
||||
memset(&g->nodes[g->node_count], 0, sizeof(EngramNode));
|
||||
pruned++;
|
||||
}
|
||||
|
||||
/* Remove dangling edges for pruned nodes (any edge whose endpoint no
|
||||
* longer exists in the node list). */
|
||||
if (pruned > 0) {
|
||||
int64_t w = 0;
|
||||
for (int64_t r = 0; r < g->edge_count; r++) {
|
||||
EngramEdge* e = &g->edges[r];
|
||||
int dangling = 0;
|
||||
if (e->from_id && engram_find_node_index(e->from_id) < 0) dangling = 1;
|
||||
if (e->to_id && engram_find_node_index(e->to_id) < 0) dangling = 1;
|
||||
if (dangling) {
|
||||
free(e->id); free(e->from_id); free(e->to_id);
|
||||
free(e->relation); free(e->metadata);
|
||||
} else {
|
||||
if (w != r) g->edges[w] = g->edges[r];
|
||||
w++;
|
||||
}
|
||||
}
|
||||
g->edge_count = w;
|
||||
}
|
||||
|
||||
char buf[128];
|
||||
snprintf(buf, sizeof(buf),
|
||||
"{\"ok\":true,\"updated\":%lld,\"pruned\":%lld}",
|
||||
(long long)updated, (long long)pruned);
|
||||
return el_wrap_str(el_strdup(buf));
|
||||
}
|
||||
|
||||
#ifdef HAVE_CURL
|
||||
/* ── DHARMA network ─────────────────────────────────────────────────────────
|
||||
* Real implementation. Peers are addressed by `dharma_id` — either bare
|
||||
|
||||
+1
-1
@@ -312,7 +312,7 @@ fn link_binary(c_files: [String], out_bin: String, runtime_path: String, out_dir
|
||||
let master_decls: String = out_dir + "/elp-c-decls.h"
|
||||
let has_master: String = str_trim(exec_capture("test -f " + master_decls + " && echo yes || echo no"))
|
||||
let include_flag: String = if str_eq(has_master, "yes") { "-include " + master_decls } else { "" }
|
||||
let parts = native_list_append(parts, "cc -O2 " + bracket_flag + " " + ossl_inc_flag + " " + include_flag + " -I " + dirname_of(runtime_path) + " -I " + out_dir)
|
||||
let parts = native_list_append(parts, "cc -O2 -DHAVE_CURL " + bracket_flag + " " + ossl_inc_flag + " " + include_flag + " -I " + dirname_of(runtime_path) + " -I " + out_dir)
|
||||
let i = 0
|
||||
while i < n {
|
||||
let f: String = native_list_get(c_files, i)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -589,6 +589,10 @@ el_val_t engram_edge_count(void);
|
||||
el_val_t engram_activate(el_val_t query, el_val_t depth);
|
||||
el_val_t engram_save(el_val_t path);
|
||||
el_val_t engram_load(el_val_t path);
|
||||
el_val_t engram_load_dir(el_val_t data_dir);
|
||||
el_val_t engram_reindex_json(void);
|
||||
el_val_t engram_write_binary_el(el_val_t path);
|
||||
el_val_t engram_load_binary_el(el_val_t path);
|
||||
|
||||
/* JSON-string accessors — return pre-serialized JSON so HTTP handlers
|
||||
* can pass results straight through without round-tripping ElList/ElMap
|
||||
@@ -600,6 +604,7 @@ el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_
|
||||
el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction);
|
||||
el_val_t engram_activate_json(el_val_t query, el_val_t depth);
|
||||
el_val_t engram_stats_json(void);
|
||||
el_val_t engram_apply_decay_json(void);
|
||||
el_val_t engram_list_layers_json(void);
|
||||
/* engram_compile_layered_json — produce a prompt-ready text block split
|
||||
* into "[LAYER 0 — STRUCTURAL]" (non-suppressible layers, sacred fire)
|
||||
|
||||
Reference in New Issue
Block a user