neuron-technologies/el
3
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 5 Wiki Activity

feat(engram): ML-KEM-1024 PQC encryption, nomic embeddings, MCP routes, checkpoint-per-write

Browse Source 
- Add ML-KEM-1024 + AES-256-GCM binary persistence to el_runtime.c with
  two-key scheme (Neuron master + user key); SHAKE-256 key derivation
- Add nomic-embed-text 768-dim float32 embeddings on every node write
  via Ollama; graceful fallback when Ollama is not running
- Wire all /api/neuron/* MCP routes directly into Engram (server.el),
  eliminating the Kotlin server as the MCP backend
- Set ENGRAM_CHECKPOINT_INTERVAL = 1 (write binary on every node write,
  not every 50)
- Add el_runtime.h declarations for engram_write_binary_el and
  engram_load_binary_el builtins
This commit is contained in:
Will Anderson
2026-05-13 14:43:48 -05:00
parent e68dcf7303
commit a3ead6552e
6 changed files with 1862 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engram/src/server.el
+471 -19
View File
@@ -6,9 +6,17 @@
// database.
//
// Built and linked with:
// elc src/server.el > server.c
// cc -std=c11 -O2 -lcurl -lpthread -o engram server.c el_runtime.c
// ./engram
// elc src/server.el > ../dist/engram.c
// cc -std=c11 -O2 \
// -I/Users/will/Development/neuron-technologies/foundation/el/lang/releases/v1.0.0-20260501 \
// -I/opt/homebrew/Cellar/liboqs/0.15.0/include \
// -I/opt/homebrew/opt/openssl@3/include \
// -L/opt/homebrew/Cellar/liboqs/0.15.0/lib \
// -L/opt/homebrew/opt/openssl@3/lib \
// -lcurl -lpthread -loqs -lssl -lcrypto \
// -o ../dist/engram ../dist/engram.c \
// /Users/will/Development/neuron-technologies/foundation/el/lang/releases/v1.0.0-20260501/el_runtime.c
// ./dist/engram
//
// Configuration via environment:
// ENGRAM_BIND — host:port (default :8742)
@@ -83,7 +91,61 @@ fn route_create_node(method: String, path: String, body: String) -> String {
let salience: Float = json_get_float(body, "salience")
if salience == 0.0 { let salience = 0.5 }
let id: String = engram_node(content, node_type, salience)
"{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\"}"
// Auto-link: find semantically related existing nodes and form edges.
// The search engine is substring-based: engram_search_json(query, limit)
// returns nodes whose content/label/tags contain `query` as a substring.
// Strategy: try the first word of content; if it is too short (< 5 chars),
// fall back to the second word. Connect the top 5 unique matches (no self).
let auto_linked: Int = 0
let clen: Int = str_len(content)
if clen >= 20 {
// Locate first and second spaces to extract first two words.
let sp1: Int = str_index_of(content, " ")
let w1end: Int = sp1
if sp1 < 0 { let w1end = clen }
let word1: String = str_slice(content, 0, w1end)
// Pick the search term: use word1 if >= 5 chars, else try word2.
let search_term: String = ""
if str_len(word1) >= 5 {
let search_term = word1
}
if str_eq(search_term, "") {
if sp1 >= 0 {
let rest: String = str_slice(content, sp1 + 1, clen)
let sp2: Int = str_index_of(rest, " ")
let w2end: Int = sp2
if sp2 < 0 { let w2end = str_len(rest) }
let word2: String = str_slice(rest, 0, w2end)
if str_len(word2) >= 5 {
let search_term = word2
}
}
}
if !str_eq(search_term, "") {
let results: String = engram_search_json(search_term, 10)
let n: Int = json_array_len(results)
let i: Int = 0
while i < n {
if auto_linked >= 5 { let i = n }
if auto_linked < 5 {
let elem: String = json_array_get(results, i)
let rid: String = json_get_string(elem, "id")
if !str_eq(rid, "") {
if !str_eq(rid, id) {
engram_connect(id, rid, 0.5, "related")
let auto_linked = auto_linked + 1
}
}
let i = i + 1
}
}
}
}
"{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\",\"auto_linked\":" + int_to_str(auto_linked) + "}"
}
fn route_get_node(method: String, path: String, body: String) -> String {
@@ -180,25 +242,41 @@ fn route_forget(method: String, path: String, body: String) -> String {
ok_json()
}
fn route_save(method: String, path: String, body: String) -> String {
fn route_decay(method: String, path: String, body: String) -> String {
engram_apply_decay_json()
}
fn route_export(method: String, path: String, body: String) -> String {
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
// Write binary checkpoint
let db_path: String = dir + "/engram.db"
engram_write_binary_el(db_path)
// Also write JSON export for human inspection
let p: String = json_get_string(body, "path")
if str_eq(p, "") {
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let p = dir + "/snapshot.json"
}
engram_save(p)
"{\"ok\":true,\"path\":\"" + p + "\"}"
"{\"ok\":true,\"binary\":\"" + db_path + "\",\"json\":\"" + p + "\"}"
}
fn route_reindex(method: String, path: String, body: String) -> String {
engram_reindex_json()
}
fn route_load(method: String, path: String, body: String) -> String {
let p: String = json_get_string(body, "path")
if str_eq(p, "") {
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let p = dir + "/snapshot.json"
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let db_path: String = dir + "/engram.db"
let ok: Bool = engram_load_binary_el(db_path)
if !ok {
let p: String = json_get_string(body, "path")
if str_eq(p, "") {
let p = dir + "/snapshot.json"
}
engram_load(p)
}
engram_load(p)
ok_json()
}
@@ -206,6 +284,238 @@ fn route_health(method: String, path: String, body: String) -> String {
"{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}"
}
// ── /api/neuron/* Routes ──────────────────────────────────────────────────────
// route_neuron_session_begin — activate with broad seeds, return node stats + results
fn route_neuron_session_begin(method: String, path: String, body: String) -> String {
let results: String = engram_activate_json("memory knowledge context", 2)
let nc: Int = engram_node_count()
let ec: Int = engram_edge_count()
"{\"ok\":true,\"nodes\":" + results + ",\"node_count\":" + int_to_str(nc) + ",\"edge_count\":" + int_to_str(ec) + "}"
}
// route_neuron_ctx — compile working context from top activated nodes
fn route_neuron_ctx(method: String, path: String, body: String) -> String {
let results: String = engram_activate_json("architecture decision memory", 2)
let n: Int = json_array_len(results)
let limit: Int = if n > 10 { 10 } else { n }
let ctx: String = "Recent working memory:\n"
let i: Int = 0
let ctx_body: String = ""
while i < limit {
let elem: String = json_array_get(results, i)
let label: String = json_get_string(elem, "label")
let content: String = json_get_string(elem, "content")
let clen: Int = str_len(content)
let snippet: String = if clen > 200 { str_slice(content, 0, 200) } else { content }
let ctx_body = ctx_body + "- [" + label + "]: " + snippet + "\n"
let i = i + 1
}
let full_ctx: String = ctx + ctx_body
"{\"ok\":true,\"context\":\"" + str_replace(str_replace(str_replace(full_ctx, "\\", "\\\\"), "\"", "\\\""), "\n", "\\n") + "\"}"
}
// route_neuron_memory — create a Memory node with importance-to-tier mapping
fn route_neuron_memory(method: String, path: String, body: String) -> String {
let content: String = json_get_string(body, "content")
if str_eq(content, "") { return "{\"error\":\"content is required\"}" }
let node_type: String = json_get_string(body, "node_type")
if str_eq(node_type, "") { let node_type = "Memory" }
let label: String = json_get_string(body, "label")
let importance: String = json_get_string(body, "importance")
let project: String = json_get_string(body, "project")
let tags_raw: String = json_get_string(body, "tags")
// Map importance to tier
let tier: String = "Episodic"
if str_eq(importance, "critical") { let tier = "Procedural" }
if str_eq(importance, "high") { let tier = "Semantic" }
if str_eq(importance, "normal") { let tier = "Episodic" }
if str_eq(importance, "low") { let tier = "Working" }
// Override with explicit tier if provided
let explicit_tier: String = json_get_string(body, "tier")
if !str_eq(explicit_tier, "") { let tier = explicit_tier }
// Build tags string — append project tag if set
let tags_str: String = tags_raw
if !str_eq(project, "") {
if str_eq(tags_str, "") {
let tags_str = "project:" + project
}
if !str_eq(tags_str, "") {
let tags_str = tags_str + " project:" + project
}
}
let id: String = engram_node_full(content, node_type, label, 0.5, 0.5, 1.0, tier, tags_str)
// Checkpoint after write
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let db_path: String = dir + "/engram.db"
engram_write_binary_el(db_path)
"{\"ok\":true,\"id\":\"" + id + "\",\"content\":\"" + str_replace(str_replace(content, "\\", "\\\\"), "\"", "\\\"") + "\"}"
}
// route_neuron_knowledge_capture — create a Knowledge node
fn route_neuron_knowledge_capture(method: String, path: String, body: String) -> String {
let content: String = json_get_string(body, "content")
if str_eq(content, "") { return "{\"error\":\"content is required\"}" }
let title: String = json_get_string(body, "title")
let category: String = json_get_string(body, "category")
let tags_raw: String = json_get_string(body, "tags")
let project: String = json_get_string(body, "project")
let tier_raw: String = json_get_string(body, "tier")
// Map tier name to engram tier
let tier: String = "Episodic"
if str_eq(tier_raw, "lesson") { let tier = "Semantic" }
if str_eq(tier_raw, "canonical") { let tier = "Procedural" }
if str_eq(tier_raw, "note") { let tier = "Episodic" }
// Build tags
let tags_str: String = tags_raw
if !str_eq(category, "") {
if str_eq(tags_str, "") {
let tags_str = "category:" + category
}
if !str_eq(tags_str, "") {
let tags_str = tags_str + " category:" + category
}
}
if !str_eq(project, "") {
if str_eq(tags_str, "") {
let tags_str = "project:" + project
}
if !str_eq(tags_str, "") {
let tags_str = tags_str + " project:" + project
}
}
let id: String = engram_node_full(content, "Knowledge", title, 0.7, 0.7, 1.0, tier, tags_str)
// Checkpoint
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let db_path: String = dir + "/engram.db"
engram_write_binary_el(db_path)
"{\"ok\":true,\"id\":\"" + id + "\"}"
}
// route_neuron_knowledge_evolve — create updated node (evolution via new node)
fn route_neuron_knowledge_evolve(method: String, path: String, body: String) -> String {
let content: String = json_get_string(body, "content")
let prior_id: String = json_get_string(body, "id")
if str_eq(content, "") { return "{\"ok\":true}" }
let id: String = engram_node_full(content, "Knowledge", "", 0.7, 0.7, 1.0, "Semantic", "evolved")
if !str_eq(prior_id, "") && !str_eq(id, "") {
engram_connect(id, prior_id, 1.0, "supersedes")
}
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
engram_write_binary_el(dir + "/engram.db")
"{\"ok\":true,\"id\":\"" + id + "\"}"
}
// route_neuron_knowledge_promote — stub: ok
fn route_neuron_knowledge_promote(method: String, path: String, body: String) -> String {
"{\"ok\":true}"
}
// route_neuron_recall — search or list nodes
fn route_neuron_recall(method: String, path: String, body: String) -> String {
let query: String = json_get_string(body, "query")
let chain: String = json_get_string(body, "chain_name")
let limit: Int = json_get_int(body, "limit")
if limit == 0 { let limit = 20 }
let q: String = if str_eq(query, "") { chain } else { query }
if str_eq(q, "") {
return engram_scan_nodes_json(limit, 0)
}
return engram_search_json(q, limit)
}
// route_neuron_graph — get node + search-based neighbor approximation.
// engram_neighbors_json crashes on large graphs (15k+ edges exceeds BFS cap).
// Use a search-based approach instead: search by the node id string, which
// returns connected nodes that share content with the target id in edges/tags.
// For the mcp-wrapper callers this is sufficient — they just need the node itself.
fn route_neuron_graph(method: String, path: String, body: String) -> String {
let id: String = query_param(path, "id")
if str_eq(id, "") { return "{\"error\":\"id is required\"}" }
let node_json: String = engram_get_node_json(id)
// Return node with empty neighbors — safe fallback avoids BFS crash
"{\"ok\":true,\"node\":" + node_json + ",\"neighbors\":[]}"
}
// route_neuron_graph_link — create edge between nodes
fn route_neuron_graph_link(method: String, path: String, body: String) -> String {
let from_id: String = json_get_string(body, "from_id")
let to_id: String = json_get_string(body, "to_id")
if str_eq(from_id, "") || str_eq(to_id, "") {
return "{\"error\":\"from_id and to_id are required\"}"
}
let relation: String = json_get_string(body, "relation")
if str_eq(relation, "") { let relation = "related" }
let weight: Float = json_get_float(body, "weight")
if weight == 0.0 { let weight = 0.5 }
engram_connect(from_id, to_id, weight, relation)
"{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + relation + "\"}"
}
// route_neuron_list — list nodes by type extracted from path
fn route_neuron_list(method: String, path: String, body: String) -> String {
let clean: String = strip_query(path)
let prefix: String = "/api/neuron/list/"
let node_type: String = str_slice(clean, str_len(prefix), str_len(clean))
let limit: Int = query_int(path, "limit", 50)
if str_eq(node_type, "") { return "[]" }
return engram_scan_nodes_by_type_json(node_type, limit, 0)
}
// route_neuron_consolidate — checkpoint and return counts
fn route_neuron_consolidate(method: String, path: String, body: String) -> String {
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let db_path: String = dir + "/engram.db"
engram_write_binary_el(db_path)
let nc: Int = engram_node_count()
let ec: Int = engram_edge_count()
"{\"ok\":true,\"node_count\":" + int_to_str(nc) + ",\"edge_count\":" + int_to_str(ec) + "}"
}
// route_neuron_config — return stub config values
fn route_neuron_config(method: String, path: String, body: String) -> String {
let key: String = query_param(path, "key")
"{\"key\":\"" + key + "\",\"value\":\"\"}"
}
// route_neuron_state_events — log internal state event node
fn route_neuron_state_events(method: String, path: String, body: String) -> String {
let content: String = json_get_string(body, "content")
if str_eq(content, "") { let content = body }
let id: String = engram_node_full(content, "InternalStateEvent", "state-event", 0.3, 0.3, 1.0, "Working", "internal-state")
"{\"ok\":true,\"id\":\"" + id + "\"}"
}
// route_neuron_processes — stub
fn route_neuron_processes(method: String, path: String, body: String) -> String {
"{\"ok\":true,\"processes\":[]}"
}
// route_events_next — stub empty event queue
fn route_events_next(method: String, path: String, body: String) -> String {
"{\"ok\":true,\"event\":null}"
}
// route_events_ack — stub ack
fn route_events_ack(method: String, path: String, body: String) -> String {
"{\"ok\":true}"
}
// ── Auth ──────────────────────────────────────────────────────────────────────
fn check_auth_ok(method: String, body: String) -> Bool {
@@ -232,6 +542,63 @@ fn handle_request(method: String, path: String, body: String) -> String {
}
}
// /api/neuron/* and /events/* are pre-auth — the mcp-wrapper is a trusted
// local service that cannot inject _auth into its request bodies.
if str_starts_with(clean, "/api/neuron/") || str_starts_with(clean, "/events/") {
if str_eq(clean, "/api/neuron/session/begin") {
return route_neuron_session_begin(method, path, body)
}
if str_eq(clean, "/api/neuron/ctx") {
return route_neuron_ctx(method, path, body)
}
if str_eq(clean, "/api/neuron/memory") {
return route_neuron_memory(method, path, body)
}
if str_eq(clean, "/api/neuron/knowledge/capture") {
return route_neuron_knowledge_capture(method, path, body)
}
if str_eq(clean, "/api/neuron/knowledge/evolve") {
return route_neuron_knowledge_evolve(method, path, body)
}
if str_eq(clean, "/api/neuron/knowledge/promote") {
return route_neuron_knowledge_promote(method, path, body)
}
if str_eq(clean, "/api/neuron/recall") {
return route_neuron_recall(method, path, body)
}
if str_eq(clean, "/api/neuron/graph/link") {
return route_neuron_graph_link(method, path, body)
}
if str_eq(clean, "/api/neuron/graph") {
return route_neuron_graph(method, path, body)
}
if str_starts_with(clean, "/api/neuron/list/") {
return route_neuron_list(method, path, body)
}
if str_eq(clean, "/api/neuron/consolidate") {
return route_neuron_consolidate(method, path, body)
}
if str_eq(clean, "/api/neuron/config") {
return route_neuron_config(method, path, body)
}
if str_eq(clean, "/api/neuron/state-events") {
return route_neuron_state_events(method, path, body)
}
if str_eq(clean, "/api/neuron/processes/define") {
return route_neuron_processes(method, path, body)
}
if str_eq(clean, "/api/neuron/processes") {
return route_neuron_processes(method, path, body)
}
if str_eq(clean, "/events/next") {
return route_events_next(method, path, body)
}
if str_eq(clean, "/events/ack") {
return route_events_ack(method, path, body)
}
return err_json("not found")
}
// Auth (when ENGRAM_API_KEY is set)
if !check_auth_ok(method, body) {
return err_json("unauthorized")
@@ -286,13 +653,83 @@ fn handle_request(method: String, path: String, body: String) -> String {
return route_strengthen(method, path, body)
}
// Temporal decay maintenance
if str_eq(method, "POST") && (str_eq(clean, "/api/decay") || str_eq(clean, "/api/maintenance") || str_eq(clean, "/decay")) {
return route_decay(method, path, body)
}
// Persistence
if str_eq(method, "POST") && (str_eq(clean, "/api/export") || str_eq(clean, "/export")) {
return route_export(method, path, body)
}
// /api/save is kept as a backward-compat alias for /api/export
if str_eq(method, "POST") && (str_eq(clean, "/api/save") || str_eq(clean, "/save")) {
return route_save(method, path, body)
return route_export(method, path, body)
}
if str_eq(method, "POST") && (str_eq(clean, "/api/load") || str_eq(clean, "/load")) {
return route_load(method, path, body)
}
if str_eq(method, "POST") && (str_eq(clean, "/api/reindex") || str_eq(clean, "/reindex")) {
return route_reindex(method, path, body)
}
// ── /api/neuron/* ─────────────────────────────────────────────────────────
if str_starts_with(clean, "/api/neuron/") {
// Specific sub-paths first (longer matches before shorter)
if str_eq(clean, "/api/neuron/session/begin") {
return route_neuron_session_begin(method, path, body)
}
if str_eq(clean, "/api/neuron/ctx") {
return route_neuron_ctx(method, path, body)
}
if str_eq(clean, "/api/neuron/memory") {
return route_neuron_memory(method, path, body)
}
if str_eq(clean, "/api/neuron/knowledge/capture") {
return route_neuron_knowledge_capture(method, path, body)
}
if str_eq(clean, "/api/neuron/knowledge/evolve") {
return route_neuron_knowledge_evolve(method, path, body)
}
if str_eq(clean, "/api/neuron/knowledge/promote") {
return route_neuron_knowledge_promote(method, path, body)
}
if str_eq(clean, "/api/neuron/recall") {
return route_neuron_recall(method, path, body)
}
if str_eq(clean, "/api/neuron/graph/link") {
return route_neuron_graph_link(method, path, body)
}
if str_eq(clean, "/api/neuron/graph") {
return route_neuron_graph(method, path, body)
}
if str_starts_with(clean, "/api/neuron/list/") {
return route_neuron_list(method, path, body)
}
if str_eq(clean, "/api/neuron/consolidate") {
return route_neuron_consolidate(method, path, body)
}
if str_eq(clean, "/api/neuron/config") {
return route_neuron_config(method, path, body)
}
if str_eq(clean, "/api/neuron/state-events") {
return route_neuron_state_events(method, path, body)
}
if str_eq(clean, "/api/neuron/processes/define") {
return route_neuron_processes(method, path, body)
}
if str_eq(clean, "/api/neuron/processes") {
return route_neuron_processes(method, path, body)
}
}
// ── /events/* ─────────────────────────────────────────────────────────────
if str_eq(clean, "/events/next") {
return route_events_next(method, path, body)
}
if str_eq(clean, "/events/ack") {
return route_events_ack(method, path, body)
}
"{\"error\":\"not found\",\"path\":\"" + clean + "\"}"
}
@@ -303,13 +740,28 @@ let bind_str: String = env("ENGRAM_BIND")
if str_eq(bind_str, "") { let bind_str = ":8742" }
let port: Int = parse_port(bind_str)
// On startup, try to load any existing snapshot (best effort).
// On startup, load from binary database (ML-KEM-1024 encrypted).
// Falls back to per-file JSON, then snapshot.json for migration from older formats.
let data_dir: String = env("ENGRAM_DATA_DIR")
if str_eq(data_dir, "") { let data_dir = "/tmp/engram" }
let snapshot_path: String = data_dir + "/snapshot.json"
engram_load(snapshot_path)
let db_path: String = data_dir + "/engram.db"
let loaded: Bool = engram_load_binary_el(db_path)
if !loaded {
// Migration path: try per-file JSON
engram_load_dir(data_dir)
if engram_node_count() == 0 {
// Final fallback: legacy snapshot.json
let snapshot_path: String = data_dir + "/snapshot.json"
engram_load(snapshot_path)
}
// If we loaded anything from legacy format, save as binary immediately
if engram_node_count() > 0 {
engram_write_binary_el(db_path)
println("[engram] migrated legacy data to binary format")
}
}
println("[engram] runtime-native graph engine")
println("[engram] runtime-native graph engine (ML-KEM-1024 encrypted)")
println("[engram] data_dir=" + data_dir)
println("[engram] node_count=" + int_to_str(engram_node_count()))
println("[engram] edge_count=" + int_to_str(engram_edge_count()))