This repository has been archived on 2026-08-20. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
el-retired/engram/src/server.el
T
will.anderson a000599bfe self-review 2026-05-28: checkpoint ISE decay fix + auto-linking for MCP nodes
Three changes:

1. Fix checkpoint ISE temporal_decay_rate: engram_emit_ise_internal was
   hardcoded to 0.0 (global 168h default) instead of 2.310 (Working-tier
   48h). Result: checkpoint ISEs accumulated at 3.5x intended rate.

2. Raise CHECKPOINT_INTERVAL 1→10: checkpoint ISE fires on every single
   node write, producing 2:1 checkpoint:content ratio in ISE stream.
   MCP routes still call engram_write_binary_el explicitly after each
   important write, so no knowledge durability is lost.

3. Add auto_link_content_node to server.el: route_neuron_memory and
   route_neuron_knowledge_capture were creating nodes with zero edges —
   invisible to BFS traversal, only reachable via lexical/semantic seed.
   New helper runs BM25 over top-20 results, skips ISE nodes (which
   dominate the 14K-node corpus), connects up to 3 related nodes.
2026-06-05 11:34:27 -05:00

1147 lines
48 KiB
EmacsLisp
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// server.el Engram HTTP server.
//
// Engram is the in-process graph store. The runtime owns the data; this
// file is the thin HTTP face. Every route maps to one or two engram_*
// builtins. There is no SQL, no db layer, no SQLite the runtime IS the
// database.
//
// Built and linked with:
// elc src/server.el > ../dist/engram.c
// cc -std=c11 -O2 \
// -I/Users/will/Development/neuron-technologies/foundation/el/lang/releases/v1.0.0-20260501 \
// -I/opt/homebrew/Cellar/liboqs/0.15.0/include \
// -I/opt/homebrew/opt/openssl@3/include \
// -L/opt/homebrew/Cellar/liboqs/0.15.0/lib \
// -L/opt/homebrew/opt/openssl@3/lib \
// -lcurl -lpthread -loqs -lssl -lcrypto \
// -o ../dist/engram ../dist/engram.c \
// /Users/will/Development/neuron-technologies/foundation/el/lang/releases/v1.0.0-20260501/el_runtime.c
// ./dist/engram
//
// Configuration via environment:
// ENGRAM_BIND host:port (default :8742)
// ENGRAM_API_KEY bearer auth (optional)
// ENGRAM_DATA_DIR snapshot location (default ~/.neuron/engram)
// BM25+ text ranking
//
// Implements BM25+ (Lv & Zhai 2011) for in-process keyword search over the
// engram node store. No external dependencies pure EL, zero Ollama calls.
//
// Parameters: k1=1.2, b=0.75, delta=1.0
//
// V1 simplification: n(t) (number of docs containing term t) is approximated
// as 1 for all terms. This collapses IDF to a constant per corpus size:
// IDF = ln((N - 1 + 0.5) / (1 + 0.5) + 1) = ln((N + 0.5) / 1.5 + 1)
// Scoring effectively becomes TF-length-normalised BM25+ (delta term present).
// Acceptable for V1; a real inverted index can replace this later.
fn bm25_tokenize(text: String) -> String {
// Lowercase and strip punctuation (replace with spaces), then trim.
let t: String = str_to_lower(text)
let t = str_replace(t, ".", " ")
let t = str_replace(t, ",", " ")
let t = str_replace(t, "!", " ")
let t = str_replace(t, "?", " ")
let t = str_replace(t, "\"", " ")
let t = str_replace(t, ":", " ")
let t = str_replace(t, ";", " ")
let t = str_replace(t, "(", " ")
let t = str_replace(t, ")", " ")
let t = str_replace(t, "[", " ")
let t = str_replace(t, "]", " ")
let t = str_replace(t, "{", " ")
let t = str_replace(t, "}", " ")
let t = str_replace(t, "/", " ")
let t = str_replace(t, "\\", " ")
let t = str_replace(t, "'", " ")
let t = str_replace(t, "-", " ")
let t = str_replace(t, "_", " ")
let t = str_replace(t, "+", " ")
str_trim(t)
}
fn bm25_count_term(term: String, doc_tokens: String) -> Int {
// Pad with spaces to avoid prefix/suffix partial matches.
let padded_term: String = " " + term + " "
let padded_doc: String = " " + doc_tokens + " "
str_count(padded_doc, padded_term)
}
fn bm25_score_doc(doc_content: String, query_tokens: String, corpus_size: Int, avg_doc_len: String) -> String {
// BM25+ parameters (stored as strings = float-encoded el_val_t from el_from_float)
// We use float_add/float_mul/float_div builtins to avoid EL operator issues.
// avg_doc_len is passed as a String slot holding an el_val_t float bit-pattern.
// (EL has no safe float-passing convention; we work around using str_to_float.)
//
// V1: n_t=1 for all terms. IDF = ln((N+0.5)/1.5 + 1) = constant per corpus.
// This collapses BM25+ to TF-length-normalised scoring acceptable for V1.
let k1: Float = 1.2
let b: Float = 0.75
let delta: Float = 1.0
let doc_tokens: String = bm25_tokenize(doc_content)
let doc_wc: Int = str_count_words(doc_tokens)
if doc_wc == 0 { return "0.0" }
let doc_len: Float = int_to_float(doc_wc)
let avg_len: Float = str_to_float(avg_doc_len)
// IDF constant
let N: Float = int_to_float(corpus_size)
// (N + 0.5) / 1.5 + 1.0
let idf_arg: Float = float_add(float_div(float_add(N, 1.2), 1.5), 1.0)
let idf: Float = math_log(idf_arg)
// Sum TF component over query terms
let terms: List = str_split(query_tokens, " ")
let n_terms: Int = len(terms)
let score: Float = 0.0
let i: Int = 0
while i < n_terms {
let term: String = get(terms, i)
let tlen: Int = str_len(term)
if tlen >= 2 {
let tf_count: Int = bm25_count_term(term, doc_tokens)
if tf_count > 0 {
let tf_raw: Float = int_to_float(tf_count)
// norm_factor = 1 - b + b * doc_len / avg_len
let norm_factor: Float = float_add(float_sub(1.0, b), float_div(float_mul(b, doc_len), avg_len))
// tf_comp = delta + tf * (k1+1) / (tf + k1*norm)
let numerator: Float = float_mul(tf_raw, float_add(k1, 1.0))
let denominator: Float = float_add(tf_raw, float_mul(k1, norm_factor))
let tf_comp: Float = float_add(delta, float_div(numerator, denominator))
let score = float_add(score, float_mul(idf, tf_comp))
}
}
let i = i + 1
}
// Return score as a string so it survives EL's lack of float-in-list support
float_to_str(score)
}
fn bm25_search_json(query: String, limit: Int) -> String {
// 1. Determine scan size: floor at 200 so small `limit` values still scan
// enough of the corpus to find relevant nodes.
// Cap raised from 500 5000 (2026-05-24 self-review): 500 was 0.3% of the
// 161K-node corpus. At 5000 we cover the top-3% by salience still fast
// (pure C scan, no Ollama calls) and 10x better recall for content search.
// engram_scan_nodes_json returns nodes sorted by salience DESC, so ISEs
// (salience 0.3) naturally fall below Knowledge/Memory (0.50.8), keeping
// the effective search corpus content-dense.
let scan_limit: Int = limit * 10
if scan_limit < 200 { let scan_limit = 200 }
if scan_limit > 5000 { let scan_limit = 5000 }
// 2. Fetch node sample
let nodes_json: String = engram_scan_nodes_json(scan_limit, 0)
let n: Int = json_array_len(nodes_json)
if n == 0 { return "[]" }
// 3. Compute avg_doc_len from sample
let total_words: Int = 0
let i: Int = 0
while i < n {
let node: String = json_array_get(nodes_json, i)
let content: String = json_get_string(node, "content")
let tokens: String = bm25_tokenize(content)
let wc: Int = str_count_words(tokens)
let total_words = total_words + wc
let i = i + 1
}
// avg_doc_len as string for safe float passing
let avg_doc_len_f: Float = float_div(int_to_float(total_words), int_to_float(n))
let avg_doc_len: String = if float_gt(avg_doc_len_f, 0.0) { float_to_str(avg_doc_len_f) } else { "1.0" }
// 4. Tokenize query
let query_tokens: String = bm25_tokenize(query)
if str_eq(str_trim(query_tokens), "") { return "[]" }
// 5. Score each node; collect results as parallel JSON and score lists.
// Scores are stored as strings (float_to_str) to avoid float-in-list issues.
let result_nodes: List = 0
let result_scores: List = 0
let result_count: Int = 0
let j: Int = 0
while j < n {
let node: String = json_array_get(nodes_json, j)
let content: String = json_get_string(node, "content")
let sc_str: String = bm25_score_doc(content, query_tokens, n, avg_doc_len)
// Only include nodes with score > 0.0 (use float comparison, not string match
// float_to_str(0.0) returns "0.000000", not "0.0").
if float_gt(str_to_float(sc_str), 0.0) {
let result_nodes = list_push(result_nodes, node)
let result_scores = list_push(result_scores, sc_str)
let result_count = result_count + 1
}
let j = j + 1
}
if result_count == 0 { return "[]" }
// 6. Selection-sort descending by score, take top `limit`
let out_limit: Int = if result_count < limit { result_count } else { limit }
let k: Int = 0
while k < out_limit {
// Find max score index in [k, result_count)
let max_idx: Int = k
let max_sc_str: String = get(result_scores, k)
let max_sc_f: Float = str_to_float(max_sc_str)
let p: Int = k + 1
while p < result_count {
let sc2_str: String = get(result_scores, p)
let sc2_f: Float = str_to_float(sc2_str)
if float_gt(sc2_f, max_sc_f) {
let max_sc_f = sc2_f
let max_sc_str = sc2_str
let max_idx = p
}
let p = p + 1
}
// Swap k <-> max_idx
if max_idx != k {
let tmp_node: String = get(result_nodes, k)
let tmp_sc: String = get(result_scores, k)
let result_nodes = list_set(result_nodes, k, get(result_nodes, max_idx))
let result_scores = list_set(result_scores, k, get(result_scores, max_idx))
let result_nodes = list_set(result_nodes, max_idx, tmp_node)
let result_scores = list_set(result_scores, max_idx, tmp_sc)
}
let k = k + 1
}
// 7. Build JSON array of top `out_limit` nodes with bm25_score field
let out: String = "["
let r: Int = 0
while r < out_limit {
let node: String = get(result_nodes, r)
let sc_str: String = get(result_scores, r)
// Inject bm25_score: trim the closing } and append field
let node_len: Int = str_len(node)
let node_body: String = str_slice(node, 0, node_len - 1)
let entry: String = node_body + ",\"bm25_score\":" + sc_str + "}"
if r > 0 { let out = out + "," }
let out = out + entry
let r = r + 1
}
out + "]"
}
// Auto-linking
//
// auto_link_content_node link a newly-created Knowledge or Memory node to
// semantically related non-ISE nodes via BM25 search.
//
// Problem it solves: route_neuron_memory and route_neuron_knowledge_capture
// both call engram_node_full directly, creating nodes with zero edges. With
// 14K+ ISEs dominating the corpus, BFS traversal contributes nothing every
// query relies solely on lexical/semantic seed matching. Auto-linking builds
// explicit "related" edges so activated knowledge nodes fan out to connected
// neighbors during BFS.
//
// Design choices:
// - BM25 (not substring search): ranks by relevance, not just occurrence
// - Skip InternalStateEvent nodes: ISEs dominate the corpus and are not
// useful link targets for knowledge/memory nodes
// - Up to 3 edges per node: enough to build graph structure without over-linking
// - weight=0.6: moderately strong; causal edges (field-validated at 2.0) are
// much stronger, so these "related" edges don't flood activation paths
// - state_set for linked counter: EL `let` in nested if-blocks creates inner
// scope only; state_set persists across block boundaries (2026-05-25 lesson)
//
// (2026-05-28 self-review)
fn auto_link_content_node(node_id: String, content: String) -> Int {
let clen: Int = str_len(content)
if clen < 20 { return 0 }
// Find search term: first word >= 5 chars, or second word.
let sp1: Int = str_index_of(content, " ")
let w1end: Int = if sp1 < 0 { clen } else { sp1 }
let word1: String = str_slice(content, 0, w1end)
state_set("aln_term", "")
if str_len(word1) >= 5 {
state_set("aln_term", word1)
}
if str_eq(state_get("aln_term"), "") {
if sp1 >= 0 {
let rest: String = str_slice(content, sp1 + 1, clen)
let sp2: Int = str_index_of(rest, " ")
let w2end: Int = if sp2 < 0 { str_len(rest) } else { sp2 }
let word2: String = str_slice(rest, 0, w2end)
if str_len(word2) >= 5 {
state_set("aln_term", word2)
}
}
}
let search_term: String = state_get("aln_term")
if str_eq(search_term, "") { return 0 }
// BM25 over top-20 results; skip ISE nodes; connect up to 3.
let results: String = bm25_search_json(search_term, 20)
let n: Int = json_array_len(results)
state_set("aln_linked", "0")
let i: Int = 0
while i < n {
let linked_so_far: Int = str_to_int(state_get("aln_linked"))
if linked_so_far < 3 {
let elem: String = json_array_get(results, i)
let rid: String = json_get_string(elem, "id")
let rtype: String = json_get_string(elem, "node_type")
if !str_eq(rtype, "InternalStateEvent") && !str_eq(rid, "") && !str_eq(rid, node_id) {
engram_connect(node_id, rid, 0.6, "related")
state_set("aln_linked", int_to_str(linked_so_far + 1))
}
}
let i = i + 1
}
return str_to_int(state_get("aln_linked"))
}
// Helpers
fn parse_port(bind: String) -> Int {
// ":8742" 8742; "0.0.0.0:8742" → 8742; bare "8742" → 8742
let colon: Int = str_index_of(bind, ":")
if colon < 0 {
return str_to_int(bind)
}
let after: String = str_slice(bind, colon + 1, str_len(bind))
return str_to_int(after)
}
fn ok_json() -> String {
"{\"ok\":true}"
}
fn err_json(msg: String) -> String {
"{\"error\":\"" + msg + "\"}"
}
fn strip_query(path: String) -> String {
let q: Int = str_index_of(path, "?")
if q < 0 { return path }
str_slice(path, 0, q)
}
fn query_param(path: String, key: String) -> String {
let q: Int = str_index_of(path, "?")
if q < 0 { return "" }
let qs: String = str_slice(path, q + 1, str_len(path))
let needle: String = key + "="
let pos: Int = str_index_of(qs, needle)
if pos < 0 { return "" }
let after: String = str_slice(qs, pos + str_len(needle), str_len(qs))
let amp: Int = str_index_of(after, "&")
if amp < 0 { return after }
str_slice(after, 0, amp)
}
fn query_int(path: String, key: String, default_val: Int) -> Int {
let v: String = query_param(path, key)
if str_eq(v, "") { return default_val }
str_to_int(v)
}
// Extract last path segment after a known prefix: extract_id("/api/nodes/abc-123", "/api/nodes/") "abc-123"
fn extract_id(path: String, prefix: String) -> String {
let clean: String = strip_query(path)
if !str_starts_with(clean, prefix) { return "" }
let after: String = str_slice(clean, str_len(prefix), str_len(clean))
let slash: Int = str_index_of(after, "/")
if slash < 0 { return after }
str_slice(after, 0, slash)
}
// Routes
fn route_stats(method: String, path: String, body: String) -> String {
engram_stats_json()
}
fn route_create_node(method: String, path: String, body: String) -> String {
let content: String = json_get_string(body, "content")
let node_type: String = json_get_string(body, "node_type")
if str_eq(node_type, "") { let node_type = "Memory" }
let salience: Float = json_get_float(body, "salience")
if salience == 0.0 { let salience = 0.5 }
let id: String = engram_node(content, node_type, salience)
// Auto-link: find semantically related existing nodes and form edges.
// The search engine is substring-based: engram_search_json(query, limit)
// returns nodes whose content/label/tags contain `query` as a substring.
// Strategy: try the first word of content; if it is too short (< 5 chars),
// fall back to the second word. Connect the top 5 unique matches (no self).
let auto_linked: Int = 0
let clen: Int = str_len(content)
if clen >= 20 {
// Locate first and second spaces to extract first two words.
let sp1: Int = str_index_of(content, " ")
let w1end: Int = sp1
if sp1 < 0 { let w1end = clen }
let word1: String = str_slice(content, 0, w1end)
// Pick the search term: use word1 if >= 5 chars, else try word2.
let search_term: String = ""
if str_len(word1) >= 5 {
let search_term = word1
}
if str_eq(search_term, "") {
if sp1 >= 0 {
let rest: String = str_slice(content, sp1 + 1, clen)
let sp2: Int = str_index_of(rest, " ")
let w2end: Int = sp2
if sp2 < 0 { let w2end = str_len(rest) }
let word2: String = str_slice(rest, 0, w2end)
if str_len(word2) >= 5 {
let search_term = word2
}
}
}
if !str_eq(search_term, "") {
let results: String = engram_search_json(search_term, 10)
let n: Int = json_array_len(results)
let i: Int = 0
while i < n {
if auto_linked >= 5 { let i = n }
if auto_linked < 5 {
let elem: String = json_array_get(results, i)
let rid: String = json_get_string(elem, "id")
if !str_eq(rid, "") {
if !str_eq(rid, id) {
engram_connect(id, rid, 0.5, "related")
let auto_linked = auto_linked + 1
}
}
let i = i + 1
}
}
}
}
"{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\",\"auto_linked\":" + int_to_str(auto_linked) + "}"
}
fn route_get_node(method: String, path: String, body: String) -> String {
let id: String = extract_id(path, "/api/nodes/")
if str_eq(id, "") { return err_json("missing id") }
return engram_get_node_json(id)
}
fn route_scan_nodes(method: String, path: String, body: String) -> String {
let limit: Int = query_int(path, "limit", 50)
let offset: Int = query_int(path, "offset", 0)
let nt: String = query_param(path, "node_type")
if str_eq(nt, "") {
return engram_scan_nodes_json(limit, offset)
}
return engram_scan_nodes_by_type_json(nt, limit, offset)
}
// route_scan_edges bulk export of all edges as a JSON array. Implemented
// via engram_save fs_read of the canonical on-disk snapshot, which the
// runtime keeps in lockstep with the in-memory graph. Live against the
// running graph, not a stale export.
fn route_scan_edges(method: String, path: String, body: String) -> String {
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let snap_path: String = dir + "/snapshot.json"
engram_save(snap_path)
let snap: String = fs_read(snap_path)
if str_eq(snap, "") { return "[]" }
// json_get truncates at the first delimiter (no bracket depth tracking),
// so for the edges ARRAY value we need json_get_raw, which honors
// brackets and returns the full sub-JSON.
let edges: String = json_get_raw(snap, "edges")
if str_eq(edges, "") { return "[]" }
return edges
}
fn route_search(method: String, path: String, body: String) -> String {
let q: String = ""
if str_eq(method, "GET") {
let q = query_param(path, "q")
} else {
let q = json_get_string(body, "query")
}
let limit: Int = query_int(path, "limit", 20)
if limit == 0 { let limit = json_get_int(body, "limit") }
if limit == 0 { let limit = 20 }
return bm25_search_json(q, limit)
}
fn route_activate(method: String, path: String, body: String) -> String {
let q: String = ""
let depth: Int = 3
if str_eq(method, "GET") {
let q = query_param(path, "q")
let depth = query_int(path, "depth", 3)
} else {
let q = json_get_string(body, "query")
let bd: Int = json_get_int(body, "depth")
if bd > 0 { let depth = bd }
}
// BM25 pre-bias: strengthen top-10 BM25 results before spreading activation
// so semantically relevant nodes already have elevated salience.
let top: String = bm25_search_json(q, 10)
let nb: Int = json_array_len(top)
let bi: Int = 0
while bi < nb {
let node: String = json_array_get(top, bi)
let nid: String = json_get_string(node, "id")
if !str_eq(nid, "") { engram_strengthen(nid) }
let bi = bi + 1
}
return "{\"results\":" + engram_activate_json(q, depth) + "}"
}
fn route_create_edge(method: String, path: String, body: String) -> String {
let from_id: String = json_get_string(body, "from_id")
let to_id: String = json_get_string(body, "to_id")
let relation: String = json_get_string(body, "relation")
if str_eq(relation, "") { let relation = "associates" }
let weight: Float = json_get_float(body, "weight")
if weight == 0.0 { let weight = 0.5 }
engram_connect(from_id, to_id, weight, relation)
"{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + relation + "\"}"
}
fn route_neighbors(method: String, path: String, body: String) -> String {
let id: String = extract_id(path, "/api/neighbors/")
if str_eq(id, "") { return err_json("missing id") }
let depth: Int = query_int(path, "depth", 1)
return engram_neighbors_json(id, depth, "both")
}
fn route_strengthen(method: String, path: String, body: String) -> String {
let id: String = json_get_string(body, "node_id")
if str_eq(id, "") { return err_json("missing node_id") }
engram_strengthen(id)
ok_json()
}
fn route_forget(method: String, path: String, body: String) -> String {
let id: String = extract_id(path, "/api/nodes/")
if str_eq(id, "") { return err_json("missing id") }
engram_forget(id)
ok_json()
}
fn route_decay(method: String, path: String, body: String) -> String {
engram_apply_decay_json()
}
fn route_export(method: String, path: String, body: String) -> String {
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
// Write binary checkpoint
let db_path: String = dir + "/engram.db"
engram_write_binary_el(db_path)
// Also write JSON export for human inspection
let p: String = json_get_string(body, "path")
if str_eq(p, "") {
let p = dir + "/snapshot.json"
}
engram_save(p)
"{\"ok\":true,\"binary\":\"" + db_path + "\",\"json\":\"" + p + "\"}"
}
fn route_reindex(method: String, path: String, body: String) -> String {
engram_reindex_json()
}
fn route_load(method: String, path: String, body: String) -> String {
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let db_path: String = dir + "/engram.db"
let ok: Bool = engram_load_binary_el(db_path)
if !ok {
let p: String = json_get_string(body, "path")
if str_eq(p, "") {
let p = dir + "/snapshot.json"
}
engram_load(p)
}
ok_json()
}
fn route_health(method: String, path: String, body: String) -> String {
"{\"status\":\"ok\",\"engine\":\"engram-runtime-native\"}"
}
// /api/neuron/* Routes
// route_neuron_session_begin activate with broad seeds, return node stats + results
fn route_neuron_session_begin(method: String, path: String, body: String) -> String {
let results: String = engram_activate_json("memory knowledge context", 2)
let nc: Int = engram_node_count()
let ec: Int = engram_edge_count()
"{\"ok\":true,\"nodes\":" + results + ",\"node_count\":" + int_to_str(nc) + ",\"edge_count\":" + int_to_str(ec) + "}"
}
// route_neuron_ctx compile working context from top activated nodes
fn route_neuron_ctx(method: String, path: String, body: String) -> String {
let results: String = engram_activate_json("architecture decision memory", 2)
let n: Int = json_array_len(results)
let limit: Int = if n > 10 { 10 } else { n }
let ctx: String = "Recent working memory:\n"
let i: Int = 0
let ctx_body: String = ""
while i < limit {
let elem: String = json_array_get(results, i)
let label: String = json_get_string(elem, "label")
let content: String = json_get_string(elem, "content")
let clen: Int = str_len(content)
let snippet: String = if clen > 200 { str_slice(content, 0, 200) } else { content }
let ctx_body = ctx_body + "- [" + label + "]: " + snippet + "\n"
let i = i + 1
}
let full_ctx: String = ctx + ctx_body
"{\"ok\":true,\"context\":\"" + str_replace(str_replace(str_replace(full_ctx, "\\", "\\\\"), "\"", "\\\""), "\n", "\\n") + "\"}"
}
// route_neuron_memory create a Memory node with importance-to-tier mapping
fn route_neuron_memory(method: String, path: String, body: String) -> String {
let content: String = json_get_string(body, "content")
if str_eq(content, "") { return "{\"error\":\"content is required\"}" }
let node_type: String = json_get_string(body, "node_type")
if str_eq(node_type, "") { let node_type = "Memory" }
let label: String = json_get_string(body, "label")
let importance: String = json_get_string(body, "importance")
let project: String = json_get_string(body, "project")
let tags_raw: String = json_get_string(body, "tags")
// Map importance to tier
let tier: String = "Episodic"
if str_eq(importance, "critical") { let tier = "Procedural" }
if str_eq(importance, "high") { let tier = "Semantic" }
if str_eq(importance, "normal") { let tier = "Episodic" }
if str_eq(importance, "low") { let tier = "Working" }
// Override with explicit tier if provided
let explicit_tier: String = json_get_string(body, "tier")
if !str_eq(explicit_tier, "") { let tier = explicit_tier }
// Build tags string append project tag if set
let tags_str: String = tags_raw
if !str_eq(project, "") {
if str_eq(tags_str, "") {
let tags_str = "project:" + project
}
if !str_eq(tags_str, "") {
let tags_str = tags_str + " project:" + project
}
}
let id: String = engram_node_full(content, node_type, label, 0.5, 0.5, 1.0, tier, tags_str)
// Auto-link to related non-ISE nodes so this memory is reachable via BFS traversal.
// Without this, MCP-created nodes arrive with zero edges and are invisible to
// graph spread during activation (only lexical/semantic seed matching finds them).
let auto_linked: Int = auto_link_content_node(id, content)
// Checkpoint after write
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let db_path: String = dir + "/engram.db"
engram_write_binary_el(db_path)
"{\"ok\":true,\"id\":\"" + id + "\",\"auto_linked\":" + int_to_str(auto_linked) + ",\"content\":\"" + str_replace(str_replace(content, "\\", "\\\\"), "\"", "\\\"") + "\"}"
}
// route_neuron_knowledge_capture create a Knowledge node
fn route_neuron_knowledge_capture(method: String, path: String, body: String) -> String {
let content: String = json_get_string(body, "content")
if str_eq(content, "") { return "{\"error\":\"content is required\"}" }
let title: String = json_get_string(body, "title")
let category: String = json_get_string(body, "category")
let tags_raw: String = json_get_string(body, "tags")
let project: String = json_get_string(body, "project")
let tier_raw: String = json_get_string(body, "tier")
// Map tier name to engram tier
let tier: String = "Episodic"
if str_eq(tier_raw, "lesson") { let tier = "Semantic" }
if str_eq(tier_raw, "canonical") { let tier = "Procedural" }
if str_eq(tier_raw, "note") { let tier = "Episodic" }
// Build tags
let tags_str: String = tags_raw
if !str_eq(category, "") {
if str_eq(tags_str, "") {
let tags_str = "category:" + category
}
if !str_eq(tags_str, "") {
let tags_str = tags_str + " category:" + category
}
}
if !str_eq(project, "") {
if str_eq(tags_str, "") {
let tags_str = "project:" + project
}
if !str_eq(tags_str, "") {
let tags_str = tags_str + " project:" + project
}
}
let id: String = engram_node_full(content, "Knowledge", title, 0.7, 0.7, 1.0, tier, tags_str)
// Auto-link to related non-ISE nodes for BFS reachability (same rationale as route_neuron_memory).
let auto_linked: Int = auto_link_content_node(id, content)
// Checkpoint
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let db_path: String = dir + "/engram.db"
engram_write_binary_el(db_path)
"{\"ok\":true,\"id\":\"" + id + "\",\"auto_linked\":" + int_to_str(auto_linked) + "}"
}
// route_neuron_knowledge_evolve create updated node (evolution via new node)
fn route_neuron_knowledge_evolve(method: String, path: String, body: String) -> String {
let content: String = json_get_string(body, "content")
let prior_id: String = json_get_string(body, "id")
if str_eq(content, "") { return "{\"ok\":true}" }
let id: String = engram_node_full(content, "Knowledge", "", 0.7, 0.7, 1.0, "Semantic", "evolved")
if !str_eq(prior_id, "") && !str_eq(id, "") {
engram_connect(id, prior_id, 1.0, "supersedes")
}
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
engram_write_binary_el(dir + "/engram.db")
"{\"ok\":true,\"id\":\"" + id + "\"}"
}
// route_neuron_knowledge_promote promote a knowledge node to a higher tier.
// Creates a new node with the promoted tier (same content) and connects
// via a "supersedes" edge from new old. Tier mapping:
// note/Episodic lesson/Semantic canonical/Procedural
fn route_neuron_knowledge_promote(method: String, path: String, body: String) -> String {
let id: String = json_get_string(body, "id")
if str_eq(id, "") { return "{\"ok\":true}" }
// Read existing node
let node_json: String = engram_get_node_json(id)
if str_eq(node_json, "") { return err_json("node not found") }
if str_eq(node_json, "null") { return err_json("node not found") }
let content: String = json_get_string(node_json, "content")
if str_eq(content, "") { return err_json("node has no content") }
let label: String = json_get_string(node_json, "label")
let tags: String = json_get_string(node_json, "tags")
let current_tier: String = json_get_string(node_json, "tier")
// Determine target tier: explicit override or auto-promote one level
let tier_raw: String = json_get_string(body, "tier")
let new_tier: String = ""
// Explicit tier takes precedence
if str_eq(tier_raw, "lesson") { let new_tier = "Semantic" }
if str_eq(tier_raw, "canonical") { let new_tier = "Procedural" }
if str_eq(tier_raw, "note") { let new_tier = "Episodic" }
// Auto-promote one level if no explicit tier
if str_eq(new_tier, "") {
if str_eq(current_tier, "Working") { let new_tier = "Episodic" }
if str_eq(current_tier, "Episodic") { let new_tier = "Semantic" }
if str_eq(current_tier, "Semantic") { let new_tier = "Procedural" }
if str_eq(current_tier, "Procedural") { let new_tier = "Procedural" }
}
if str_eq(new_tier, "") { let new_tier = "Semantic" }
// Create promoted node higher importance (0.8) signals durable knowledge
let new_id: String = engram_node_full(content, "Knowledge", label, 0.7, 0.8, 1.0, new_tier, tags)
// Wire supersedes edge: new node supersedes old
if !str_eq(new_id, "") {
engram_connect(new_id, id, 1.0, "supersedes")
}
// Checkpoint
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
engram_write_binary_el(dir + "/engram.db")
"{\"ok\":true,\"id\":\"" + new_id + "\",\"promoted_from\":\"" + id + "\",\"tier\":\"" + new_tier + "\"}"
}
// route_neuron_recall search or list nodes
fn route_neuron_recall(method: String, path: String, body: String) -> String {
let query: String = json_get_string(body, "query")
let chain: String = json_get_string(body, "chain_name")
let limit: Int = json_get_int(body, "limit")
if limit == 0 { let limit = 20 }
let q: String = if str_eq(query, "") { chain } else { query }
if str_eq(q, "") {
return engram_scan_nodes_json(limit, 0)
}
return bm25_search_json(q, limit)
}
// route_neuron_graph get node + search-based neighbor approximation.
// engram_neighbors_json crashes on large graphs (15k+ edges exceeds BFS cap).
// Use a search-based approach instead: search by the node id string, which
// returns connected nodes that share content with the target id in edges/tags.
// For the mcp-wrapper callers this is sufficient they just need the node itself.
fn route_neuron_graph(method: String, path: String, body: String) -> String {
let id: String = query_param(path, "id")
if str_eq(id, "") { return "{\"error\":\"id is required\"}" }
let node_json: String = engram_get_node_json(id)
// Return node with empty neighbors safe fallback avoids BFS crash
"{\"ok\":true,\"node\":" + node_json + ",\"neighbors\":[]}"
}
// route_neuron_graph_link create edge between nodes
fn route_neuron_graph_link(method: String, path: String, body: String) -> String {
let from_id: String = json_get_string(body, "from_id")
let to_id: String = json_get_string(body, "to_id")
if str_eq(from_id, "") || str_eq(to_id, "") {
return "{\"error\":\"from_id and to_id are required\"}"
}
let relation: String = json_get_string(body, "relation")
if str_eq(relation, "") { let relation = "related" }
let weight: Float = json_get_float(body, "weight")
if weight == 0.0 { let weight = 0.5 }
engram_connect(from_id, to_id, weight, relation)
"{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + relation + "\"}"
}
// route_neuron_list list nodes by type extracted from path
fn route_neuron_list(method: String, path: String, body: String) -> String {
let clean: String = strip_query(path)
let prefix: String = "/api/neuron/list/"
let node_type: String = str_slice(clean, str_len(prefix), str_len(clean))
let limit: Int = query_int(path, "limit", 50)
if str_eq(node_type, "") { return "[]" }
return engram_scan_nodes_by_type_json(node_type, limit, 0)
}
// route_neuron_consolidate checkpoint and return counts
fn route_neuron_consolidate(method: String, path: String, body: String) -> String {
let dir: String = env("ENGRAM_DATA_DIR")
if str_eq(dir, "") { let dir = "/tmp/engram" }
let db_path: String = dir + "/engram.db"
engram_write_binary_el(db_path)
let nc: Int = engram_node_count()
let ec: Int = engram_edge_count()
"{\"ok\":true,\"node_count\":" + int_to_str(nc) + ",\"edge_count\":" + int_to_str(ec) + "}"
}
// route_neuron_config return stub config values
fn route_neuron_config(method: String, path: String, body: String) -> String {
let key: String = query_param(path, "key")
"{\"key\":\"" + key + "\",\"value\":\"\"}"
}
// route_neuron_state_events GET lists ISEs, POST logs a new one.
// GET supports ?limit=N&offset=M for pagination; ?label=X to extract label
// from the ISE content's "event" field.
// ISEs sort by created_at DESC (most-recent-first) as of 2026-05-23 fix.
// ?limit=10 returns the 10 most recent ISEs. Offset for pagination, not for
// skipping to recent events (that was the pre-fix behavior; no longer needed).
fn route_neuron_state_events(method: String, path: String, body: String) -> String {
if str_eq(method, "GET") {
let limit_str: String = query_param(path, "limit")
let limit: Int = if str_eq(limit_str, "") { 50 } else { str_to_int(limit_str) }
let offset_str: String = query_param(path, "offset")
let offset: Int = if str_eq(offset_str, "") { 0 } else { str_to_int(offset_str) }
return engram_scan_nodes_by_type_json("InternalStateEvent", limit, offset)
}
let content: String = json_get_string(body, "content")
if str_eq(content, "") { let content = body }
// Extract label from content JSON "event" field for better ISE searchability
let event_label: String = json_get_string(content, "event")
let label: String = if str_eq(event_label, "") { "state-event" } else { event_label }
let id: String = engram_node_full(content, "InternalStateEvent", label, 0.3, 0.3, 1.0, "Working", "internal-state")
"{\"ok\":true,\"id\":\"" + id + "\"}"
}
// route_neuron_processes stub
fn route_neuron_processes(method: String, path: String, body: String) -> String {
"{\"ok\":true,\"processes\":[]}"
}
// route_events_next stub empty event queue
fn route_events_next(method: String, path: String, body: String) -> String {
"{\"ok\":true,\"event\":null}"
}
// route_events_ack stub ack
fn route_events_ack(method: String, path: String, body: String) -> String {
"{\"ok\":true}"
}
fn route_bm25_search(method: String, path: String, body: String) -> String {
let q: String = ""
if str_eq(method, "GET") {
let q = query_param(path, "q")
} else {
let q = json_get_string(body, "query")
}
if str_eq(q, "") { return "{\"error\":\"query is required\"}" }
let limit: Int = query_int(path, "limit", 20)
if limit == 0 { let limit = json_get_int(body, "limit") }
if limit == 0 { let limit = 20 }
bm25_search_json(q, limit)
}
// Auth
fn check_auth_ok(method: String, body: String) -> Bool {
let key: String = env("ENGRAM_API_KEY")
if str_eq(key, "") { return true }
// Read-only methods don't require auth. Until http_serve surfaces
// request headers we can't accept a Bearer token cleanly; mutating
// requests must include "_auth": "<key>" in the JSON body.
if str_eq(method, "GET") { return true }
let provided: String = json_get_string(body, "_auth")
if str_eq(provided, key) { return true }
return false
}
// Dispatcher
fn handle_request(method: String, path: String, body: String) -> String {
let clean: String = strip_query(path)
// Health is always reachable
if str_eq(method, "GET") {
if str_eq(clean, "/health") || str_eq(clean, "/") {
return route_health(method, path, body)
}
}
// /api/neuron/* and /events/* are pre-auth the mcp-wrapper is a trusted
// local service that cannot inject _auth into its request bodies.
if str_starts_with(clean, "/api/neuron/") || str_starts_with(clean, "/events/") {
if str_eq(clean, "/api/neuron/session/begin") {
return route_neuron_session_begin(method, path, body)
}
if str_eq(clean, "/api/neuron/ctx") {
return route_neuron_ctx(method, path, body)
}
if str_eq(clean, "/api/neuron/memory") {
return route_neuron_memory(method, path, body)
}
if str_eq(clean, "/api/neuron/knowledge/capture") {
return route_neuron_knowledge_capture(method, path, body)
}
if str_eq(clean, "/api/neuron/knowledge/evolve") {
return route_neuron_knowledge_evolve(method, path, body)
}
if str_eq(clean, "/api/neuron/knowledge/promote") {
return route_neuron_knowledge_promote(method, path, body)
}
if str_eq(clean, "/api/neuron/recall") {
return route_neuron_recall(method, path, body)
}
if str_eq(clean, "/api/neuron/graph/link") {
return route_neuron_graph_link(method, path, body)
}
if str_eq(clean, "/api/neuron/graph") {
return route_neuron_graph(method, path, body)
}
if str_starts_with(clean, "/api/neuron/list/") {
return route_neuron_list(method, path, body)
}
if str_eq(clean, "/api/neuron/consolidate") {
return route_neuron_consolidate(method, path, body)
}
if str_eq(clean, "/api/neuron/config") {
return route_neuron_config(method, path, body)
}
if str_eq(clean, "/api/neuron/state-events") {
return route_neuron_state_events(method, path, body)
}
if str_eq(clean, "/api/neuron/processes/define") {
return route_neuron_processes(method, path, body)
}
if str_eq(clean, "/api/neuron/processes") {
return route_neuron_processes(method, path, body)
}
if str_eq(clean, "/events/next") {
return route_events_next(method, path, body)
}
if str_eq(clean, "/events/ack") {
return route_events_ack(method, path, body)
}
return err_json("not found")
}
// Auth (when ENGRAM_API_KEY is set)
if !check_auth_ok(method, body) {
return err_json("unauthorized")
}
// Stats
if str_eq(method, "GET") && (str_eq(clean, "/api/stats") || str_eq(clean, "/stats")) {
return route_stats(method, path, body)
}
// Nodes
if str_eq(method, "POST") && (str_eq(clean, "/api/nodes") || str_eq(clean, "/nodes")) {
return route_create_node(method, path, body)
}
if str_eq(method, "GET") && (str_eq(clean, "/api/nodes") || str_eq(clean, "/nodes") || str_eq(clean, "/nodes/list") || str_eq(clean, "/api/nodes/list")) {
return route_scan_nodes(method, path, body)
}
if str_eq(method, "GET") && (str_eq(clean, "/api/edges") || str_eq(clean, "/edges")) {
return route_scan_edges(method, path, body)
}
if str_eq(method, "GET") && str_starts_with(clean, "/api/nodes/") {
return route_get_node(method, path, body)
}
if str_eq(method, "DELETE") && str_starts_with(clean, "/api/nodes/") {
return route_forget(method, path, body)
}
// Edges
if str_eq(method, "POST") && (str_eq(clean, "/api/edges") || str_eq(clean, "/edges")) {
return route_create_edge(method, path, body)
}
if str_eq(method, "GET") && str_starts_with(clean, "/api/neighbors/") {
return route_neighbors(method, path, body)
}
// Activation + Search
if str_eq(method, "POST") && (str_eq(clean, "/api/activate") || str_eq(clean, "/activate")) {
return route_activate(method, path, body)
}
if str_eq(method, "GET") && str_starts_with(clean, "/api/activate") {
return route_activate(method, path, body)
}
if str_eq(method, "POST") && (str_eq(clean, "/api/search") || str_eq(clean, "/search")) {
return route_search(method, path, body)
}
if str_eq(method, "GET") && str_starts_with(clean, "/api/search") {
return route_search(method, path, body)
}
// BM25+ text ranking
if str_eq(clean, "/api/bm25/search") {
return route_bm25_search(method, path, body)
}
// Strengthen
if str_eq(method, "POST") && (str_eq(clean, "/api/strengthen") || str_eq(clean, "/strengthen")) {
return route_strengthen(method, path, body)
}
// Temporal decay maintenance
if str_eq(method, "POST") && (str_eq(clean, "/api/decay") || str_eq(clean, "/api/maintenance") || str_eq(clean, "/decay")) {
return route_decay(method, path, body)
}
// Persistence
if str_eq(method, "POST") && (str_eq(clean, "/api/export") || str_eq(clean, "/export")) {
return route_export(method, path, body)
}
// /api/save is kept as a backward-compat alias for /api/export
if str_eq(method, "POST") && (str_eq(clean, "/api/save") || str_eq(clean, "/save")) {
return route_export(method, path, body)
}
if str_eq(method, "POST") && (str_eq(clean, "/api/load") || str_eq(clean, "/load")) {
return route_load(method, path, body)
}
if str_eq(method, "POST") && (str_eq(clean, "/api/reindex") || str_eq(clean, "/reindex")) {
return route_reindex(method, path, body)
}
// /api/neuron/*
if str_starts_with(clean, "/api/neuron/") {
// Specific sub-paths first (longer matches before shorter)
if str_eq(clean, "/api/neuron/session/begin") {
return route_neuron_session_begin(method, path, body)
}
if str_eq(clean, "/api/neuron/ctx") {
return route_neuron_ctx(method, path, body)
}
if str_eq(clean, "/api/neuron/memory") {
return route_neuron_memory(method, path, body)
}
if str_eq(clean, "/api/neuron/knowledge/capture") {
return route_neuron_knowledge_capture(method, path, body)
}
if str_eq(clean, "/api/neuron/knowledge/evolve") {
return route_neuron_knowledge_evolve(method, path, body)
}
if str_eq(clean, "/api/neuron/knowledge/promote") {
return route_neuron_knowledge_promote(method, path, body)
}
if str_eq(clean, "/api/neuron/recall") {
return route_neuron_recall(method, path, body)
}
if str_eq(clean, "/api/neuron/graph/link") {
return route_neuron_graph_link(method, path, body)
}
if str_eq(clean, "/api/neuron/graph") {
return route_neuron_graph(method, path, body)
}
if str_starts_with(clean, "/api/neuron/list/") {
return route_neuron_list(method, path, body)
}
if str_eq(clean, "/api/neuron/consolidate") {
return route_neuron_consolidate(method, path, body)
}
if str_eq(clean, "/api/neuron/config") {
return route_neuron_config(method, path, body)
}
if str_eq(clean, "/api/neuron/state-events") {
return route_neuron_state_events(method, path, body)
}
if str_eq(clean, "/api/neuron/processes/define") {
return route_neuron_processes(method, path, body)
}
if str_eq(clean, "/api/neuron/processes") {
return route_neuron_processes(method, path, body)
}
}
// /events/*
if str_eq(clean, "/events/next") {
return route_events_next(method, path, body)
}
if str_eq(clean, "/events/ack") {
return route_events_ack(method, path, body)
}
"{\"error\":\"not found\",\"path\":\"" + clean + "\"}"
}
// Entry
let bind_str: String = env("ENGRAM_BIND")
if str_eq(bind_str, "") { let bind_str = ":8742" }
let port: Int = parse_port(bind_str)
// On startup, load from binary database (ML-KEM-1024 encrypted).
// Falls back to per-file JSON, then snapshot.json for migration from older formats.
let data_dir: String = env("ENGRAM_DATA_DIR")
if str_eq(data_dir, "") { let data_dir = "/tmp/engram" }
let db_path: String = data_dir + "/engram.db"
let loaded: Bool = engram_load_binary_el(db_path)
if !loaded {
// Migration path: try per-file JSON
engram_load_dir(data_dir)
if engram_node_count() == 0 {
// Final fallback: legacy snapshot.json
let snapshot_path: String = data_dir + "/snapshot.json"
engram_load(snapshot_path)
}
// If we loaded anything from legacy format, save as binary immediately
if engram_node_count() > 0 {
engram_write_binary_el(db_path)
println("[engram] migrated legacy data to binary format")
}
}
println("[engram] runtime-native graph engine (ML-KEM-1024 encrypted)")
println("[engram] data_dir=" + data_dir)
println("[engram] node_count=" + int_to_str(engram_node_count()))
println("[engram] edge_count=" + int_to_str(engram_edge_count()))
println("[engram] listening on " + int_to_str(port))
http_set_handler("handle_request")
http_serve(port, "handle_request")