971b21751a
Yesterday's eligibility-trace fix made Hebbian consolidation numerically real: hebb_max 0.000799 -> 0.4725, and 1,198 hebbian-associate edges formed in 23h48m. This morning's census found where they went: nowhere. soul daemon (in-process graph): 42,426 edges, 1,198 hebbian engram server (:8742, durable): 41,213 edges, 49 hebbian Two processes, two graphs, one direction of travel. The soul pulls from the server every 10 min (GET /api/sync) and never pushes. It cannot fall back on saving its own copy either: soul.el sets soul_snapshot_path only inside `if is_genesis && safe_to_seed`, and safe_to_seed is unconditionally false whenever ENGRAM_URL is set -- because the server owns persistence and a soul writing snapshot.json would clobber it. That guard is correct. The consequence was not: mem_save() has never once executed. The soul is the ONLY process running idle cognition, so it is where essentially all co-activation happens -- and it was throwing away every association it learned, every restart, silently. The mechanism worked and the learning still evaporated. Consolidation is now a message, not a file. Fast volatile store hands each newly-formed association to the slow durable store over the API the server already exposes; only edges past ENGRAM_HEBB_LINK_MIN are ever queued, so what crosses the process boundary already earned it. - el_runtime.c: 512-slot overwrite-oldest write-back ring; enqueue at edge formation; engram_hebb_drain_json() pops a postable JSON batch. Drops and drains are counted, not silent -- a consolidation path that quietly discards is the exact failure this entry exists to correct. - server.el: POST /api/edges/batch. persist_canonical() writes the full 60MB snapshot per call, and route_create_edge calls it per edge -- correct for one interactive edge, ruinous for bulk (~840MB/beat to persist 14 associations). Batch connects all, snapshots once. Same durability, 1/N the writes. - act-stats: hebb_wb_pending / _drained / _dropped. pending climbing with drained flat = drain not called; drained climbing with sent 0 = POST refused. Both failure modes are now visible in the stream instead of in an autopsy. Verified live: batch route accepts valid entries, skips malformed ones without aborting the batch, and enforces _auth. All 1,256 learned associations are now in the canonical store; the soul booted at 42,431 edges with hebb_max 0.4941 carried across the restart for the first time.
692 lines
34 KiB
EmacsLisp
692 lines
34 KiB
EmacsLisp
// server.el — Engram HTTP server.
|
|
//
|
|
// Engram is the in-process graph store. The runtime owns the data; this
|
|
// file is the thin HTTP face. Every route maps to one or two engram_*
|
|
// builtins. There is no SQL, no db layer, no SQLite — the runtime IS the
|
|
// database.
|
|
//
|
|
// Built and linked with:
|
|
// elc src/server.el > server.c
|
|
// cc -std=c11 -O2 -lcurl -lpthread -o engram server.c el_runtime.c
|
|
// ./engram
|
|
//
|
|
// Configuration via environment:
|
|
// ENGRAM_BIND — host:port (default :8742)
|
|
// ENGRAM_API_KEY — bearer auth (optional)
|
|
// ENGRAM_DATA_DIR — snapshot location (default ~/.neuron/engram)
|
|
|
|
// ── Helpers ───────────────────────────────────────────────────────────────────
|
|
|
|
fn parse_port(bind: String) -> Int {
|
|
// ":8742" → 8742; "0.0.0.0:8742" → 8742; bare "8742" → 8742
|
|
let colon: Int = str_index_of(bind, ":")
|
|
if colon < 0 {
|
|
return str_to_int(bind)
|
|
}
|
|
let after: String = str_slice(bind, colon + 1, str_len(bind))
|
|
return str_to_int(after)
|
|
}
|
|
|
|
fn ok_json() -> String {
|
|
"{\"ok\":true}"
|
|
}
|
|
|
|
fn err_json(msg: String) -> String {
|
|
"{\"error\":\"" + msg + "\"}"
|
|
}
|
|
|
|
fn strip_query(path: String) -> String {
|
|
let q: Int = str_index_of(path, "?")
|
|
if q < 0 { return path }
|
|
str_slice(path, 0, q)
|
|
}
|
|
|
|
fn query_param(path: String, key: String) -> String {
|
|
let q: Int = str_index_of(path, "?")
|
|
if q < 0 { return "" }
|
|
let qs: String = str_slice(path, q + 1, str_len(path))
|
|
let needle: String = key + "="
|
|
let pos: Int = str_index_of(qs, needle)
|
|
if pos < 0 { return "" }
|
|
let after: String = str_slice(qs, pos + str_len(needle), str_len(qs))
|
|
let amp: Int = str_index_of(after, "&")
|
|
if amp < 0 { return after }
|
|
str_slice(after, 0, amp)
|
|
}
|
|
|
|
fn query_int(path: String, key: String, default_val: Int) -> Int {
|
|
let v: String = query_param(path, key)
|
|
if str_eq(v, "") { return default_val }
|
|
str_to_int(v)
|
|
}
|
|
|
|
// Extract last path segment after a known prefix: extract_id("/api/nodes/abc-123", "/api/nodes/") → "abc-123"
|
|
fn extract_id(path: String, prefix: String) -> String {
|
|
let clean: String = strip_query(path)
|
|
if !str_starts_with(clean, prefix) { return "" }
|
|
let after: String = str_slice(clean, str_len(prefix), str_len(clean))
|
|
let slash: Int = str_index_of(after, "/")
|
|
if slash < 0 { return after }
|
|
str_slice(after, 0, slash)
|
|
}
|
|
|
|
// ── Routes ────────────────────────────────────────────────────────────────────
|
|
|
|
fn route_stats(method: String, path: String, body: String) -> String {
|
|
engram_stats_json()
|
|
}
|
|
|
|
// route_act_stats — GET /api/act-stats
|
|
// (2026-08-04 self-review) engram_act_stats_json() has existed since the
|
|
// 2026-07-27 review but was reachable ONLY through the soul daemon's heartbeat
|
|
// binding. Every activation-layer gauge — WM evictions, breakthroughs, embedder
|
|
// breaker state, context drift, and now the Hebbian counters — was therefore
|
|
// invisible unless the soul happened to be running and its ISEs were read back
|
|
// out of the store. Diagnosing the activation layer required a working soul,
|
|
// which is exactly backwards: the lower layer should be observable on its own.
|
|
// This review needed it to verify link formation and could not get at it. One
|
|
// line of plumbing, and the whole activation layer becomes directly diagnosable.
|
|
fn route_act_stats(method: String, path: String, body: String) -> String {
|
|
engram_act_stats_json()
|
|
}
|
|
|
|
// (2026-07-18 self-review) Scoping sweep: `let` inside an if-block creates an
|
|
// inner scope only — it does NOT mutate the outer binding (documented with
|
|
// evidence in awareness.el, 2026-05-25). Every default/reassignment below used
|
|
// that broken pattern, so defaults never applied: nodes were created with
|
|
// node_type="" and salience=0.0, /api/search and /api/activate ALWAYS ran with
|
|
// q="" regardless of input, edges defaulted to relation=""/weight=0.0, and
|
|
// save/load with no "path" hit engram_save(""). Rewritten to the
|
|
// `let x = if cond { a } else { b }` expression form (the pattern the newer
|
|
// routes route_emit_ise/route_capture_knowledge already use correctly).
|
|
// persist_canonical — save the canonical snapshot after a durable write.
|
|
//
|
|
// WHY (2026-07-22 self-review): the 2026-07-21 fix correctly stopped READ
|
|
// routes from writing the canonical snapshot.json — but nothing was left
|
|
// that saved it on WRITE. Every mutation (node create, edge create,
|
|
// knowledge capture, forget, merge) lived only in RAM until someone POSTed
|
|
// /api/save manually; a process restart silently discarded everything since
|
|
// the last manual save. Observed live: two engram restarts during the
|
|
// 2026-07-22 review reverted the store to a ~17h-old snapshot, destroying
|
|
// same-day writes. Reads must never write the canonical; writes must always
|
|
// persist it. ISE telemetry is deliberately excluded (48h-pruned, loss-
|
|
// tolerant, ~2/min — snapshotting the whole store per heartbeat is waste;
|
|
// any durable write that follows persists the pruning too).
|
|
fn persist_canonical() -> Int {
|
|
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
|
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
|
engram_save(dir + "/snapshot.json")
|
|
return 1
|
|
}
|
|
|
|
// INCOMPLETE-ROUTE FIX (2026-07-24 self-review): this route silently dropped
|
|
// label, importance, tier, and tags — engram_node() defaults label to content
|
|
// and importance to 0.5, so every node created over HTTP lost its metadata.
|
|
// Observed live: the soul's boot-counter write-back landed with
|
|
// label="soul:boot_count:99" (content), importance 0.5, no tags. Honor the
|
|
// full field set via engram_node_full when any of them is supplied.
|
|
// PRESENCE-AWARE DEFAULTS (2026-08-01 self-review): the old pattern
|
|
// `if x == 0.0 { default }` made a legitimate 0.0 unrepresentable — a caller
|
|
// setting salience/importance/weight to zero silently got 0.5. json_get_raw
|
|
// returns "" when the key is ABSENT and the raw token when present, so
|
|
// absence and zero are now distinguishable. Also: confidence was hardcoded
|
|
// to 1.0 regardless of input — every HTTP-created node claimed full
|
|
// epistemic confidence. Now honored from the payload (default 1.0).
|
|
fn route_create_node(method: String, path: String, body: String) -> String {
|
|
let content: String = json_get_string(body, "content")
|
|
let nt_raw: String = json_get_string(body, "node_type")
|
|
let node_type: String = if str_eq(nt_raw, "") { "Memory" } else { nt_raw }
|
|
let sal_present: String = json_get_raw(body, "salience")
|
|
let salience: Float = if str_eq(sal_present, "") { 0.5 } else { json_get_float(body, "salience") }
|
|
let label_raw: String = json_get_string(body, "label")
|
|
let label: String = if str_eq(label_raw, "") { content } else { label_raw }
|
|
let imp_present: String = json_get_raw(body, "importance")
|
|
let importance: Float = if str_eq(imp_present, "") { 0.5 } else { json_get_float(body, "importance") }
|
|
let conf_present: String = json_get_raw(body, "confidence")
|
|
let confidence: Float = if str_eq(conf_present, "") { 1.0 } else { json_get_float(body, "confidence") }
|
|
let tier_raw: String = json_get_string(body, "tier")
|
|
let tier: String = if str_eq(tier_raw, "") { "Working" } else { tier_raw }
|
|
let tags: String = json_get_string(body, "tags")
|
|
// NO el_from_float WRAPPER (2026-08-01 self-review): salience/importance/
|
|
// confidence are already Float (el_val_t) values — json_get_float and
|
|
// Float literals both encode. Wrapping them in el_from_float AGAIN
|
|
// reinterpreted the boxed bits as a raw double, producing garbage that
|
|
// failed engram_decode_score's range check and clamped every HTTP-created
|
|
// node to defaults (salience 0.9 in → 0.5 stored; confidence 0.6 in → 1.0
|
|
// stored — verified live). route_emit_ise always passed Floats bare and
|
|
// its 0.3/0.3/0.8 stored correctly; this call now does the same.
|
|
let id: String = engram_node_full(
|
|
content, node_type, label,
|
|
salience, importance, confidence,
|
|
tier, tags
|
|
)
|
|
let saved: Int = persist_canonical()
|
|
"{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\"}"
|
|
}
|
|
|
|
fn route_get_node(method: String, path: String, body: String) -> String {
|
|
let id: String = extract_id(path, "/api/nodes/")
|
|
if str_eq(id, "") { return err_json("missing id") }
|
|
return engram_get_node_json(id)
|
|
}
|
|
|
|
fn route_scan_nodes(method: String, path: String, body: String) -> String {
|
|
let limit: Int = query_int(path, "limit", 50)
|
|
let offset: Int = query_int(path, "offset", 0)
|
|
let nt: String = query_param(path, "node_type")
|
|
if str_eq(nt, "") {
|
|
return engram_scan_nodes_json(limit, offset)
|
|
}
|
|
return engram_scan_nodes_by_type_json(nt, limit, offset)
|
|
}
|
|
|
|
// route_scan_edges — bulk export of all edges as a JSON array. Implemented
|
|
// via engram_save → fs_read of a SCRATCH export path. (2026-07-21 self-review:
|
|
// previously this saved over the canonical snapshot.json on every GET — if the
|
|
// process ever booted with a partial/empty store, the first read request
|
|
// clobbered the good snapshot. Read routes must never write the canonical path.)
|
|
fn route_scan_edges(method: String, path: String, body: String) -> String {
|
|
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
|
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
|
let snap_path: String = dir + "/.scan-export.json"
|
|
engram_save(snap_path)
|
|
let snap: String = fs_read(snap_path)
|
|
if str_eq(snap, "") { return "[]" }
|
|
// json_get truncates at the first delimiter (no bracket depth tracking),
|
|
// so for the edges ARRAY value we need json_get_raw, which honors
|
|
// brackets and returns the full sub-JSON.
|
|
let edges: String = json_get_raw(snap, "edges")
|
|
if str_eq(edges, "") { return "[]" }
|
|
return edges
|
|
}
|
|
|
|
fn route_search(method: String, path: String, body: String) -> String {
|
|
let q: String = if str_eq(method, "GET") { query_param(path, "q") } else { json_get_string(body, "query") }
|
|
let lim_url: Int = query_int(path, "limit", 0)
|
|
let lim_body: Int = json_get_int(body, "limit")
|
|
let lim_either: Int = if lim_url > 0 { lim_url } else { lim_body }
|
|
let limit: Int = if lim_either > 0 { lim_either } else { 20 }
|
|
return engram_search_json(q, limit)
|
|
}
|
|
|
|
fn route_activate(method: String, path: String, body: String) -> String {
|
|
let q: String = if str_eq(method, "GET") { query_param(path, "q") } else { json_get_string(body, "query") }
|
|
// Guard: engram_activate with an empty query matches zero seeds, which
|
|
// zeroes ALL carried working-memory weights (documented in awareness.el
|
|
// perceive()). Never let an empty activation through to wipe WM.
|
|
if str_eq(q, "") { return err_json("missing query") }
|
|
let d_raw: Int = if str_eq(method, "GET") { query_int(path, "depth", 3) } else { json_get_int(body, "depth") }
|
|
let depth: Int = if d_raw > 0 { d_raw } else { 3 }
|
|
return "{\"results\":" + engram_activate_json(q, depth) + "}"
|
|
}
|
|
|
|
fn route_create_edge(method: String, path: String, body: String) -> String {
|
|
let from_id: String = json_get_string(body, "from_id")
|
|
let to_id: String = json_get_string(body, "to_id")
|
|
let rel_raw: String = json_get_string(body, "relation")
|
|
let relation: String = if str_eq(rel_raw, "") { "associates" } else { rel_raw }
|
|
// Presence-aware (2026-08-01): weight 0.0 is a legitimate edge weight
|
|
// (dormant association); only default when the key is absent.
|
|
let w_present: String = json_get_raw(body, "weight")
|
|
let weight: Float = if str_eq(w_present, "") { 0.5 } else { json_get_float(body, "weight") }
|
|
engram_connect(from_id, to_id, weight, relation)
|
|
let saved: Int = persist_canonical()
|
|
"{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + relation + "\"}"
|
|
}
|
|
|
|
// route_create_edges_batch — POST /api/edges/batch {"edges":[{from_id,to_id,relation,weight}, ...]}
|
|
//
|
|
// WHY THIS EXISTS (2026-08-07 self-review). persist_canonical() writes the
|
|
// FULL canonical snapshot — 60MB at current graph size — and route_create_edge
|
|
// calls it once per edge. That is correct for the interactive one-edge case and
|
|
// ruinous for any bulk write: the soul's Hebbian consolidation path delivers
|
|
// ~14 associations per 8-minute heartbeat, which through the single-edge route
|
|
// would be ~840MB of disk writes per beat, ~150GB/day, to persist 14 edges.
|
|
//
|
|
// The fix is not to weaken durability — it is to make the unit of durability
|
|
// the BATCH. Connect every edge, then snapshot exactly once. Same guarantee
|
|
// (nothing acknowledged is lost to a restart), 1/N the writes. Empty or
|
|
// malformed entries are skipped rather than aborting the batch: a consolidation
|
|
// payload is best-effort by design, and one bad id should not cost the other 13.
|
|
//
|
|
// Returns the accepted count so the caller can tell delivery from silence.
|
|
fn route_create_edges_batch(method: String, path: String, body: String) -> String {
|
|
let arr: String = json_get_raw(body, "edges")
|
|
if str_eq(arr, "") { return err_json("missing edges array") }
|
|
let n: Int = json_array_len(arr)
|
|
if n == 0 { return "{\"ok\":true,\"accepted\":0,\"skipped\":0}" }
|
|
let i: Int = 0
|
|
let accepted: Int = 0
|
|
let skipped: Int = 0
|
|
while i < n {
|
|
let item: String = json_array_get(arr, i)
|
|
let from_id: String = json_get_string(item, "from_id")
|
|
let to_id: String = json_get_string(item, "to_id")
|
|
if str_eq(from_id, "") || str_eq(to_id, "") {
|
|
let skipped = skipped + 1
|
|
} else {
|
|
let rel_raw: String = json_get_string(item, "relation")
|
|
let relation: String = if str_eq(rel_raw, "") { "associates" } else { rel_raw }
|
|
let w_present: String = json_get_raw(item, "weight")
|
|
let weight: Float = if str_eq(w_present, "") { 0.5 } else { json_get_float(item, "weight") }
|
|
engram_connect(from_id, to_id, weight, relation)
|
|
let accepted = accepted + 1
|
|
}
|
|
let i = i + 1
|
|
}
|
|
// ONE snapshot for the whole batch — the entire point of this route.
|
|
// Skip it when nothing was accepted: an all-malformed payload must not
|
|
// trigger a 60MB write.
|
|
if accepted > 0 {
|
|
let saved: Int = persist_canonical()
|
|
}
|
|
return "{\"ok\":true,\"accepted\":" + int_to_str(accepted) + ",\"skipped\":" + int_to_str(skipped) + "}"
|
|
}
|
|
|
|
fn route_neighbors(method: String, path: String, body: String) -> String {
|
|
let id: String = extract_id(path, "/api/neighbors/")
|
|
if str_eq(id, "") { return err_json("missing id") }
|
|
let depth: Int = query_int(path, "depth", 1)
|
|
return engram_neighbors_json(id, depth, "both")
|
|
}
|
|
|
|
fn route_strengthen(method: String, path: String, body: String) -> String {
|
|
let id: String = json_get_string(body, "node_id")
|
|
if str_eq(id, "") { return err_json("missing node_id") }
|
|
engram_strengthen(id)
|
|
let saved: Int = persist_canonical()
|
|
ok_json()
|
|
}
|
|
|
|
fn route_forget(method: String, path: String, body: String) -> String {
|
|
let id: String = extract_id(path, "/api/nodes/")
|
|
if str_eq(id, "") { return err_json("missing id") }
|
|
engram_forget(id)
|
|
let saved: Int = persist_canonical()
|
|
ok_json()
|
|
}
|
|
|
|
fn route_save(method: String, path: String, body: String) -> String {
|
|
let p_raw: String = json_get_string(body, "path")
|
|
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
|
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
|
let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw }
|
|
engram_save(p)
|
|
"{\"ok\":true,\"path\":\"" + p + "\"}"
|
|
}
|
|
|
|
fn route_load(method: String, path: String, body: String) -> String {
|
|
let p_raw: String = json_get_string(body, "path")
|
|
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
|
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
|
let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw }
|
|
engram_load(p)
|
|
ok_json()
|
|
}
|
|
|
|
// (2026-08-01 self-review) Health previously returned a hardcoded literal —
|
|
// it reported "ok" even when the snapshot failed to load and the store was
|
|
// empty. Now reports live counts so a monitor can distinguish "up and
|
|
// loaded" from "up and hollow" (node_count=0 after boot = failed load).
|
|
fn route_health(method: String, path: String, body: String) -> String {
|
|
"{\"status\":\"ok\",\"engine\":\"engram-runtime-native\",\"node_count\":" + int_to_str(engram_node_count()) + ",\"edge_count\":" + int_to_str(engram_edge_count()) + "}"
|
|
}
|
|
|
|
// route_embed_backfill — GET/POST /api/embed-backfill?n=48
|
|
//
|
|
// (2026-07-25 self-review) The lazy embedding backfill runs only inside
|
|
// engram_activate, and nothing in production calls /api/activate on this
|
|
// store — the soul's curiosity loop activates its own in-process graph.
|
|
// After a restart from a snapshot without vectors, embedded_count stalled
|
|
// at 93/12175 and would never recover. This route lets the soul's
|
|
// heartbeat pump the backfill explicitly (48/min clears a 12k backlog in
|
|
// ~4h). Persists the canonical snapshot whenever new vectors were
|
|
// generated — the 2026-07-25 regression happened precisely because 3747
|
|
// in-RAM embeddings were never snapshotted before a restart. Self-
|
|
// limiting: once coverage is full, embedded=0 and no save occurs.
|
|
fn route_embed_backfill(method: String, path: String, body: String) -> String {
|
|
let n: Int = query_int(path, "n", 32)
|
|
let result: String = engram_embed_backfill(n)
|
|
let done: Float = json_get_float(result, "embedded")
|
|
if done > 0.0 {
|
|
let saved: Int = persist_canonical()
|
|
}
|
|
return result
|
|
}
|
|
|
|
// route_sync — return a snapshot of non-ISE/non-Working nodes for the soul daemon
|
|
// to merge into its in-process graph via engram_load_merge.
|
|
//
|
|
// The soul calls GET /api/sync every SOUL_REFRESH_MS (default 10 min) to pull
|
|
// new Knowledge/Memory/BacklogItem nodes from the authoritative HTTP Engram into
|
|
// its in-process working store. Previously this returned 404 "not found", causing
|
|
// the soul to write the error JSON to a temp file and attempt an empty merge.
|
|
//
|
|
// Strategy: save the current snapshot to disk, read it back, return the full
|
|
// snapshot JSON. The soul's engram_load_merge handles large files gracefully
|
|
// (it skips nodes already present by ID). Auth-exempt: same-host internal call.
|
|
// (2026-06-27 self-review: added this route to fix silent 10-min sync failures)
|
|
fn route_sync(method: String, path: String, body: String) -> String {
|
|
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
|
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
|
// 2026-07-21 self-review: export to a scratch path, never the canonical
|
|
// snapshot.json — read routes must not be able to clobber the good snapshot.
|
|
let snap_path: String = dir + "/.sync-export.json"
|
|
engram_save(snap_path)
|
|
let snap: String = fs_read(snap_path)
|
|
// 2026-08-02 self-review: this used to return {"nodes":[],"edges":[]} when
|
|
// the export/read failed. The soul's sync_ok test (awareness.el) only
|
|
// checks for "" and "{}", so that placeholder PASSED as a healthy sync:
|
|
// soul.last_sync_ok_ts got stamped, sync_age_ms stayed green, the
|
|
// sync_empty warn ISE never fired, and engram_sync reported added:0
|
|
// forever. A totally broken sync was indistinguishable from a quiet
|
|
// healthy one — the exact failure class this route was added to fix in
|
|
// the first place (see 2026-06-27 note above). Return a real error so the
|
|
// failure is loud on both sides.
|
|
if str_eq(snap, "") { return err_json("sync export failed: snapshot unreadable") }
|
|
return snap
|
|
}
|
|
|
|
// route_load_merge — POST /api/load-merge {"path": "..."} — merge a snapshot
|
|
// file into the live store WITHOUT resetting it (engram_load_merge skips nodes
|
|
// already present by id). Added 2026-07-21 self-review to restore the 244 kn-
|
|
// identity Knowledge nodes lost from the snapshot lineage between 05-13 and
|
|
// 07-13. Requires an explicit path: refuses to run without one so it can never
|
|
// be triggered accidentally against a default.
|
|
fn route_load_merge(method: String, path: String, body: String) -> String {
|
|
let p: String = json_get_string(body, "path")
|
|
if str_eq(p, "") { return err_json("path is required") }
|
|
if str_eq(fs_read(p), "") { return err_json("file missing or empty") }
|
|
let before_n: Int = engram_node_count()
|
|
let before_e: Int = engram_edge_count()
|
|
engram_load_merge(p)
|
|
let added_n: Int = engram_node_count() - before_n
|
|
let added_e: Int = engram_edge_count() - before_e
|
|
let saved: Int = persist_canonical()
|
|
"{\"ok\":true,\"nodes_added\":" + int_to_str(added_n) + ",\"edges_added\":" + int_to_str(added_e) + ",\"node_count\":" + int_to_str(engram_node_count()) + "}"
|
|
}
|
|
|
|
// route_emit_ise — write an InternalStateEvent node from the soul daemon.
|
|
//
|
|
// Endpoint: POST /api/neuron/state-events
|
|
// Body: {"content": "<json-string>"}
|
|
//
|
|
// Auth: exempt (internal endpoint, soul daemon on same host, no _auth needed).
|
|
// The soul's ise_post() sends {"content":"..."} without _auth; enforcing auth
|
|
// here would silently drop all heartbeat/curiosity ISEs. Unauthenticated POST
|
|
// to this endpoint is acceptable: ISE writes are observability-only, append-only,
|
|
// and come from a trusted process on localhost.
|
|
//
|
|
// Salience/importance set to match engram_node_full ISE defaults used by the
|
|
// in-process fallback path in awareness.el (salience=0.3, importance=0.3,
|
|
// confidence=0.8, tier=Episodic).
|
|
// (2026-06-26 self-review: added this route after discovering ise_post was
|
|
// silently failing — the soul posts here but the endpoint didn't exist.)
|
|
//
|
|
// Retention (2026-07-16 self-review): an earlier comment here claimed ISEs
|
|
// got temporal_decay_rate=1.617 — that was never implemented (engram_node_full
|
|
// hardcodes 0.0), and per-node decay only dampens activation anyway; it never
|
|
// removes nodes. By 2026-07-16 ISEs were 75% of the store (10,175 of 13,522
|
|
// nodes, ~4,300/day, unbounded). ISEs are already WM-excluded in
|
|
// engram_activate, so the fix is retention, not decay: every insert calls
|
|
// engram_prune_telemetry(), a single O(nodes+edges) compaction pass that
|
|
// removes ISEs older than ENGRAM_ISE_RETENTION_MS (default 48h), protecting
|
|
// "session-start" labels and self_review events as durable history. At
|
|
// ~3 ISEs/min this bounds telemetry at ~8.6k nodes instead of growing forever.
|
|
fn route_emit_ise(method: String, path: String, body: String) -> String {
|
|
let content: String = json_get_string(body, "content")
|
|
if str_eq(content, "") { return err_json("missing content") }
|
|
let sal: Float = 0.3
|
|
let imp: Float = 0.3
|
|
let conf: Float = 0.8
|
|
let id: String = engram_node_full(
|
|
content, "InternalStateEvent", "state-event",
|
|
sal, imp, conf,
|
|
"Episodic", "[\"internal-state\",\"InternalStateEvent\"]"
|
|
)
|
|
let ret_raw: String = env("ENGRAM_ISE_RETENTION_MS")
|
|
let ret_ms: Int = if str_eq(ret_raw, "") { 172800000 } else { str_to_int(ret_raw) }
|
|
let pruned: Int = engram_prune_telemetry(ret_ms)
|
|
"{\"ok\":true,\"id\":\"" + id + "\",\"pruned\":" + int_to_str(pruned) + "}"
|
|
}
|
|
|
|
// ── Knowledge capture ─────────────────────────────────────────────────────────
|
|
//
|
|
// route_capture_knowledge — direct Knowledge-node capture over HTTP.
|
|
//
|
|
// Endpoint: POST /api/neuron/knowledge/capture (auth required: "_auth" in body)
|
|
// Body: {"content": "...", "title": "...", "category": "...",
|
|
// "tier": "note|lesson|canonical", "tags": [...], "project": "...",
|
|
// "_auth": "<key>"}
|
|
//
|
|
// WHY (2026-07-15 self-review): the world-ingestor integrator was designed
|
|
// against this endpoint (its MCP-unavailable fallback), but the route never
|
|
// existed — every direct push 404'd, and because the auth gate ran before
|
|
// routing, the failure surfaced as {"error":"unauthorized"} and was
|
|
// misdiagnosed for two weeks while world knowledge silently dropped.
|
|
// POST /api/nodes was no substitute: it discards label/tags/tier, which
|
|
// makes captured knowledge invisible to tag-scoped search and curiosity.
|
|
//
|
|
// The incoming knowledge tier (note/lesson/canonical) is preserved as a
|
|
// "tier:<x>" tag rather than mapped onto Engram's cognitive tiers — Knowledge
|
|
// nodes land in Semantic (stable reference), and the epistemic tier stays
|
|
// queryable without inventing a lossy mapping.
|
|
fn route_capture_knowledge(method: String, path: String, body: String) -> String {
|
|
let content: String = json_get_string(body, "content")
|
|
if str_eq(content, "") { return err_json("missing content") }
|
|
let title: String = json_get_string(body, "title")
|
|
let label: String = if str_eq(title, "") { str_slice(content, 0, 60) } else { title }
|
|
let category_raw: String = json_get_string(body, "category")
|
|
let category: String = if str_eq(category_raw, "") { "other" } else { category_raw }
|
|
let ktier_raw: String = json_get_string(body, "tier")
|
|
let ktier: String = if str_eq(ktier_raw, "") { "note" } else { ktier_raw }
|
|
let project: String = json_get_string(body, "project")
|
|
let tags_raw: String = json_get_raw(body, "tags")
|
|
let tags_base: String = if str_eq(tags_raw, "") { "[]" } else { tags_raw }
|
|
// Merge category/tier/project markers into the tag array. Search matches
|
|
// against the tags string, so these make captures findable by facet.
|
|
let base_len: Int = str_len(tags_base)
|
|
let head: String = str_slice(tags_base, 0, base_len - 1)
|
|
let sep: String = if str_eq(head, "[") { "" } else { "," }
|
|
let safe_cat: String = str_replace(category, "\"", "'")
|
|
let safe_tier: String = str_replace(ktier, "\"", "'")
|
|
let safe_proj: String = str_replace(project, "\"", "'")
|
|
let proj_tag: String = if str_eq(safe_proj, "") { "" } else { ",\"project:" + safe_proj + "\"" }
|
|
let tags: String = head + sep + "\"category:" + safe_cat + "\",\"tier:" + safe_tier + "\"" + proj_tag + "]"
|
|
let sal: Float = 0.5
|
|
let imp: Float = 0.5
|
|
let conf: Float = 0.9
|
|
let id: String = engram_node_full(
|
|
content, "Knowledge", label,
|
|
sal, imp, conf,
|
|
"Semantic", tags
|
|
)
|
|
let saved: Int = persist_canonical()
|
|
"{\"ok\":true,\"id\":\"" + id + "\"}"
|
|
}
|
|
|
|
// route_similarity — GET /api/similarity?a=<id>&b=<id>
|
|
//
|
|
// (2026-08-01 self-review) engram_cosine_sim was added 2026-07-24
|
|
// (bl-b2d1c944) with the stated purpose of exposing semantic distance to
|
|
// "EL code and the introspection API" — but it had ZERO callers anywhere:
|
|
// no route, no soul-daemon use. The activation path uses embeddings
|
|
// internally (semantic seeding, Pass-2 additive term), but there was no way
|
|
// to probe pairwise node similarity from outside. This closes that: cosine
|
|
// in [-1,1], or -2 when either node is missing or not yet embedded (so
|
|
// "not comparable" is distinguishable from "genuinely orthogonal" 0.0).
|
|
fn route_similarity(method: String, path: String, body: String) -> String {
|
|
let a: String = query_param(path, "a")
|
|
let b: String = query_param(path, "b")
|
|
if str_eq(a, "") { return err_json("missing a") }
|
|
if str_eq(b, "") { return err_json("missing b") }
|
|
let sim: Float = engram_cosine_sim(a, b)
|
|
"{\"a\":\"" + a + "\",\"b\":\"" + b + "\",\"cosine\":" + float_to_str(sim) + "}"
|
|
}
|
|
|
|
// ── Auth ──────────────────────────────────────────────────────────────────────
|
|
|
|
fn check_auth_ok(method: String, body: String) -> Bool {
|
|
let key: String = env("ENGRAM_API_KEY")
|
|
if str_eq(key, "") { return true }
|
|
// Read-only methods don't require auth. Until http_serve surfaces
|
|
// request headers we can't accept a Bearer token cleanly; mutating
|
|
// requests must include "_auth": "<key>" in the JSON body.
|
|
if str_eq(method, "GET") { return true }
|
|
let provided: String = json_get_string(body, "_auth")
|
|
if str_eq(provided, key) { return true }
|
|
return false
|
|
}
|
|
|
|
// ── Dispatcher ────────────────────────────────────────────────────────────────
|
|
|
|
fn handle_request(method: String, path: String, body: String) -> String {
|
|
let clean: String = strip_query(path)
|
|
|
|
// Health is always reachable
|
|
if str_eq(method, "GET") {
|
|
if str_eq(clean, "/health") || str_eq(clean, "/") {
|
|
return route_health(method, path, body)
|
|
}
|
|
}
|
|
|
|
// ISE posting is auth-exempt (internal soul daemon, same host, no _auth key)
|
|
if str_eq(method, "POST") && str_eq(clean, "/api/neuron/state-events") {
|
|
return route_emit_ise(method, path, body)
|
|
}
|
|
|
|
// Auth (when ENGRAM_API_KEY is set)
|
|
if !check_auth_ok(method, body) {
|
|
return err_json("unauthorized")
|
|
}
|
|
|
|
// Knowledge capture (auth enforced above; the world-ingestor integrator
|
|
// and any headless session without MCP push knowledge through this)
|
|
if str_eq(method, "POST") && str_eq(clean, "/api/neuron/knowledge/capture") {
|
|
return route_capture_knowledge(method, path, body)
|
|
}
|
|
|
|
// Stats
|
|
if str_eq(method, "GET") && (str_eq(clean, "/api/stats") || str_eq(clean, "/stats")) {
|
|
return route_stats(method, path, body)
|
|
}
|
|
if str_eq(method, "GET") && (str_eq(clean, "/api/act-stats") || str_eq(clean, "/act-stats")) {
|
|
return route_act_stats(method, path, body)
|
|
}
|
|
|
|
// Nodes
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/nodes") || str_eq(clean, "/nodes")) {
|
|
return route_create_node(method, path, body)
|
|
}
|
|
if str_eq(method, "GET") && (str_eq(clean, "/api/nodes") || str_eq(clean, "/nodes") || str_eq(clean, "/nodes/list") || str_eq(clean, "/api/nodes/list")) {
|
|
return route_scan_nodes(method, path, body)
|
|
}
|
|
if str_eq(method, "GET") && (str_eq(clean, "/api/edges") || str_eq(clean, "/edges")) {
|
|
return route_scan_edges(method, path, body)
|
|
}
|
|
if str_eq(method, "GET") && str_starts_with(clean, "/api/nodes/") {
|
|
return route_get_node(method, path, body)
|
|
}
|
|
if str_eq(method, "DELETE") && str_starts_with(clean, "/api/nodes/") {
|
|
return route_forget(method, path, body)
|
|
}
|
|
|
|
// Edges
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/edges") || str_eq(clean, "/edges")) {
|
|
return route_create_edge(method, path, body)
|
|
}
|
|
// Batch edge write — one snapshot for the whole payload. Must be tested
|
|
// BEFORE nothing else claims it; the exact-match on "/api/edges" above
|
|
// does not catch "/api/edges/batch", so order is not load-bearing here,
|
|
// but keeping the two adjacent keeps them from drifting apart.
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/edges/batch") || str_eq(clean, "/edges/batch")) {
|
|
return route_create_edges_batch(method, path, body)
|
|
}
|
|
if str_eq(method, "GET") && str_starts_with(clean, "/api/neighbors/") {
|
|
return route_neighbors(method, path, body)
|
|
}
|
|
|
|
// Activation + Search
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/activate") || str_eq(clean, "/activate")) {
|
|
return route_activate(method, path, body)
|
|
}
|
|
if str_eq(method, "GET") && str_starts_with(clean, "/api/activate") {
|
|
return route_activate(method, path, body)
|
|
}
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/search") || str_eq(clean, "/search")) {
|
|
return route_search(method, path, body)
|
|
}
|
|
if str_eq(method, "GET") && str_starts_with(clean, "/api/search") {
|
|
return route_search(method, path, body)
|
|
}
|
|
|
|
// Strengthen
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/strengthen") || str_eq(clean, "/strengthen")) {
|
|
return route_strengthen(method, path, body)
|
|
}
|
|
|
|
// Persistence
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/save") || str_eq(clean, "/save")) {
|
|
return route_save(method, path, body)
|
|
}
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/load") || str_eq(clean, "/load")) {
|
|
return route_load(method, path, body)
|
|
}
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/load-merge") || str_eq(clean, "/load-merge")) {
|
|
return route_load_merge(method, path, body)
|
|
}
|
|
|
|
// Sync — soul daemon periodic pull of non-ISE knowledge into in-process graph
|
|
if str_eq(method, "GET") && str_eq(clean, "/api/sync") {
|
|
return route_sync(method, path, body)
|
|
}
|
|
|
|
// Embedding backfill — pumped by the soul heartbeat (2026-07-25)
|
|
if str_eq(clean, "/api/embed-backfill") {
|
|
return route_embed_backfill(method, path, body)
|
|
}
|
|
|
|
// Semantic similarity probe (2026-08-01)
|
|
if str_eq(method, "GET") && str_starts_with(clean, "/api/similarity") {
|
|
return route_similarity(method, path, body)
|
|
}
|
|
|
|
"{\"error\":\"not found\",\"path\":\"" + clean + "\"}"
|
|
}
|
|
|
|
// ── Entry ─────────────────────────────────────────────────────────────────────
|
|
|
|
let bind_raw: String = env("ENGRAM_BIND")
|
|
let bind_str: String = if str_eq(bind_raw, "") { ":8742" } else { bind_raw }
|
|
let port: Int = parse_port(bind_str)
|
|
|
|
// On startup, try to load any existing snapshot (best effort).
|
|
let data_dir_raw: String = env("ENGRAM_DATA_DIR")
|
|
let data_dir: String = if str_eq(data_dir_raw, "") { "/tmp/engram" } else { data_dir_raw }
|
|
let snapshot_path: String = data_dir + "/snapshot.json"
|
|
engram_load(snapshot_path)
|
|
|
|
// 2026-07-21 self-review boot guard: if the snapshot file has content but the
|
|
// load produced 0 nodes, something is wrong (corrupt file / parse failure).
|
|
// Preserve the evidence and warn loudly — and since read routes no longer write
|
|
// the canonical path, a bad boot can no longer clobber the good snapshot.
|
|
let boot_snap: String = fs_read(snapshot_path)
|
|
if !str_eq(boot_snap, "") {
|
|
if engram_node_count() == 0 {
|
|
println("[engram] WARNING: snapshot.json is non-empty but load produced 0 nodes — preserving copy at snapshot.failed-load.json")
|
|
fs_write(data_dir + "/snapshot.failed-load.json", boot_snap)
|
|
} else {
|
|
// Good load: keep a boot-time backup of the snapshot as loaded.
|
|
fs_write(data_dir + "/snapshot.boot-backup.json", boot_snap)
|
|
}
|
|
}
|
|
|
|
println("[engram] runtime-native graph engine")
|
|
println("[engram] data_dir=" + data_dir)
|
|
println("[engram] node_count=" + int_to_str(engram_node_count()))
|
|
println("[engram] edge_count=" + int_to_str(engram_edge_count()))
|
|
println("[engram] listening on " + int_to_str(port))
|
|
|
|
http_set_handler("handle_request")
|
|
http_serve(port, "handle_request")
|