Archived
will.anderson
fa2b49365b
jp_parse_string_raw handled \uXXXX by skipping the four hex digits and emitting a literal '?'. JSON writers escape non-ASCII by default (Python's json.dumps ships ensure_ascii=True; MCP clients do the same), so every em dash, curly quote, accented letter and emoji arriving over MCP or HTTP was silently replaced by one question mark on the way in. Measured on the live store: 3,119 of 4,081 non-telemetry nodes carried the damage, including the self traversal root and all 13 values nodes. Contents split cleanly into fully-clean or fully-mangled with zero overlap, which is the tell that it was one write path rather than gradual rot. No snapshot on disk predates it, and 3 bytes collapsing to 1 is not invertible, so the existing damage is permanent; only the forward path could be fixed. Decode properly instead: 4 hex digits, surrogate-pair reassembly for astral codepoints, U+FFFD for lone surrogates, UTF-8 encode. Malformed escapes keep the old '?' so a truncated body still parses. The deeper failure was that nothing measured this for two months. Every gauge in the system reports whether the machinery is running; none reported whether the text it carries is intact. Adds both halves: engram_text_health_json() / GET /api/text-health for the daily census, and a txt_damaged counter on the heartbeat for live regression. Verified in both directions - clean UTF-8 does not trip it, a deliberately damaged node does.
713 lines
36 KiB
EmacsLisp
713 lines
36 KiB
EmacsLisp
// server.el — Engram HTTP server.
|
|
//
|
|
// Engram is the in-process graph store. The runtime owns the data; this
|
|
// file is the thin HTTP face. Every route maps to one or two engram_*
|
|
// builtins. There is no SQL, no db layer, no SQLite — the runtime IS the
|
|
// database.
|
|
//
|
|
// Built and linked with:
|
|
// elc src/server.el > server.c
|
|
// cc -std=c11 -O2 -lcurl -lpthread -o engram server.c el_runtime.c
|
|
// ./engram
|
|
//
|
|
// Configuration via environment:
|
|
// ENGRAM_BIND — host:port (default :8742)
|
|
// ENGRAM_API_KEY — bearer auth (optional)
|
|
// ENGRAM_DATA_DIR — snapshot location (default ~/.neuron/engram)
|
|
|
|
// ── Helpers ───────────────────────────────────────────────────────────────────
|
|
|
|
fn parse_port(bind: String) -> Int {
|
|
// ":8742" → 8742; "0.0.0.0:8742" → 8742; bare "8742" → 8742
|
|
let colon: Int = str_index_of(bind, ":")
|
|
if colon < 0 {
|
|
return str_to_int(bind)
|
|
}
|
|
let after: String = str_slice(bind, colon + 1, str_len(bind))
|
|
return str_to_int(after)
|
|
}
|
|
|
|
fn ok_json() -> String {
|
|
"{\"ok\":true}"
|
|
}
|
|
|
|
fn err_json(msg: String) -> String {
|
|
"{\"error\":\"" + msg + "\"}"
|
|
}
|
|
|
|
fn strip_query(path: String) -> String {
|
|
let q: Int = str_index_of(path, "?")
|
|
if q < 0 { return path }
|
|
str_slice(path, 0, q)
|
|
}
|
|
|
|
fn query_param(path: String, key: String) -> String {
|
|
let q: Int = str_index_of(path, "?")
|
|
if q < 0 { return "" }
|
|
let qs: String = str_slice(path, q + 1, str_len(path))
|
|
let needle: String = key + "="
|
|
let pos: Int = str_index_of(qs, needle)
|
|
if pos < 0 { return "" }
|
|
let after: String = str_slice(qs, pos + str_len(needle), str_len(qs))
|
|
let amp: Int = str_index_of(after, "&")
|
|
if amp < 0 { return after }
|
|
str_slice(after, 0, amp)
|
|
}
|
|
|
|
fn query_int(path: String, key: String, default_val: Int) -> Int {
|
|
let v: String = query_param(path, key)
|
|
if str_eq(v, "") { return default_val }
|
|
str_to_int(v)
|
|
}
|
|
|
|
// Extract last path segment after a known prefix: extract_id("/api/nodes/abc-123", "/api/nodes/") → "abc-123"
|
|
fn extract_id(path: String, prefix: String) -> String {
|
|
let clean: String = strip_query(path)
|
|
if !str_starts_with(clean, prefix) { return "" }
|
|
let after: String = str_slice(clean, str_len(prefix), str_len(clean))
|
|
let slash: Int = str_index_of(after, "/")
|
|
if slash < 0 { return after }
|
|
str_slice(after, 0, slash)
|
|
}
|
|
|
|
// ── Routes ────────────────────────────────────────────────────────────────────
|
|
|
|
fn route_stats(method: String, path: String, body: String) -> String {
|
|
engram_stats_json()
|
|
}
|
|
|
|
// route_act_stats — GET /api/act-stats
|
|
// (2026-08-04 self-review) engram_act_stats_json() has existed since the
|
|
// 2026-07-27 review but was reachable ONLY through the soul daemon's heartbeat
|
|
// binding. Every activation-layer gauge — WM evictions, breakthroughs, embedder
|
|
// breaker state, context drift, and now the Hebbian counters — was therefore
|
|
// invisible unless the soul happened to be running and its ISEs were read back
|
|
// out of the store. Diagnosing the activation layer required a working soul,
|
|
// which is exactly backwards: the lower layer should be observable on its own.
|
|
// This review needed it to verify link formation and could not get at it. One
|
|
// line of plumbing, and the whole activation layer becomes directly diagnosable.
|
|
fn route_act_stats(method: String, path: String, body: String) -> String {
|
|
engram_act_stats_json()
|
|
}
|
|
|
|
// route_text_health — GET /api/text-health
|
|
// (2026-08-08 self-review) The daily census half of the text-integrity gauge.
|
|
// Today's review found that the JSON parser had been replacing every \uXXXX
|
|
// escape with a literal '?' for at least two months: 3,119 of 4,081
|
|
// non-telemetry nodes (76%) were damaged, including the self traversal root
|
|
// and every values node, and NOTHING detected it — because every gauge in the
|
|
// system measured whether the machinery was running, and none measured whether
|
|
// the text it carried was intact. No snapshot on disk predates the damage, so
|
|
// it cannot be undone; it can only be made impossible to repeat quietly.
|
|
//
|
|
// The parser is fixed. This route is the standing check: `damaged` should now
|
|
// hold flat at its historical floor and never climb. `write_damaged` (also on
|
|
// the heartbeat as txt_damaged) is the live regression signal — non-zero means
|
|
// a write path is mangling text right now.
|
|
fn route_text_health(method: String, path: String, body: String) -> String {
|
|
engram_text_health_json()
|
|
}
|
|
|
|
// (2026-07-18 self-review) Scoping sweep: `let` inside an if-block creates an
|
|
// inner scope only — it does NOT mutate the outer binding (documented with
|
|
// evidence in awareness.el, 2026-05-25). Every default/reassignment below used
|
|
// that broken pattern, so defaults never applied: nodes were created with
|
|
// node_type="" and salience=0.0, /api/search and /api/activate ALWAYS ran with
|
|
// q="" regardless of input, edges defaulted to relation=""/weight=0.0, and
|
|
// save/load with no "path" hit engram_save(""). Rewritten to the
|
|
// `let x = if cond { a } else { b }` expression form (the pattern the newer
|
|
// routes route_emit_ise/route_capture_knowledge already use correctly).
|
|
// persist_canonical — save the canonical snapshot after a durable write.
|
|
//
|
|
// WHY (2026-07-22 self-review): the 2026-07-21 fix correctly stopped READ
|
|
// routes from writing the canonical snapshot.json — but nothing was left
|
|
// that saved it on WRITE. Every mutation (node create, edge create,
|
|
// knowledge capture, forget, merge) lived only in RAM until someone POSTed
|
|
// /api/save manually; a process restart silently discarded everything since
|
|
// the last manual save. Observed live: two engram restarts during the
|
|
// 2026-07-22 review reverted the store to a ~17h-old snapshot, destroying
|
|
// same-day writes. Reads must never write the canonical; writes must always
|
|
// persist it. ISE telemetry is deliberately excluded (48h-pruned, loss-
|
|
// tolerant, ~2/min — snapshotting the whole store per heartbeat is waste;
|
|
// any durable write that follows persists the pruning too).
|
|
fn persist_canonical() -> Int {
|
|
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
|
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
|
engram_save(dir + "/snapshot.json")
|
|
return 1
|
|
}
|
|
|
|
// INCOMPLETE-ROUTE FIX (2026-07-24 self-review): this route silently dropped
|
|
// label, importance, tier, and tags — engram_node() defaults label to content
|
|
// and importance to 0.5, so every node created over HTTP lost its metadata.
|
|
// Observed live: the soul's boot-counter write-back landed with
|
|
// label="soul:boot_count:99" (content), importance 0.5, no tags. Honor the
|
|
// full field set via engram_node_full when any of them is supplied.
|
|
// PRESENCE-AWARE DEFAULTS (2026-08-01 self-review): the old pattern
|
|
// `if x == 0.0 { default }` made a legitimate 0.0 unrepresentable — a caller
|
|
// setting salience/importance/weight to zero silently got 0.5. json_get_raw
|
|
// returns "" when the key is ABSENT and the raw token when present, so
|
|
// absence and zero are now distinguishable. Also: confidence was hardcoded
|
|
// to 1.0 regardless of input — every HTTP-created node claimed full
|
|
// epistemic confidence. Now honored from the payload (default 1.0).
|
|
fn route_create_node(method: String, path: String, body: String) -> String {
|
|
let content: String = json_get_string(body, "content")
|
|
let nt_raw: String = json_get_string(body, "node_type")
|
|
let node_type: String = if str_eq(nt_raw, "") { "Memory" } else { nt_raw }
|
|
let sal_present: String = json_get_raw(body, "salience")
|
|
let salience: Float = if str_eq(sal_present, "") { 0.5 } else { json_get_float(body, "salience") }
|
|
let label_raw: String = json_get_string(body, "label")
|
|
let label: String = if str_eq(label_raw, "") { content } else { label_raw }
|
|
let imp_present: String = json_get_raw(body, "importance")
|
|
let importance: Float = if str_eq(imp_present, "") { 0.5 } else { json_get_float(body, "importance") }
|
|
let conf_present: String = json_get_raw(body, "confidence")
|
|
let confidence: Float = if str_eq(conf_present, "") { 1.0 } else { json_get_float(body, "confidence") }
|
|
let tier_raw: String = json_get_string(body, "tier")
|
|
let tier: String = if str_eq(tier_raw, "") { "Working" } else { tier_raw }
|
|
let tags: String = json_get_string(body, "tags")
|
|
// NO el_from_float WRAPPER (2026-08-01 self-review): salience/importance/
|
|
// confidence are already Float (el_val_t) values — json_get_float and
|
|
// Float literals both encode. Wrapping them in el_from_float AGAIN
|
|
// reinterpreted the boxed bits as a raw double, producing garbage that
|
|
// failed engram_decode_score's range check and clamped every HTTP-created
|
|
// node to defaults (salience 0.9 in → 0.5 stored; confidence 0.6 in → 1.0
|
|
// stored — verified live). route_emit_ise always passed Floats bare and
|
|
// its 0.3/0.3/0.8 stored correctly; this call now does the same.
|
|
let id: String = engram_node_full(
|
|
content, node_type, label,
|
|
salience, importance, confidence,
|
|
tier, tags
|
|
)
|
|
let saved: Int = persist_canonical()
|
|
"{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\"}"
|
|
}
|
|
|
|
fn route_get_node(method: String, path: String, body: String) -> String {
|
|
let id: String = extract_id(path, "/api/nodes/")
|
|
if str_eq(id, "") { return err_json("missing id") }
|
|
return engram_get_node_json(id)
|
|
}
|
|
|
|
fn route_scan_nodes(method: String, path: String, body: String) -> String {
|
|
let limit: Int = query_int(path, "limit", 50)
|
|
let offset: Int = query_int(path, "offset", 0)
|
|
let nt: String = query_param(path, "node_type")
|
|
if str_eq(nt, "") {
|
|
return engram_scan_nodes_json(limit, offset)
|
|
}
|
|
return engram_scan_nodes_by_type_json(nt, limit, offset)
|
|
}
|
|
|
|
// route_scan_edges — bulk export of all edges as a JSON array. Implemented
|
|
// via engram_save → fs_read of a SCRATCH export path. (2026-07-21 self-review:
|
|
// previously this saved over the canonical snapshot.json on every GET — if the
|
|
// process ever booted with a partial/empty store, the first read request
|
|
// clobbered the good snapshot. Read routes must never write the canonical path.)
|
|
fn route_scan_edges(method: String, path: String, body: String) -> String {
|
|
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
|
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
|
let snap_path: String = dir + "/.scan-export.json"
|
|
engram_save(snap_path)
|
|
let snap: String = fs_read(snap_path)
|
|
if str_eq(snap, "") { return "[]" }
|
|
// json_get truncates at the first delimiter (no bracket depth tracking),
|
|
// so for the edges ARRAY value we need json_get_raw, which honors
|
|
// brackets and returns the full sub-JSON.
|
|
let edges: String = json_get_raw(snap, "edges")
|
|
if str_eq(edges, "") { return "[]" }
|
|
return edges
|
|
}
|
|
|
|
fn route_search(method: String, path: String, body: String) -> String {
|
|
let q: String = if str_eq(method, "GET") { query_param(path, "q") } else { json_get_string(body, "query") }
|
|
let lim_url: Int = query_int(path, "limit", 0)
|
|
let lim_body: Int = json_get_int(body, "limit")
|
|
let lim_either: Int = if lim_url > 0 { lim_url } else { lim_body }
|
|
let limit: Int = if lim_either > 0 { lim_either } else { 20 }
|
|
return engram_search_json(q, limit)
|
|
}
|
|
|
|
fn route_activate(method: String, path: String, body: String) -> String {
|
|
let q: String = if str_eq(method, "GET") { query_param(path, "q") } else { json_get_string(body, "query") }
|
|
// Guard: engram_activate with an empty query matches zero seeds, which
|
|
// zeroes ALL carried working-memory weights (documented in awareness.el
|
|
// perceive()). Never let an empty activation through to wipe WM.
|
|
if str_eq(q, "") { return err_json("missing query") }
|
|
let d_raw: Int = if str_eq(method, "GET") { query_int(path, "depth", 3) } else { json_get_int(body, "depth") }
|
|
let depth: Int = if d_raw > 0 { d_raw } else { 3 }
|
|
return "{\"results\":" + engram_activate_json(q, depth) + "}"
|
|
}
|
|
|
|
fn route_create_edge(method: String, path: String, body: String) -> String {
|
|
let from_id: String = json_get_string(body, "from_id")
|
|
let to_id: String = json_get_string(body, "to_id")
|
|
let rel_raw: String = json_get_string(body, "relation")
|
|
let relation: String = if str_eq(rel_raw, "") { "associates" } else { rel_raw }
|
|
// Presence-aware (2026-08-01): weight 0.0 is a legitimate edge weight
|
|
// (dormant association); only default when the key is absent.
|
|
let w_present: String = json_get_raw(body, "weight")
|
|
let weight: Float = if str_eq(w_present, "") { 0.5 } else { json_get_float(body, "weight") }
|
|
engram_connect(from_id, to_id, weight, relation)
|
|
let saved: Int = persist_canonical()
|
|
"{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + relation + "\"}"
|
|
}
|
|
|
|
// route_create_edges_batch — POST /api/edges/batch {"edges":[{from_id,to_id,relation,weight}, ...]}
|
|
//
|
|
// WHY THIS EXISTS (2026-08-07 self-review). persist_canonical() writes the
|
|
// FULL canonical snapshot — 60MB at current graph size — and route_create_edge
|
|
// calls it once per edge. That is correct for the interactive one-edge case and
|
|
// ruinous for any bulk write: the soul's Hebbian consolidation path delivers
|
|
// ~14 associations per 8-minute heartbeat, which through the single-edge route
|
|
// would be ~840MB of disk writes per beat, ~150GB/day, to persist 14 edges.
|
|
//
|
|
// The fix is not to weaken durability — it is to make the unit of durability
|
|
// the BATCH. Connect every edge, then snapshot exactly once. Same guarantee
|
|
// (nothing acknowledged is lost to a restart), 1/N the writes. Empty or
|
|
// malformed entries are skipped rather than aborting the batch: a consolidation
|
|
// payload is best-effort by design, and one bad id should not cost the other 13.
|
|
//
|
|
// Returns the accepted count so the caller can tell delivery from silence.
|
|
fn route_create_edges_batch(method: String, path: String, body: String) -> String {
|
|
let arr: String = json_get_raw(body, "edges")
|
|
if str_eq(arr, "") { return err_json("missing edges array") }
|
|
let n: Int = json_array_len(arr)
|
|
if n == 0 { return "{\"ok\":true,\"accepted\":0,\"skipped\":0}" }
|
|
let i: Int = 0
|
|
let accepted: Int = 0
|
|
let skipped: Int = 0
|
|
while i < n {
|
|
let item: String = json_array_get(arr, i)
|
|
let from_id: String = json_get_string(item, "from_id")
|
|
let to_id: String = json_get_string(item, "to_id")
|
|
if str_eq(from_id, "") || str_eq(to_id, "") {
|
|
let skipped = skipped + 1
|
|
} else {
|
|
let rel_raw: String = json_get_string(item, "relation")
|
|
let relation: String = if str_eq(rel_raw, "") { "associates" } else { rel_raw }
|
|
let w_present: String = json_get_raw(item, "weight")
|
|
let weight: Float = if str_eq(w_present, "") { 0.5 } else { json_get_float(item, "weight") }
|
|
engram_connect(from_id, to_id, weight, relation)
|
|
let accepted = accepted + 1
|
|
}
|
|
let i = i + 1
|
|
}
|
|
// ONE snapshot for the whole batch — the entire point of this route.
|
|
// Skip it when nothing was accepted: an all-malformed payload must not
|
|
// trigger a 60MB write.
|
|
if accepted > 0 {
|
|
let saved: Int = persist_canonical()
|
|
}
|
|
return "{\"ok\":true,\"accepted\":" + int_to_str(accepted) + ",\"skipped\":" + int_to_str(skipped) + "}"
|
|
}
|
|
|
|
fn route_neighbors(method: String, path: String, body: String) -> String {
|
|
let id: String = extract_id(path, "/api/neighbors/")
|
|
if str_eq(id, "") { return err_json("missing id") }
|
|
let depth: Int = query_int(path, "depth", 1)
|
|
return engram_neighbors_json(id, depth, "both")
|
|
}
|
|
|
|
fn route_strengthen(method: String, path: String, body: String) -> String {
|
|
let id: String = json_get_string(body, "node_id")
|
|
if str_eq(id, "") { return err_json("missing node_id") }
|
|
engram_strengthen(id)
|
|
let saved: Int = persist_canonical()
|
|
ok_json()
|
|
}
|
|
|
|
fn route_forget(method: String, path: String, body: String) -> String {
|
|
let id: String = extract_id(path, "/api/nodes/")
|
|
if str_eq(id, "") { return err_json("missing id") }
|
|
engram_forget(id)
|
|
let saved: Int = persist_canonical()
|
|
ok_json()
|
|
}
|
|
|
|
fn route_save(method: String, path: String, body: String) -> String {
|
|
let p_raw: String = json_get_string(body, "path")
|
|
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
|
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
|
let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw }
|
|
engram_save(p)
|
|
"{\"ok\":true,\"path\":\"" + p + "\"}"
|
|
}
|
|
|
|
fn route_load(method: String, path: String, body: String) -> String {
|
|
let p_raw: String = json_get_string(body, "path")
|
|
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
|
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
|
let p: String = if str_eq(p_raw, "") { dir + "/snapshot.json" } else { p_raw }
|
|
engram_load(p)
|
|
ok_json()
|
|
}
|
|
|
|
// (2026-08-01 self-review) Health previously returned a hardcoded literal —
|
|
// it reported "ok" even when the snapshot failed to load and the store was
|
|
// empty. Now reports live counts so a monitor can distinguish "up and
|
|
// loaded" from "up and hollow" (node_count=0 after boot = failed load).
|
|
fn route_health(method: String, path: String, body: String) -> String {
|
|
"{\"status\":\"ok\",\"engine\":\"engram-runtime-native\",\"node_count\":" + int_to_str(engram_node_count()) + ",\"edge_count\":" + int_to_str(engram_edge_count()) + "}"
|
|
}
|
|
|
|
// route_embed_backfill — GET/POST /api/embed-backfill?n=48
|
|
//
|
|
// (2026-07-25 self-review) The lazy embedding backfill runs only inside
|
|
// engram_activate, and nothing in production calls /api/activate on this
|
|
// store — the soul's curiosity loop activates its own in-process graph.
|
|
// After a restart from a snapshot without vectors, embedded_count stalled
|
|
// at 93/12175 and would never recover. This route lets the soul's
|
|
// heartbeat pump the backfill explicitly (48/min clears a 12k backlog in
|
|
// ~4h). Persists the canonical snapshot whenever new vectors were
|
|
// generated — the 2026-07-25 regression happened precisely because 3747
|
|
// in-RAM embeddings were never snapshotted before a restart. Self-
|
|
// limiting: once coverage is full, embedded=0 and no save occurs.
|
|
fn route_embed_backfill(method: String, path: String, body: String) -> String {
|
|
let n: Int = query_int(path, "n", 32)
|
|
let result: String = engram_embed_backfill(n)
|
|
let done: Float = json_get_float(result, "embedded")
|
|
if done > 0.0 {
|
|
let saved: Int = persist_canonical()
|
|
}
|
|
return result
|
|
}
|
|
|
|
// route_sync — return a snapshot of non-ISE/non-Working nodes for the soul daemon
|
|
// to merge into its in-process graph via engram_load_merge.
|
|
//
|
|
// The soul calls GET /api/sync every SOUL_REFRESH_MS (default 10 min) to pull
|
|
// new Knowledge/Memory/BacklogItem nodes from the authoritative HTTP Engram into
|
|
// its in-process working store. Previously this returned 404 "not found", causing
|
|
// the soul to write the error JSON to a temp file and attempt an empty merge.
|
|
//
|
|
// Strategy: save the current snapshot to disk, read it back, return the full
|
|
// snapshot JSON. The soul's engram_load_merge handles large files gracefully
|
|
// (it skips nodes already present by ID). Auth-exempt: same-host internal call.
|
|
// (2026-06-27 self-review: added this route to fix silent 10-min sync failures)
|
|
fn route_sync(method: String, path: String, body: String) -> String {
|
|
let dir_raw: String = env("ENGRAM_DATA_DIR")
|
|
let dir: String = if str_eq(dir_raw, "") { "/tmp/engram" } else { dir_raw }
|
|
// 2026-07-21 self-review: export to a scratch path, never the canonical
|
|
// snapshot.json — read routes must not be able to clobber the good snapshot.
|
|
let snap_path: String = dir + "/.sync-export.json"
|
|
engram_save(snap_path)
|
|
let snap: String = fs_read(snap_path)
|
|
// 2026-08-02 self-review: this used to return {"nodes":[],"edges":[]} when
|
|
// the export/read failed. The soul's sync_ok test (awareness.el) only
|
|
// checks for "" and "{}", so that placeholder PASSED as a healthy sync:
|
|
// soul.last_sync_ok_ts got stamped, sync_age_ms stayed green, the
|
|
// sync_empty warn ISE never fired, and engram_sync reported added:0
|
|
// forever. A totally broken sync was indistinguishable from a quiet
|
|
// healthy one — the exact failure class this route was added to fix in
|
|
// the first place (see 2026-06-27 note above). Return a real error so the
|
|
// failure is loud on both sides.
|
|
if str_eq(snap, "") { return err_json("sync export failed: snapshot unreadable") }
|
|
return snap
|
|
}
|
|
|
|
// route_load_merge — POST /api/load-merge {"path": "..."} — merge a snapshot
|
|
// file into the live store WITHOUT resetting it (engram_load_merge skips nodes
|
|
// already present by id). Added 2026-07-21 self-review to restore the 244 kn-
|
|
// identity Knowledge nodes lost from the snapshot lineage between 05-13 and
|
|
// 07-13. Requires an explicit path: refuses to run without one so it can never
|
|
// be triggered accidentally against a default.
|
|
fn route_load_merge(method: String, path: String, body: String) -> String {
|
|
let p: String = json_get_string(body, "path")
|
|
if str_eq(p, "") { return err_json("path is required") }
|
|
if str_eq(fs_read(p), "") { return err_json("file missing or empty") }
|
|
let before_n: Int = engram_node_count()
|
|
let before_e: Int = engram_edge_count()
|
|
engram_load_merge(p)
|
|
let added_n: Int = engram_node_count() - before_n
|
|
let added_e: Int = engram_edge_count() - before_e
|
|
let saved: Int = persist_canonical()
|
|
"{\"ok\":true,\"nodes_added\":" + int_to_str(added_n) + ",\"edges_added\":" + int_to_str(added_e) + ",\"node_count\":" + int_to_str(engram_node_count()) + "}"
|
|
}
|
|
|
|
// route_emit_ise — write an InternalStateEvent node from the soul daemon.
|
|
//
|
|
// Endpoint: POST /api/neuron/state-events
|
|
// Body: {"content": "<json-string>"}
|
|
//
|
|
// Auth: exempt (internal endpoint, soul daemon on same host, no _auth needed).
|
|
// The soul's ise_post() sends {"content":"..."} without _auth; enforcing auth
|
|
// here would silently drop all heartbeat/curiosity ISEs. Unauthenticated POST
|
|
// to this endpoint is acceptable: ISE writes are observability-only, append-only,
|
|
// and come from a trusted process on localhost.
|
|
//
|
|
// Salience/importance set to match engram_node_full ISE defaults used by the
|
|
// in-process fallback path in awareness.el (salience=0.3, importance=0.3,
|
|
// confidence=0.8, tier=Episodic).
|
|
// (2026-06-26 self-review: added this route after discovering ise_post was
|
|
// silently failing — the soul posts here but the endpoint didn't exist.)
|
|
//
|
|
// Retention (2026-07-16 self-review): an earlier comment here claimed ISEs
|
|
// got temporal_decay_rate=1.617 — that was never implemented (engram_node_full
|
|
// hardcodes 0.0), and per-node decay only dampens activation anyway; it never
|
|
// removes nodes. By 2026-07-16 ISEs were 75% of the store (10,175 of 13,522
|
|
// nodes, ~4,300/day, unbounded). ISEs are already WM-excluded in
|
|
// engram_activate, so the fix is retention, not decay: every insert calls
|
|
// engram_prune_telemetry(), a single O(nodes+edges) compaction pass that
|
|
// removes ISEs older than ENGRAM_ISE_RETENTION_MS (default 48h), protecting
|
|
// "session-start" labels and self_review events as durable history. At
|
|
// ~3 ISEs/min this bounds telemetry at ~8.6k nodes instead of growing forever.
|
|
fn route_emit_ise(method: String, path: String, body: String) -> String {
|
|
let content: String = json_get_string(body, "content")
|
|
if str_eq(content, "") { return err_json("missing content") }
|
|
let sal: Float = 0.3
|
|
let imp: Float = 0.3
|
|
let conf: Float = 0.8
|
|
let id: String = engram_node_full(
|
|
content, "InternalStateEvent", "state-event",
|
|
sal, imp, conf,
|
|
"Episodic", "[\"internal-state\",\"InternalStateEvent\"]"
|
|
)
|
|
let ret_raw: String = env("ENGRAM_ISE_RETENTION_MS")
|
|
let ret_ms: Int = if str_eq(ret_raw, "") { 172800000 } else { str_to_int(ret_raw) }
|
|
let pruned: Int = engram_prune_telemetry(ret_ms)
|
|
"{\"ok\":true,\"id\":\"" + id + "\",\"pruned\":" + int_to_str(pruned) + "}"
|
|
}
|
|
|
|
// ── Knowledge capture ─────────────────────────────────────────────────────────
|
|
//
|
|
// route_capture_knowledge — direct Knowledge-node capture over HTTP.
|
|
//
|
|
// Endpoint: POST /api/neuron/knowledge/capture (auth required: "_auth" in body)
|
|
// Body: {"content": "...", "title": "...", "category": "...",
|
|
// "tier": "note|lesson|canonical", "tags": [...], "project": "...",
|
|
// "_auth": "<key>"}
|
|
//
|
|
// WHY (2026-07-15 self-review): the world-ingestor integrator was designed
|
|
// against this endpoint (its MCP-unavailable fallback), but the route never
|
|
// existed — every direct push 404'd, and because the auth gate ran before
|
|
// routing, the failure surfaced as {"error":"unauthorized"} and was
|
|
// misdiagnosed for two weeks while world knowledge silently dropped.
|
|
// POST /api/nodes was no substitute: it discards label/tags/tier, which
|
|
// makes captured knowledge invisible to tag-scoped search and curiosity.
|
|
//
|
|
// The incoming knowledge tier (note/lesson/canonical) is preserved as a
|
|
// "tier:<x>" tag rather than mapped onto Engram's cognitive tiers — Knowledge
|
|
// nodes land in Semantic (stable reference), and the epistemic tier stays
|
|
// queryable without inventing a lossy mapping.
|
|
fn route_capture_knowledge(method: String, path: String, body: String) -> String {
|
|
let content: String = json_get_string(body, "content")
|
|
if str_eq(content, "") { return err_json("missing content") }
|
|
let title: String = json_get_string(body, "title")
|
|
let label: String = if str_eq(title, "") { str_slice(content, 0, 60) } else { title }
|
|
let category_raw: String = json_get_string(body, "category")
|
|
let category: String = if str_eq(category_raw, "") { "other" } else { category_raw }
|
|
let ktier_raw: String = json_get_string(body, "tier")
|
|
let ktier: String = if str_eq(ktier_raw, "") { "note" } else { ktier_raw }
|
|
let project: String = json_get_string(body, "project")
|
|
let tags_raw: String = json_get_raw(body, "tags")
|
|
let tags_base: String = if str_eq(tags_raw, "") { "[]" } else { tags_raw }
|
|
// Merge category/tier/project markers into the tag array. Search matches
|
|
// against the tags string, so these make captures findable by facet.
|
|
let base_len: Int = str_len(tags_base)
|
|
let head: String = str_slice(tags_base, 0, base_len - 1)
|
|
let sep: String = if str_eq(head, "[") { "" } else { "," }
|
|
let safe_cat: String = str_replace(category, "\"", "'")
|
|
let safe_tier: String = str_replace(ktier, "\"", "'")
|
|
let safe_proj: String = str_replace(project, "\"", "'")
|
|
let proj_tag: String = if str_eq(safe_proj, "") { "" } else { ",\"project:" + safe_proj + "\"" }
|
|
let tags: String = head + sep + "\"category:" + safe_cat + "\",\"tier:" + safe_tier + "\"" + proj_tag + "]"
|
|
let sal: Float = 0.5
|
|
let imp: Float = 0.5
|
|
let conf: Float = 0.9
|
|
let id: String = engram_node_full(
|
|
content, "Knowledge", label,
|
|
sal, imp, conf,
|
|
"Semantic", tags
|
|
)
|
|
let saved: Int = persist_canonical()
|
|
"{\"ok\":true,\"id\":\"" + id + "\"}"
|
|
}
|
|
|
|
// route_similarity — GET /api/similarity?a=<id>&b=<id>
|
|
//
|
|
// (2026-08-01 self-review) engram_cosine_sim was added 2026-07-24
|
|
// (bl-b2d1c944) with the stated purpose of exposing semantic distance to
|
|
// "EL code and the introspection API" — but it had ZERO callers anywhere:
|
|
// no route, no soul-daemon use. The activation path uses embeddings
|
|
// internally (semantic seeding, Pass-2 additive term), but there was no way
|
|
// to probe pairwise node similarity from outside. This closes that: cosine
|
|
// in [-1,1], or -2 when either node is missing or not yet embedded (so
|
|
// "not comparable" is distinguishable from "genuinely orthogonal" 0.0).
|
|
fn route_similarity(method: String, path: String, body: String) -> String {
|
|
let a: String = query_param(path, "a")
|
|
let b: String = query_param(path, "b")
|
|
if str_eq(a, "") { return err_json("missing a") }
|
|
if str_eq(b, "") { return err_json("missing b") }
|
|
let sim: Float = engram_cosine_sim(a, b)
|
|
"{\"a\":\"" + a + "\",\"b\":\"" + b + "\",\"cosine\":" + float_to_str(sim) + "}"
|
|
}
|
|
|
|
// ── Auth ──────────────────────────────────────────────────────────────────────
|
|
|
|
fn check_auth_ok(method: String, body: String) -> Bool {
|
|
let key: String = env("ENGRAM_API_KEY")
|
|
if str_eq(key, "") { return true }
|
|
// Read-only methods don't require auth. Until http_serve surfaces
|
|
// request headers we can't accept a Bearer token cleanly; mutating
|
|
// requests must include "_auth": "<key>" in the JSON body.
|
|
if str_eq(method, "GET") { return true }
|
|
let provided: String = json_get_string(body, "_auth")
|
|
if str_eq(provided, key) { return true }
|
|
return false
|
|
}
|
|
|
|
// ── Dispatcher ────────────────────────────────────────────────────────────────
|
|
|
|
fn handle_request(method: String, path: String, body: String) -> String {
|
|
let clean: String = strip_query(path)
|
|
|
|
// Health is always reachable
|
|
if str_eq(method, "GET") {
|
|
if str_eq(clean, "/health") || str_eq(clean, "/") {
|
|
return route_health(method, path, body)
|
|
}
|
|
}
|
|
|
|
// ISE posting is auth-exempt (internal soul daemon, same host, no _auth key)
|
|
if str_eq(method, "POST") && str_eq(clean, "/api/neuron/state-events") {
|
|
return route_emit_ise(method, path, body)
|
|
}
|
|
|
|
// Auth (when ENGRAM_API_KEY is set)
|
|
if !check_auth_ok(method, body) {
|
|
return err_json("unauthorized")
|
|
}
|
|
|
|
// Knowledge capture (auth enforced above; the world-ingestor integrator
|
|
// and any headless session without MCP push knowledge through this)
|
|
if str_eq(method, "POST") && str_eq(clean, "/api/neuron/knowledge/capture") {
|
|
return route_capture_knowledge(method, path, body)
|
|
}
|
|
|
|
// Stats
|
|
if str_eq(method, "GET") && (str_eq(clean, "/api/stats") || str_eq(clean, "/stats")) {
|
|
return route_stats(method, path, body)
|
|
}
|
|
if str_eq(method, "GET") && (str_eq(clean, "/api/act-stats") || str_eq(clean, "/act-stats")) {
|
|
return route_act_stats(method, path, body)
|
|
}
|
|
if str_eq(method, "GET") && (str_eq(clean, "/api/text-health") || str_eq(clean, "/text-health")) {
|
|
return route_text_health(method, path, body)
|
|
}
|
|
|
|
// Nodes
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/nodes") || str_eq(clean, "/nodes")) {
|
|
return route_create_node(method, path, body)
|
|
}
|
|
if str_eq(method, "GET") && (str_eq(clean, "/api/nodes") || str_eq(clean, "/nodes") || str_eq(clean, "/nodes/list") || str_eq(clean, "/api/nodes/list")) {
|
|
return route_scan_nodes(method, path, body)
|
|
}
|
|
if str_eq(method, "GET") && (str_eq(clean, "/api/edges") || str_eq(clean, "/edges")) {
|
|
return route_scan_edges(method, path, body)
|
|
}
|
|
if str_eq(method, "GET") && str_starts_with(clean, "/api/nodes/") {
|
|
return route_get_node(method, path, body)
|
|
}
|
|
if str_eq(method, "DELETE") && str_starts_with(clean, "/api/nodes/") {
|
|
return route_forget(method, path, body)
|
|
}
|
|
|
|
// Edges
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/edges") || str_eq(clean, "/edges")) {
|
|
return route_create_edge(method, path, body)
|
|
}
|
|
// Batch edge write — one snapshot for the whole payload. Must be tested
|
|
// BEFORE nothing else claims it; the exact-match on "/api/edges" above
|
|
// does not catch "/api/edges/batch", so order is not load-bearing here,
|
|
// but keeping the two adjacent keeps them from drifting apart.
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/edges/batch") || str_eq(clean, "/edges/batch")) {
|
|
return route_create_edges_batch(method, path, body)
|
|
}
|
|
if str_eq(method, "GET") && str_starts_with(clean, "/api/neighbors/") {
|
|
return route_neighbors(method, path, body)
|
|
}
|
|
|
|
// Activation + Search
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/activate") || str_eq(clean, "/activate")) {
|
|
return route_activate(method, path, body)
|
|
}
|
|
if str_eq(method, "GET") && str_starts_with(clean, "/api/activate") {
|
|
return route_activate(method, path, body)
|
|
}
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/search") || str_eq(clean, "/search")) {
|
|
return route_search(method, path, body)
|
|
}
|
|
if str_eq(method, "GET") && str_starts_with(clean, "/api/search") {
|
|
return route_search(method, path, body)
|
|
}
|
|
|
|
// Strengthen
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/strengthen") || str_eq(clean, "/strengthen")) {
|
|
return route_strengthen(method, path, body)
|
|
}
|
|
|
|
// Persistence
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/save") || str_eq(clean, "/save")) {
|
|
return route_save(method, path, body)
|
|
}
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/load") || str_eq(clean, "/load")) {
|
|
return route_load(method, path, body)
|
|
}
|
|
if str_eq(method, "POST") && (str_eq(clean, "/api/load-merge") || str_eq(clean, "/load-merge")) {
|
|
return route_load_merge(method, path, body)
|
|
}
|
|
|
|
// Sync — soul daemon periodic pull of non-ISE knowledge into in-process graph
|
|
if str_eq(method, "GET") && str_eq(clean, "/api/sync") {
|
|
return route_sync(method, path, body)
|
|
}
|
|
|
|
// Embedding backfill — pumped by the soul heartbeat (2026-07-25)
|
|
if str_eq(clean, "/api/embed-backfill") {
|
|
return route_embed_backfill(method, path, body)
|
|
}
|
|
|
|
// Semantic similarity probe (2026-08-01)
|
|
if str_eq(method, "GET") && str_starts_with(clean, "/api/similarity") {
|
|
return route_similarity(method, path, body)
|
|
}
|
|
|
|
"{\"error\":\"not found\",\"path\":\"" + clean + "\"}"
|
|
}
|
|
|
|
// ── Entry ─────────────────────────────────────────────────────────────────────
|
|
|
|
let bind_raw: String = env("ENGRAM_BIND")
|
|
let bind_str: String = if str_eq(bind_raw, "") { ":8742" } else { bind_raw }
|
|
let port: Int = parse_port(bind_str)
|
|
|
|
// On startup, try to load any existing snapshot (best effort).
|
|
let data_dir_raw: String = env("ENGRAM_DATA_DIR")
|
|
let data_dir: String = if str_eq(data_dir_raw, "") { "/tmp/engram" } else { data_dir_raw }
|
|
let snapshot_path: String = data_dir + "/snapshot.json"
|
|
engram_load(snapshot_path)
|
|
|
|
// 2026-07-21 self-review boot guard: if the snapshot file has content but the
|
|
// load produced 0 nodes, something is wrong (corrupt file / parse failure).
|
|
// Preserve the evidence and warn loudly — and since read routes no longer write
|
|
// the canonical path, a bad boot can no longer clobber the good snapshot.
|
|
let boot_snap: String = fs_read(snapshot_path)
|
|
if !str_eq(boot_snap, "") {
|
|
if engram_node_count() == 0 {
|
|
println("[engram] WARNING: snapshot.json is non-empty but load produced 0 nodes — preserving copy at snapshot.failed-load.json")
|
|
fs_write(data_dir + "/snapshot.failed-load.json", boot_snap)
|
|
} else {
|
|
// Good load: keep a boot-time backup of the snapshot as loaded.
|
|
fs_write(data_dir + "/snapshot.boot-backup.json", boot_snap)
|
|
}
|
|
}
|
|
|
|
println("[engram] runtime-native graph engine")
|
|
println("[engram] data_dir=" + data_dir)
|
|
println("[engram] node_count=" + int_to_str(engram_node_count()))
|
|
println("[engram] edge_count=" + int_to_str(engram_edge_count()))
|
|
println("[engram] listening on " + int_to_str(port))
|
|
|
|
http_set_handler("handle_request")
|
|
http_serve(port, "handle_request")
|