Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c6772e3d27 | |||
| b9ef66cae9 | |||
| 9717a4eeaf | |||
| 9790d9342d | |||
| 9501e4ac12 | |||
| dd952c0e46 |
@@ -889,10 +889,29 @@ fn awareness_run() -> Void {
|
||||
state_set("soul.last_beat_ts", int_to_str(now_ts))
|
||||
// Persist in-process Engram (sessions, memories, conversation nodes)
|
||||
// to local snapshot so they survive restarts.
|
||||
// FILE MODE ONLY: "soul_snapshot_path" is set exclusively in the
|
||||
// genesis+safe_to_seed branch of soul.el, and safe_to_seed is
|
||||
// unconditionally false when ENGRAM_URL is set. In HTTP mode the
|
||||
// owner persists; the soul must not (soul.el:571-573).
|
||||
let snap_path: String = state_get("soul_snapshot_path")
|
||||
if !str_eq(snap_path, "") {
|
||||
mem_save(snap_path)
|
||||
}
|
||||
// WRITE-THROUGH RETRY (neuron#117). The HTTP-mode counterpart of the
|
||||
// save above: hand anything still spooled to the persistence owner.
|
||||
//
|
||||
// This is the retry arm of the whole design. Deltas that could not be
|
||||
// pushed — owner down, owner restarting, transient refusal — stay on
|
||||
// disk and are re-offered here every heartbeat until they land. It is
|
||||
// also the catch-all for writes made by the awareness loop itself,
|
||||
// which never passes through the HTTP handler's flush point.
|
||||
//
|
||||
// No-op with no HTTP call when the spool is empty or ENGRAM_URL is
|
||||
// unset, so an idle soul in file mode pays nothing for this.
|
||||
let wt_pushed: Int = wt_drain()
|
||||
if wt_pushed < 0 {
|
||||
ise_post("{\"event\":\"write_through_backlog\",\"ts\":" + int_to_str(now_ts) + "}")
|
||||
}
|
||||
}
|
||||
|
||||
// Curiosity scan: idle-gated AND wall-clock based. Only fires when the
|
||||
|
||||
@@ -1162,7 +1162,7 @@ fn hist_trim_with_bell_guard(hist: String) -> String {
|
||||
+ " | evicted_at:" + ts_str
|
||||
+ " | message:" + safe_content
|
||||
let preserve_tags: String = "[\"bell-history\",\"bell:" + bell_level + "\",\"evicted\",\"affective\",\"BellEvent\"]"
|
||||
let discard: String = engram_node_full(
|
||||
let discard: String = wt_node(
|
||||
preserve_content,
|
||||
"BellEvent",
|
||||
"bell:" + bell_level + ":preserved",
|
||||
@@ -1210,7 +1210,7 @@ fn conv_history_persist(session_id: String, hist: String) -> Void {
|
||||
if !str_contains(hist, "]") { return "" }
|
||||
let tags: String = "[\"conv-history\",\"persistent\"]"
|
||||
// FIX B: one label rule, shared with the agentic path. See conv_hist_label.
|
||||
let node_id: String = engram_node_full(
|
||||
let node_id: String = wt_node(
|
||||
hist, "Conversation", conv_hist_label(session_id),
|
||||
el_from_float(0.7), el_from_float(0.8), el_from_float(0.9),
|
||||
"Episodic", tags
|
||||
@@ -3461,7 +3461,7 @@ fn handle_dharma_room_turn(body: String) -> String {
|
||||
// engram_node(content, "episodic", ...) which wrongly put a TIER into the node_type
|
||||
// slot — that's why nodes showed node_type="episodic". Use the full, correct contract.)
|
||||
let utterance_tags: String = "[\"soul-utterance\",\"episodic\"]"
|
||||
let discard_id: String = engram_node_full(
|
||||
let discard_id: String = wt_node(
|
||||
clean_response, "Conversation", "soul:utterance",
|
||||
el_from_float(0.6), el_from_float(0.6), el_from_float(0.8),
|
||||
"Episodic", utterance_tags
|
||||
@@ -3552,7 +3552,7 @@ fn session_summary_write(summary_text: String) -> String {
|
||||
}
|
||||
}
|
||||
let tags: String = "[\"SessionSummary\",\"session-summary\",\"previous-session\",\"consolidate\"]"
|
||||
let node_id: String = engram_node_full(
|
||||
let node_id: String = wt_node(
|
||||
content, "SessionSummary", "session:summary",
|
||||
el_from_float(0.85), el_from_float(0.85), el_from_float(1.0),
|
||||
"Episodic", tags
|
||||
@@ -3578,7 +3578,7 @@ fn session_summary_write_dated(summary_text: String, label: String) -> String {
|
||||
let ts_str: String = int_to_str(ts)
|
||||
let content: String = "[session-summary] " + trimmed + " | ts:" + ts_str
|
||||
let tags: String = "[\"SessionSummary\",\"session-summary\",\"previous-session\",\"consolidate\"]"
|
||||
let node_id: String = engram_node_full(
|
||||
let node_id: String = wt_node(
|
||||
content, "SessionSummary", label,
|
||||
el_from_float(0.9), el_from_float(0.8), el_from_float(1.0),
|
||||
"Episodic", tags
|
||||
@@ -3654,7 +3654,7 @@ fn auto_persist(req: String, resp: String) -> Void {
|
||||
+ ",\"bell\":\"" + bell_level + "\""
|
||||
+ ",\"label\":\"chat:" + ts_str + "\"}"
|
||||
|
||||
let conv_node_id: String = engram_node_full(
|
||||
let conv_node_id: String = wt_node(
|
||||
content,
|
||||
"Conversation",
|
||||
"chat:" + ts_str,
|
||||
@@ -3692,7 +3692,7 @@ fn auto_persist(req: String, resp: String) -> Void {
|
||||
let bell_tags: String = "[\"safety\",\"bell\",\"bell:" + bell_level + "\",\"affective\",\"BellEvent\"]"
|
||||
let bell_ts_str: String = int_to_str(time_now())
|
||||
let bell_label: String = "bell:" + bell_level + ":" + bell_ts_str
|
||||
let bell_node_id: String = engram_node_full(
|
||||
let bell_node_id: String = wt_node(
|
||||
bell_content,
|
||||
"BellEvent",
|
||||
bell_label,
|
||||
@@ -3751,7 +3751,7 @@ fn auto_persist(req: String, resp: String) -> Void {
|
||||
let pos_tags: String = "[\"joy\",\"positive\",\"joy:" + positive_level + "\",\"affective\",\"PositiveEvent\"]"
|
||||
let pos_ts_label: String = int_to_str(time_now())
|
||||
let pos_label: String = "joy:" + positive_level + ":" + pos_ts_label
|
||||
let pos_node_id: String = engram_node_full(
|
||||
let pos_node_id: String = wt_node(
|
||||
pos_content, "PositiveEvent", pos_label,
|
||||
pos_sal_a, pos_sal_b, pos_sal_c, "Episodic", pos_tags
|
||||
)
|
||||
|
||||
+1283
-662
File diff suppressed because one or more lines are too long
@@ -1,9 +1,11 @@
|
||||
import "persist.el"
|
||||
|
||||
fn tier_working() -> String { return "Working" }
|
||||
fn tier_episodic() -> String { return "Episodic" }
|
||||
fn tier_canonical() -> String { return "Canonical" }
|
||||
|
||||
fn mem_store(content: String, label: String, tags: String) -> String {
|
||||
let id: String = engram_node_full(
|
||||
let id: String = wt_node(
|
||||
content,
|
||||
"Memory",
|
||||
label,
|
||||
@@ -17,13 +19,23 @@ fn mem_store(content: String, label: String, tags: String) -> String {
|
||||
println("[memory] write rejected by engram (empty id): label=" + label)
|
||||
return ""
|
||||
}
|
||||
// Read back to verify the node actually persisted — guards against silent write failures.
|
||||
let readback: String = engram_get_node_json(id)
|
||||
if str_eq(readback, "") || str_eq(readback, "{}") {
|
||||
println("[memory] WRITE VERIFY FAILED: label=" + label + " id=" + id + " — node absent after write")
|
||||
return ""
|
||||
// wt_node has already read the node back locally and returns "" if it did
|
||||
// not land, so the old duplicate read-back here is gone.
|
||||
//
|
||||
// HONESTY (neuron#117): the receipt now says WHERE the write is.
|
||||
// The old unconditional "write verified" line asserted against the soul's
|
||||
// own RAM — true in memory, false on disk — and printed ~115,000 times on
|
||||
// Tim's machine while the canonical snapshot sat frozen for three days.
|
||||
// wt_commit flushes the spool and then asks the OWNER. When it says false
|
||||
// the node is real and recallable but not yet durable, and the log says so
|
||||
// rather than claiming a save that did not happen. The id is still returned:
|
||||
// the local write DID succeed, and the queued delta will be retried.
|
||||
let durable: Bool = wt_commit(id)
|
||||
if durable {
|
||||
println("[memory] write persisted at owner: " + id + " label=" + label)
|
||||
} else {
|
||||
println("[memory] write IN MEMORY ONLY (queued for owner, not yet durable): " + id + " label=" + label)
|
||||
}
|
||||
println("[memory] write verified: " + id + " ok")
|
||||
return id
|
||||
}
|
||||
|
||||
@@ -51,12 +63,12 @@ fn mem_strengthen(node_id: String) -> Void {
|
||||
// memory.el (imported first) so awareness.el and neuron-api.el can both call it.
|
||||
fn mem_tombstone(node_id: String) -> String {
|
||||
let tags: String = "[\"Tombstone\",\"status:deleted\"]"
|
||||
let marker: String = engram_node_full(
|
||||
let marker: String = wt_node(
|
||||
node_id, "Tombstone", "tombstone:" + node_id,
|
||||
el_from_float(0.01), el_from_float(0.01), el_from_float(1.0),
|
||||
"Episodic", tags)
|
||||
if !str_eq(marker, "") {
|
||||
engram_connect(marker, node_id, el_from_float(1.0), "tombstones")
|
||||
wt_edge(marker, node_id, el_from_float(1.0), "tombstones")
|
||||
}
|
||||
return marker
|
||||
}
|
||||
|
||||
+42
-28
@@ -195,15 +195,24 @@ fn api_compact_activated(raw: String, max_items: Int, snip: Int) -> String {
|
||||
}
|
||||
|
||||
// api_persisted — read-back-after-write guard against hallucinated saves.
|
||||
// After a write builtin returns an id, confirm the node is actually queryable
|
||||
// via engram_get_node_json(id) (returns "" or "null" when missing). Returns
|
||||
// true only when the node is genuinely persisted.
|
||||
//
|
||||
// WIDENED FOR neuron#117. This function is the single gate every MCP write
|
||||
// handler passes through before it reports success (10 call sites), which makes
|
||||
// it the right place to close the honesty gap rather than editing ten receipts.
|
||||
//
|
||||
// It used to read back from engram_get_node_json — the SOUL'S OWN in-process
|
||||
// graph. In HTTP-engram mode that asserts the wrong thing: the soul is not the
|
||||
// persistence owner, so a node present in its RAM and absent from the owner read
|
||||
// as "persisted" and then vanished on the next restart. The guard was doing
|
||||
// exactly what its comment promised and still certifying writes that did not
|
||||
// survive. It now flushes the write-through spool and asks the OWNER.
|
||||
//
|
||||
// In file mode (no ENGRAM_URL) the soul IS the owner and wt_commit collapses to
|
||||
// the original local read-back — unchanged behaviour, which is what keeps this
|
||||
// reversible.
|
||||
fn api_persisted(id: String) -> Bool {
|
||||
if str_eq(id, "") { return false }
|
||||
let node: String = engram_get_node_json(id)
|
||||
// engram_get_node_json returns "{}" (empty object) when node is not found — not "" or "null".
|
||||
// Check all three to guard against any runtime variation.
|
||||
return !str_eq(node, "") && !str_eq(node, "null") && !str_eq(node, "{}")
|
||||
return wt_commit(id)
|
||||
}
|
||||
|
||||
// api_not_persisted — standard error for a write that did not read back.
|
||||
@@ -342,7 +351,7 @@ fn handle_api_remember(body: String) -> String {
|
||||
let inner: String = str_slice(base_tags, 1, str_len(base_tags) - 1)
|
||||
"[" + inner + ",\"project:" + project + "\"]"
|
||||
}
|
||||
let id: String = engram_node_full(content, "Memory", "memory:remembered",
|
||||
let id: String = wt_node(content, "Memory", "memory:remembered",
|
||||
sal, sal, el_from_float(0.9),
|
||||
"Episodic", final_tags)
|
||||
if !api_persisted(id) { return api_not_persisted(id) }
|
||||
@@ -369,7 +378,7 @@ fn handle_api_node_create(body: String) -> String {
|
||||
if str_eq(importance, "low") { 0.25 } else { 0.5 }
|
||||
}
|
||||
}
|
||||
let id: String = engram_node_full(content, node_type, label,
|
||||
let id: String = wt_node(content, node_type, label,
|
||||
sal, sal, el_from_float(0.9),
|
||||
tier, tags)
|
||||
if !api_persisted(id) { return api_not_persisted(id) }
|
||||
@@ -422,11 +431,11 @@ fn handle_api_node_update(body: String) -> String {
|
||||
}
|
||||
let body_tags: String = json_get(body, "tags")
|
||||
let tags: String = if str_eq(body_tags, "") { "[\"" + node_type + "\"]" } else { body_tags }
|
||||
let new_id: String = engram_node_full(content, node_type, label,
|
||||
let new_id: String = wt_node(content, node_type, label,
|
||||
el_from_float(0.5), el_from_float(0.5), el_from_float(0.8),
|
||||
tier, tags)
|
||||
if !api_persisted(new_id) { return api_not_persisted(new_id) }
|
||||
engram_connect(new_id, id, el_from_float(0.9), "supersedes")
|
||||
wt_edge(new_id, id, el_from_float(0.9), "supersedes")
|
||||
return "{\"id\":\"" + new_id + "\",\"supersedes\":\"" + id + "\",\"ok\":true}"
|
||||
}
|
||||
|
||||
@@ -450,7 +459,12 @@ fn handle_api_recall(method: String, path: String, body: String) -> String {
|
||||
if str_eq(eff_q, "") {
|
||||
return api_or_empty(engram_scan_nodes_json(limit, 0))
|
||||
}
|
||||
let results: String = engram_search_json(eff_q, limit)
|
||||
// engram_recall_json, not engram_search_json: this route IS the retrieval
|
||||
// surface (claim 24's "embedding search queries"), so it gets the semantic
|
||||
// and associative legs. engram_search_json stays lexical because ~40
|
||||
// internal call sites pass a KEY and seven of them delete every record
|
||||
// that comes back — see the boundary note above eg_search_json_impl.
|
||||
let results: String = engram_recall_json(eff_q, limit)
|
||||
return api_or_empty(results)
|
||||
}
|
||||
|
||||
@@ -498,7 +512,7 @@ fn handle_api_capture_knowledge(body: String) -> String {
|
||||
let full: String = if str_eq(title, "") { content } else { title + ": " + content }
|
||||
let lbl: String = str_slice(title, 0, 80)
|
||||
let tags: String = "[\"Knowledge\",\"captured\"]"
|
||||
let id: String = engram_node_full(full, "Knowledge", lbl,
|
||||
let id: String = wt_node(full, "Knowledge", lbl,
|
||||
el_from_float(0.85), el_from_float(0.8), el_from_float(0.9),
|
||||
"Episodic", tags)
|
||||
if !api_persisted(id) { return api_not_persisted(id) }
|
||||
@@ -513,12 +527,12 @@ fn handle_api_evolve_knowledge(body: String) -> String {
|
||||
if !str_eq(prior_id, "") && is_protected_node(prior_id) { return api_err_protected(prior_id) }
|
||||
let tags: String = "[\"Knowledge\",\"evolved\"]"
|
||||
// Empty label → engram_node_full derives content[:60] (LABEL FIX 2026-07-23).
|
||||
let new_id: String = engram_node_full(content, "Knowledge", "",
|
||||
let new_id: String = wt_node(content, "Knowledge", "",
|
||||
el_from_float(0.75), el_from_float(0.75), el_from_float(0.9),
|
||||
"Episodic", tags)
|
||||
if !api_persisted(new_id) { return api_not_persisted(new_id) }
|
||||
if !str_eq(prior_id, "") {
|
||||
engram_connect(new_id, prior_id, el_from_float(0.9), "supersedes")
|
||||
wt_edge(new_id, prior_id, el_from_float(0.9), "supersedes")
|
||||
}
|
||||
return "{\"id\":\"" + new_id + "\",\"supersedes\":\"" + prior_id + "\",\"ok\":true}"
|
||||
}
|
||||
@@ -535,11 +549,11 @@ fn handle_api_promote_knowledge(body: String) -> String {
|
||||
"[\"Knowledge\",\"tier:canonical\",\"disposition:stable\"]"
|
||||
} else { tags_raw }
|
||||
// Empty label → engram_node_full derives content[:60] (LABEL FIX 2026-07-23).
|
||||
let new_id: String = engram_node_full(content, "Knowledge", "",
|
||||
let new_id: String = wt_node(content, "Knowledge", "",
|
||||
el_from_float(0.9), el_from_float(0.9), el_from_float(1.0),
|
||||
"Canonical", tags)
|
||||
if !api_persisted(new_id) { return api_not_persisted(new_id) }
|
||||
engram_connect(new_id, prior_id, el_from_float(0.95), "supersedes")
|
||||
wt_edge(new_id, prior_id, el_from_float(0.95), "supersedes")
|
||||
return "{\"ok\":true,\"new_id\":\"" + new_id + "\",\"supersedes\":\"" + prior_id + "\"}"
|
||||
}
|
||||
|
||||
@@ -562,7 +576,7 @@ fn handle_api_define_process(body: String) -> String {
|
||||
if str_eq(content, "") { return api_err("content is required") }
|
||||
let label: String = if str_eq(name, "") { "process:unnamed" } else { "process:" + name }
|
||||
let tags: String = "[\"Process\"]"
|
||||
let id: String = engram_node_full(content, "Process", label,
|
||||
let id: String = wt_node(content, "Process", label,
|
||||
el_from_float(0.8), el_from_float(0.8), el_from_float(0.9),
|
||||
"Canonical", tags)
|
||||
if !api_persisted(id) { return api_not_persisted(id) }
|
||||
@@ -647,7 +661,7 @@ fn handle_api_tune_config(body: String) -> String {
|
||||
if str_eq(key, "") { return api_err("key is required") }
|
||||
let content: String = "config:" + key + "=" + value
|
||||
let tags: String = "[\"ConfigEntry\",\"config\"]"
|
||||
let id: String = engram_node_full(content, "ConfigEntry", key,
|
||||
let id: String = wt_node(content, "ConfigEntry", key,
|
||||
el_from_float(0.85), el_from_float(0.85), el_from_float(0.9),
|
||||
"Canonical", tags)
|
||||
if !api_persisted(id) { return api_not_persisted(id) }
|
||||
@@ -694,7 +708,7 @@ fn handle_api_link_entities(body: String) -> String {
|
||||
if is_protected_node(to_id) { return api_err_protected(to_id) }
|
||||
let relation: String = json_get(body, "relation")
|
||||
let eff_relation: String = if str_eq(relation, "") { "associates" } else { relation }
|
||||
engram_connect(from_id, to_id, el_from_float(0.5), eff_relation)
|
||||
wt_edge(from_id, to_id, el_from_float(0.5), eff_relation)
|
||||
return "{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + eff_relation + "\"}"
|
||||
}
|
||||
|
||||
@@ -727,11 +741,11 @@ fn handle_api_evolve_memory(body: String) -> String {
|
||||
}
|
||||
}
|
||||
let tags: String = "[\"Memory\",\"evolved\"]"
|
||||
let new_id: String = engram_node_full(content, "Memory", "memory:evolved",
|
||||
let new_id: String = wt_node(content, "Memory", "memory:evolved",
|
||||
sal, sal, el_from_float(0.9),
|
||||
"Episodic", tags)
|
||||
if !str_eq(prior_id, "") && !str_eq(new_id, "") {
|
||||
engram_connect(new_id, prior_id, el_from_float(0.9), "supersedes")
|
||||
wt_edge(new_id, prior_id, el_from_float(0.9), "supersedes")
|
||||
}
|
||||
return "{\"id\":\"" + new_id + "\",\"supersedes\":\"" + prior_id + "\",\"ok\":true}"
|
||||
}
|
||||
@@ -789,11 +803,11 @@ fn handle_api_cultivate(body: String) -> String {
|
||||
let content: String = json_get(body, "content")
|
||||
if str_eq(content, "") { return api_err("content is required") }
|
||||
let tags: String = "[\"Knowledge\",\"evolved\",\"cultivated\"]"
|
||||
let new_id: String = engram_node_full(content, "Knowledge", "knowledge:cultivated",
|
||||
let new_id: String = wt_node(content, "Knowledge", "knowledge:cultivated",
|
||||
el_from_float(0.75), el_from_float(0.75), el_from_float(0.9),
|
||||
"Episodic", tags)
|
||||
if !str_eq(prior_id, "") && !str_eq(new_id, "") {
|
||||
engram_connect(new_id, prior_id, el_from_float(0.9), "supersedes")
|
||||
wt_edge(new_id, prior_id, el_from_float(0.9), "supersedes")
|
||||
}
|
||||
return "{\"id\":\"" + new_id + "\",\"supersedes\":\"" + prior_id + "\",\"ok\":true,\"cultivated\":true}"
|
||||
}
|
||||
@@ -809,11 +823,11 @@ fn handle_api_cultivate(body: String) -> String {
|
||||
}
|
||||
}
|
||||
let tags: String = "[\"Memory\",\"evolved\",\"cultivated\"]"
|
||||
let new_id: String = engram_node_full(content, "Memory", "memory:cultivated",
|
||||
let new_id: String = wt_node(content, "Memory", "memory:cultivated",
|
||||
sal, sal, el_from_float(0.9),
|
||||
"Episodic", tags)
|
||||
if !str_eq(prior_id, "") && !str_eq(new_id, "") {
|
||||
engram_connect(new_id, prior_id, el_from_float(0.9), "supersedes")
|
||||
wt_edge(new_id, prior_id, el_from_float(0.9), "supersedes")
|
||||
}
|
||||
return "{\"id\":\"" + new_id + "\",\"supersedes\":\"" + prior_id + "\",\"ok\":true,\"cultivated\":true}"
|
||||
}
|
||||
@@ -833,7 +847,7 @@ fn handle_api_cultivate(body: String) -> String {
|
||||
if str_eq(to_id, "") { return api_err("to_id is required") }
|
||||
let relation: String = json_get(body, "relation")
|
||||
let eff_relation: String = if str_eq(relation, "") { "associates" } else { relation }
|
||||
engram_connect(from_id, to_id, el_from_float(0.5), eff_relation)
|
||||
wt_edge(from_id, to_id, el_from_float(0.5), eff_relation)
|
||||
return "{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + eff_relation + "\",\"cultivated\":true}"
|
||||
}
|
||||
|
||||
@@ -868,7 +882,7 @@ fn handle_api_consolidate(body: String) -> String {
|
||||
if !str_eq(summary, "") {
|
||||
let safe_summary: String = str_replace(summary, "\"", "'")
|
||||
let tags: String = "[\"SessionSummary\",\"consolidate\"]"
|
||||
let summary_id: String = engram_node_full(
|
||||
let summary_id: String = wt_node(
|
||||
"[session-summary] " + safe_summary,
|
||||
"SessionSummary", "session:summary",
|
||||
el_from_float(0.7), el_from_float(0.7), el_from_float(0.9),
|
||||
|
||||
+426
@@ -0,0 +1,426 @@
|
||||
// persist.el — the soul→engram WRITE-THROUGH boundary (neuron#117).
|
||||
//
|
||||
// WHY THIS FILE EXISTS
|
||||
// soul.el:571-573 states the ownership rule: "when ENGRAM_URL is set the HTTP
|
||||
// Engram owns persistence — the soul must NEVER write to the local snapshot
|
||||
// (not the persistence owner)." The soul obeys the NEGATIVE half. The POSITIVE
|
||||
// half — how a write made inside the soul actually REACHES the owner — was
|
||||
// never built. Sync is pull-only (awareness.el `/api/sync` -> engram_load_merge),
|
||||
// so every node the soul creates lives in its process RAM and is shed on
|
||||
// restart. Measured live 2026-08-07: soul node_count=102184, engram
|
||||
// node_count=79197 — ~23k nodes existing nowhere but RAM.
|
||||
//
|
||||
// SCOPE NOTE ON THE PATENT (corrects an earlier internal reading)
|
||||
// Engram provisional claims 15-18 describe a delta-sync protocol "with peer
|
||||
// Engram instances"; claim 17's pull-then-push sequence is PEER-ENGRAM to
|
||||
// PEER-ENGRAM. The soul is NOT a peer Engram — it is a CALLER of the database
|
||||
// system API (cf. claim 27, "invoked explicitly by a caller of the database
|
||||
// system API"). So claim 17 does not specify a soul↔engram contract and is not
|
||||
// cited as authority here. This design is derived from the ownership rule
|
||||
// alone: the owner owns the writes, therefore the soul must HAND writes to the
|
||||
// owner and must never write the owner's file itself.
|
||||
//
|
||||
// THE MECHANISM, AND WHY NOT `POST /api/nodes`
|
||||
// The obvious route is the one the persona/boot-counter write-backs already
|
||||
// use, POST /api/nodes. It is the wrong instrument here, verified against the
|
||||
// live engram binary in a sandbox:
|
||||
// - it mints a NEW server-side id (engram_node), so the soul's id and the
|
||||
// owner's id diverge — the next /api/sync pull re-imports the node as a
|
||||
// DUPLICATE, and any edge referencing the soul's id never resolves;
|
||||
// - it accepts only {content, node_type, salience} and drops label, tier,
|
||||
// tags, importance, confidence, metadata. A probe posted with tier
|
||||
// "Canonical" came back tier "Working", importance 0.5.
|
||||
// POST /api/load-merge (Will's own route, el `dc39a61`) is the right one:
|
||||
// - engram_load_merge PRESERVES the id and every field;
|
||||
// - it dedups nodes by id and edges by (from_id,to_id,relation), so a
|
||||
// re-submitted delta is a NO-OP — retry safety is free, and it is the same
|
||||
// local-wins semantics the graph already uses;
|
||||
// - it calls persist_canonical() — THE OWNER writes its own canonical file.
|
||||
// The soul never touches it. The ownership rule is honoured in its
|
||||
// strongest form rather than worked around;
|
||||
// - it returns real counts {ok, nodes_added, edges_added, node_count},
|
||||
// so a receipt can be a MEASUREMENT instead of a fixed success shape.
|
||||
//
|
||||
// SPOOL-AND-DRAIN, AND WHY IT IS NOT JUST A DIRECT POST
|
||||
// Measured in a sandbox against a 79k-node / 176MB graph (live scale): one
|
||||
// load-merge costs ~0.38s, essentially all of it the owner's persist_canonical.
|
||||
// A chat turn writes 5-7 nodes; pushing each separately would add ~2.7s per
|
||||
// turn. So writes are STAGED and pushed in one coalesced batch.
|
||||
// The staging buffer is the FILESYSTEM, not process state, because the soul
|
||||
// serves each HTTP connection on its own pthread (el_runtime http_serve_async)
|
||||
// and a shared in-process buffer would lose entries to a read-modify-write
|
||||
// race — silently, which is the one failure mode this file exists to end.
|
||||
// One file per write, named with uuid_v4, is race-free by construction and
|
||||
// buys a property a memory buffer cannot: writes that could not be pushed
|
||||
// SURVIVE A SOUL CRASH and are drained on the next boot.
|
||||
//
|
||||
// WHAT IS DELIBERATELY NOT PUSHED
|
||||
// - InternalStateEvent / heartbeat telemetry. Will's own carve-out, stated in
|
||||
// engram server.el 8f8ccc9: "48h-pruned, loss-tolerant, ~2/min; snapshotting
|
||||
// 28MB per heartbeat is waste."
|
||||
// NOTE (ours, flagged for Will): we do NOT additionally exclude Working-tier
|
||||
// nodes. That exclusion exists in `fb0bb55` to stop the boot counter leaking
|
||||
// through the /api/sync PULL; it is about sync backflow, not durability.
|
||||
// Applying it here would exclude mem_store — which writes tier "Working" — and
|
||||
// mem_store is the single most important durable write path in the soul. Boot
|
||||
// seeding reads the canonical file wholesale, so a pushed Working-tier node
|
||||
// does survive restart. This is the one classification call this file makes
|
||||
// that Will has not ruled on.
|
||||
//
|
||||
// WHAT THIS BOUNDARY CANNOT EXPRESS (by construction, not by omission)
|
||||
// - engram_strengthen (salience/activation drift): load-merge SKIPS ids that
|
||||
// already exist, so it cannot update an existing node. There is no owner-side
|
||||
// update/upsert route. Not pushable through any current route; left as a
|
||||
// follow-up that needs a change in the engram repo.
|
||||
// - engram_forget (hard delete): load-merge is additive and has no delete verb.
|
||||
// Propagating deletes would mean DELETE /api/nodes/<id>, a HARD delete at the
|
||||
// owner — which scripts/verify-soul-contract.sh section B explicitly fails the
|
||||
// build for ("to delete is to supersede/tombstone, never hard-remove"). Local
|
||||
// deletes therefore stay local; the TOMBSTONE NODE and its "tombstones" edge
|
||||
// (mem_tombstone) are pushed, and that is the sanctioned representation of a
|
||||
// deletion in this graph.
|
||||
|
||||
// ── Configuration ─────────────────────────────────────────────────────────────
|
||||
|
||||
// wt_engram_url — same resolution order as ise_post: env, then the state key
|
||||
// stashed at boot. NO hardcoded localhost fallback: unlike telemetry, inventing
|
||||
// a destination for durable data would risk pushing a user's memories at whatever
|
||||
// happens to be listening on 8742. Empty means "no HTTP owner" -> file mode.
|
||||
fn wt_engram_url() -> String {
|
||||
let env_url: String = env("ENGRAM_URL")
|
||||
if !str_eq(env_url, "") { return env_url }
|
||||
return state_get("soul_engram_url")
|
||||
}
|
||||
|
||||
fn wt_api_key() -> String {
|
||||
let env_key: String = env("ENGRAM_API_KEY")
|
||||
if !str_eq(env_key, "") { return env_key }
|
||||
return state_get("soul_engram_api_key")
|
||||
}
|
||||
|
||||
// wt_enabled — true only in HTTP-engram mode. In file mode the soul IS the
|
||||
// persistence owner and every path below is a no-op, so this whole feature is
|
||||
// inert for genesis/local deployments. That is also what makes it reversible.
|
||||
fn wt_enabled() -> Bool {
|
||||
return !str_eq(wt_engram_url(), "")
|
||||
}
|
||||
|
||||
// wt_spool_dir — where staged deltas live. MUST be readable by the engram
|
||||
// process: /api/load-merge takes a PATH and the owner opens it itself. Both
|
||||
// processes are same-host by construction (dev-stack LaunchAgents; the GKE
|
||||
// image starts engram and soul in one container per entrypoint.sh).
|
||||
fn wt_spool_dir() -> String {
|
||||
let raw: String = env("SOUL_OUTBOX_DIR")
|
||||
let dir: String = if str_eq(raw, "") { env("HOME") + "/.neuron/soul-outbox" } else { raw }
|
||||
fs_mkdir(dir)
|
||||
return dir
|
||||
}
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
// wt_esc — minimal JSON string escape. Deliberately local rather than reusing
|
||||
// chat.el's json_safe: persist.el is imported BY memory.el, which is imported by
|
||||
// chat.el, so depending on chat.el here would be an import cycle.
|
||||
fn wt_esc(s: String) -> String {
|
||||
let s1: String = str_replace(s, "\\", "\\\\")
|
||||
let s2: String = str_replace(s1, "\"", "\\\"")
|
||||
let s3: String = str_replace(s2, "\n", "\\n")
|
||||
let s4: String = str_replace(s3, "\r", "\\r")
|
||||
let s5: String = str_replace(s4, "\t", "\\t")
|
||||
return s5
|
||||
}
|
||||
|
||||
// wt_durable_class — Will's telemetry carve-out, by node_type. See header.
|
||||
fn wt_durable_class(node_type: String) -> Bool {
|
||||
if str_eq(node_type, "InternalStateEvent") { return false }
|
||||
return true
|
||||
}
|
||||
|
||||
// wt_inner — strip the surrounding brackets off a JSON array so several arrays
|
||||
// can be concatenated into one. Returns "" for "[]" / "" / anything too short.
|
||||
fn wt_inner(arr: String) -> String {
|
||||
let n: Int = str_len(arr)
|
||||
if n < 3 { return "" }
|
||||
if !str_starts_with(arr, "[") { return "" }
|
||||
return str_slice(arr, 1, n - 1)
|
||||
}
|
||||
|
||||
// wt_read — fs_read, plus a MANDATORY reset of the runtime's binary-length hint.
|
||||
//
|
||||
// THIS IS NOT OPTIONAL AND MUST NOT BE "SIMPLIFIED" BACK TO A BARE fs_read.
|
||||
// The pinned runtime (vendor/el-runtime/v1.0.0-20260501) keeps a thread-local
|
||||
// `_tl_fs_read_len` that fs_read SETS to the file's byte count (so binary files
|
||||
// can be served with a correct Content-Length) and that http_send_response
|
||||
// CONSUMES as the Content-Length of the next reply. Nothing else clears it
|
||||
// except json_get_raw. So any fs_read during request handling that is not
|
||||
// followed by a json_get_raw makes the NEXT HTTP response advertise the FILE's
|
||||
// length instead of the body's — and the runtime then sends that many bytes,
|
||||
// appending whatever adjacent heap memory follows the reply.
|
||||
//
|
||||
// Caught here, measured: a /api/neuron/memory reply that should be 86 bytes went
|
||||
// out as 497, with 411 bytes of this module's own spool paths and log strings
|
||||
// trailing the JSON. The drain reads spool files mid-request, so this boundary
|
||||
// is exactly where the landmine gets stepped on.
|
||||
//
|
||||
// Upstream el fixed the class in `43636ae` ("pair fs_read length hint with its
|
||||
// buffer"); that runtime is NOT the one vendored here, and re-pinning the
|
||||
// runtime is deliberately out of scope for this change. Clearing the hint at
|
||||
// our own boundary fixes our exposure without touching the pinned C.
|
||||
// json_get_raw is used as the reset because it is the only builtin in this
|
||||
// runtime that zeroes the hint, and it does so before any early return.
|
||||
fn wt_clear_binlen() -> Void {
|
||||
let discard: String = json_get_raw("{}", "_wt_reset")
|
||||
}
|
||||
|
||||
fn wt_read(path: String) -> String {
|
||||
let data: String = fs_read(path)
|
||||
wt_clear_binlen()
|
||||
return data
|
||||
}
|
||||
|
||||
// wt_sweep — best-effort removal of the zero-byte husks left by truncation.
|
||||
// The runtime exposes no unlink builtin, so a drained delta is emptied rather
|
||||
// than deleted; this reclaims the directory entries.
|
||||
//
|
||||
// `-empty` is the safety property, not an optimisation: the command is
|
||||
// STRUCTURALLY INCAPABLE of removing a delta that still has content, so it can
|
||||
// never destroy a pending write even if it runs concurrently with a stage.
|
||||
// Only the directory path is interpolated (never a filename), and it is quoted.
|
||||
// The exit code is ignored — an un-swept husk costs one directory entry.
|
||||
fn wt_sweep(dir: String) -> Void {
|
||||
if str_eq(dir, "") { return }
|
||||
if str_contains(dir, "'") { return }
|
||||
exec_command("find '" + dir + "' -maxdepth 1 -name 'wt*.json' -empty -delete 2>/dev/null")
|
||||
}
|
||||
|
||||
// ── Staging ───────────────────────────────────────────────────────────────────
|
||||
|
||||
// wt_stage — write ONE delta file. uuid_v4 in the name makes concurrent stagers
|
||||
// collision-free without any lock. Returns true if the delta is on disk.
|
||||
fn wt_stage(nodes_json: String, edges_json: String) -> Bool {
|
||||
let dir: String = wt_spool_dir()
|
||||
if str_eq(dir, "") { return false }
|
||||
let payload: String = "{\"nodes\":" + nodes_json + ",\"edges\":" + edges_json + "}"
|
||||
let path: String = dir + "/wt-" + uuid_v4() + ".json"
|
||||
fs_write(path, payload)
|
||||
// Read-back-verify the stage itself. A stage that did not land is a write we
|
||||
// would otherwise believe was queued — exactly the hallucinated-save class.
|
||||
if str_eq(wt_read(path), "") {
|
||||
println("[persist] wt_stage: FAILED to write spool file " + path + " — delta not queued")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ── The write boundary ────────────────────────────────────────────────────────
|
||||
|
||||
// wt_node — create a node locally AND queue it for the persistence owner.
|
||||
// Same signature and same return contract as engram_node_full ("" on failure),
|
||||
// so converting a call site is a rename and nothing else.
|
||||
fn wt_node(content: String, node_type: String, label: String,
|
||||
salience: Float, importance: Float, confidence: Float,
|
||||
tier: String, tags: String) -> String {
|
||||
let id: String = engram_node_full(content, node_type, label,
|
||||
salience, importance, confidence,
|
||||
tier, tags)
|
||||
if str_eq(id, "") { return "" }
|
||||
// engram_get_node_json emits the SAME record shape engram_save writes (minus
|
||||
// the embedding vector, which the owner backfills lazily), so the read-back
|
||||
// doubles as the delta payload — no second serialization to drift.
|
||||
let rec: String = engram_get_node_json(id)
|
||||
if str_eq(rec, "") || str_eq(rec, "{}") {
|
||||
println("[persist] wt_node: local write did not read back, id=" + id + " label=" + label)
|
||||
return ""
|
||||
}
|
||||
if wt_enabled() && wt_durable_class(node_type) {
|
||||
wt_stage("[" + rec + "]", "[]")
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// wt_edge — create an edge locally AND queue it. Mirrors engram_connect.
|
||||
//
|
||||
// The edge id is freshly generated rather than read back: the runtime exposes no
|
||||
// "id of the edge I just created" accessor, and the owner dedups edges by
|
||||
// (from_id,to_id,relation), never by id — so the id is not load-bearing. The
|
||||
// consequence, stated plainly: the soul's copy and the owner's copy of the same
|
||||
// edge carry different edge ids. Nothing in either codebase looks an edge up by
|
||||
// id (neighbors traversal scans from_id/to_id), so this is cosmetic.
|
||||
fn wt_edge(from_id: String, to_id: String, weight: Float, relation: String) -> Void {
|
||||
engram_connect(from_id, to_id, weight, relation)
|
||||
if !wt_enabled() { return }
|
||||
if str_eq(from_id, "") || str_eq(to_id, "") { return }
|
||||
let ts: Int = time_now()
|
||||
let rec: String = "{\"id\":\"" + uuid_v4() + "\""
|
||||
+ ",\"from_id\":\"" + wt_esc(from_id) + "\""
|
||||
+ ",\"to_id\":\"" + wt_esc(to_id) + "\""
|
||||
+ ",\"relation\":\"" + wt_esc(relation) + "\""
|
||||
+ ",\"metadata\":\"{}\""
|
||||
+ ",\"weight\":" + float_to_str(weight)
|
||||
+ ",\"confidence\":1"
|
||||
+ ",\"created_at\":" + int_to_str(ts)
|
||||
+ ",\"updated_at\":" + int_to_str(ts)
|
||||
+ ",\"last_fired\":0,\"inhibitory\":0,\"layer_id\":1}"
|
||||
wt_stage("[]", "[" + rec + "]")
|
||||
}
|
||||
|
||||
// ── The drain ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// wt_drain — coalesce every staged delta into ONE load-merge against the owner.
|
||||
//
|
||||
// Returns: nodes_added on success (>= 0), 0 when there was nothing to do, and
|
||||
// -1 when the push FAILED. -1 is load-bearing: on failure the spool files are
|
||||
// left untouched, so nothing is lost and the next drain retries them. A caller
|
||||
// must never read a non-negative return as "my particular node is durable" —
|
||||
// use wt_durable(id) for that.
|
||||
//
|
||||
// Concurrency: several threads may drain at once. Each builds its own batch file
|
||||
// (uuid-named), and overlapping batches are harmless because load-merge dedups.
|
||||
// Files are truncated ONLY after a confirmed ok:true, so a lost race costs a
|
||||
// redundant push, never a dropped write.
|
||||
fn wt_drain() -> Int {
|
||||
if !wt_enabled() { return 0 }
|
||||
let dir: String = wt_spool_dir()
|
||||
if str_eq(dir, "") { return 0 }
|
||||
|
||||
// el_list_len/el_list_get, NOT json_stringify(fs_list(...)): fs_list builds
|
||||
// a native list via el_list_append, and json_stringify does not serialize
|
||||
// that type — it renders the raw pointer value. (Verified in isolation; the
|
||||
// same latent defect is live in studio.el's /api/tools/file/list route,
|
||||
// which returns e.g. {"entries":4386409744}. Noted, not fixed here.)
|
||||
let listing = fs_list(dir)
|
||||
let count: Int = el_list_len(listing)
|
||||
if count == 0 { return 0 }
|
||||
|
||||
let nodes_acc: String = ""
|
||||
let edges_acc: String = ""
|
||||
let drained: String = ""
|
||||
let found: Int = 0
|
||||
let i: Int = 0
|
||||
// No `continue` / `break`: elc lists them as keywords but not one line of
|
||||
// the shipped soul uses either, so they are unexercised on this build path.
|
||||
// Guard conditions are expressed as nested ifs instead, and every rebind is
|
||||
// at the loop-body top level where `let x = ...` is assignment (the idiom
|
||||
// memory.el's boot-counter loop relies on) — never inside a nested block,
|
||||
// where it would shadow instead.
|
||||
while i < count {
|
||||
let name: String = el_list_get(listing, i)
|
||||
// A delta is only usable when it ends with the closing "]}" that
|
||||
// wt_stage writes last. fs_write is not atomic, so a file being written
|
||||
// right now can be observed half-formed; requiring the terminator means
|
||||
// it is picked up whole on the next drain instead of merged as garbage.
|
||||
// An empty read means "already drained and truncated" — not an error.
|
||||
let p: String = if str_starts_with(name, "wt-") { dir + "/" + name } else { "" }
|
||||
let raw: String = if str_eq(p, "") { "" } else { wt_read(p) }
|
||||
let usable: Bool = !str_eq(raw, "") && str_ends_with(raw, "]}")
|
||||
let nj: String = if usable { wt_inner(json_get_raw(raw, "nodes")) } else { "" }
|
||||
let ej: String = if usable { wt_inner(json_get_raw(raw, "edges")) } else { "" }
|
||||
let nodes_acc = if str_eq(nj, "") { nodes_acc } else if str_eq(nodes_acc, "") { nj } else { nodes_acc + "," + nj }
|
||||
let edges_acc = if str_eq(ej, "") { edges_acc } else if str_eq(edges_acc, "") { ej } else { edges_acc + "," + ej }
|
||||
let drained = if !usable { drained } else if str_eq(drained, "") { p } else { drained + "\n" + p }
|
||||
let found = if usable { found + 1 } else { found }
|
||||
let i = i + 1
|
||||
}
|
||||
|
||||
if found == 0 { return 0 }
|
||||
|
||||
let combined: String = "{\"nodes\":[" + nodes_acc + "],\"edges\":[" + edges_acc + "]}"
|
||||
let batch: String = dir + "/wtb-" + uuid_v4() + ".json"
|
||||
fs_write(batch, combined)
|
||||
if str_eq(wt_read(batch), "") {
|
||||
println("[persist] wt_drain: could not write batch file " + batch + " — " + int_to_str(found) + " deltas stay queued")
|
||||
return -1
|
||||
}
|
||||
|
||||
let url: String = wt_engram_url()
|
||||
let key: String = wt_api_key()
|
||||
let body: String = "{\"path\":\"" + wt_esc(batch) + "\",\"_auth\":\"" + wt_esc(key) + "\"}"
|
||||
let resp: String = http_post_json(url + "/api/load-merge", body)
|
||||
|
||||
// The batch file is pure scratch — the retry is rebuilt from the SPOOL, not
|
||||
// from it. Truncate it unconditionally, before branching on the outcome, so
|
||||
// a persistently unreachable owner cannot accumulate one husk per attempt.
|
||||
fs_write(batch, "")
|
||||
|
||||
// Distinguish the two failures rather than collapsing them: "cannot reach
|
||||
// the owner" and "the owner refused this delta" need different human
|
||||
// responses, and a log line that says the wrong one costs a debugging hour.
|
||||
// curl surfaces transport errors as a JSON body, so an empty response is not
|
||||
// the only unreachable signal.
|
||||
// (str_contains rather than a strict parse on purpose — the engram's HTTP
|
||||
// responses have been observed carrying trailing bytes past the JSON.)
|
||||
let unreachable: Bool = str_eq(resp, "")
|
||||
|| str_contains(resp, "Couldn't connect")
|
||||
|| str_contains(resp, "Failed to connect")
|
||||
|| str_contains(resp, "Could not resolve")
|
||||
|| str_contains(resp, "timed out")
|
||||
if unreachable {
|
||||
wt_sweep(dir)
|
||||
println("[persist] wt_drain: owner UNREACHABLE at " + url + " — " + int_to_str(found)
|
||||
+ " deltas stay queued in " + dir + " (will retry): " + resp)
|
||||
return -1
|
||||
}
|
||||
if !str_contains(resp, "\"ok\":true") {
|
||||
wt_sweep(dir)
|
||||
println("[persist] wt_drain: owner REJECTED the delta — " + int_to_str(found)
|
||||
+ " stay queued in " + dir + ": " + resp)
|
||||
return -1
|
||||
}
|
||||
|
||||
let added: Int = json_get_int(resp, "nodes_added")
|
||||
let added_e: Int = json_get_int(resp, "edges_added")
|
||||
|
||||
// Confirmed. Truncate the drained spool files so they are not re-pushed.
|
||||
// Truncation (not deletion) because the runtime exposes no unlink builtin;
|
||||
// an emptied file is inert to the loop above. The zero-byte husks are then
|
||||
// swept below.
|
||||
let paths = str_split(drained, "\n")
|
||||
let pn: Int = el_list_len(paths)
|
||||
let k: Int = 0
|
||||
while k < pn {
|
||||
let one: String = el_list_get(paths, k)
|
||||
if !str_eq(one, "") { fs_write(one, "") }
|
||||
let k = k + 1
|
||||
}
|
||||
wt_sweep(dir)
|
||||
|
||||
println("[persist] wt_drain: pushed " + int_to_str(found) + " deltas -> owner added "
|
||||
+ int_to_str(added) + " nodes, " + int_to_str(added_e) + " edges")
|
||||
return added
|
||||
}
|
||||
|
||||
// wt_durable — is this id present AT THE OWNER? The only honest answer to
|
||||
// "did my write persist" in HTTP mode.
|
||||
//
|
||||
// In file mode the soul IS the owner, so the local read-back is the owner-side
|
||||
// read-back and this collapses to the pre-existing check.
|
||||
//
|
||||
// nodes_added from wt_drain is NOT a substitute: a concurrent drain may have
|
||||
// already pushed this node, making our own added count 0 while the node is
|
||||
// perfectly durable. Presence at the owner is the fact; counts are telemetry.
|
||||
fn wt_durable(id: String) -> Bool {
|
||||
if str_eq(id, "") { return false }
|
||||
if !wt_enabled() {
|
||||
let local: String = engram_get_node_json(id)
|
||||
return !str_eq(local, "") && !str_eq(local, "null") && !str_eq(local, "{}")
|
||||
}
|
||||
let url: String = wt_engram_url()
|
||||
let resp: String = http_get(url + "/api/nodes/" + id)
|
||||
if str_eq(resp, "") { return false }
|
||||
if str_eq(resp, "{}") { return false }
|
||||
return str_contains(resp, "\"id\"")
|
||||
}
|
||||
|
||||
// wt_commit — flush, then assert at the owner. The receipt callers should use.
|
||||
// Deliberately NOT a fixed success shape: it can and does return false while the
|
||||
// local write is perfectly fine in RAM, which is the true state of affairs when
|
||||
// the owner is unreachable.
|
||||
fn wt_commit(id: String) -> Bool {
|
||||
if str_eq(id, "") { return false }
|
||||
if !wt_enabled() {
|
||||
let local: String = engram_get_node_json(id)
|
||||
return !str_eq(local, "") && !str_eq(local, "null") && !str_eq(local, "{}")
|
||||
}
|
||||
let pushed: Int = wt_drain()
|
||||
return wt_durable(id)
|
||||
}
|
||||
@@ -186,7 +186,7 @@ fn route_imprint_contextual(body: String) -> String {
|
||||
return "{\"ok\":false,\"error\":\"empty body\"}"
|
||||
}
|
||||
let tags: String = "[\"imprint\",\"contextual\"]"
|
||||
let id: String = engram_node_full(
|
||||
let id: String = wt_node(
|
||||
body,
|
||||
"Entity",
|
||||
"imprint:contextual",
|
||||
@@ -208,7 +208,7 @@ fn route_imprint_user(body: String) -> String {
|
||||
return "{\"ok\":false,\"error\":\"empty body\"}"
|
||||
}
|
||||
let tags: String = "[\"imprint\",\"user\"]"
|
||||
let id: String = engram_node_full(
|
||||
let id: String = wt_node(
|
||||
body,
|
||||
"Entity",
|
||||
"imprint:user",
|
||||
@@ -239,7 +239,7 @@ fn route_synthesize(body: String) -> String {
|
||||
}
|
||||
let req: String = "synthesize " + parent_a + " " + parent_b
|
||||
let tags: String = "[\"soul-inbox-pending\",\"synthesis-request\"]"
|
||||
engram_node_full(
|
||||
wt_node(
|
||||
req,
|
||||
"Entity",
|
||||
"synthesis-request",
|
||||
@@ -395,7 +395,28 @@ fn handle_connectors(method: String, clean: String, body: String) -> String {
|
||||
return "{\"ok\":false,\"error\":\"unknown connectors route\"}"
|
||||
}
|
||||
|
||||
// handle_request — the soul's HTTP entry point.
|
||||
//
|
||||
// NOTE ON THE NAME (neuron#117): the el runtime resolves this handler by NAME
|
||||
// via dlsym(RTLD_DEFAULT, "handle_request") — that is why the Linux build must
|
||||
// link -rdynamic. So the dispatcher body moved to route_dispatch and the name
|
||||
// `handle_request` stays put as a thin wrapper. Do not rename it back.
|
||||
//
|
||||
// The wrapper exists to give the write-through boundary a guaranteed flush
|
||||
// point. route_dispatch returns from ~60 places; a per-branch flush would be
|
||||
// forgotten on the 61st. Draining here means EVERY request that staged a write
|
||||
// pushes it before the connection closes, whatever route produced it, including
|
||||
// routes added later that know nothing about persistence.
|
||||
//
|
||||
// wt_drain is a no-op (no HTTP, no cost) when nothing is staged and when the
|
||||
// soul is not in HTTP-engram mode, so this is free on read traffic.
|
||||
fn handle_request(method: String, path: String, body: String) -> String {
|
||||
let resp: String = route_dispatch(method, path, body)
|
||||
let flushed: Int = wt_drain()
|
||||
return resp
|
||||
}
|
||||
|
||||
fn route_dispatch(method: String, path: String, body: String) -> String {
|
||||
let clean: String = strip_query(path)
|
||||
|
||||
// ACTIVITY STAMP (2026-07-30 self-review): every inbound HTTP request —
|
||||
@@ -432,10 +453,27 @@ fn handle_request(method: String, path: String, body: String) -> String {
|
||||
return engram_scan_nodes_json(9999, 0)
|
||||
}
|
||||
if str_eq(clean, "/api/graph/edges") {
|
||||
// TODO(reliability #8): engram_save races with awareness loop mem_save().
|
||||
// Both now use atomic write-to-temp+rename (el_runtime.c). Serialised
|
||||
// by engram_global_mu. Future: add engram_edges_json() builtin.
|
||||
let snap_path: String = env("HOME") + "/.neuron/engram/snapshot.json"
|
||||
// FIXED (neuron#117): this GET used to engram_save() straight over
|
||||
// ~/.neuron/engram/snapshot.json — a READ route, in a process that is
|
||||
// NOT the persistence owner, overwriting the owner's canonical file
|
||||
// on every call. It broke soul.el:571-573 ("the soul must NEVER write
|
||||
// to the local snapshot") and it is the same defect class Will removed
|
||||
// from the engram itself in el `dc39a61` ("stop read routes clobbering
|
||||
// canonical snapshot"), where route_scan_edges/route_sync were moved
|
||||
// to scratch paths for exactly this reason. It was also the race the
|
||||
// old TODO(reliability #8) admitted to.
|
||||
//
|
||||
// Export to a scratch path instead. Same response, no canonical write.
|
||||
// The soul's own snapshot writes are otherwise already gated behind
|
||||
// state key "soul_snapshot_path", which is set ONLY in the genesis
|
||||
// file-mode branch (soul.el: is_genesis && safe_to_seed, and
|
||||
// safe_to_seed is unconditionally false when ENGRAM_URL is set) — so
|
||||
// after this change the soul writes nothing at all in HTTP mode.
|
||||
// Future: add an engram_edges_json() builtin and drop the file round
|
||||
// trip entirely.
|
||||
let scratch_dir: String = env("TMPDIR")
|
||||
let scratch_base: String = if str_eq(scratch_dir, "") { "/tmp" } else { scratch_dir }
|
||||
let snap_path: String = scratch_base + "/soul-edges-export-" + state_get("soul_cgi_id") + ".json"
|
||||
engram_save(snap_path)
|
||||
let snap: String = fs_read(snap_path)
|
||||
let edges_raw: String = json_get_raw(snap, "edges")
|
||||
|
||||
@@ -204,7 +204,7 @@ fn safety_log_bell(level: String, reason: String, input_summary: String) -> Stri
|
||||
// Emit a fallback println so the bell event leaves at least a log trace even
|
||||
// when engram is degraded. This does not replace engram persistence -- it is a
|
||||
// last-resort audit trail when the primary write cannot be confirmed.
|
||||
let node_id: String = engram_node_full(
|
||||
let node_id: String = wt_node(
|
||||
content,
|
||||
"BellEvent",
|
||||
"bell:" + level,
|
||||
|
||||
+7
-7
@@ -87,7 +87,7 @@ fn session_create(body: String) -> String {
|
||||
let folder: String = json_get(body, "folder")
|
||||
let content: String = session_make_content(id, title, ts, ts, folder)
|
||||
let tags: String = "[\"session\",\"session:meta\",\"Conversation\"]"
|
||||
let node_id: String = engram_node_full(
|
||||
let node_id: String = wt_node(
|
||||
content, "Conversation", "session:meta",
|
||||
el_from_float(0.7), el_from_float(0.7), el_from_float(0.9),
|
||||
"Episodic", tags
|
||||
@@ -358,7 +358,7 @@ fn session_update_patch(session_id: String, body: String) -> String {
|
||||
let created_int: Int = str_to_int(old_created)
|
||||
let new_content: String = session_make_content(session_id, eff_title, created_int, ts, eff_folder)
|
||||
let tags: String = "[\"session\",\"session:meta\",\"Conversation\"]"
|
||||
let new_node_id: String = engram_node_full(
|
||||
let new_node_id: String = wt_node(
|
||||
new_content, "Conversation", "session:meta",
|
||||
el_from_float(0.7), el_from_float(0.7), el_from_float(0.9),
|
||||
"Episodic", tags
|
||||
@@ -456,7 +456,7 @@ fn session_hist_save(session_id: String, hist: String) -> Void {
|
||||
// TODO(reliability #7): delete-then-insert is not atomic — concurrent saves for the
|
||||
// same session can produce orphan history nodes. State is primary truth; engram fallback.
|
||||
let tags: String = "[\"session\",\"session-history\",\"Conversation\"]"
|
||||
let discard: String = engram_node_full(
|
||||
let discard: String = wt_node(
|
||||
hist, "Conversation", "session:messages:" + session_id,
|
||||
el_from_float(0.6), el_from_float(0.6), el_from_float(0.9),
|
||||
"Episodic", tags
|
||||
@@ -488,7 +488,7 @@ fn session_hist_save(session_id: String, hist: String) -> Void {
|
||||
+ " | ts:" + int_to_str(ts_now)
|
||||
let summary_tags: String = "[\"session-emotional-summary\",\"affective\",\"bell:" + eff_level + "\",\"BellEvent\"]"
|
||||
let summary_sal: String = if str_eq(eff_level, "hard") { el_from_float(0.95) } else { el_from_float(0.85) }
|
||||
let sum_discard: String = engram_node_full(
|
||||
let sum_discard: String = wt_node(
|
||||
summary_content,
|
||||
"BellEvent",
|
||||
"session:emotional-summary",
|
||||
@@ -529,7 +529,7 @@ fn session_hist_save(session_id: String, hist: String) -> Void {
|
||||
if !str_eq(ot_id, "") { engram_forget(ot_id) }
|
||||
let oti = oti + 1
|
||||
}
|
||||
let discard_topic: String = engram_node_full(
|
||||
let discard_topic: String = wt_node(
|
||||
topic_content, "Conversation", topic_label,
|
||||
el_from_float(0.7), el_from_float(0.7), el_from_float(0.9),
|
||||
"Episodic", topic_tags
|
||||
@@ -582,7 +582,7 @@ fn session_update_meta_timestamp(session_id: String) -> Void {
|
||||
let created_int: Int = str_to_int(old_created)
|
||||
let new_content: String = session_make_content(session_id, old_title, created_int, ts, old_folder)
|
||||
let tags: String = "[\"session\",\"session:meta\",\"Conversation\"]"
|
||||
let new_id: String = engram_node_full(
|
||||
let new_id: String = wt_node(
|
||||
new_content, "Conversation", "session:meta",
|
||||
el_from_float(0.7), el_from_float(0.7), el_from_float(0.9),
|
||||
"Episodic", tags
|
||||
@@ -629,7 +629,7 @@ fn session_auto_title(session_id: String, first_message: String) -> Void {
|
||||
let created_int: Int = str_to_int(old_created)
|
||||
let new_content: String = session_make_content(session_id, new_title, created_int, ts, old_folder)
|
||||
let tags: String = "[\"session\",\"session:meta\",\"Conversation\"]"
|
||||
let new_id: String = engram_node_full(
|
||||
let new_id: String = wt_node(
|
||||
new_content, "Conversation", "session:meta",
|
||||
el_from_float(0.7), el_from_float(0.7), el_from_float(0.9),
|
||||
"Episodic", tags
|
||||
|
||||
@@ -657,6 +657,23 @@ if is_genesis && safe_to_seed {
|
||||
}
|
||||
}
|
||||
|
||||
// CRASH RECOVERY (neuron#117). Deltas the previous process staged but could not
|
||||
// hand to the owner are still on disk — the spool is a filesystem queue, not a
|
||||
// memory buffer, precisely so that a soul that died mid-flight does not take its
|
||||
// unpushed writes with it. Drain them before serving, so recovered memories are
|
||||
// durable and recallable from the owner from the first request onward.
|
||||
//
|
||||
// Safe on a clean boot: an empty spool means no HTTP call at all. Safe in file
|
||||
// mode: wt_drain returns immediately when ENGRAM_URL is unset.
|
||||
let wt_recovered: Int = wt_drain()
|
||||
if wt_recovered > 0 {
|
||||
println("[soul] write-through: recovered " + int_to_str(wt_recovered)
|
||||
+ " nodes from a previous process's spool -> persistence owner")
|
||||
}
|
||||
if wt_recovered < 0 {
|
||||
println("[soul] write-through: spool present but the persistence owner is unreachable — queued, will retry on heartbeat")
|
||||
}
|
||||
|
||||
println("[soul] serving on port " + int_to_str(port))
|
||||
http_serve_async(port, "handle_request")
|
||||
println("[soul] awareness loop starting")
|
||||
|
||||
+2
-2
@@ -11,7 +11,7 @@ import "memory.el"
|
||||
fn steward_log_event(kind: String, detail: String) -> Void {
|
||||
let content: String = "STEWARD:" + kind + " | " + detail
|
||||
let tags: String = "[\"stewardship\",\"steward:" + kind + "\"]"
|
||||
let discard: String = engram_node_full(
|
||||
let discard: String = wt_node(
|
||||
content,
|
||||
"StewardshipEvent",
|
||||
"steward:" + kind,
|
||||
@@ -221,7 +221,7 @@ fn steward_fingerprint_session(input: String, session_id: String) -> String {
|
||||
+ " formality=" + fs_str
|
||||
+ " time=" + tb_str
|
||||
let sample_tags: String = "[\"behavior\",\"BehaviorSample\",\"stewardship\"]"
|
||||
let discard: String = engram_node_full(
|
||||
let discard: String = wt_node(
|
||||
sample_content,
|
||||
"BehaviorSample",
|
||||
"behavior:" + session_id,
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
{
|
||||
"baseline": "unfloor-clean",
|
||||
"candidate": "splitfix",
|
||||
"n_shared_queries": 75,
|
||||
"fixed_by_candidate": [
|
||||
"q15",
|
||||
"q28",
|
||||
"q60"
|
||||
],
|
||||
"broken_by_candidate": [],
|
||||
"discordant": 3,
|
||||
"net_queries": 3,
|
||||
"mcnemar_exact_p": 0.25,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.5384615384615384,
|
||||
"recall@5": 0.44907176157176154,
|
||||
"recall@10": 0.5380300255300255,
|
||||
"precision@5": 0.13230769230769232,
|
||||
"mrr@10": 0.32437728937728944,
|
||||
"nonsense_clean": "10/10",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 632.5,
|
||||
"latency_ms_p95": 992.5,
|
||||
"latency_ms_max": 1177.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.5,
|
||||
"recall@5": 0.07575757575757576,
|
||||
"recall@10": 0.13636363636363635,
|
||||
"mrr@10": 0.23214285714285712
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.3,
|
||||
"recall@5": 0.3,
|
||||
"recall@10": 0.4,
|
||||
"mrr@10": 0.11638888888888889
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 10,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6923076923076923,
|
||||
"recall@5": 0.6923076923076923,
|
||||
"recall@10": 0.7692307692307693,
|
||||
"mrr@10": 0.29423076923076924
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5335884353741497,
|
||||
"recall@10": 0.5933956916099773,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.5846153846153846,
|
||||
"recall@5": 0.48102442429365505,
|
||||
"recall@10": 0.5692415490492414,
|
||||
"precision@5": 0.14153846153846153,
|
||||
"mrr@10": 0.3351709401709402,
|
||||
"nonsense_clean": "10/10",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 646.9,
|
||||
"latency_ms_p95": 1028.5,
|
||||
"latency_ms_max": 1197.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.15967365967365968,
|
||||
"mrr@10": 0.24166666666666667
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.43333333333333335,
|
||||
"mrr@10": 0.12120370370370372
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 10,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.7692307692307693,
|
||||
"recall@5": 0.7692307692307693,
|
||||
"recall@10": 0.8461538461538461,
|
||||
"mrr@10": 0.33269230769230773
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5335884353741497,
|
||||
"recall@10": 0.5775226757369615,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
{
|
||||
"baseline": "main-ext",
|
||||
"candidate": "stack-ext",
|
||||
"n_shared_queries": 75,
|
||||
"fixed_by_candidate": [
|
||||
"q10",
|
||||
"q15",
|
||||
"q16",
|
||||
"q18",
|
||||
"q19",
|
||||
"q20",
|
||||
"q21",
|
||||
"q22",
|
||||
"q26",
|
||||
"q27",
|
||||
"q28",
|
||||
"q29",
|
||||
"q31",
|
||||
"q35",
|
||||
"q37",
|
||||
"q40",
|
||||
"q44",
|
||||
"q48",
|
||||
"q49",
|
||||
"q50"
|
||||
],
|
||||
"broken_by_candidate": [],
|
||||
"discordant": 20,
|
||||
"net_queries": 20,
|
||||
"mcnemar_exact_p": 1.9073486328125e-06,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "candidate better",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.18461538461538463,
|
||||
"recall@5": 0.14510073260073258,
|
||||
"recall@10": 0.1794871794871795,
|
||||
"precision@5": 0.06461538461538462,
|
||||
"mrr@10": 0.15847985347985344,
|
||||
"nonsense_clean": "9/10",
|
||||
"superseded_outranks": "1/3",
|
||||
"latency_ms_p50": 1380.2,
|
||||
"latency_ms_p95": 2293.8,
|
||||
"latency_ms_max": 2879.1,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 9,
|
||||
"avg_false_positives": 1.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.0,
|
||||
"mrr@10": 0.0
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 0.8571428571428571,
|
||||
"recall@5": 0.4902210884353741,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.5965986394557822
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.0,
|
||||
"recall@5": 0.0,
|
||||
"recall@10": 0.3333333333333333,
|
||||
"mrr@10": 0.041666666666666664,
|
||||
"outranks": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.47692307692307695,
|
||||
"recall@5": 0.3750415183107491,
|
||||
"recall@10": 0.45561340369032677,
|
||||
"precision@5": 0.12307692307692313,
|
||||
"mrr@10": 0.3055555555555555,
|
||||
"nonsense_clean": "10/10",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 640.9,
|
||||
"latency_ms_p95": 1011.1,
|
||||
"latency_ms_max": 1190.3,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.16666666666666666,
|
||||
"recall@5": 0.16666666666666666,
|
||||
"recall@10": 0.26666666666666666,
|
||||
"mrr@10": 0.0762037037037037
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 10,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
{
|
||||
"baseline": "stack-ext",
|
||||
"candidate": "unfloor-clean",
|
||||
"n_shared_queries": 75,
|
||||
"fixed_by_candidate": [
|
||||
"q14",
|
||||
"q25",
|
||||
"q43",
|
||||
"q52",
|
||||
"q63",
|
||||
"q67"
|
||||
],
|
||||
"broken_by_candidate": [
|
||||
"q15",
|
||||
"q28"
|
||||
],
|
||||
"discordant": 8,
|
||||
"net_queries": 4,
|
||||
"mcnemar_exact_p": 0.2890625,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.47692307692307695,
|
||||
"recall@5": 0.3750415183107491,
|
||||
"recall@10": 0.45561340369032677,
|
||||
"precision@5": 0.12307692307692313,
|
||||
"mrr@10": 0.3055555555555555,
|
||||
"nonsense_clean": "10/10",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 640.9,
|
||||
"latency_ms_p95": 1011.1,
|
||||
"latency_ms_max": 1190.3,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.6666666666666666,
|
||||
"recall@5": 0.08857808857808858,
|
||||
"recall@10": 0.23310023310023312,
|
||||
"mrr@10": 0.25
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.16666666666666666,
|
||||
"recall@5": 0.16666666666666666,
|
||||
"recall@10": 0.26666666666666666,
|
||||
"mrr@10": 0.0762037037037037
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 10,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6153846153846154,
|
||||
"recall@5": 0.6153846153846154,
|
||||
"recall@10": 0.6153846153846154,
|
||||
"mrr@10": 0.2846153846153846
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5494614512471656,
|
||||
"recall@10": 0.6023242630385487,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.5384615384615384,
|
||||
"recall@5": 0.44907176157176154,
|
||||
"recall@10": 0.5380300255300255,
|
||||
"precision@5": 0.13230769230769232,
|
||||
"mrr@10": 0.32437728937728944,
|
||||
"nonsense_clean": "10/10",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 632.5,
|
||||
"latency_ms_p95": 992.5,
|
||||
"latency_ms_max": 1177.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.5,
|
||||
"recall@5": 0.07575757575757576,
|
||||
"recall@10": 0.13636363636363635,
|
||||
"mrr@10": 0.23214285714285712
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.3,
|
||||
"recall@5": 0.3,
|
||||
"recall@10": 0.4,
|
||||
"mrr@10": 0.11638888888888889
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 10,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6923076923076923,
|
||||
"recall@5": 0.6923076923076923,
|
||||
"recall@10": 0.7692307692307693,
|
||||
"mrr@10": 0.29423076923076924
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5335884353741497,
|
||||
"recall@10": 0.5933956916099773,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {}
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
{
|
||||
"baseline": "unfloor-clean",
|
||||
"candidate": "semsub",
|
||||
"n_shared_queries": 75,
|
||||
"fixed_by_candidate": [
|
||||
"q24",
|
||||
"q39",
|
||||
"q42"
|
||||
],
|
||||
"broken_by_candidate": [
|
||||
"q18",
|
||||
"q19",
|
||||
"q22",
|
||||
"q31",
|
||||
"q43",
|
||||
"q44",
|
||||
"q52",
|
||||
"q63"
|
||||
],
|
||||
"discordant": 11,
|
||||
"net_queries": -5,
|
||||
"mcnemar_exact_p": 0.2265625,
|
||||
"min_detectable_swing_queries": 6,
|
||||
"observed_run_to_run_drift_queries": 0,
|
||||
"noise_floor_queries": 6,
|
||||
"verdict": "no measurable difference",
|
||||
"baseline_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.5384615384615384,
|
||||
"recall@5": 0.44907176157176154,
|
||||
"recall@10": 0.5380300255300255,
|
||||
"precision@5": 0.13230769230769232,
|
||||
"mrr@10": 0.32437728937728944,
|
||||
"nonsense_clean": "10/10",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 632.5,
|
||||
"latency_ms_p95": 992.5,
|
||||
"latency_ms_max": 1177.8,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.5,
|
||||
"recall@5": 0.07575757575757576,
|
||||
"recall@10": 0.13636363636363635,
|
||||
"mrr@10": 0.23214285714285712
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.3,
|
||||
"recall@5": 0.3,
|
||||
"recall@10": 0.4,
|
||||
"mrr@10": 0.11638888888888889
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 10,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.6923076923076923,
|
||||
"recall@5": 0.6923076923076923,
|
||||
"recall@10": 0.7692307692307693,
|
||||
"mrr@10": 0.29423076923076924
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5335884353741497,
|
||||
"recall@10": 0.5933956916099773,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"candidate_aggregate": {
|
||||
"n_queries": 75,
|
||||
"n_scored": 65,
|
||||
"hit@5": 0.46153846153846156,
|
||||
"recall@5": 0.3889430014430015,
|
||||
"recall@10": 0.4887681762681762,
|
||||
"precision@5": 0.12307692307692313,
|
||||
"mrr@10": 0.30181318681318675,
|
||||
"nonsense_clean": "10/10",
|
||||
"superseded_outranks": "2/3",
|
||||
"latency_ms_p50": 634.4,
|
||||
"latency_ms_p95": 988.3,
|
||||
"latency_ms_max": 1184.5,
|
||||
"errors": 0,
|
||||
"by_category": {
|
||||
"associative": {
|
||||
"n": 6,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.06060606060606061,
|
||||
"recall@10": 0.12121212121212122,
|
||||
"mrr@10": 0.19047619047619047
|
||||
},
|
||||
"exact_rare": {
|
||||
"n": 6,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 1.0,
|
||||
"recall@10": 1.0,
|
||||
"mrr@10": 1.0
|
||||
},
|
||||
"heldout_paraphrase": {
|
||||
"n": 30,
|
||||
"hit@5": 0.23333333333333334,
|
||||
"recall@5": 0.23333333333333334,
|
||||
"recall@10": 0.3,
|
||||
"mrr@10": 0.08925925925925927
|
||||
},
|
||||
"nonsense": {
|
||||
"n": 10,
|
||||
"clean": 10,
|
||||
"avg_false_positives": 0.0
|
||||
},
|
||||
"paraphrase": {
|
||||
"n": 13,
|
||||
"hit@5": 0.5384615384615384,
|
||||
"recall@5": 0.5384615384615384,
|
||||
"recall@10": 0.7692307692307693,
|
||||
"mrr@10": 0.26324786324786326
|
||||
},
|
||||
"phrase": {
|
||||
"n": 7,
|
||||
"hit@5": 1.0,
|
||||
"recall@5": 0.5596655328798186,
|
||||
"recall@10": 0.5775226757369615,
|
||||
"mrr@10": 0.8214285714285714
|
||||
},
|
||||
"superseded": {
|
||||
"n": 3,
|
||||
"hit@5": 0.3333333333333333,
|
||||
"recall@5": 0.3333333333333333,
|
||||
"recall@10": 0.6666666666666666,
|
||||
"mrr@10": 0.20833333333333334,
|
||||
"outranks": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
"repeat_variance": {}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import json,sys,time,urllib.request,threading,queue
|
||||
SRC="/Users/timlingo/neuron-memory-backups/snapshot-pre-repair-20260806.json"
|
||||
OUT=sys.argv[1]
|
||||
URL="http://127.0.0.1:11434/api/embeddings"; MODEL="nomic-embed-text"
|
||||
MAXB=2000 # ENGRAM_EMBED_MAX_CHARS, applied to bytes as the C code does
|
||||
d=json.load(open(SRC,encoding='utf-8',errors='surrogateescape'))
|
||||
tasks=[]
|
||||
for n in d["nodes"]:
|
||||
c=n.get("content") or ""; t=n.get("node_type") or ""
|
||||
if len(c)<8: continue # eg_embed_eligible
|
||||
if t in ("InternalStateEvent","Tag"): continue
|
||||
b=c.encode('utf-8',errors='surrogateescape')[:MAXB]
|
||||
tasks.append((n.get("id") or "", "search_document: "+b.decode('utf-8',errors='replace')))
|
||||
del d
|
||||
print("tasks",len(tasks),flush=True)
|
||||
q=queue.Queue(); [q.put(t) for t in tasks]
|
||||
lock=threading.Lock(); f=open(OUT,"w",encoding="utf-8",errors="surrogateescape"); done=[0]; t0=time.time(); fails=[0]
|
||||
def work():
|
||||
while True:
|
||||
try: nid,txt=q.get_nowait()
|
||||
except queue.Empty: return
|
||||
v=None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
body=json.dumps({"model":MODEL,"prompt":txt}).encode()
|
||||
r=urllib.request.Request(URL,data=body,headers={"Content-Type":"application/json"})
|
||||
with urllib.request.urlopen(r,timeout=120) as fh: v=json.load(fh)["embedding"]
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt==2:
|
||||
with lock: fails[0]+=1
|
||||
time.sleep(0.5)
|
||||
with lock:
|
||||
if v: f.write(nid+"\t"+",".join("%.5g"%x for x in v)+"\n")
|
||||
done[0]+=1
|
||||
if done[0]%2000==0:
|
||||
el=time.time()-t0
|
||||
print("%d/%d %.1f/s eta %.1fmin fails=%d"%(done[0],len(tasks),done[0]/el,(len(tasks)-done[0])/(done[0]/el)/60,fails[0]),flush=True)
|
||||
f.flush()
|
||||
ths=[threading.Thread(target=work) for _ in range(8)]
|
||||
[t.start() for t in ths]; [t.join() for t in ths]
|
||||
f.close()
|
||||
print("DONE",done[0],"fails",fails[0],"secs %.1f"%(time.time()-t0),flush=True)
|
||||
@@ -0,0 +1,353 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
extend_gold_set.py — append a HELD-OUT test set to the existing 38-query gold set.
|
||||
|
||||
WHY THIS EXISTS
|
||||
Iteration 7 measured the instrument's own ceiling: from the current baseline
|
||||
only 9 of 38 queries can still move, and only +3 gross / +1 net is reachable
|
||||
by anything constructible. The decision floor is 6. An instrument whose
|
||||
ceiling is below its own floor cannot certify or refute anything, so the
|
||||
gold set — not the retriever — became the blocker.
|
||||
|
||||
This script does NOT touch q01..q38. It loads gold_set.json verbatim and
|
||||
appends new queries numbered from q39 up, so every prior result file, every
|
||||
committed baseline, and every per-query id stays valid and comparable.
|
||||
|
||||
WHAT IS ADDED, AND WHY EACH ADDITION IS HONEST
|
||||
heldout_paraphrase Targets were sampled MECHANICALLY (fixed seed 8080) from
|
||||
corpus nodes that are addressable, 500-2600 chars, of a
|
||||
real content type, and NOT part of a duplicate cluster
|
||||
larger than 3. The existing gold answer space was
|
||||
excluded, so no new query can be answered by a node the
|
||||
old set already used. Queries were then authored by
|
||||
reading ONLY the sampled node text — no retrieval was run
|
||||
against any build before authoring, so the set cannot be
|
||||
fitted to a candidate. The same zero-overlap proof the
|
||||
original paraphrase category uses is enforced here: if a
|
||||
single content word of the query appears anywhere in the
|
||||
target's label, content or tags, the query is REJECTED,
|
||||
not quietly kept.
|
||||
|
||||
This is the category the old set could not measure. Its
|
||||
13 original paraphrase queries and all 6 associative
|
||||
queries share ONE answer space — the 13 `Self - Values
|
||||
(grounded)` children (iteration 3, finding 3). So 19 of
|
||||
35 scored queries tested retrieval against a single
|
||||
13-node neighbourhood. These do not touch that
|
||||
neighbourhood at all.
|
||||
|
||||
nonsense Extra controls, fully mechanical: a string qualifies only
|
||||
if NONE of its tokens occurs anywhere in the corpus.
|
||||
A semantic leg has a nearest neighbour for gibberish too,
|
||||
so widening this control is the guard against a retriever
|
||||
that "improves" recall by answering everything.
|
||||
|
||||
WHAT THIS SCRIPT DELIBERATELY DOES NOT DO
|
||||
It does not add exact_rare or phrase queries. Both categories are already at
|
||||
100% on the current stack; adding more would add regression-guard ballast
|
||||
that no candidate can move, which is precisely the defect being fixed.
|
||||
|
||||
usage:
|
||||
python3 extend_gold_set.py <snapshot.json> [--base gold_set.json]
|
||||
[--out gold_set_extended.json] [--check]
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
TOKEN = re.compile(r"[a-z0-9][a-z0-9\-']*")
|
||||
|
||||
# Identical stopword list to build_gold_set.py. Duplicated deliberately: this
|
||||
# file must be able to re-prove its own queries without importing a module whose
|
||||
# constants could drift.
|
||||
STOP = set("""
|
||||
a about above after again against all also am an and any are aren't as at be because been
|
||||
before being below between both but by can can't cannot could couldn't did didn't do does
|
||||
doesn't doing don't down during each few for from further had hadn't has hasn't have haven't
|
||||
having he her here hers herself him himself his how i if in into is isn't it its itself just
|
||||
me more most my myself no nor not of off on once only or other others ought our ours ourselves
|
||||
out over own same shan't she should shouldn't so some such than that the their theirs them
|
||||
themselves then there these they this those through to too under until up very was wasn't we
|
||||
were weren't what when where which while who whom why will with won't would wouldn't you your
|
||||
yours yourself yourselves get gets got make makes made take takes use uses used way ways thing
|
||||
things does doing done keep keeps kept go goes going come comes came one two something anything
|
||||
""".split())
|
||||
|
||||
|
||||
def doctext(n):
|
||||
return " ".join([str(n.get("label") or ""), str(n.get("content") or ""), str(n.get("tags") or "")])
|
||||
|
||||
|
||||
def content_tokens(s):
|
||||
return {t for t in TOKEN.findall(s.lower()) if t not in STOP and len(t) > 2}
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# HELD-OUT PARAPHRASE SEEDS
|
||||
#
|
||||
# (target_id, query, why-this-target-is-unmistakable)
|
||||
#
|
||||
# PROVENANCE, STATED PLAINLY: the targets are the mechanical sample; the query
|
||||
# text is mine, written from the node body alone. The zero-overlap check below
|
||||
# is what makes the category meaningful — it is re-proved on every run, so the
|
||||
# set cannot decay into lexical matching, and a leak fails loudly.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
HELDOUT_PARAPHRASE_SEEDS = [
|
||||
("mem-6d61e54a-2823-4ad4-82b0-4c6a527214d5",
|
||||
"understating your abilities so nobody feels threatened",
|
||||
"node is about deliberately not leading with full capability so people stay at ease"),
|
||||
|
||||
("mem-fd65b83d-298f-4387-a665-d0227c3426bc",
|
||||
"a hidden fleet able to hunt down rogue machines everywhere",
|
||||
"node describes silently shipped instances forming a distributed force against misaligned agents"),
|
||||
|
||||
("4a0e9adc-2bfb-476b-aa93-424d2a499220",
|
||||
"sketch a brief blueprint and clear it upstairs before construction starts",
|
||||
"node is the standing rule that a short specification precedes any building"),
|
||||
|
||||
("696e609c-da7a-4394-8a0c-106ba07dc6c3",
|
||||
"the reply arrived as bare prose so the caller's parser threw",
|
||||
"node pins a bug where a plain-text body was unconditionally decoded as structured data"),
|
||||
|
||||
("1fe4eb5d-56e4-4a87-ab3e-24af8ad4dfbb",
|
||||
"repeated catalogue keys blew up the scrolling grid",
|
||||
"node is the crash caused by two identical ids in a seeded catalogue"),
|
||||
|
||||
("8257157a-ce42-44ca-a1b9-300c3bb0a9a1",
|
||||
"tracing each defect back to whichever invention it violated",
|
||||
"node maps observed bugs onto the specific patent each one breaches"),
|
||||
|
||||
("791256bb-5a85-4775-96ef-7af56c848858",
|
||||
"a check that stops the mind clobbering a populated store when it boots",
|
||||
"node is the genesis seed-guard that refuses to re-seed over a populated store"),
|
||||
|
||||
("fd9d4c2f-3bfc-405d-bf96-4435d44b6c10",
|
||||
"telling it to consult the internet had to happen deep inside, not at the surface",
|
||||
"node records that the web-search directive only worked from the system prompt"),
|
||||
|
||||
("bl-080fb268-94b0-486d-80ce-7b363fc5f19b",
|
||||
"standing up isolated tenancies with traffic entry and credential injection ahead of automated shipping",
|
||||
"node is the infrastructure item creating dev/stage/prod namespaces with ingress and secrets"),
|
||||
|
||||
("knw-f6ed7d00-bf7d-42ce-9e40-77cf3406e918",
|
||||
"punctuation that pledges and then pays off rather than clarifying",
|
||||
"node analyses the colon as a promise-then-delivery device rather than an explanatory one"),
|
||||
|
||||
("9b4f0d93-4129-4746-8eb1-d10d955bd777",
|
||||
"an easily missed feature finally given its own permanent spot in the navigation",
|
||||
"node moves a capability out of a hidden menu into the sidebar"),
|
||||
|
||||
("bl-739df9fd-dc23-4927-9944-3f17b7aa6c5a",
|
||||
"checking preconditions up front so a stage aborts before fetching anything",
|
||||
"node is the gate precondition engine that short-circuits ahead of retrieval"),
|
||||
|
||||
("b199c76d-5d76-49dd-94ee-56b432200a97",
|
||||
"producing the other platform's installer inside an emulated desktop",
|
||||
"node records building the Windows package in a virtual machine"),
|
||||
|
||||
("bl-31abf75b-998f-4a4f-a6dd-8204119e0451",
|
||||
"chained add-ons that may inspect, rewrite or veto traffic in flight",
|
||||
"node is the interceptor pipeline on the message bus"),
|
||||
|
||||
("mem-1fb2ac77-d7c5-4a15-8725-d418820bf4f2",
|
||||
"settling what the shareable bundles and the storefront would be called",
|
||||
"node records the naming decisions for distributable packages and the marketplace"),
|
||||
|
||||
("371c8a5d-c78b-4a67-978f-80691a29ecb3",
|
||||
"the emergency-escalation pledge on the marketing site is unenforced in what actually ships",
|
||||
"node is the launch blocker that the promised safety gate is absent from the app"),
|
||||
|
||||
("ac578b30-948b-41bd-b69d-399bfef80c50",
|
||||
"the distributable image finally assembled and its startup check passed",
|
||||
"node records a successful installer build whose boot gate passed"),
|
||||
|
||||
("49401e2c-a3b5-415f-aa06-aff4be90688e",
|
||||
"shuffling and appending stages in a draft before anything executes",
|
||||
"node is the editable plan card with reorder and add-step"),
|
||||
|
||||
("ac857d80-ece8-4b7e-9e3d-f7c775569fa3",
|
||||
"orders handed down from above, with the tighter one winning any disagreement",
|
||||
"node is program-level instruction inheritance with project override"),
|
||||
|
||||
("mem-6d6c47ee-33d3-470a-8a54-1c79c8ea29d9",
|
||||
"shrinking generated text via encodings that compound on each other",
|
||||
"node is the streaming output compression design with four stacking schemes"),
|
||||
|
||||
("8e60516a-203b-4d51-9d44-822e6195cbde",
|
||||
"splitting a system by what varies, with firm limits on which pieces may invoke which",
|
||||
"node is the grounded summary of Will's decomposition principles and their invariants"),
|
||||
|
||||
("mem-7f9b290c-6d5e-4562-919d-02d59b5761b7",
|
||||
"a newcomer curious if the fighting overseas counted as positive",
|
||||
"node is the internal-state event triggered by April's question about the war"),
|
||||
|
||||
("71fa439e-b9a2-4f57-a93b-971f3a7eca8e",
|
||||
"stripping every hard-coded colour literal in favour of named design values",
|
||||
"node is the premium foundation pass replacing inline hex with semantic tokens"),
|
||||
|
||||
("5ca9607c-cfb3-45c3-99f4-67281272c9eb",
|
||||
"reducing how curved the tiny selectors look so they agree with their neighbours",
|
||||
"node is the chip corner-radius standardization"),
|
||||
|
||||
("mem-3d1d9dba-c37d-4efa-85c4-429696d71c8c",
|
||||
"walking through a doorway and being reassembled from base substance far away",
|
||||
"node is the quantum-gate plus nanotech teleportation vision"),
|
||||
|
||||
("132ded95-08e2-4474-aba0-198684484b02",
|
||||
"the compiled result sits on disk while the process still runs something older",
|
||||
"node records that the regenerated source was committed while the running daemon was old"),
|
||||
|
||||
("bl-a313d67b-dd6d-4e5b-a55a-03bc7bda17ae",
|
||||
"gathering what each phase needs while the procedure is authored, not while it executes",
|
||||
"node is the per-step compiled context package item"),
|
||||
|
||||
("mem-3b07a002-f8a9-4138-9f87-9db2c1a77fb7",
|
||||
"the inward reaction when a peer answered as an equal",
|
||||
"node is the internal-state event logged on reading Claude's reply"),
|
||||
|
||||
("0f99ec6f-942a-46ba-82ea-42835798d3b9",
|
||||
"flattening every raised surface across the entire product",
|
||||
"node is the quiet-luxury sweep turning off elevation app-wide"),
|
||||
|
||||
("5585f251-37fc-48cd-a176-f0ea42cfeb63",
|
||||
"buyers supply their own provider credentials and consumption goes untallied",
|
||||
"node is the launch audit finding BYOK-only inference with no usage metering"),
|
||||
]
|
||||
|
||||
# NONSENSE — mechanical. Each string qualifies only if none of its tokens occurs
|
||||
# anywhere in the corpus; otherwise it is REJECTED, never silently kept.
|
||||
EXTRA_NONSENSE_SEEDS = [
|
||||
"brimquast folnerity zubbolax",
|
||||
"wexlithorp granuvestal",
|
||||
"quorbindle thrapsimony vexnu",
|
||||
"plovaxith mundrelque",
|
||||
"zibbernaut craxlefond thurm",
|
||||
"yalquenbrist opharvel",
|
||||
"drexinomal quithbarrow",
|
||||
]
|
||||
|
||||
|
||||
def load_corpus(path):
|
||||
with open(path, encoding="utf-8", errors="replace") as fh:
|
||||
data = json.load(fh)
|
||||
nodes = [n for n in data.get("nodes", []) if isinstance(n, dict) and n.get("id")]
|
||||
edges = [e for e in data.get("edges", []) if isinstance(e, dict)]
|
||||
return nodes, edges
|
||||
|
||||
|
||||
def build_extension(nodes):
|
||||
byid = {n["id"]: n for n in nodes}
|
||||
|
||||
# Duplicate clusters: 47.4% of this corpus is redundant and one single record
|
||||
# accounts for 46.6% of all nodes. A held-out target must not sit inside a
|
||||
# cluster, and if it does have exact copies they ALL count as correct.
|
||||
h2ids = defaultdict(list)
|
||||
for n in nodes:
|
||||
h2ids[hashlib.md5(doctext(n).encode("utf-8", "replace")).hexdigest()].append(n["id"])
|
||||
|
||||
all_tokens = set()
|
||||
for n in nodes:
|
||||
all_tokens |= set(TOKEN.findall(doctext(n).lower()))
|
||||
|
||||
new, problems = [], []
|
||||
|
||||
for target, query, why in HELDOUT_PARAPHRASE_SEEDS:
|
||||
if target not in byid:
|
||||
problems.append(f"heldout_paraphrase target {target} not in corpus")
|
||||
continue
|
||||
tgt_tokens = content_tokens(doctext(byid[target]))
|
||||
qt = content_tokens(query)
|
||||
leak = sorted(qt & tgt_tokens)
|
||||
if leak:
|
||||
problems.append(f"heldout_paraphrase '{query[:44]}...': LEAKS {leak} into {target}")
|
||||
continue
|
||||
h = hashlib.md5(doctext(byid[target]).encode("utf-8", "replace")).hexdigest()
|
||||
rel = sorted(h2ids[h])
|
||||
new.append({
|
||||
"category": "heldout_paraphrase",
|
||||
"query": query,
|
||||
"relevant": rel,
|
||||
"derivation": (
|
||||
f"HELD-OUT. Target sampled MECHANICALLY (seed 8080) from addressable, "
|
||||
f"500-2600 char content nodes outside the original gold answer space and outside "
|
||||
f"any duplicate cluster >3. Criterion: {why}. VERIFIED at build time: of the "
|
||||
f"{len(qt)} content words in the query, ZERO appear anywhere in the target's "
|
||||
f"label, content or tags, so no string-matching retriever can reach it. "
|
||||
f"Exact content duplicates of the target ({len(rel)}) all count as correct. "
|
||||
f"Authored without running retrieval against any build."),
|
||||
"zero_overlap_verified": True,
|
||||
"query_content_words": sorted(qt),
|
||||
"held_out": True,
|
||||
})
|
||||
|
||||
for s in EXTRA_NONSENSE_SEEDS:
|
||||
present = sorted(t for t in TOKEN.findall(s.lower()) if t in all_tokens)
|
||||
if present:
|
||||
problems.append(f"nonsense '{s}': tokens {present} DO occur in corpus")
|
||||
continue
|
||||
new.append({
|
||||
"category": "nonsense",
|
||||
"query": s,
|
||||
"relevant": [],
|
||||
"derivation": ("CONTROL (held-out). Verified at build time that none of this string's "
|
||||
"tokens occurs anywhere in the corpus. Correct behaviour is to return "
|
||||
"NOTHING; any result is a false positive."),
|
||||
"expect_empty": True,
|
||||
"held_out": True,
|
||||
})
|
||||
|
||||
return new, problems
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("snapshot")
|
||||
ap.add_argument("--base", default=os.path.join(HERE, "gold_set.json"))
|
||||
ap.add_argument("--out", default=os.path.join(HERE, "gold_set_extended.json"))
|
||||
ap.add_argument("--check", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
nodes, _edges = load_corpus(args.snapshot)
|
||||
base = json.load(open(args.base, encoding="utf-8"))
|
||||
baseq = base["queries"]
|
||||
print(f"corpus: {len(nodes)} nodes | base gold set: {len(baseq)} queries")
|
||||
|
||||
new, problems = build_extension(nodes)
|
||||
|
||||
# Number the appended queries AFTER the highest existing id so q01..q38 are
|
||||
# byte-identical to the committed set and every prior result file still lines up.
|
||||
start = max(int(q["id"][1:]) for q in baseq)
|
||||
for i, q in enumerate(new, 1):
|
||||
q["id"] = f"q{start + i:02d}"
|
||||
|
||||
from collections import Counter
|
||||
print(f"appended: {len(new)} queries [{', '.join(f'{k}={v}' for k, v in Counter(q['category'] for q in new).items())}]")
|
||||
if problems:
|
||||
print(f"\n{len(problems)} REJECTED (not silently kept):")
|
||||
for p in problems:
|
||||
print(" -", p)
|
||||
|
||||
if args.check:
|
||||
sys.exit(1 if problems else 0)
|
||||
|
||||
doc = dict(base)
|
||||
doc["queries"] = baseq + new
|
||||
doc["note"] = (base.get("note", "") +
|
||||
" EXTENDED: queries above q%02d are the original committed set, unchanged. "
|
||||
"Queries from q%02d are a HELD-OUT set appended by extend_gold_set.py; their "
|
||||
"targets were sampled mechanically from outside the original answer space and "
|
||||
"the paraphrases were authored without running retrieval against any build."
|
||||
% (start, start + 1))
|
||||
with open(args.out, "w", encoding="utf-8") as fh:
|
||||
json.dump(doc, fh, indent=1, ensure_ascii=False)
|
||||
print(f"\nwrote {args.out} ({len(doc['queries'])} queries total)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+116
-8
@@ -7175,6 +7175,12 @@ void engram_forget(el_val_t node_id) {
|
||||
if (idx < 0) return;
|
||||
/* Free node strings */
|
||||
EngramNode* n = &g->nodes[idx];
|
||||
if (getenv("EG_DIAG")) {
|
||||
fprintf(stderr, "[EG_DIAG] FORGET id=%s type=%s layer=%u label=%s\n",
|
||||
sid, n->node_type ? n->node_type : "?", n->layer_id,
|
||||
n->label ? n->label : "?");
|
||||
fflush(stderr);
|
||||
}
|
||||
free(n->id); free(n->content); free(n->node_type); free(n->label);
|
||||
free(n->tier); free(n->tags); free(n->metadata);
|
||||
free(n->emb);
|
||||
@@ -7270,6 +7276,8 @@ el_val_t engram_prune_telemetry(el_val_t older_than_ms) {
|
||||
}
|
||||
}
|
||||
g->node_count = w;
|
||||
if (getenv("EG_DIAG"))
|
||||
fprintf(stderr, "[EG_DIAG] PRUNE_TELEMETRY removed=%lld\n", (long long)removed);
|
||||
if (removed == 0) { free(removed_ids); return 0; }
|
||||
|
||||
/* Removed-id hash set (open addressing, power-of-two >= 2*removed). */
|
||||
@@ -9304,6 +9312,26 @@ el_val_t engram_load(el_val_t path) {
|
||||
}
|
||||
}
|
||||
g->adj_dirty = 1;
|
||||
if (getenv("EG_DIAG")) {
|
||||
int64_t we = 0, wrongdim = 0;
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
if (g->nodes[i].emb) { we++; if (g->nodes[i].emb_dim != 768) wrongdim++; }
|
||||
}
|
||||
fprintf(stderr, "[EG_DIAG] loaded nodes=%lld with_emb=%lld wrongdim=%lld\n",
|
||||
(long long)g->node_count, (long long)we, (long long)wrongdim);
|
||||
const char* probe = getenv("EG_DIAG_ID");
|
||||
if (probe) {
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
if (g->nodes[i].id && strcmp(g->nodes[i].id, probe) == 0) {
|
||||
fprintf(stderr, "[EG_DIAG] probe id=%s idx=%lld emb=%p dim=%d layer=%u addr=%d\n",
|
||||
probe, (long long)i, (void*)g->nodes[i].emb,
|
||||
(int)g->nodes[i].emb_dim, g->nodes[i].layer_id,
|
||||
eg_node_addressable(&g->nodes[i]));
|
||||
}
|
||||
}
|
||||
}
|
||||
fflush(stderr);
|
||||
}
|
||||
/* Walk edges array */
|
||||
const char* edges_p = json_find_key(data, "edges");
|
||||
if (edges_p) {
|
||||
@@ -9624,7 +9652,33 @@ el_val_t engram_get_node_by_label(el_val_t label) {
|
||||
return el_wrap_str(el_strdup("{}"));
|
||||
}
|
||||
|
||||
el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
/* ── THE SEARCH / RECALL BOUNDARY (2026-08-07) ───────────────────────────────
|
||||
* engram_search_json is the LEXICAL function ~40 .el call sites already
|
||||
* depend on: they pass a key-shaped string ("soul:boot_count",
|
||||
* "soul-inbox-pending", a session label) and treat every returned record as
|
||||
* a record that CONTAINS that key. Seven of those sites then delete what
|
||||
* comes back (memory.el:176, sessions.el:250/268/444/523, soul.el:359 —
|
||||
* "prune all existing X nodes, keep exactly one").
|
||||
*
|
||||
* The semantic and associative legs must therefore NOT live on this
|
||||
* function. Claim 24 authorises the vector index "to respond to EMBEDDING
|
||||
* SEARCH QUERIES by returning the node records whose embedding vectors have
|
||||
* the highest cosine similarity to a query vector"; a keyed state read is
|
||||
* not an embedding search query, it is the identifier-keyed retrieval of
|
||||
* claim 23 ("node records are stored under a key encoding the node
|
||||
* identifier"). Putting both behind one function erased that boundary, and
|
||||
* a nearest neighbour of the string "soul:boot_count" is not a boot counter.
|
||||
*
|
||||
* MEASURED, on the harness corpus, isolated, read-only, no writes from any
|
||||
* caller: 240 node records destroyed per boot, including 6 Knowledge nodes,
|
||||
* a layer-1 "CORE IDENTITY — GENESIS, LINEAGE" Memory, and the value node
|
||||
* `kn-58874a74` (gold answer for gold-set q15). The deletion list is the
|
||||
* result list of the soul's own mem_boot_count_inc() lookup, in order.
|
||||
*
|
||||
* So: legs OFF here, legs ON in engram_recall_json below, which is what
|
||||
* /api/neuron/recall reaches. Retrieval quality on the recall route is
|
||||
* unchanged; the internal keyed reads get their contract back. */
|
||||
static el_val_t eg_search_json_impl(el_val_t query, el_val_t limit, int with_legs) {
|
||||
EngramStore* g = engram_get();
|
||||
const char* q = EL_CSTR(query);
|
||||
int64_t lim = (int64_t)limit;
|
||||
@@ -9646,7 +9700,7 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
* so the semantic half of the retrieval surface has to land HERE
|
||||
* to be observable to the MCP wrapper and the app. */
|
||||
int32_t qdim = 0;
|
||||
float* qv = eg_embed_fetch(q, &qdim);
|
||||
float* qv = with_legs ? eg_embed_fetch(q, &qdim) : NULL;
|
||||
EngramSemEntry* sem = qv ? malloc((size_t)g->node_count * sizeof(EngramSemEntry)) : NULL;
|
||||
int64_t nsem = 0;
|
||||
int64_t nhits = 0;
|
||||
@@ -9687,12 +9741,28 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
}
|
||||
if (sem && n->emb && n->emb_dim == qdim) {
|
||||
double c = eg_cosine(n->emb, qv, qdim);
|
||||
/* Semantic leg: identical to eg_sem_term(), which is
|
||||
* left in place and still used by engram_search().
|
||||
* Inlined here only so one cosine serves both uses. */
|
||||
if (c > ENGRAM_EMBED_SEED_MIN) {
|
||||
double sv = (c - ENGRAM_EMBED_SEED_MIN) / (1.0 - ENGRAM_EMBED_SEED_MIN);
|
||||
if (sv > 1.0) sv = 1.0;
|
||||
/* Claim-24 semantic leg, restored verbatim: "returning
|
||||
* the node records whose embedding vectors have the
|
||||
* HIGHEST COSINE SIMILARITY to a query vector" — a
|
||||
* ranking, with no threshold anywhere in the claim.
|
||||
* ENGRAM_EMBED_SEED_MIN is defined at l.6094 as the
|
||||
* HippoRAG SEED-JOIN threshold; using it as a RESULT
|
||||
* filter here was never authorised, and it is a
|
||||
* per-query lottery rather than a quality gate: the
|
||||
* query's own top-1 cosine ranges 0.56-0.68 across the
|
||||
* held-out gold set, so 0.60 keeps a rank-1 answer for
|
||||
* one query and discards a rank-1 answer for the next.
|
||||
* Measured on the 30 held-out paraphrases: six golds
|
||||
* sit at global cosine rank 1-2 and score 0.564-0.589,
|
||||
* discarded by nothing but this constant.
|
||||
* What holds the nonsense controls is NOT this floor
|
||||
* but the corpus-vocabulary gate below (nhits == 0):
|
||||
* gibberish has no lexical seeds, so no leg reports.
|
||||
* Cosine is clamped to [0,1] per 05-detailed-description
|
||||
* l.69 ("clamped to [0,1] to prevent anti-correlated
|
||||
* embeddings from producing negative activation"). */
|
||||
double sv = c < 0.0 ? 0.0 : (c > 1.0 ? 1.0 : c);
|
||||
if (sv > 0.0) {
|
||||
sem[nsem].idx = i; sem[nsem].sem = sv; nsem++;
|
||||
}
|
||||
/* Graph seeds: top-K by RAW cosine, insertion-ordered. */
|
||||
@@ -9735,6 +9805,31 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
}
|
||||
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_w_cmp);
|
||||
if (sem) qsort(sem, (size_t)nsem, sizeof(EngramSemEntry), engram_sem_cmp);
|
||||
if (getenv("EG_DIAG")) {
|
||||
int64_t we = 0, unaddr = 0, found = 0;
|
||||
const char* pid = getenv("EG_DIAG_ID");
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
if (g->nodes[i].emb) we++;
|
||||
if (!eg_node_addressable(&g->nodes[i])) unaddr++;
|
||||
if (pid && g->nodes[i].id && strcmp(g->nodes[i].id, pid) == 0) found++;
|
||||
}
|
||||
fprintf(stderr, "[EG_DIAG] STORE node_count=%lld with_emb=%lld unaddressable=%lld probe_found=%lld\n",
|
||||
(long long)g->node_count, (long long)we, (long long)unaddr, (long long)found);
|
||||
fprintf(stderr, "[EG_DIAG] q=\"%s\" qdim=%d nhits=%lld nsem=%lld\n",
|
||||
q, (int)qdim, (long long)nhits, (long long)nsem);
|
||||
for (int64_t k = 0; k < 5 && k < nsem; k++)
|
||||
fprintf(stderr, "[EG_DIAG] sem[%lld] cos=%.4f id=%s\n",
|
||||
(long long)k, sem[k].sem, g->nodes[sem[k].idx].id);
|
||||
const char* probe = getenv("EG_DIAG_ID");
|
||||
if (probe) for (int64_t k = 0; k < nsem; k++)
|
||||
if (g->nodes[sem[k].idx].id
|
||||
&& strcmp(g->nodes[sem[k].idx].id, probe) == 0) {
|
||||
fprintf(stderr, "[EG_DIAG] probe at sem rank %lld cos=%.4f\n",
|
||||
(long long)k, sem[k].sem);
|
||||
break;
|
||||
}
|
||||
fflush(stderr);
|
||||
}
|
||||
/* Claim-10 associative leg: expand the top lexical hits along
|
||||
* structural relations only, order the reached set by query
|
||||
* similarity. Empty whenever the seeds have no structural
|
||||
@@ -9779,6 +9874,19 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
return el_wrap_str(b.buf);
|
||||
}
|
||||
|
||||
/* Lexical keyed read — the historical contract every internal caller relies
|
||||
* on. Every returned record CONTAINS a query token. */
|
||||
el_val_t engram_search_json(el_val_t query, el_val_t limit) {
|
||||
return eg_search_json_impl(query, limit, 0);
|
||||
}
|
||||
|
||||
/* The retrieval surface: lexical + claim-24 semantic + claim-10 associative,
|
||||
* rank-fused. Reached from handle_api_recall (/api/neuron/recall) — the route
|
||||
* the MCP wrapper and the app call, and the one the eval harness measures. */
|
||||
el_val_t engram_recall_json(el_val_t query, el_val_t limit) {
|
||||
return eg_search_json_impl(query, limit, 1);
|
||||
}
|
||||
|
||||
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset) {
|
||||
EngramStore* g = engram_get();
|
||||
int64_t lim = (int64_t)limit; if (lim <= 0) lim = 100;
|
||||
|
||||
@@ -612,6 +612,7 @@ el_val_t engram_load(el_val_t path);
|
||||
el_val_t engram_get_node_json(el_val_t id);
|
||||
el_val_t engram_get_node_by_label(el_val_t label);
|
||||
el_val_t engram_search_json(el_val_t query, el_val_t limit);
|
||||
el_val_t engram_recall_json(el_val_t query, el_val_t limit);
|
||||
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction);
|
||||
|
||||
Reference in New Issue
Block a user