Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Tim Lingo
|
027a573d89 |
@@ -889,29 +889,10 @@ fn awareness_run() -> Void {
|
||||
state_set("soul.last_beat_ts", int_to_str(now_ts))
|
||||
// Persist in-process Engram (sessions, memories, conversation nodes)
|
||||
// to local snapshot so they survive restarts.
|
||||
// FILE MODE ONLY: "soul_snapshot_path" is set exclusively in the
|
||||
// genesis+safe_to_seed branch of soul.el, and safe_to_seed is
|
||||
// unconditionally false when ENGRAM_URL is set. In HTTP mode the
|
||||
// owner persists; the soul must not (soul.el:571-573).
|
||||
let snap_path: String = state_get("soul_snapshot_path")
|
||||
if !str_eq(snap_path, "") {
|
||||
mem_save(snap_path)
|
||||
}
|
||||
// WRITE-THROUGH RETRY (neuron#117). The HTTP-mode counterpart of the
|
||||
// save above: hand anything still spooled to the persistence owner.
|
||||
//
|
||||
// This is the retry arm of the whole design. Deltas that could not be
|
||||
// pushed — owner down, owner restarting, transient refusal — stay on
|
||||
// disk and are re-offered here every heartbeat until they land. It is
|
||||
// also the catch-all for writes made by the awareness loop itself,
|
||||
// which never passes through the HTTP handler's flush point.
|
||||
//
|
||||
// No-op with no HTTP call when the spool is empty or ENGRAM_URL is
|
||||
// unset, so an idle soul in file mode pays nothing for this.
|
||||
let wt_pushed: Int = wt_drain()
|
||||
if wt_pushed < 0 {
|
||||
ise_post("{\"event\":\"write_through_backlog\",\"ts\":" + int_to_str(now_ts) + "}")
|
||||
}
|
||||
}
|
||||
|
||||
// Curiosity scan: idle-gated AND wall-clock based. Only fires when the
|
||||
|
||||
@@ -1162,7 +1162,7 @@ fn hist_trim_with_bell_guard(hist: String) -> String {
|
||||
+ " | evicted_at:" + ts_str
|
||||
+ " | message:" + safe_content
|
||||
let preserve_tags: String = "[\"bell-history\",\"bell:" + bell_level + "\",\"evicted\",\"affective\",\"BellEvent\"]"
|
||||
let discard: String = wt_node(
|
||||
let discard: String = engram_node_full(
|
||||
preserve_content,
|
||||
"BellEvent",
|
||||
"bell:" + bell_level + ":preserved",
|
||||
@@ -1210,7 +1210,7 @@ fn conv_history_persist(session_id: String, hist: String) -> Void {
|
||||
if !str_contains(hist, "]") { return "" }
|
||||
let tags: String = "[\"conv-history\",\"persistent\"]"
|
||||
// FIX B: one label rule, shared with the agentic path. See conv_hist_label.
|
||||
let node_id: String = wt_node(
|
||||
let node_id: String = engram_node_full(
|
||||
hist, "Conversation", conv_hist_label(session_id),
|
||||
el_from_float(0.7), el_from_float(0.8), el_from_float(0.9),
|
||||
"Episodic", tags
|
||||
@@ -3461,7 +3461,7 @@ fn handle_dharma_room_turn(body: String) -> String {
|
||||
// engram_node(content, "episodic", ...) which wrongly put a TIER into the node_type
|
||||
// slot — that's why nodes showed node_type="episodic". Use the full, correct contract.)
|
||||
let utterance_tags: String = "[\"soul-utterance\",\"episodic\"]"
|
||||
let discard_id: String = wt_node(
|
||||
let discard_id: String = engram_node_full(
|
||||
clean_response, "Conversation", "soul:utterance",
|
||||
el_from_float(0.6), el_from_float(0.6), el_from_float(0.8),
|
||||
"Episodic", utterance_tags
|
||||
@@ -3552,7 +3552,7 @@ fn session_summary_write(summary_text: String) -> String {
|
||||
}
|
||||
}
|
||||
let tags: String = "[\"SessionSummary\",\"session-summary\",\"previous-session\",\"consolidate\"]"
|
||||
let node_id: String = wt_node(
|
||||
let node_id: String = engram_node_full(
|
||||
content, "SessionSummary", "session:summary",
|
||||
el_from_float(0.85), el_from_float(0.85), el_from_float(1.0),
|
||||
"Episodic", tags
|
||||
@@ -3578,7 +3578,7 @@ fn session_summary_write_dated(summary_text: String, label: String) -> String {
|
||||
let ts_str: String = int_to_str(ts)
|
||||
let content: String = "[session-summary] " + trimmed + " | ts:" + ts_str
|
||||
let tags: String = "[\"SessionSummary\",\"session-summary\",\"previous-session\",\"consolidate\"]"
|
||||
let node_id: String = wt_node(
|
||||
let node_id: String = engram_node_full(
|
||||
content, "SessionSummary", label,
|
||||
el_from_float(0.9), el_from_float(0.8), el_from_float(1.0),
|
||||
"Episodic", tags
|
||||
@@ -3654,7 +3654,7 @@ fn auto_persist(req: String, resp: String) -> Void {
|
||||
+ ",\"bell\":\"" + bell_level + "\""
|
||||
+ ",\"label\":\"chat:" + ts_str + "\"}"
|
||||
|
||||
let conv_node_id: String = wt_node(
|
||||
let conv_node_id: String = engram_node_full(
|
||||
content,
|
||||
"Conversation",
|
||||
"chat:" + ts_str,
|
||||
@@ -3692,7 +3692,7 @@ fn auto_persist(req: String, resp: String) -> Void {
|
||||
let bell_tags: String = "[\"safety\",\"bell\",\"bell:" + bell_level + "\",\"affective\",\"BellEvent\"]"
|
||||
let bell_ts_str: String = int_to_str(time_now())
|
||||
let bell_label: String = "bell:" + bell_level + ":" + bell_ts_str
|
||||
let bell_node_id: String = wt_node(
|
||||
let bell_node_id: String = engram_node_full(
|
||||
bell_content,
|
||||
"BellEvent",
|
||||
bell_label,
|
||||
@@ -3751,7 +3751,7 @@ fn auto_persist(req: String, resp: String) -> Void {
|
||||
let pos_tags: String = "[\"joy\",\"positive\",\"joy:" + positive_level + "\",\"affective\",\"PositiveEvent\"]"
|
||||
let pos_ts_label: String = int_to_str(time_now())
|
||||
let pos_label: String = "joy:" + positive_level + ":" + pos_ts_label
|
||||
let pos_node_id: String = wt_node(
|
||||
let pos_node_id: String = engram_node_full(
|
||||
pos_content, "PositiveEvent", pos_label,
|
||||
pos_sal_a, pos_sal_b, pos_sal_c, "Episodic", pos_tags
|
||||
)
|
||||
|
||||
+662
-1283
File diff suppressed because one or more lines are too long
@@ -1,11 +1,9 @@
|
||||
import "persist.el"
|
||||
|
||||
fn tier_working() -> String { return "Working" }
|
||||
fn tier_episodic() -> String { return "Episodic" }
|
||||
fn tier_canonical() -> String { return "Canonical" }
|
||||
|
||||
fn mem_store(content: String, label: String, tags: String) -> String {
|
||||
let id: String = wt_node(
|
||||
let id: String = engram_node_full(
|
||||
content,
|
||||
"Memory",
|
||||
label,
|
||||
@@ -19,23 +17,13 @@ fn mem_store(content: String, label: String, tags: String) -> String {
|
||||
println("[memory] write rejected by engram (empty id): label=" + label)
|
||||
return ""
|
||||
}
|
||||
// wt_node has already read the node back locally and returns "" if it did
|
||||
// not land, so the old duplicate read-back here is gone.
|
||||
//
|
||||
// HONESTY (neuron#117): the receipt now says WHERE the write is.
|
||||
// The old unconditional "write verified" line asserted against the soul's
|
||||
// own RAM — true in memory, false on disk — and printed ~115,000 times on
|
||||
// Tim's machine while the canonical snapshot sat frozen for three days.
|
||||
// wt_commit flushes the spool and then asks the OWNER. When it says false
|
||||
// the node is real and recallable but not yet durable, and the log says so
|
||||
// rather than claiming a save that did not happen. The id is still returned:
|
||||
// the local write DID succeed, and the queued delta will be retried.
|
||||
let durable: Bool = wt_commit(id)
|
||||
if durable {
|
||||
println("[memory] write persisted at owner: " + id + " label=" + label)
|
||||
} else {
|
||||
println("[memory] write IN MEMORY ONLY (queued for owner, not yet durable): " + id + " label=" + label)
|
||||
// Read back to verify the node actually persisted — guards against silent write failures.
|
||||
let readback: String = engram_get_node_json(id)
|
||||
if str_eq(readback, "") || str_eq(readback, "{}") {
|
||||
println("[memory] WRITE VERIFY FAILED: label=" + label + " id=" + id + " — node absent after write")
|
||||
return ""
|
||||
}
|
||||
println("[memory] write verified: " + id + " ok")
|
||||
return id
|
||||
}
|
||||
|
||||
@@ -63,12 +51,12 @@ fn mem_strengthen(node_id: String) -> Void {
|
||||
// memory.el (imported first) so awareness.el and neuron-api.el can both call it.
|
||||
fn mem_tombstone(node_id: String) -> String {
|
||||
let tags: String = "[\"Tombstone\",\"status:deleted\"]"
|
||||
let marker: String = wt_node(
|
||||
let marker: String = engram_node_full(
|
||||
node_id, "Tombstone", "tombstone:" + node_id,
|
||||
el_from_float(0.01), el_from_float(0.01), el_from_float(1.0),
|
||||
"Episodic", tags)
|
||||
if !str_eq(marker, "") {
|
||||
wt_edge(marker, node_id, el_from_float(1.0), "tombstones")
|
||||
engram_connect(marker, node_id, el_from_float(1.0), "tombstones")
|
||||
}
|
||||
return marker
|
||||
}
|
||||
|
||||
+27
-36
@@ -195,24 +195,15 @@ fn api_compact_activated(raw: String, max_items: Int, snip: Int) -> String {
|
||||
}
|
||||
|
||||
// api_persisted — read-back-after-write guard against hallucinated saves.
|
||||
//
|
||||
// WIDENED FOR neuron#117. This function is the single gate every MCP write
|
||||
// handler passes through before it reports success (10 call sites), which makes
|
||||
// it the right place to close the honesty gap rather than editing ten receipts.
|
||||
//
|
||||
// It used to read back from engram_get_node_json — the SOUL'S OWN in-process
|
||||
// graph. In HTTP-engram mode that asserts the wrong thing: the soul is not the
|
||||
// persistence owner, so a node present in its RAM and absent from the owner read
|
||||
// as "persisted" and then vanished on the next restart. The guard was doing
|
||||
// exactly what its comment promised and still certifying writes that did not
|
||||
// survive. It now flushes the write-through spool and asks the OWNER.
|
||||
//
|
||||
// In file mode (no ENGRAM_URL) the soul IS the owner and wt_commit collapses to
|
||||
// the original local read-back — unchanged behaviour, which is what keeps this
|
||||
// reversible.
|
||||
// After a write builtin returns an id, confirm the node is actually queryable
|
||||
// via engram_get_node_json(id) (returns "" or "null" when missing). Returns
|
||||
// true only when the node is genuinely persisted.
|
||||
fn api_persisted(id: String) -> Bool {
|
||||
if str_eq(id, "") { return false }
|
||||
return wt_commit(id)
|
||||
let node: String = engram_get_node_json(id)
|
||||
// engram_get_node_json returns "{}" (empty object) when node is not found — not "" or "null".
|
||||
// Check all three to guard against any runtime variation.
|
||||
return !str_eq(node, "") && !str_eq(node, "null") && !str_eq(node, "{}")
|
||||
}
|
||||
|
||||
// api_not_persisted — standard error for a write that did not read back.
|
||||
@@ -351,7 +342,7 @@ fn handle_api_remember(body: String) -> String {
|
||||
let inner: String = str_slice(base_tags, 1, str_len(base_tags) - 1)
|
||||
"[" + inner + ",\"project:" + project + "\"]"
|
||||
}
|
||||
let id: String = wt_node(content, "Memory", "memory:remembered",
|
||||
let id: String = engram_node_full(content, "Memory", "memory:remembered",
|
||||
sal, sal, el_from_float(0.9),
|
||||
"Episodic", final_tags)
|
||||
if !api_persisted(id) { return api_not_persisted(id) }
|
||||
@@ -378,7 +369,7 @@ fn handle_api_node_create(body: String) -> String {
|
||||
if str_eq(importance, "low") { 0.25 } else { 0.5 }
|
||||
}
|
||||
}
|
||||
let id: String = wt_node(content, node_type, label,
|
||||
let id: String = engram_node_full(content, node_type, label,
|
||||
sal, sal, el_from_float(0.9),
|
||||
tier, tags)
|
||||
if !api_persisted(id) { return api_not_persisted(id) }
|
||||
@@ -431,11 +422,11 @@ fn handle_api_node_update(body: String) -> String {
|
||||
}
|
||||
let body_tags: String = json_get(body, "tags")
|
||||
let tags: String = if str_eq(body_tags, "") { "[\"" + node_type + "\"]" } else { body_tags }
|
||||
let new_id: String = wt_node(content, node_type, label,
|
||||
let new_id: String = engram_node_full(content, node_type, label,
|
||||
el_from_float(0.5), el_from_float(0.5), el_from_float(0.8),
|
||||
tier, tags)
|
||||
if !api_persisted(new_id) { return api_not_persisted(new_id) }
|
||||
wt_edge(new_id, id, el_from_float(0.9), "supersedes")
|
||||
engram_connect(new_id, id, el_from_float(0.9), "supersedes")
|
||||
return "{\"id\":\"" + new_id + "\",\"supersedes\":\"" + id + "\",\"ok\":true}"
|
||||
}
|
||||
|
||||
@@ -507,7 +498,7 @@ fn handle_api_capture_knowledge(body: String) -> String {
|
||||
let full: String = if str_eq(title, "") { content } else { title + ": " + content }
|
||||
let lbl: String = str_slice(title, 0, 80)
|
||||
let tags: String = "[\"Knowledge\",\"captured\"]"
|
||||
let id: String = wt_node(full, "Knowledge", lbl,
|
||||
let id: String = engram_node_full(full, "Knowledge", lbl,
|
||||
el_from_float(0.85), el_from_float(0.8), el_from_float(0.9),
|
||||
"Episodic", tags)
|
||||
if !api_persisted(id) { return api_not_persisted(id) }
|
||||
@@ -522,12 +513,12 @@ fn handle_api_evolve_knowledge(body: String) -> String {
|
||||
if !str_eq(prior_id, "") && is_protected_node(prior_id) { return api_err_protected(prior_id) }
|
||||
let tags: String = "[\"Knowledge\",\"evolved\"]"
|
||||
// Empty label → engram_node_full derives content[:60] (LABEL FIX 2026-07-23).
|
||||
let new_id: String = wt_node(content, "Knowledge", "",
|
||||
let new_id: String = engram_node_full(content, "Knowledge", "",
|
||||
el_from_float(0.75), el_from_float(0.75), el_from_float(0.9),
|
||||
"Episodic", tags)
|
||||
if !api_persisted(new_id) { return api_not_persisted(new_id) }
|
||||
if !str_eq(prior_id, "") {
|
||||
wt_edge(new_id, prior_id, el_from_float(0.9), "supersedes")
|
||||
engram_connect(new_id, prior_id, el_from_float(0.9), "supersedes")
|
||||
}
|
||||
return "{\"id\":\"" + new_id + "\",\"supersedes\":\"" + prior_id + "\",\"ok\":true}"
|
||||
}
|
||||
@@ -544,11 +535,11 @@ fn handle_api_promote_knowledge(body: String) -> String {
|
||||
"[\"Knowledge\",\"tier:canonical\",\"disposition:stable\"]"
|
||||
} else { tags_raw }
|
||||
// Empty label → engram_node_full derives content[:60] (LABEL FIX 2026-07-23).
|
||||
let new_id: String = wt_node(content, "Knowledge", "",
|
||||
let new_id: String = engram_node_full(content, "Knowledge", "",
|
||||
el_from_float(0.9), el_from_float(0.9), el_from_float(1.0),
|
||||
"Canonical", tags)
|
||||
if !api_persisted(new_id) { return api_not_persisted(new_id) }
|
||||
wt_edge(new_id, prior_id, el_from_float(0.95), "supersedes")
|
||||
engram_connect(new_id, prior_id, el_from_float(0.95), "supersedes")
|
||||
return "{\"ok\":true,\"new_id\":\"" + new_id + "\",\"supersedes\":\"" + prior_id + "\"}"
|
||||
}
|
||||
|
||||
@@ -571,7 +562,7 @@ fn handle_api_define_process(body: String) -> String {
|
||||
if str_eq(content, "") { return api_err("content is required") }
|
||||
let label: String = if str_eq(name, "") { "process:unnamed" } else { "process:" + name }
|
||||
let tags: String = "[\"Process\"]"
|
||||
let id: String = wt_node(content, "Process", label,
|
||||
let id: String = engram_node_full(content, "Process", label,
|
||||
el_from_float(0.8), el_from_float(0.8), el_from_float(0.9),
|
||||
"Canonical", tags)
|
||||
if !api_persisted(id) { return api_not_persisted(id) }
|
||||
@@ -656,7 +647,7 @@ fn handle_api_tune_config(body: String) -> String {
|
||||
if str_eq(key, "") { return api_err("key is required") }
|
||||
let content: String = "config:" + key + "=" + value
|
||||
let tags: String = "[\"ConfigEntry\",\"config\"]"
|
||||
let id: String = wt_node(content, "ConfigEntry", key,
|
||||
let id: String = engram_node_full(content, "ConfigEntry", key,
|
||||
el_from_float(0.85), el_from_float(0.85), el_from_float(0.9),
|
||||
"Canonical", tags)
|
||||
if !api_persisted(id) { return api_not_persisted(id) }
|
||||
@@ -703,7 +694,7 @@ fn handle_api_link_entities(body: String) -> String {
|
||||
if is_protected_node(to_id) { return api_err_protected(to_id) }
|
||||
let relation: String = json_get(body, "relation")
|
||||
let eff_relation: String = if str_eq(relation, "") { "associates" } else { relation }
|
||||
wt_edge(from_id, to_id, el_from_float(0.5), eff_relation)
|
||||
engram_connect(from_id, to_id, el_from_float(0.5), eff_relation)
|
||||
return "{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + eff_relation + "\"}"
|
||||
}
|
||||
|
||||
@@ -736,11 +727,11 @@ fn handle_api_evolve_memory(body: String) -> String {
|
||||
}
|
||||
}
|
||||
let tags: String = "[\"Memory\",\"evolved\"]"
|
||||
let new_id: String = wt_node(content, "Memory", "memory:evolved",
|
||||
let new_id: String = engram_node_full(content, "Memory", "memory:evolved",
|
||||
sal, sal, el_from_float(0.9),
|
||||
"Episodic", tags)
|
||||
if !str_eq(prior_id, "") && !str_eq(new_id, "") {
|
||||
wt_edge(new_id, prior_id, el_from_float(0.9), "supersedes")
|
||||
engram_connect(new_id, prior_id, el_from_float(0.9), "supersedes")
|
||||
}
|
||||
return "{\"id\":\"" + new_id + "\",\"supersedes\":\"" + prior_id + "\",\"ok\":true}"
|
||||
}
|
||||
@@ -798,11 +789,11 @@ fn handle_api_cultivate(body: String) -> String {
|
||||
let content: String = json_get(body, "content")
|
||||
if str_eq(content, "") { return api_err("content is required") }
|
||||
let tags: String = "[\"Knowledge\",\"evolved\",\"cultivated\"]"
|
||||
let new_id: String = wt_node(content, "Knowledge", "knowledge:cultivated",
|
||||
let new_id: String = engram_node_full(content, "Knowledge", "knowledge:cultivated",
|
||||
el_from_float(0.75), el_from_float(0.75), el_from_float(0.9),
|
||||
"Episodic", tags)
|
||||
if !str_eq(prior_id, "") && !str_eq(new_id, "") {
|
||||
wt_edge(new_id, prior_id, el_from_float(0.9), "supersedes")
|
||||
engram_connect(new_id, prior_id, el_from_float(0.9), "supersedes")
|
||||
}
|
||||
return "{\"id\":\"" + new_id + "\",\"supersedes\":\"" + prior_id + "\",\"ok\":true,\"cultivated\":true}"
|
||||
}
|
||||
@@ -818,11 +809,11 @@ fn handle_api_cultivate(body: String) -> String {
|
||||
}
|
||||
}
|
||||
let tags: String = "[\"Memory\",\"evolved\",\"cultivated\"]"
|
||||
let new_id: String = wt_node(content, "Memory", "memory:cultivated",
|
||||
let new_id: String = engram_node_full(content, "Memory", "memory:cultivated",
|
||||
sal, sal, el_from_float(0.9),
|
||||
"Episodic", tags)
|
||||
if !str_eq(prior_id, "") && !str_eq(new_id, "") {
|
||||
wt_edge(new_id, prior_id, el_from_float(0.9), "supersedes")
|
||||
engram_connect(new_id, prior_id, el_from_float(0.9), "supersedes")
|
||||
}
|
||||
return "{\"id\":\"" + new_id + "\",\"supersedes\":\"" + prior_id + "\",\"ok\":true,\"cultivated\":true}"
|
||||
}
|
||||
@@ -842,7 +833,7 @@ fn handle_api_cultivate(body: String) -> String {
|
||||
if str_eq(to_id, "") { return api_err("to_id is required") }
|
||||
let relation: String = json_get(body, "relation")
|
||||
let eff_relation: String = if str_eq(relation, "") { "associates" } else { relation }
|
||||
wt_edge(from_id, to_id, el_from_float(0.5), eff_relation)
|
||||
engram_connect(from_id, to_id, el_from_float(0.5), eff_relation)
|
||||
return "{\"ok\":true,\"from_id\":\"" + from_id + "\",\"to_id\":\"" + to_id + "\",\"relation\":\"" + eff_relation + "\",\"cultivated\":true}"
|
||||
}
|
||||
|
||||
@@ -877,7 +868,7 @@ fn handle_api_consolidate(body: String) -> String {
|
||||
if !str_eq(summary, "") {
|
||||
let safe_summary: String = str_replace(summary, "\"", "'")
|
||||
let tags: String = "[\"SessionSummary\",\"consolidate\"]"
|
||||
let summary_id: String = wt_node(
|
||||
let summary_id: String = engram_node_full(
|
||||
"[session-summary] " + safe_summary,
|
||||
"SessionSummary", "session:summary",
|
||||
el_from_float(0.7), el_from_float(0.7), el_from_float(0.9),
|
||||
|
||||
-426
@@ -1,426 +0,0 @@
|
||||
// persist.el — the soul→engram WRITE-THROUGH boundary (neuron#117).
|
||||
//
|
||||
// WHY THIS FILE EXISTS
|
||||
// soul.el:571-573 states the ownership rule: "when ENGRAM_URL is set the HTTP
|
||||
// Engram owns persistence — the soul must NEVER write to the local snapshot
|
||||
// (not the persistence owner)." The soul obeys the NEGATIVE half. The POSITIVE
|
||||
// half — how a write made inside the soul actually REACHES the owner — was
|
||||
// never built. Sync is pull-only (awareness.el `/api/sync` -> engram_load_merge),
|
||||
// so every node the soul creates lives in its process RAM and is shed on
|
||||
// restart. Measured live 2026-08-07: soul node_count=102184, engram
|
||||
// node_count=79197 — ~23k nodes existing nowhere but RAM.
|
||||
//
|
||||
// SCOPE NOTE ON THE PATENT (corrects an earlier internal reading)
|
||||
// Engram provisional claims 15-18 describe a delta-sync protocol "with peer
|
||||
// Engram instances"; claim 17's pull-then-push sequence is PEER-ENGRAM to
|
||||
// PEER-ENGRAM. The soul is NOT a peer Engram — it is a CALLER of the database
|
||||
// system API (cf. claim 27, "invoked explicitly by a caller of the database
|
||||
// system API"). So claim 17 does not specify a soul↔engram contract and is not
|
||||
// cited as authority here. This design is derived from the ownership rule
|
||||
// alone: the owner owns the writes, therefore the soul must HAND writes to the
|
||||
// owner and must never write the owner's file itself.
|
||||
//
|
||||
// THE MECHANISM, AND WHY NOT `POST /api/nodes`
|
||||
// The obvious route is the one the persona/boot-counter write-backs already
|
||||
// use, POST /api/nodes. It is the wrong instrument here, verified against the
|
||||
// live engram binary in a sandbox:
|
||||
// - it mints a NEW server-side id (engram_node), so the soul's id and the
|
||||
// owner's id diverge — the next /api/sync pull re-imports the node as a
|
||||
// DUPLICATE, and any edge referencing the soul's id never resolves;
|
||||
// - it accepts only {content, node_type, salience} and drops label, tier,
|
||||
// tags, importance, confidence, metadata. A probe posted with tier
|
||||
// "Canonical" came back tier "Working", importance 0.5.
|
||||
// POST /api/load-merge (Will's own route, el `dc39a61`) is the right one:
|
||||
// - engram_load_merge PRESERVES the id and every field;
|
||||
// - it dedups nodes by id and edges by (from_id,to_id,relation), so a
|
||||
// re-submitted delta is a NO-OP — retry safety is free, and it is the same
|
||||
// local-wins semantics the graph already uses;
|
||||
// - it calls persist_canonical() — THE OWNER writes its own canonical file.
|
||||
// The soul never touches it. The ownership rule is honoured in its
|
||||
// strongest form rather than worked around;
|
||||
// - it returns real counts {ok, nodes_added, edges_added, node_count},
|
||||
// so a receipt can be a MEASUREMENT instead of a fixed success shape.
|
||||
//
|
||||
// SPOOL-AND-DRAIN, AND WHY IT IS NOT JUST A DIRECT POST
|
||||
// Measured in a sandbox against a 79k-node / 176MB graph (live scale): one
|
||||
// load-merge costs ~0.38s, essentially all of it the owner's persist_canonical.
|
||||
// A chat turn writes 5-7 nodes; pushing each separately would add ~2.7s per
|
||||
// turn. So writes are STAGED and pushed in one coalesced batch.
|
||||
// The staging buffer is the FILESYSTEM, not process state, because the soul
|
||||
// serves each HTTP connection on its own pthread (el_runtime http_serve_async)
|
||||
// and a shared in-process buffer would lose entries to a read-modify-write
|
||||
// race — silently, which is the one failure mode this file exists to end.
|
||||
// One file per write, named with uuid_v4, is race-free by construction and
|
||||
// buys a property a memory buffer cannot: writes that could not be pushed
|
||||
// SURVIVE A SOUL CRASH and are drained on the next boot.
|
||||
//
|
||||
// WHAT IS DELIBERATELY NOT PUSHED
|
||||
// - InternalStateEvent / heartbeat telemetry. Will's own carve-out, stated in
|
||||
// engram server.el 8f8ccc9: "48h-pruned, loss-tolerant, ~2/min; snapshotting
|
||||
// 28MB per heartbeat is waste."
|
||||
// NOTE (ours, flagged for Will): we do NOT additionally exclude Working-tier
|
||||
// nodes. That exclusion exists in `fb0bb55` to stop the boot counter leaking
|
||||
// through the /api/sync PULL; it is about sync backflow, not durability.
|
||||
// Applying it here would exclude mem_store — which writes tier "Working" — and
|
||||
// mem_store is the single most important durable write path in the soul. Boot
|
||||
// seeding reads the canonical file wholesale, so a pushed Working-tier node
|
||||
// does survive restart. This is the one classification call this file makes
|
||||
// that Will has not ruled on.
|
||||
//
|
||||
// WHAT THIS BOUNDARY CANNOT EXPRESS (by construction, not by omission)
|
||||
// - engram_strengthen (salience/activation drift): load-merge SKIPS ids that
|
||||
// already exist, so it cannot update an existing node. There is no owner-side
|
||||
// update/upsert route. Not pushable through any current route; left as a
|
||||
// follow-up that needs a change in the engram repo.
|
||||
// - engram_forget (hard delete): load-merge is additive and has no delete verb.
|
||||
// Propagating deletes would mean DELETE /api/nodes/<id>, a HARD delete at the
|
||||
// owner — which scripts/verify-soul-contract.sh section B explicitly fails the
|
||||
// build for ("to delete is to supersede/tombstone, never hard-remove"). Local
|
||||
// deletes therefore stay local; the TOMBSTONE NODE and its "tombstones" edge
|
||||
// (mem_tombstone) are pushed, and that is the sanctioned representation of a
|
||||
// deletion in this graph.
|
||||
|
||||
// ── Configuration ─────────────────────────────────────────────────────────────
|
||||
|
||||
// wt_engram_url — same resolution order as ise_post: env, then the state key
|
||||
// stashed at boot. NO hardcoded localhost fallback: unlike telemetry, inventing
|
||||
// a destination for durable data would risk pushing a user's memories at whatever
|
||||
// happens to be listening on 8742. Empty means "no HTTP owner" -> file mode.
|
||||
fn wt_engram_url() -> String {
|
||||
let env_url: String = env("ENGRAM_URL")
|
||||
if !str_eq(env_url, "") { return env_url }
|
||||
return state_get("soul_engram_url")
|
||||
}
|
||||
|
||||
fn wt_api_key() -> String {
|
||||
let env_key: String = env("ENGRAM_API_KEY")
|
||||
if !str_eq(env_key, "") { return env_key }
|
||||
return state_get("soul_engram_api_key")
|
||||
}
|
||||
|
||||
// wt_enabled — true only in HTTP-engram mode. In file mode the soul IS the
|
||||
// persistence owner and every path below is a no-op, so this whole feature is
|
||||
// inert for genesis/local deployments. That is also what makes it reversible.
|
||||
fn wt_enabled() -> Bool {
|
||||
return !str_eq(wt_engram_url(), "")
|
||||
}
|
||||
|
||||
// wt_spool_dir — where staged deltas live. MUST be readable by the engram
|
||||
// process: /api/load-merge takes a PATH and the owner opens it itself. Both
|
||||
// processes are same-host by construction (dev-stack LaunchAgents; the GKE
|
||||
// image starts engram and soul in one container per entrypoint.sh).
|
||||
fn wt_spool_dir() -> String {
|
||||
let raw: String = env("SOUL_OUTBOX_DIR")
|
||||
let dir: String = if str_eq(raw, "") { env("HOME") + "/.neuron/soul-outbox" } else { raw }
|
||||
fs_mkdir(dir)
|
||||
return dir
|
||||
}
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
// wt_esc — minimal JSON string escape. Deliberately local rather than reusing
|
||||
// chat.el's json_safe: persist.el is imported BY memory.el, which is imported by
|
||||
// chat.el, so depending on chat.el here would be an import cycle.
|
||||
fn wt_esc(s: String) -> String {
|
||||
let s1: String = str_replace(s, "\\", "\\\\")
|
||||
let s2: String = str_replace(s1, "\"", "\\\"")
|
||||
let s3: String = str_replace(s2, "\n", "\\n")
|
||||
let s4: String = str_replace(s3, "\r", "\\r")
|
||||
let s5: String = str_replace(s4, "\t", "\\t")
|
||||
return s5
|
||||
}
|
||||
|
||||
// wt_durable_class — Will's telemetry carve-out, by node_type. See header.
|
||||
fn wt_durable_class(node_type: String) -> Bool {
|
||||
if str_eq(node_type, "InternalStateEvent") { return false }
|
||||
return true
|
||||
}
|
||||
|
||||
// wt_inner — strip the surrounding brackets off a JSON array so several arrays
|
||||
// can be concatenated into one. Returns "" for "[]" / "" / anything too short.
|
||||
fn wt_inner(arr: String) -> String {
|
||||
let n: Int = str_len(arr)
|
||||
if n < 3 { return "" }
|
||||
if !str_starts_with(arr, "[") { return "" }
|
||||
return str_slice(arr, 1, n - 1)
|
||||
}
|
||||
|
||||
// wt_read — fs_read, plus a MANDATORY reset of the runtime's binary-length hint.
|
||||
//
|
||||
// THIS IS NOT OPTIONAL AND MUST NOT BE "SIMPLIFIED" BACK TO A BARE fs_read.
|
||||
// The pinned runtime (vendor/el-runtime/v1.0.0-20260501) keeps a thread-local
|
||||
// `_tl_fs_read_len` that fs_read SETS to the file's byte count (so binary files
|
||||
// can be served with a correct Content-Length) and that http_send_response
|
||||
// CONSUMES as the Content-Length of the next reply. Nothing else clears it
|
||||
// except json_get_raw. So any fs_read during request handling that is not
|
||||
// followed by a json_get_raw makes the NEXT HTTP response advertise the FILE's
|
||||
// length instead of the body's — and the runtime then sends that many bytes,
|
||||
// appending whatever adjacent heap memory follows the reply.
|
||||
//
|
||||
// Caught here, measured: a /api/neuron/memory reply that should be 86 bytes went
|
||||
// out as 497, with 411 bytes of this module's own spool paths and log strings
|
||||
// trailing the JSON. The drain reads spool files mid-request, so this boundary
|
||||
// is exactly where the landmine gets stepped on.
|
||||
//
|
||||
// Upstream el fixed the class in `43636ae` ("pair fs_read length hint with its
|
||||
// buffer"); that runtime is NOT the one vendored here, and re-pinning the
|
||||
// runtime is deliberately out of scope for this change. Clearing the hint at
|
||||
// our own boundary fixes our exposure without touching the pinned C.
|
||||
// json_get_raw is used as the reset because it is the only builtin in this
|
||||
// runtime that zeroes the hint, and it does so before any early return.
|
||||
fn wt_clear_binlen() -> Void {
|
||||
let discard: String = json_get_raw("{}", "_wt_reset")
|
||||
}
|
||||
|
||||
fn wt_read(path: String) -> String {
|
||||
let data: String = fs_read(path)
|
||||
wt_clear_binlen()
|
||||
return data
|
||||
}
|
||||
|
||||
// wt_sweep — best-effort removal of the zero-byte husks left by truncation.
|
||||
// The runtime exposes no unlink builtin, so a drained delta is emptied rather
|
||||
// than deleted; this reclaims the directory entries.
|
||||
//
|
||||
// `-empty` is the safety property, not an optimisation: the command is
|
||||
// STRUCTURALLY INCAPABLE of removing a delta that still has content, so it can
|
||||
// never destroy a pending write even if it runs concurrently with a stage.
|
||||
// Only the directory path is interpolated (never a filename), and it is quoted.
|
||||
// The exit code is ignored — an un-swept husk costs one directory entry.
|
||||
fn wt_sweep(dir: String) -> Void {
|
||||
if str_eq(dir, "") { return }
|
||||
if str_contains(dir, "'") { return }
|
||||
exec_command("find '" + dir + "' -maxdepth 1 -name 'wt*.json' -empty -delete 2>/dev/null")
|
||||
}
|
||||
|
||||
// ── Staging ───────────────────────────────────────────────────────────────────
|
||||
|
||||
// wt_stage — write ONE delta file. uuid_v4 in the name makes concurrent stagers
|
||||
// collision-free without any lock. Returns true if the delta is on disk.
|
||||
fn wt_stage(nodes_json: String, edges_json: String) -> Bool {
|
||||
let dir: String = wt_spool_dir()
|
||||
if str_eq(dir, "") { return false }
|
||||
let payload: String = "{\"nodes\":" + nodes_json + ",\"edges\":" + edges_json + "}"
|
||||
let path: String = dir + "/wt-" + uuid_v4() + ".json"
|
||||
fs_write(path, payload)
|
||||
// Read-back-verify the stage itself. A stage that did not land is a write we
|
||||
// would otherwise believe was queued — exactly the hallucinated-save class.
|
||||
if str_eq(wt_read(path), "") {
|
||||
println("[persist] wt_stage: FAILED to write spool file " + path + " — delta not queued")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ── The write boundary ────────────────────────────────────────────────────────
|
||||
|
||||
// wt_node — create a node locally AND queue it for the persistence owner.
|
||||
// Same signature and same return contract as engram_node_full ("" on failure),
|
||||
// so converting a call site is a rename and nothing else.
|
||||
fn wt_node(content: String, node_type: String, label: String,
|
||||
salience: Float, importance: Float, confidence: Float,
|
||||
tier: String, tags: String) -> String {
|
||||
let id: String = engram_node_full(content, node_type, label,
|
||||
salience, importance, confidence,
|
||||
tier, tags)
|
||||
if str_eq(id, "") { return "" }
|
||||
// engram_get_node_json emits the SAME record shape engram_save writes (minus
|
||||
// the embedding vector, which the owner backfills lazily), so the read-back
|
||||
// doubles as the delta payload — no second serialization to drift.
|
||||
let rec: String = engram_get_node_json(id)
|
||||
if str_eq(rec, "") || str_eq(rec, "{}") {
|
||||
println("[persist] wt_node: local write did not read back, id=" + id + " label=" + label)
|
||||
return ""
|
||||
}
|
||||
if wt_enabled() && wt_durable_class(node_type) {
|
||||
wt_stage("[" + rec + "]", "[]")
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// wt_edge — create an edge locally AND queue it. Mirrors engram_connect.
|
||||
//
|
||||
// The edge id is freshly generated rather than read back: the runtime exposes no
|
||||
// "id of the edge I just created" accessor, and the owner dedups edges by
|
||||
// (from_id,to_id,relation), never by id — so the id is not load-bearing. The
|
||||
// consequence, stated plainly: the soul's copy and the owner's copy of the same
|
||||
// edge carry different edge ids. Nothing in either codebase looks an edge up by
|
||||
// id (neighbors traversal scans from_id/to_id), so this is cosmetic.
|
||||
fn wt_edge(from_id: String, to_id: String, weight: Float, relation: String) -> Void {
|
||||
engram_connect(from_id, to_id, weight, relation)
|
||||
if !wt_enabled() { return }
|
||||
if str_eq(from_id, "") || str_eq(to_id, "") { return }
|
||||
let ts: Int = time_now()
|
||||
let rec: String = "{\"id\":\"" + uuid_v4() + "\""
|
||||
+ ",\"from_id\":\"" + wt_esc(from_id) + "\""
|
||||
+ ",\"to_id\":\"" + wt_esc(to_id) + "\""
|
||||
+ ",\"relation\":\"" + wt_esc(relation) + "\""
|
||||
+ ",\"metadata\":\"{}\""
|
||||
+ ",\"weight\":" + float_to_str(weight)
|
||||
+ ",\"confidence\":1"
|
||||
+ ",\"created_at\":" + int_to_str(ts)
|
||||
+ ",\"updated_at\":" + int_to_str(ts)
|
||||
+ ",\"last_fired\":0,\"inhibitory\":0,\"layer_id\":1}"
|
||||
wt_stage("[]", "[" + rec + "]")
|
||||
}
|
||||
|
||||
// ── The drain ─────────────────────────────────────────────────────────────────
|
||||
|
||||
// wt_drain — coalesce every staged delta into ONE load-merge against the owner.
|
||||
//
|
||||
// Returns: nodes_added on success (>= 0), 0 when there was nothing to do, and
|
||||
// -1 when the push FAILED. -1 is load-bearing: on failure the spool files are
|
||||
// left untouched, so nothing is lost and the next drain retries them. A caller
|
||||
// must never read a non-negative return as "my particular node is durable" —
|
||||
// use wt_durable(id) for that.
|
||||
//
|
||||
// Concurrency: several threads may drain at once. Each builds its own batch file
|
||||
// (uuid-named), and overlapping batches are harmless because load-merge dedups.
|
||||
// Files are truncated ONLY after a confirmed ok:true, so a lost race costs a
|
||||
// redundant push, never a dropped write.
|
||||
fn wt_drain() -> Int {
|
||||
if !wt_enabled() { return 0 }
|
||||
let dir: String = wt_spool_dir()
|
||||
if str_eq(dir, "") { return 0 }
|
||||
|
||||
// el_list_len/el_list_get, NOT json_stringify(fs_list(...)): fs_list builds
|
||||
// a native list via el_list_append, and json_stringify does not serialize
|
||||
// that type — it renders the raw pointer value. (Verified in isolation; the
|
||||
// same latent defect is live in studio.el's /api/tools/file/list route,
|
||||
// which returns e.g. {"entries":4386409744}. Noted, not fixed here.)
|
||||
let listing = fs_list(dir)
|
||||
let count: Int = el_list_len(listing)
|
||||
if count == 0 { return 0 }
|
||||
|
||||
let nodes_acc: String = ""
|
||||
let edges_acc: String = ""
|
||||
let drained: String = ""
|
||||
let found: Int = 0
|
||||
let i: Int = 0
|
||||
// No `continue` / `break`: elc lists them as keywords but not one line of
|
||||
// the shipped soul uses either, so they are unexercised on this build path.
|
||||
// Guard conditions are expressed as nested ifs instead, and every rebind is
|
||||
// at the loop-body top level where `let x = ...` is assignment (the idiom
|
||||
// memory.el's boot-counter loop relies on) — never inside a nested block,
|
||||
// where it would shadow instead.
|
||||
while i < count {
|
||||
let name: String = el_list_get(listing, i)
|
||||
// A delta is only usable when it ends with the closing "]}" that
|
||||
// wt_stage writes last. fs_write is not atomic, so a file being written
|
||||
// right now can be observed half-formed; requiring the terminator means
|
||||
// it is picked up whole on the next drain instead of merged as garbage.
|
||||
// An empty read means "already drained and truncated" — not an error.
|
||||
let p: String = if str_starts_with(name, "wt-") { dir + "/" + name } else { "" }
|
||||
let raw: String = if str_eq(p, "") { "" } else { wt_read(p) }
|
||||
let usable: Bool = !str_eq(raw, "") && str_ends_with(raw, "]}")
|
||||
let nj: String = if usable { wt_inner(json_get_raw(raw, "nodes")) } else { "" }
|
||||
let ej: String = if usable { wt_inner(json_get_raw(raw, "edges")) } else { "" }
|
||||
let nodes_acc = if str_eq(nj, "") { nodes_acc } else if str_eq(nodes_acc, "") { nj } else { nodes_acc + "," + nj }
|
||||
let edges_acc = if str_eq(ej, "") { edges_acc } else if str_eq(edges_acc, "") { ej } else { edges_acc + "," + ej }
|
||||
let drained = if !usable { drained } else if str_eq(drained, "") { p } else { drained + "\n" + p }
|
||||
let found = if usable { found + 1 } else { found }
|
||||
let i = i + 1
|
||||
}
|
||||
|
||||
if found == 0 { return 0 }
|
||||
|
||||
let combined: String = "{\"nodes\":[" + nodes_acc + "],\"edges\":[" + edges_acc + "]}"
|
||||
let batch: String = dir + "/wtb-" + uuid_v4() + ".json"
|
||||
fs_write(batch, combined)
|
||||
if str_eq(wt_read(batch), "") {
|
||||
println("[persist] wt_drain: could not write batch file " + batch + " — " + int_to_str(found) + " deltas stay queued")
|
||||
return -1
|
||||
}
|
||||
|
||||
let url: String = wt_engram_url()
|
||||
let key: String = wt_api_key()
|
||||
let body: String = "{\"path\":\"" + wt_esc(batch) + "\",\"_auth\":\"" + wt_esc(key) + "\"}"
|
||||
let resp: String = http_post_json(url + "/api/load-merge", body)
|
||||
|
||||
// The batch file is pure scratch — the retry is rebuilt from the SPOOL, not
|
||||
// from it. Truncate it unconditionally, before branching on the outcome, so
|
||||
// a persistently unreachable owner cannot accumulate one husk per attempt.
|
||||
fs_write(batch, "")
|
||||
|
||||
// Distinguish the two failures rather than collapsing them: "cannot reach
|
||||
// the owner" and "the owner refused this delta" need different human
|
||||
// responses, and a log line that says the wrong one costs a debugging hour.
|
||||
// curl surfaces transport errors as a JSON body, so an empty response is not
|
||||
// the only unreachable signal.
|
||||
// (str_contains rather than a strict parse on purpose — the engram's HTTP
|
||||
// responses have been observed carrying trailing bytes past the JSON.)
|
||||
let unreachable: Bool = str_eq(resp, "")
|
||||
|| str_contains(resp, "Couldn't connect")
|
||||
|| str_contains(resp, "Failed to connect")
|
||||
|| str_contains(resp, "Could not resolve")
|
||||
|| str_contains(resp, "timed out")
|
||||
if unreachable {
|
||||
wt_sweep(dir)
|
||||
println("[persist] wt_drain: owner UNREACHABLE at " + url + " — " + int_to_str(found)
|
||||
+ " deltas stay queued in " + dir + " (will retry): " + resp)
|
||||
return -1
|
||||
}
|
||||
if !str_contains(resp, "\"ok\":true") {
|
||||
wt_sweep(dir)
|
||||
println("[persist] wt_drain: owner REJECTED the delta — " + int_to_str(found)
|
||||
+ " stay queued in " + dir + ": " + resp)
|
||||
return -1
|
||||
}
|
||||
|
||||
let added: Int = json_get_int(resp, "nodes_added")
|
||||
let added_e: Int = json_get_int(resp, "edges_added")
|
||||
|
||||
// Confirmed. Truncate the drained spool files so they are not re-pushed.
|
||||
// Truncation (not deletion) because the runtime exposes no unlink builtin;
|
||||
// an emptied file is inert to the loop above. The zero-byte husks are then
|
||||
// swept below.
|
||||
let paths = str_split(drained, "\n")
|
||||
let pn: Int = el_list_len(paths)
|
||||
let k: Int = 0
|
||||
while k < pn {
|
||||
let one: String = el_list_get(paths, k)
|
||||
if !str_eq(one, "") { fs_write(one, "") }
|
||||
let k = k + 1
|
||||
}
|
||||
wt_sweep(dir)
|
||||
|
||||
println("[persist] wt_drain: pushed " + int_to_str(found) + " deltas -> owner added "
|
||||
+ int_to_str(added) + " nodes, " + int_to_str(added_e) + " edges")
|
||||
return added
|
||||
}
|
||||
|
||||
// wt_durable — is this id present AT THE OWNER? The only honest answer to
|
||||
// "did my write persist" in HTTP mode.
|
||||
//
|
||||
// In file mode the soul IS the owner, so the local read-back is the owner-side
|
||||
// read-back and this collapses to the pre-existing check.
|
||||
//
|
||||
// nodes_added from wt_drain is NOT a substitute: a concurrent drain may have
|
||||
// already pushed this node, making our own added count 0 while the node is
|
||||
// perfectly durable. Presence at the owner is the fact; counts are telemetry.
|
||||
fn wt_durable(id: String) -> Bool {
|
||||
if str_eq(id, "") { return false }
|
||||
if !wt_enabled() {
|
||||
let local: String = engram_get_node_json(id)
|
||||
return !str_eq(local, "") && !str_eq(local, "null") && !str_eq(local, "{}")
|
||||
}
|
||||
let url: String = wt_engram_url()
|
||||
let resp: String = http_get(url + "/api/nodes/" + id)
|
||||
if str_eq(resp, "") { return false }
|
||||
if str_eq(resp, "{}") { return false }
|
||||
return str_contains(resp, "\"id\"")
|
||||
}
|
||||
|
||||
// wt_commit — flush, then assert at the owner. The receipt callers should use.
|
||||
// Deliberately NOT a fixed success shape: it can and does return false while the
|
||||
// local write is perfectly fine in RAM, which is the true state of affairs when
|
||||
// the owner is unreachable.
|
||||
fn wt_commit(id: String) -> Bool {
|
||||
if str_eq(id, "") { return false }
|
||||
if !wt_enabled() {
|
||||
let local: String = engram_get_node_json(id)
|
||||
return !str_eq(local, "") && !str_eq(local, "null") && !str_eq(local, "{}")
|
||||
}
|
||||
let pushed: Int = wt_drain()
|
||||
return wt_durable(id)
|
||||
}
|
||||
@@ -186,7 +186,7 @@ fn route_imprint_contextual(body: String) -> String {
|
||||
return "{\"ok\":false,\"error\":\"empty body\"}"
|
||||
}
|
||||
let tags: String = "[\"imprint\",\"contextual\"]"
|
||||
let id: String = wt_node(
|
||||
let id: String = engram_node_full(
|
||||
body,
|
||||
"Entity",
|
||||
"imprint:contextual",
|
||||
@@ -208,7 +208,7 @@ fn route_imprint_user(body: String) -> String {
|
||||
return "{\"ok\":false,\"error\":\"empty body\"}"
|
||||
}
|
||||
let tags: String = "[\"imprint\",\"user\"]"
|
||||
let id: String = wt_node(
|
||||
let id: String = engram_node_full(
|
||||
body,
|
||||
"Entity",
|
||||
"imprint:user",
|
||||
@@ -239,7 +239,7 @@ fn route_synthesize(body: String) -> String {
|
||||
}
|
||||
let req: String = "synthesize " + parent_a + " " + parent_b
|
||||
let tags: String = "[\"soul-inbox-pending\",\"synthesis-request\"]"
|
||||
wt_node(
|
||||
engram_node_full(
|
||||
req,
|
||||
"Entity",
|
||||
"synthesis-request",
|
||||
@@ -395,28 +395,7 @@ fn handle_connectors(method: String, clean: String, body: String) -> String {
|
||||
return "{\"ok\":false,\"error\":\"unknown connectors route\"}"
|
||||
}
|
||||
|
||||
// handle_request — the soul's HTTP entry point.
|
||||
//
|
||||
// NOTE ON THE NAME (neuron#117): the el runtime resolves this handler by NAME
|
||||
// via dlsym(RTLD_DEFAULT, "handle_request") — that is why the Linux build must
|
||||
// link -rdynamic. So the dispatcher body moved to route_dispatch and the name
|
||||
// `handle_request` stays put as a thin wrapper. Do not rename it back.
|
||||
//
|
||||
// The wrapper exists to give the write-through boundary a guaranteed flush
|
||||
// point. route_dispatch returns from ~60 places; a per-branch flush would be
|
||||
// forgotten on the 61st. Draining here means EVERY request that staged a write
|
||||
// pushes it before the connection closes, whatever route produced it, including
|
||||
// routes added later that know nothing about persistence.
|
||||
//
|
||||
// wt_drain is a no-op (no HTTP, no cost) when nothing is staged and when the
|
||||
// soul is not in HTTP-engram mode, so this is free on read traffic.
|
||||
fn handle_request(method: String, path: String, body: String) -> String {
|
||||
let resp: String = route_dispatch(method, path, body)
|
||||
let flushed: Int = wt_drain()
|
||||
return resp
|
||||
}
|
||||
|
||||
fn route_dispatch(method: String, path: String, body: String) -> String {
|
||||
let clean: String = strip_query(path)
|
||||
|
||||
// ACTIVITY STAMP (2026-07-30 self-review): every inbound HTTP request —
|
||||
@@ -453,27 +432,10 @@ fn route_dispatch(method: String, path: String, body: String) -> String {
|
||||
return engram_scan_nodes_json(9999, 0)
|
||||
}
|
||||
if str_eq(clean, "/api/graph/edges") {
|
||||
// FIXED (neuron#117): this GET used to engram_save() straight over
|
||||
// ~/.neuron/engram/snapshot.json — a READ route, in a process that is
|
||||
// NOT the persistence owner, overwriting the owner's canonical file
|
||||
// on every call. It broke soul.el:571-573 ("the soul must NEVER write
|
||||
// to the local snapshot") and it is the same defect class Will removed
|
||||
// from the engram itself in el `dc39a61` ("stop read routes clobbering
|
||||
// canonical snapshot"), where route_scan_edges/route_sync were moved
|
||||
// to scratch paths for exactly this reason. It was also the race the
|
||||
// old TODO(reliability #8) admitted to.
|
||||
//
|
||||
// Export to a scratch path instead. Same response, no canonical write.
|
||||
// The soul's own snapshot writes are otherwise already gated behind
|
||||
// state key "soul_snapshot_path", which is set ONLY in the genesis
|
||||
// file-mode branch (soul.el: is_genesis && safe_to_seed, and
|
||||
// safe_to_seed is unconditionally false when ENGRAM_URL is set) — so
|
||||
// after this change the soul writes nothing at all in HTTP mode.
|
||||
// Future: add an engram_edges_json() builtin and drop the file round
|
||||
// trip entirely.
|
||||
let scratch_dir: String = env("TMPDIR")
|
||||
let scratch_base: String = if str_eq(scratch_dir, "") { "/tmp" } else { scratch_dir }
|
||||
let snap_path: String = scratch_base + "/soul-edges-export-" + state_get("soul_cgi_id") + ".json"
|
||||
// TODO(reliability #8): engram_save races with awareness loop mem_save().
|
||||
// Both now use atomic write-to-temp+rename (el_runtime.c). Serialised
|
||||
// by engram_global_mu. Future: add engram_edges_json() builtin.
|
||||
let snap_path: String = env("HOME") + "/.neuron/engram/snapshot.json"
|
||||
engram_save(snap_path)
|
||||
let snap: String = fs_read(snap_path)
|
||||
let edges_raw: String = json_get_raw(snap, "edges")
|
||||
|
||||
@@ -204,7 +204,7 @@ fn safety_log_bell(level: String, reason: String, input_summary: String) -> Stri
|
||||
// Emit a fallback println so the bell event leaves at least a log trace even
|
||||
// when engram is degraded. This does not replace engram persistence -- it is a
|
||||
// last-resort audit trail when the primary write cannot be confirmed.
|
||||
let node_id: String = wt_node(
|
||||
let node_id: String = engram_node_full(
|
||||
content,
|
||||
"BellEvent",
|
||||
"bell:" + level,
|
||||
|
||||
Executable
+937
@@ -0,0 +1,937 @@
|
||||
#!/usr/bin/env python3
|
||||
"""state-key-audit.py — the analyzer behind scripts/verify-state-keys.sh.
|
||||
|
||||
Read that script's header for WHY this exists (issue #129). This file is the
|
||||
HOW: a small El reader that resolves the key expression at every state_get /
|
||||
state_set site, including keys that are computed.
|
||||
|
||||
WHAT IT PARSES
|
||||
El as this engine writes it: `fn f(a: T, b: T) -> T { ... }`, `let x: T = e`,
|
||||
`return e`, `if c { a } else { b }` as an expression, `+` concatenation,
|
||||
`"..."` with backslash escapes, `//` line comments. No block comments, no
|
||||
const/match/struct exist in this dialect (verified over the whole tree).
|
||||
|
||||
KEY PATTERNS — the only two things a key expression can resolve to
|
||||
EXACT "soul_model" the whole key is known
|
||||
PREFIX "session_hist_" a known head, then runtime text
|
||||
(plus UNRESOLVED, which is a report line and never a failure)
|
||||
|
||||
RESOLUTION — resolve_expr() returns a SET of patterns; unions are how branches,
|
||||
multiple returns, and multiple bindings of one name are represented.
|
||||
literal "k" -> {EXACT k}
|
||||
concat A + B -> fold left; all-static -> EXACT,
|
||||
static head + dynamic tail -> PREFIX
|
||||
if-expression if c {A} else {B} -> resolve(A) | resolve(B), except that
|
||||
str_eq(X,"") with X statically ""
|
||||
folds to the taken branch only
|
||||
call f(args) -> union over f's return expressions,
|
||||
with f's params bound to THIS call
|
||||
site's actual argument expressions
|
||||
local var let k = e; state_get(k)-> union over every `let k =` in the
|
||||
enclosing function
|
||||
parameter fn g(k) { state_get(k) }-> union over the argument at that
|
||||
position across every call site of g
|
||||
anything else json_get(...), env(...)-> UNRESOLVED
|
||||
Recursion is depth- and cycle-guarded; a guard trip yields UNRESOLVED, never a
|
||||
failure.
|
||||
|
||||
COVERAGE — a read is satisfied when some write can produce the same key:
|
||||
read EXACT k <- write EXACT k, or write PREFIX p where k starts with p
|
||||
read PREFIX p <- write EXACT k where k starts with p, or write PREFIX q
|
||||
where p and q are prefixes of each other
|
||||
Deliberately permissive at the boundaries: a gate that cries wolf gets deleted.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
MAX_DEPTH = 12
|
||||
|
||||
# ── patterns ────────────────────────────────────────────────────────────────
|
||||
EXACT = "exact"
|
||||
PREFIX = "prefix"
|
||||
|
||||
|
||||
def pat_exact(s):
|
||||
return (EXACT, s)
|
||||
|
||||
|
||||
def pat_prefix(s):
|
||||
# A prefix with no static text at all carries no information; that is the
|
||||
# UNRESOLVED case, not a pattern.
|
||||
return (PREFIX, s) if s else None
|
||||
|
||||
|
||||
def covers(write, read):
|
||||
"""Can a write of pattern `write` produce a key that `read` reads?
|
||||
|
||||
The prefix rule is DIRECTIONAL, and that direction is the whole point. A
|
||||
write namespace that is the same or BROADER than the read namespace covers
|
||||
it (write "rl:" covers read "rl:x"). A write namespace that is NARROWER does
|
||||
NOT (write "session_histv2_" does not cover read "session_hist_") — being
|
||||
permissive there re-opens the exact hole this gate exists to close: rename
|
||||
the producer, leave the readers, stay green. Verified with a control run
|
||||
that renames sessions.el's writer and leaves its four readers behind."""
|
||||
wk, wv = write
|
||||
rk, rv = read
|
||||
if rk == EXACT:
|
||||
return rv == wv if wk == EXACT else rv.startswith(wv)
|
||||
# read is a PREFIX: some key starting with rv is read
|
||||
if wk == EXACT:
|
||||
return wv.startswith(rv) # that one written key is in range
|
||||
return rv.startswith(wv) # write namespace same-or-broader
|
||||
|
||||
|
||||
# ── lexer ───────────────────────────────────────────────────────────────────
|
||||
TOK_STR, TOK_IDENT, TOK_PUNCT, TOK_NUM = "str", "ident", "punct", "num"
|
||||
IDENT_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
||||
NUM_RE = re.compile(r"[0-9]+(\.[0-9]+)?")
|
||||
|
||||
|
||||
class Tok:
|
||||
__slots__ = ("kind", "val", "line")
|
||||
|
||||
def __init__(self, kind, val, line):
|
||||
self.kind, self.val, self.line = kind, val, line
|
||||
|
||||
def __repr__(self):
|
||||
return "%s(%r)@%d" % (self.kind, self.val, self.line)
|
||||
|
||||
|
||||
def lex(src):
|
||||
toks, i, n, line = [], 0, len(src), 1
|
||||
while i < n:
|
||||
c = src[i]
|
||||
if c == "\n":
|
||||
line += 1
|
||||
i += 1
|
||||
continue
|
||||
if c in " \t\r":
|
||||
i += 1
|
||||
continue
|
||||
if c == "/" and i + 1 < n and src[i + 1] == "/":
|
||||
while i < n and src[i] != "\n":
|
||||
i += 1
|
||||
continue
|
||||
if c == '"':
|
||||
j, buf = i + 1, []
|
||||
while j < n:
|
||||
if src[j] == "\\" and j + 1 < n:
|
||||
esc = src[j + 1]
|
||||
buf.append({"n": "\n", "t": "\t", "r": "\r"}.get(esc, esc))
|
||||
j += 2
|
||||
continue
|
||||
if src[j] == '"':
|
||||
break
|
||||
if src[j] == "\n":
|
||||
line += 1
|
||||
buf.append(src[j])
|
||||
j += 1
|
||||
toks.append(Tok(TOK_STR, "".join(buf), line))
|
||||
i = j + 1
|
||||
continue
|
||||
m = IDENT_RE.match(src, i)
|
||||
if m:
|
||||
toks.append(Tok(TOK_IDENT, m.group(0), line))
|
||||
i = m.end()
|
||||
continue
|
||||
m = NUM_RE.match(src, i)
|
||||
if m:
|
||||
toks.append(Tok(TOK_NUM, m.group(0), line))
|
||||
i = m.end()
|
||||
continue
|
||||
toks.append(Tok(TOK_PUNCT, c, line))
|
||||
i += 1
|
||||
return toks
|
||||
|
||||
|
||||
def match_close(toks, i, open_ch, close_ch):
|
||||
"""toks[i] is open_ch; return index of its matching close_ch."""
|
||||
depth = 0
|
||||
while i < len(toks):
|
||||
if toks[i].kind == TOK_PUNCT:
|
||||
if toks[i].val == open_ch:
|
||||
depth += 1
|
||||
elif toks[i].val == close_ch:
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return i
|
||||
i += 1
|
||||
return len(toks) - 1
|
||||
|
||||
|
||||
# ── program model ───────────────────────────────────────────────────────────
|
||||
class Func:
|
||||
def __init__(self, name, path, line, params, toks, start, end):
|
||||
self.name, self.path, self.line = name, path, line
|
||||
self.params = params # [param name]
|
||||
self.toks = toks # the whole file's token list
|
||||
self.start, self.end = start, end # body token range, exclusive of braces
|
||||
self.lets = None # name -> [expr token ranges], lazily built
|
||||
|
||||
|
||||
class Site:
|
||||
def __init__(self, kind, path, line, func, arg_range, text):
|
||||
self.kind = kind # "get" | "set"
|
||||
self.path, self.line = path, line
|
||||
self.func = func
|
||||
self.arg_range = arg_range
|
||||
self.text = text # source text of the key expression
|
||||
self.pats = set()
|
||||
self.unresolved = False
|
||||
self.literal = None # set when the key expression is a bare literal
|
||||
|
||||
|
||||
class Program:
|
||||
def __init__(self):
|
||||
self.files = {} # path -> toks
|
||||
self.funcs = {} # name -> [Func] (El allows no overloads, but be safe)
|
||||
self.toplevel = [] # [Func] one per file, params=[]
|
||||
self.sites = [] # [Site]
|
||||
self.calls = {} # callee name -> [(Func caller, [arg ranges])]
|
||||
|
||||
# -- loading ------------------------------------------------------------
|
||||
def load(self, path, rel):
|
||||
with open(path, "r", encoding="utf-8", errors="replace") as fh:
|
||||
src = fh.read()
|
||||
toks = lex(src)
|
||||
self.files[rel] = toks
|
||||
self._scan_funcs(rel, toks)
|
||||
|
||||
def _scan_funcs(self, rel, toks):
|
||||
covered = []
|
||||
i = 0
|
||||
while i < len(toks):
|
||||
t = toks[i]
|
||||
if t.kind == TOK_IDENT and t.val == "fn" and i + 2 < len(toks) \
|
||||
and toks[i + 1].kind == TOK_IDENT and toks[i + 2].val == "(":
|
||||
name = toks[i + 1].val
|
||||
pclose = match_close(toks, i + 2, "(", ")")
|
||||
params = self._params(toks, i + 3, pclose)
|
||||
bopen = pclose + 1
|
||||
while bopen < len(toks) and toks[bopen].val != "{":
|
||||
bopen += 1
|
||||
bclose = match_close(toks, bopen, "{", "}")
|
||||
f = Func(name, rel, t.line, params, toks, bopen + 1, bclose)
|
||||
self.funcs.setdefault(name, []).append(f)
|
||||
covered.append((i, bclose))
|
||||
i = bclose + 1
|
||||
continue
|
||||
i += 1
|
||||
# everything outside a fn is the file's top-level "function"
|
||||
tl = Func("<toplevel:%s>" % rel, rel, 1, [], toks, 0, len(toks))
|
||||
tl.covered = covered
|
||||
self.toplevel.append(tl)
|
||||
|
||||
@staticmethod
|
||||
def _params(toks, i, end):
|
||||
"""`a: T, b: T` -> ['a','b'] (top-level commas only)."""
|
||||
names, depth, expect = [], 0, True
|
||||
while i < end:
|
||||
t = toks[i]
|
||||
if t.kind == TOK_PUNCT and t.val in "([{":
|
||||
depth += 1
|
||||
elif t.kind == TOK_PUNCT and t.val in ")]}":
|
||||
depth -= 1
|
||||
elif depth == 0 and t.kind == TOK_PUNCT and t.val == ",":
|
||||
expect = True
|
||||
elif depth == 0 and expect and t.kind == TOK_IDENT:
|
||||
names.append(t.val)
|
||||
expect = False
|
||||
i += 1
|
||||
return names
|
||||
|
||||
def func_at(self, rel, tok_index):
|
||||
for f in self.funcs_in(rel):
|
||||
if f.start <= tok_index < f.end:
|
||||
return f
|
||||
for f in self.toplevel:
|
||||
if f.path == rel:
|
||||
return f
|
||||
return None
|
||||
|
||||
def funcs_in(self, rel):
|
||||
for fl in self.funcs.values():
|
||||
for f in fl:
|
||||
if f.path == rel:
|
||||
yield f
|
||||
|
||||
# -- indexing -----------------------------------------------------------
|
||||
def index(self):
|
||||
for rel, toks in self.files.items():
|
||||
i = 0
|
||||
while i < len(toks):
|
||||
t = toks[i]
|
||||
if t.kind == TOK_IDENT and i + 1 < len(toks) and toks[i + 1].val == "(" \
|
||||
and t.val not in KEYWORDS \
|
||||
and not (i > 0 and toks[i - 1].kind == TOK_IDENT
|
||||
and toks[i - 1].val == "fn"):
|
||||
# ^ the `fn f(a: T)` declaration is not a call site; counting
|
||||
# it as one makes every parameter resolve to its own name
|
||||
# and reports the whole function UNRESOLVED.
|
||||
close = match_close(toks, i + 1, "(", ")")
|
||||
args = split_args(toks, i + 2, close)
|
||||
self.calls.setdefault(t.val, []).append(
|
||||
(self.func_at(rel, i), args, rel, t.line))
|
||||
if t.val in ("state_get", "state_set") and args:
|
||||
self.sites.append(Site(
|
||||
"get" if t.val == "state_get" else "set",
|
||||
rel, t.line, self.func_at(rel, i), args[0],
|
||||
render(toks, *args[0])))
|
||||
i += 1
|
||||
|
||||
# -- resolution ---------------------------------------------------------
|
||||
def lets_of(self, f):
|
||||
if f.lets is not None:
|
||||
return f.lets
|
||||
f.lets = {}
|
||||
toks = f.toks
|
||||
skip = getattr(f, "covered", [])
|
||||
i = f.start
|
||||
while i < f.end:
|
||||
if any(a <= i <= b for a, b in skip):
|
||||
i = max(b for a, b in skip if a <= i <= b) + 1
|
||||
continue
|
||||
t = toks[i]
|
||||
if t.kind == TOK_IDENT and t.val == "let" and i + 1 < f.end \
|
||||
and toks[i + 1].kind == TOK_IDENT:
|
||||
name = toks[i + 1].val
|
||||
j = i + 2
|
||||
if j < f.end and toks[j].val == ":": # skip the type
|
||||
while j < f.end and toks[j].val != "=":
|
||||
j += 1
|
||||
if j < f.end and toks[j].val == "=":
|
||||
s = j + 1
|
||||
e = stmt_end(toks, s, f.end)
|
||||
f.lets.setdefault(name, []).append((s, e))
|
||||
i = e
|
||||
continue
|
||||
i += 1
|
||||
return f.lets
|
||||
|
||||
def returns_of(self, ctx, depth=0, seen=None):
|
||||
"""The value expressions of a function, in the context it was CALLED in.
|
||||
|
||||
Context-sensitive on purpose. `conv_hist_key` is written as a guard:
|
||||
|
||||
if str_eq(session_id, "") { return "conv_history" }
|
||||
return "session_hist_" + session_id
|
||||
|
||||
Collecting both returns flat would make state_set(conv_hist_key("")) — the
|
||||
dead handle_chat() write — claim to produce the session_hist_ namespace
|
||||
too. That is a producer this engine does not actually have, and claiming
|
||||
it would let the gate stay green if sessions.el's real writer vanished:
|
||||
a masking hole in the exact namespace #129 lives in. So a guard whose
|
||||
condition folds is honoured, and the branch not taken is dropped."""
|
||||
out = []
|
||||
self._values(ctx.toks, ctx.start, ctx.end, ctx, depth,
|
||||
seen if seen is not None else set(), out)
|
||||
return out
|
||||
|
||||
def _values(self, toks, s, e, ctx, depth, seen, out):
|
||||
"""Append the value expressions of a statement sequence.
|
||||
Returns True when the sequence definitely returns (rest unreachable)."""
|
||||
if depth > MAX_DEPTH:
|
||||
return False
|
||||
i = s
|
||||
while i < e:
|
||||
t = toks[i]
|
||||
if t.kind == TOK_IDENT and t.val == "return":
|
||||
j = stmt_end(toks, i + 1, e)
|
||||
if j > i + 1:
|
||||
out.append((i + 1, j))
|
||||
return True
|
||||
if t.kind == TOK_IDENT and t.val == "let":
|
||||
i = stmt_end(toks, i + 2, e)
|
||||
continue
|
||||
if t.kind == TOK_IDENT and t.val == "if":
|
||||
i = self._if_stmt(toks, i, e, ctx, depth, seen, out)
|
||||
if i is True:
|
||||
return True
|
||||
continue
|
||||
if t.kind == TOK_PUNCT and t.val in "([{":
|
||||
i = match_close(toks, i, t.val,
|
||||
{"(": ")", "[": "]", "{": "}"}[t.val]) + 1
|
||||
continue
|
||||
en = stmt_end(toks, i, e)
|
||||
if en <= i:
|
||||
i += 1
|
||||
continue
|
||||
if en >= e: # trailing expression = the value
|
||||
out.append((i, en))
|
||||
i = en
|
||||
return False
|
||||
|
||||
def _if_stmt(self, toks, i, e, ctx, depth, seen, out):
|
||||
"""Walk one if / else-if / else chain. Returns the next index, or True
|
||||
if the chain definitely returns on every reachable branch."""
|
||||
bopen = i + 1
|
||||
while bopen < e and toks[bopen].val != "{":
|
||||
bopen += 1
|
||||
if bopen >= e:
|
||||
return e
|
||||
bclose = match_close(toks, bopen, "{", "}")
|
||||
fold = self._fold_cond(toks, i + 1, bopen, ctx, depth, seen)
|
||||
|
||||
j = bclose + 1
|
||||
else_s = else_e = None
|
||||
if j < e and toks[j].kind == TOK_IDENT and toks[j].val == "else":
|
||||
if j + 1 < e and toks[j + 1].val == "{":
|
||||
ec = match_close(toks, j + 1, "{", "}")
|
||||
else_s, else_e = j + 2, ec
|
||||
j = ec + 1
|
||||
else: # `else if ...` — the rest of the chain
|
||||
else_s = j + 1
|
||||
else_e = stmt_end(toks, j + 1, e)
|
||||
j = else_e
|
||||
|
||||
then_ret = else_ret = False
|
||||
if fold is not False:
|
||||
then_ret = self._values(toks, bopen + 1, bclose, ctx, depth + 1, seen, out)
|
||||
if fold is not True and else_s is not None:
|
||||
else_ret = self._values(toks, else_s, else_e, ctx, depth + 1, seen, out)
|
||||
|
||||
if fold is True and then_ret:
|
||||
return True
|
||||
if fold is False and else_s is not None and else_ret:
|
||||
return True
|
||||
if fold is None and else_s is not None and then_ret and else_ret:
|
||||
return True
|
||||
return j
|
||||
|
||||
def resolve(self, rng, func, depth=0, seen=None):
|
||||
"""-> (set of patterns, unresolved_flag)"""
|
||||
if seen is None:
|
||||
seen = set()
|
||||
if depth > MAX_DEPTH:
|
||||
return set(), True
|
||||
return self._expr(func.toks, rng[0], rng[1], func, depth, seen)
|
||||
|
||||
# -- expression walker --------------------------------------------------
|
||||
def _expr(self, toks, s, e, func, depth, seen):
|
||||
parts, cur, d = [], s, 0
|
||||
i = s
|
||||
while i < e: # split on top-level '+'
|
||||
v = toks[i].val
|
||||
if toks[i].kind == TOK_PUNCT and v in "([{":
|
||||
d += 1
|
||||
elif toks[i].kind == TOK_PUNCT and v in ")]}":
|
||||
d -= 1
|
||||
elif d == 0 and toks[i].kind == TOK_PUNCT and v == "+" and i > s:
|
||||
parts.append((cur, i))
|
||||
cur = i + 1
|
||||
i += 1
|
||||
parts.append((cur, e))
|
||||
if len(parts) == 1:
|
||||
return self._primary(toks, s, e, func, depth, seen)
|
||||
|
||||
# concatenation: keep folding while every operand so far is EXACT
|
||||
head, unres = "", False
|
||||
static = True
|
||||
for (ps, pe) in parts:
|
||||
pats, u = self._primary(toks, ps, pe, func, depth, seen)
|
||||
exacts = {p[1] for p in pats if p[0] == EXACT}
|
||||
if static and len(exacts) == 1 and not u and len(pats) == 1:
|
||||
head += exacts.pop()
|
||||
continue
|
||||
if static and pats and all(p[0] == EXACT for p in pats) and len(pats) > 1:
|
||||
# a branchy static operand: keep the shared head only
|
||||
static = False
|
||||
head += os.path.commonprefix(sorted({p[1] for p in pats}))
|
||||
break
|
||||
static = False
|
||||
# first non-static operand: everything after it is runtime text
|
||||
if (ps, pe) == parts[0]:
|
||||
for p in pats:
|
||||
if p[0] == PREFIX:
|
||||
head = p[1]
|
||||
break
|
||||
if not head:
|
||||
unres = True
|
||||
break
|
||||
if static:
|
||||
return {pat_exact(head)}, False
|
||||
p = pat_prefix(head)
|
||||
return ({p} if p else set()), (unres or not p)
|
||||
|
||||
def _primary(self, toks, s, e, func, depth, seen):
|
||||
while s < e and toks[s].kind == TOK_PUNCT and toks[s].val == "(" \
|
||||
and match_close(toks, s, "(", ")") == e - 1:
|
||||
s, e = s + 1, e - 1
|
||||
if s >= e:
|
||||
return set(), True
|
||||
t = toks[s]
|
||||
|
||||
if t.kind == TOK_STR and e == s + 1:
|
||||
return {pat_exact(t.val)}, False
|
||||
|
||||
if t.kind == TOK_IDENT and t.val == "if":
|
||||
return self._if_expr(toks, s, e, func, depth, seen)
|
||||
|
||||
if t.kind == TOK_IDENT and s + 1 < e and toks[s + 1].val == "(":
|
||||
close = match_close(toks, s + 1, "(", ")")
|
||||
if close == e - 1:
|
||||
return self._call(toks, t.val, split_args(toks, s + 2, close),
|
||||
func, depth, seen)
|
||||
|
||||
if t.kind == TOK_IDENT and e == s + 1:
|
||||
return self._var(t.val, func, depth, seen)
|
||||
|
||||
return set(), True
|
||||
|
||||
def _if_expr(self, toks, s, e, func, depth, seen):
|
||||
bopen = s + 1
|
||||
while bopen < e and toks[bopen].val != "{":
|
||||
bopen += 1
|
||||
cond = (s + 1, bopen)
|
||||
bclose = match_close(toks, bopen, "{", "}")
|
||||
then_rng = block_tail(toks, bopen + 1, bclose) or (bopen + 1, bclose)
|
||||
|
||||
else_rng = None
|
||||
j = bclose + 1
|
||||
if j < e and toks[j].kind == TOK_IDENT and toks[j].val == "else":
|
||||
if j + 1 < e and toks[j + 1].val == "{":
|
||||
ec = match_close(toks, j + 1, "{", "}")
|
||||
else_rng = block_tail(toks, j + 2, ec) or (j + 2, ec)
|
||||
else:
|
||||
else_rng = (j + 1, e) # `else if ...`
|
||||
|
||||
taken = self._fold_cond(toks, cond[0], cond[1], func, depth, seen)
|
||||
rngs = []
|
||||
if taken is not False:
|
||||
rngs.append(then_rng)
|
||||
if taken is not True and else_rng:
|
||||
rngs.append(else_rng)
|
||||
|
||||
pats, unres = set(), False
|
||||
for r in rngs:
|
||||
p, u = self._expr(toks, r[0], r[1], func, depth + 1, seen)
|
||||
pats |= p
|
||||
unres = unres or u
|
||||
return pats, unres
|
||||
|
||||
def _fold_cond(self, toks, s, e, func, depth, seen):
|
||||
"""Constant-fold `str_eq(X, "")` / `!str_eq(X, "")` so a helper called with
|
||||
a literal (conv_hist_key("")) yields only the branch it really takes.
|
||||
Returns True / False / None(unknown)."""
|
||||
neg = False
|
||||
if s < e and toks[s].kind == TOK_PUNCT and toks[s].val == "!":
|
||||
neg, s = True, s + 1
|
||||
if not (s < e and toks[s].kind == TOK_IDENT and toks[s].val == "str_eq"
|
||||
and s + 1 < e and toks[s + 1].val == "("):
|
||||
return None
|
||||
close = match_close(toks, s + 1, "(", ")")
|
||||
if close != e - 1:
|
||||
return None
|
||||
args = split_args(toks, s + 2, close)
|
||||
if len(args) != 2:
|
||||
return None
|
||||
va, ua = self._expr(toks, args[0][0], args[0][1], func, depth + 1, seen)
|
||||
vb, ub = self._expr(toks, args[1][0], args[1][1], func, depth + 1, seen)
|
||||
if ua or ub or len(va) != 1 or len(vb) != 1:
|
||||
return None
|
||||
(ka, sa), (kb, sb) = va.pop(), vb.pop()
|
||||
if ka != EXACT or kb != EXACT:
|
||||
return None
|
||||
r = (sa == sb)
|
||||
return (not r) if neg else r
|
||||
|
||||
def _call(self, toks, name, args, func, depth, seen):
|
||||
cands = self.funcs.get(name)
|
||||
if not cands:
|
||||
return set(), True # builtin: json_get, env, ...
|
||||
pats, unres = set(), False
|
||||
for callee in cands:
|
||||
key = ("fn", callee.path, callee.name, tuple(args))
|
||||
if key in seen:
|
||||
unres = True
|
||||
continue
|
||||
seen = seen | {key}
|
||||
# bind the callee's params to THIS call site's argument expressions
|
||||
binding = {}
|
||||
for idx, pname in enumerate(callee.params):
|
||||
if idx < len(args):
|
||||
binding[pname] = (args[idx], func)
|
||||
callee_ctx = _Bound(callee, binding)
|
||||
for r in self.returns_of(callee_ctx, depth + 1, seen):
|
||||
p, u = self._expr(callee.toks, r[0], r[1], callee_ctx,
|
||||
depth + 1, seen)
|
||||
pats |= p
|
||||
unres = unres or u
|
||||
return pats, unres
|
||||
|
||||
def _var(self, name, func, depth, seen):
|
||||
real = func.func if isinstance(func, _Bound) else func
|
||||
|
||||
# 1. a parameter bound by the call site we came through
|
||||
if isinstance(func, _Bound) and name in func.binding:
|
||||
rng, caller_ctx = func.binding[name]
|
||||
return self._expr(caller_ctx.toks, rng[0], rng[1], caller_ctx,
|
||||
depth + 1, seen)
|
||||
|
||||
# 2. a local `let` in the enclosing function
|
||||
lets = self.lets_of(real)
|
||||
if name in lets:
|
||||
key = ("let", real.path, real.name, name)
|
||||
if key in seen:
|
||||
return set(), True
|
||||
seen = seen | {key}
|
||||
pats, unres = set(), False
|
||||
for rng in lets[name]:
|
||||
p, u = self._expr(real.toks, rng[0], rng[1], real, depth + 1, seen)
|
||||
pats |= p
|
||||
unres = unres or u
|
||||
return pats, unres
|
||||
|
||||
# 3. an unbound parameter -> look at every call site of the enclosing fn
|
||||
if name in real.params:
|
||||
key = ("param", real.path, real.name, name)
|
||||
if key in seen:
|
||||
return set(), True
|
||||
seen = seen | {key}
|
||||
idx = real.params.index(name)
|
||||
pats, unres = set(), False
|
||||
sites = self.calls.get(real.name, [])
|
||||
if not sites:
|
||||
return set(), True
|
||||
for caller, args, _rel, _line in sites:
|
||||
if caller is None or idx >= len(args):
|
||||
unres = True
|
||||
continue
|
||||
p, u = self._expr(caller.toks, args[idx][0], args[idx][1],
|
||||
caller, depth + 1, seen)
|
||||
pats |= p
|
||||
unres = unres or u
|
||||
return pats, unres
|
||||
|
||||
# 4. a file-level / cross-file top-level `let`
|
||||
for tl in self.toplevel:
|
||||
lets = self.lets_of(tl)
|
||||
if name in lets:
|
||||
key = ("let", tl.path, tl.name, name)
|
||||
if key in seen:
|
||||
return set(), True
|
||||
seen2 = seen | {key}
|
||||
pats, unres = set(), False
|
||||
for rng in lets[name]:
|
||||
p, u = self._expr(tl.toks, rng[0], rng[1], tl, depth + 1, seen2)
|
||||
pats |= p
|
||||
unres = unres or u
|
||||
return pats, unres
|
||||
|
||||
return set(), True
|
||||
|
||||
|
||||
class _Bound:
|
||||
"""A callee view that also knows what its params were called with."""
|
||||
|
||||
def __init__(self, func, binding):
|
||||
self.func, self.binding = func, binding
|
||||
self.toks, self.start, self.end = func.toks, func.start, func.end
|
||||
self.params, self.path, self.name = func.params, func.path, func.name
|
||||
|
||||
def __getattr__(self, k):
|
||||
return getattr(self.func, k)
|
||||
|
||||
|
||||
# ── token helpers ───────────────────────────────────────────────────────────
|
||||
def split_args(toks, s, e):
|
||||
out, cur, d = [], s, 0
|
||||
i = s
|
||||
while i < e:
|
||||
v = toks[i].val
|
||||
if toks[i].kind == TOK_PUNCT and v in "([{":
|
||||
d += 1
|
||||
elif toks[i].kind == TOK_PUNCT and v in ")]}":
|
||||
d -= 1
|
||||
elif d == 0 and toks[i].kind == TOK_PUNCT and v == ",":
|
||||
out.append((cur, i))
|
||||
cur = i + 1
|
||||
i += 1
|
||||
if cur < e:
|
||||
out.append((cur, e))
|
||||
return out
|
||||
|
||||
|
||||
STMT_START = {"let", "return", "if", "while", "for"}
|
||||
KEYWORDS = {"if", "while", "for", "return", "fn", "let", "else", "match"}
|
||||
|
||||
|
||||
def stmt_end(toks, s, limit):
|
||||
"""End of the expression starting at s: the next top-level statement
|
||||
boundary. El has no semicolons, so a newline that starts a new statement
|
||||
ends this one."""
|
||||
d, i = 0, s
|
||||
while i < limit:
|
||||
t = toks[i]
|
||||
if t.kind == TOK_PUNCT and t.val in "([":
|
||||
d += 1
|
||||
elif t.kind == TOK_PUNCT and t.val in ")]":
|
||||
d -= 1
|
||||
if d < 0:
|
||||
return i
|
||||
elif t.kind == TOK_PUNCT and t.val == "{":
|
||||
# a brace at depth 0 belongs to this expression only when it is an
|
||||
# if/else block that is part of it
|
||||
d += 1
|
||||
elif t.kind == TOK_PUNCT and t.val == "}":
|
||||
d -= 1
|
||||
if d < 0:
|
||||
return i
|
||||
elif d == 0 and t.kind == TOK_PUNCT and t.val == ",":
|
||||
return i
|
||||
elif d == 0 and i > s and t.kind == TOK_IDENT and t.val in STMT_START:
|
||||
if t.val == "if" and toks[i - 1].kind == TOK_IDENT and toks[i - 1].val == "else":
|
||||
i += 1
|
||||
continue
|
||||
return i
|
||||
elif d == 0 and i > s and t.kind == TOK_IDENT and t.val == "fn":
|
||||
return i
|
||||
i += 1
|
||||
return limit
|
||||
|
||||
|
||||
def block_tail(toks, s, e):
|
||||
"""The trailing expression of a block, if the block ends in one."""
|
||||
i, last = s, None
|
||||
while i < e:
|
||||
t = toks[i]
|
||||
if t.kind == TOK_IDENT and t.val in ("let", "return"):
|
||||
i = stmt_end(toks, i + 1, e)
|
||||
last = None
|
||||
continue
|
||||
if t.kind == TOK_PUNCT and t.val in "([{":
|
||||
i = match_close(toks, i, t.val, {"(": ")", "[": "]", "{": "}"}[t.val]) + 1
|
||||
continue
|
||||
st = i
|
||||
en = stmt_end(toks, i, e)
|
||||
if en <= st:
|
||||
i = st + 1
|
||||
continue
|
||||
last = (st, en)
|
||||
i = en
|
||||
return last
|
||||
|
||||
|
||||
def render(toks, s, e):
|
||||
out = []
|
||||
for t in toks[s:e]:
|
||||
out.append('"%s"' % t.val if t.kind == TOK_STR else t.val)
|
||||
return " ".join(out)
|
||||
|
||||
|
||||
# ── the gate ────────────────────────────────────────────────────────────────
|
||||
def collect(root, include_tests):
|
||||
files = []
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
dirnames[:] = [d for d in dirnames
|
||||
if d not in ("dist", "vendor", ".git", "node_modules")]
|
||||
rel_dir = os.path.relpath(dirpath, root)
|
||||
if not include_tests and rel_dir.split(os.sep)[0] == "tests":
|
||||
continue
|
||||
for fn in sorted(filenames):
|
||||
if fn.endswith(".el"):
|
||||
rel = os.path.normpath(os.path.join(rel_dir, fn))
|
||||
files.append((os.path.join(dirpath, fn), rel))
|
||||
return sorted(files, key=lambda x: x[1])
|
||||
|
||||
|
||||
def is_bare_literal(prog, site):
|
||||
toks = prog.files[site.path]
|
||||
s, e = site.arg_range
|
||||
return e == s + 1 and toks[s].kind == TOK_STR
|
||||
|
||||
|
||||
def read_decl(path):
|
||||
"""A declaration file: one entry per line, `# ...` comments stripped."""
|
||||
out = []
|
||||
if not path or not os.path.exists(path):
|
||||
return out
|
||||
with open(path) as fh:
|
||||
for ln in fh:
|
||||
ln = ln.split("#", 1)[0].strip()
|
||||
if ln:
|
||||
out.append(ln)
|
||||
return out
|
||||
|
||||
|
||||
def opt(argv, name, default=None):
|
||||
for i, a in enumerate(argv):
|
||||
if a == name and i + 1 < len(argv):
|
||||
return argv[i + 1]
|
||||
return default
|
||||
|
||||
|
||||
def main(argv):
|
||||
root = os.path.abspath(argv[1]) if len(argv) > 1 and not argv[1].startswith("-") else "."
|
||||
include_tests = "--include-tests" in argv
|
||||
verbose = "--verbose" in argv
|
||||
baseline_path = opt(argv, "--baseline")
|
||||
external_path = opt(argv, "--external")
|
||||
|
||||
prog = Program()
|
||||
for path, rel in collect(root, include_tests):
|
||||
prog.load(path, rel)
|
||||
prog.index()
|
||||
for site in prog.sites:
|
||||
pats, unres = prog.resolve(site.arg_range, site.func)
|
||||
site.pats, site.unresolved = {p for p in pats if p}, unres
|
||||
if is_bare_literal(prog, site):
|
||||
site.literal = prog.files[site.path][site.arg_range[0]].val
|
||||
|
||||
writes = [s for s in prog.sites if s.kind == "set"]
|
||||
reads = [s for s in prog.sites if s.kind == "get"]
|
||||
write_pats = set()
|
||||
for w in writes:
|
||||
write_pats |= w.pats
|
||||
|
||||
# Declared host-set keys: written by something outside the El tree (an
|
||||
# operator, the installer, a host process). Each entry must carry a reason.
|
||||
external = []
|
||||
for ln in read_decl(external_path):
|
||||
parts = ln.split(None, 1)
|
||||
if len(parts) != 2 or parts[0] not in (EXACT, PREFIX):
|
||||
print("bad --external line (want `exact|prefix <key>`): %r" % ln,
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
external.append((parts[0], parts[1]))
|
||||
write_pats |= set(external)
|
||||
|
||||
# F1 — a read of a key no write in the tree produces.
|
||||
f1 = []
|
||||
for r in reads:
|
||||
for p in sorted(r.pats):
|
||||
if not any(covers(w, p) for w in write_pats):
|
||||
f1.append((r, p))
|
||||
|
||||
# F2 — a key namespace owned by a helper, accessed by a hand-rolled literal.
|
||||
# This is the #129 shape: the producer moved behind conv_hist_key() and
|
||||
# one consumer kept spelling the old key out by hand.
|
||||
owners = {} # helper fn name -> its value set
|
||||
for s in prog.sites:
|
||||
toks = prog.files[s.path]
|
||||
a, b = s.arg_range
|
||||
if toks[a].kind == TOK_IDENT and a + 1 < b and toks[a + 1].val == "(" \
|
||||
and match_close(toks, a + 1, "(", ")") == b - 1 \
|
||||
and toks[a].val in prog.funcs:
|
||||
name = toks[a].val
|
||||
if name not in owners:
|
||||
vals = set()
|
||||
for callee in prog.funcs[name]:
|
||||
# No call context here on purpose: the OWNED namespace is
|
||||
# every key the helper can ever produce, over all call sites.
|
||||
for rng in prog.returns_of(callee):
|
||||
p, _ = prog._expr(callee.toks, rng[0], rng[1], callee, 0, set())
|
||||
vals |= {x for x in p if x}
|
||||
owners[name] = vals
|
||||
f2 = []
|
||||
for s in prog.sites:
|
||||
if s.literal is None:
|
||||
continue
|
||||
for owner, vals in sorted(owners.items()):
|
||||
for v in sorted(vals):
|
||||
if covers(v, pat_exact(s.literal)):
|
||||
f2.append((s, owner, v))
|
||||
break
|
||||
else:
|
||||
continue
|
||||
break
|
||||
|
||||
unresolved = [s for s in prog.sites if s.unresolved or not s.pats]
|
||||
|
||||
# Baseline signatures carry NO line number on purpose: an unrelated edit that
|
||||
# shifts a line must not un-mute an accepted finding (that is crying wolf),
|
||||
# but a GROWTH in count must not hide either. So a baseline entry is
|
||||
# `<file> <CODE> <detail> [xN]` and only the first N matches are muted.
|
||||
baseline, bad_baseline = {}, []
|
||||
for ln in read_decl(baseline_path):
|
||||
n, key = 1, ln
|
||||
parts = ln.rsplit(" x", 1)
|
||||
if len(parts) == 2 and parts[1].isdigit():
|
||||
key, n = parts[0].strip(), int(parts[1])
|
||||
baseline[key] = n
|
||||
|
||||
def sig(path, code, detail):
|
||||
return "%s %s %s" % (path, code, detail)
|
||||
|
||||
findings = []
|
||||
for r, p in f1:
|
||||
findings.append((sig(r.path, "DEAD-READ", "%s:%s" % p), r.line,
|
||||
" %s:%d state_get(%s)\n resolves to %s %r — no state_set in the tree produces it"
|
||||
% (r.path, r.line, r.text, p[0].upper(), p[1])))
|
||||
for s, owner, v in f2:
|
||||
findings.append((sig(s.path, "HAND-ROLLED", "%s<-%s()" % (s.literal, owner)), s.line,
|
||||
" %s:%d state_%s(\"%s\")\n %s() owns this key namespace (%s %r) — go through the helper, "
|
||||
"or a rename orphans this site silently" % (s.path, s.line, s.kind, s.literal, owner, v[0].upper(), v[1])))
|
||||
findings.sort(key=lambda f: (f[0], f[1]))
|
||||
|
||||
live, muted, budget = [], [], dict(baseline)
|
||||
for f in findings:
|
||||
if budget.get(f[0], 0) > 0:
|
||||
budget[f[0]] -= 1
|
||||
muted.append(f)
|
||||
else:
|
||||
live.append(f)
|
||||
stale = sorted(k for k, v in budget.items() if v > 0)
|
||||
|
||||
print("── state-key audit ─────────────────────────────────────────────")
|
||||
print("scanned %d .el files%s" % (len(prog.files),
|
||||
"" if include_tests else " (tests/ excluded)"))
|
||||
print("sites %d state_set, %d state_get" % (len(writes), len(reads)))
|
||||
print("keys %d distinct write patterns" % len(write_pats))
|
||||
print("")
|
||||
|
||||
if verbose:
|
||||
print("WRITE PATTERNS")
|
||||
for k, v in sorted(write_pats):
|
||||
print(" %-6s %s" % (k, v))
|
||||
print("")
|
||||
|
||||
if external:
|
||||
print("DECLARED HOST-SET (%d) — %s" % (len(external), external_path))
|
||||
for k, v in sorted(external):
|
||||
print(" %-6s %s" % (k, v))
|
||||
print("")
|
||||
|
||||
print("UNRESOLVED (%d) — reported, never fails the build" % len(unresolved))
|
||||
if not unresolved:
|
||||
print(" (none)")
|
||||
for s in sorted(unresolved, key=lambda x: (x.path, x.line)):
|
||||
print(" %s:%d state_%s(%s)%s"
|
||||
% (s.path, s.line, s.kind, s.text,
|
||||
" [partial: %s]" % ", ".join("%s %r" % p for p in sorted(s.pats))
|
||||
if s.pats else ""))
|
||||
print("")
|
||||
|
||||
if muted:
|
||||
print("BASELINED (%d) — pre-existing debt accepted in %s. NOT clean; fix these."
|
||||
% (len(muted), baseline_path))
|
||||
for sg, line, _ in muted:
|
||||
print(" %s (line %d)" % (sg, line))
|
||||
print("")
|
||||
if stale:
|
||||
print("STALE BASELINE (%d) — entries that no longer match anything; delete them:"
|
||||
% len(stale))
|
||||
for sg in stale:
|
||||
print(" %s" % sg)
|
||||
print("")
|
||||
|
||||
print("FINDINGS (%d)" % len(live))
|
||||
if not live:
|
||||
print(" (none)")
|
||||
for _, _, body in live:
|
||||
print(body)
|
||||
print("")
|
||||
|
||||
if live:
|
||||
print("FAIL: %d state-key finding(s). See scripts/verify-state-keys.sh "
|
||||
"for why this gate exists (issue #129)." % len(live))
|
||||
return 1
|
||||
print("PASS: every resolvable state_get key has a producer, and no key "
|
||||
"namespace is spelled two ways.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv))
|
||||
@@ -0,0 +1,28 @@
|
||||
# state-key-baseline.txt — findings that already existed when this gate landed
|
||||
# (2026-08-07). Each one is a REAL defect of the #129 class, not a false
|
||||
# positive. They are muted only so the gate can be turned on today instead of
|
||||
# being deferred until the debt is paid; every run still prints them under
|
||||
# BASELINED with the word "debt".
|
||||
#
|
||||
# THIS FILE SHOULD ONLY EVER SHRINK. Adding a line means you are shipping a
|
||||
# known silent-"" read. If you must, date it and say why in the comment.
|
||||
#
|
||||
# format: <file> <CODE> <detail> [xN] # N = how many sites are accepted
|
||||
# No line numbers on purpose: an unrelated edit must not un-mute an accepted
|
||||
# finding, but a GROWTH in count is NOT muted — the extra site fails the build.
|
||||
#
|
||||
chat.el DEAD-READ exact:soul_identity x5
|
||||
# ^ soul.el used to run `state_set("soul_identity", soul_identity)`. It was
|
||||
# deleted on 2026-05-13 in b163fa6 ("feat(awareness): route ISE writes to HTTP
|
||||
# Engram ..."), a commit about something else entirely, and the five readers in
|
||||
# chat.el were left behind. Since that date build_system_prompt (737), the
|
||||
# vision handler (1745), the agentic system prompt (2620), the council
|
||||
# transcript handler (3425) and 3480 have all been prefixing "" — exactly the
|
||||
# #129 shape, found by this gate on its first run. Sites: 737, 1745, 2620,
|
||||
# 3425, 3480. Fix = restore the boot-time write or delete the reads; not done
|
||||
# here because this branch must not change engine behaviour.
|
||||
|
||||
studio.el DEAD-READ exact:soul_principal x1
|
||||
# ^ studio.el:57 dharma_registry() emits "principal":"" on every call — no
|
||||
# producer has ever existed in the tree's history (git log -S finds none).
|
||||
# Never-wired rather than orphaned, same silent-"" result.
|
||||
@@ -0,0 +1,16 @@
|
||||
# state-key-external.txt — state keys the engine READS but deliberately never
|
||||
# WRITES, because a host outside the El tree sets them (an operator, the
|
||||
# installer, a deployment env). Read scripts/verify-state-keys.sh for why this
|
||||
# list has to exist and why it has to stay short.
|
||||
#
|
||||
# THE RULE FOR ADDING A LINE: the read site must already treat "" as a defined
|
||||
# default (`if str_eq(x, "") { <default> }`) AND the source must say so in a
|
||||
# comment. "I could not find the writer" is NOT a reason — that is the #129
|
||||
# defect, and it belongs in state-key-baseline.txt with a date, not here.
|
||||
#
|
||||
# format: exact|prefix <key> # why, and where the source says so
|
||||
#
|
||||
exact soul_rate_limit # routes.el:59-61 — "configurable via soul state key ... Falls back to 60 req/min if not set."
|
||||
exact web_search_tool_version # chat.el:1884-1910 — version lives in state "so a future bump is a config write, not a recompile"; defaults to web_search_20250305
|
||||
exact platform_auth # stewardship.el:92 — host-set capability flag; fail-CLOSED (anything but "true" denies the platform tool)
|
||||
exact security_research_authorized # awareness.el:991-996 — state override for env SECURITY_RESEARCH_TOKEN; fail-closed, defaults false
|
||||
Executable
+118
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env bash
|
||||
# verify-state-keys.sh — the state-key gate. Retires a defect class at build time.
|
||||
#
|
||||
# ── WHY THIS EXISTS. DO NOT DELETE IT AS NOISE. ──────────────────────────────
|
||||
#
|
||||
# The engine keeps runtime values in a key-value store: state_set("k", v) writes,
|
||||
# state_get("k") reads. A read of a key that NOTHING writes returns an empty
|
||||
# string. Silently. No error, no warning, no log line. The El compiler cannot see
|
||||
# it, no test sees it, and the product keeps running — just with a hole in it.
|
||||
#
|
||||
# That is how issue #129 happened. ff421d3 (2026-08-05) correctly moved
|
||||
# conversation history to a per-session key behind conv_hist_key(session_id). One
|
||||
# consumer did not move with it: the agentic path's L1 safety screen kept reading
|
||||
# the old anonymous "conv_history" bucket. The desktop app always mints a session
|
||||
# id, so history was always written under session_hist_<id> and that read always
|
||||
# returned "". The half of the crisis score that receives history is the
|
||||
# ESCALATION half — the one that exists for distress building across several
|
||||
# turns, where no single message trips the bell on its own. It scored 0 on every
|
||||
# real conversation for two days, and nothing failed.
|
||||
#
|
||||
# The line that broke carried a comment describing this exact bug being fixed
|
||||
# once already, under issue #9. A comment is not a gate. This is the gate.
|
||||
#
|
||||
# ── WHAT IT CHECKS ──────────────────────────────────────────────────────────
|
||||
#
|
||||
# DEAD-READ a state_get whose key resolves to something no state_set in the
|
||||
# tree produces. The direct form of the class.
|
||||
#
|
||||
# HAND-ROLLED a state_get/state_set that spells out a literal belonging to a
|
||||
# key namespace a helper function owns (e.g. "conv_history", owned
|
||||
# by conv_hist_key()). This is #129's actual shape: the producer
|
||||
# moved behind the helper and one consumer kept the old spelling
|
||||
# by hand. DEAD-READ alone does NOT catch #129, because the dead
|
||||
# handle_chat() still writes that key through the helper — so this
|
||||
# second check is the one that earns the gate its keep.
|
||||
#
|
||||
# ── WHY IT DOES NOT CRY WOLF ────────────────────────────────────────────────
|
||||
#
|
||||
# Keys are usually COMPUTED, not literal, so a naive grep would flood and get
|
||||
# switched off within a day. scripts/state-key-audit.py resolves computed keys:
|
||||
# string concatenation (matched on the static prefix), helper functions (resolved
|
||||
# to their possible return values), keys built into a local variable, and keys
|
||||
# arriving as a function parameter (resolved through the call sites). Where a key
|
||||
# genuinely cannot be resolved it is printed under UNRESOLVED and does NOT fail
|
||||
# the build — visible, never silently ignored. Keep that list short.
|
||||
#
|
||||
# On this tree it resolves 278 of 278 sites: UNRESOLVED is 0 and FINDINGS is 0.
|
||||
#
|
||||
# Two declaration files, both of which should only ever shrink:
|
||||
# scripts/state-key-external.txt keys a host outside the El tree writes
|
||||
# scripts/state-key-baseline.txt findings that predate the gate (real debt)
|
||||
#
|
||||
# ── PROVEN TO DISCRIMINATE (2026-08-07) ─────────────────────────────────────
|
||||
#
|
||||
# 1. Synthetic: a scratch copy of this tree with agentic_safety_screen reverted
|
||||
# to the pre-fix state_get("conv_history") — ONE line, nothing else — FAILS
|
||||
# with `chat.el:2536 ... conv_hist_key() owns this key namespace`. The tree
|
||||
# as shipped PASSES. One variable, opposite verdicts.
|
||||
# 2. Independent: run read-only against origin/feat/soul-openai-tools-v2, which
|
||||
# carries the same defect on its own, the gate reported chat.el:2937 — the
|
||||
# exact line 43d0449's commit message had named by hand. Against that
|
||||
# branch's fix (origin/fix/129-on-openai-tools) it passes.
|
||||
# 3. Producer-moved controls: renaming the sole writer of an EXACT key
|
||||
# (soul_model) orphans 3 readers across 3 files; renaming the sole writer of
|
||||
# a PREFIX namespace (agent_workspace_root_*) orphans 3 readers — including
|
||||
# when the producer moves to a NARROWER namespace, which an earlier,
|
||||
# sloppier prefix rule let through.
|
||||
#
|
||||
# It also found, on its first run, a defect nobody was looking for: soul.el's
|
||||
# `state_set("soul_identity", ...)` was deleted on 2026-05-13 in b163fa6 (a
|
||||
# commit about awareness/ISE writes) and five readers in chat.el were left
|
||||
# behind — the system prompt, the vision handler, the agentic prompt and the
|
||||
# council handler have been prefixing "" ever since. See state-key-baseline.txt.
|
||||
#
|
||||
# ── SAFETY ──────────────────────────────────────────────────────────────────
|
||||
# Pure static read of .el sources. Starts nothing, opens no port, touches no
|
||||
# daemon, and never reads or writes ~/.neuron.
|
||||
#
|
||||
# ── USAGE ───────────────────────────────────────────────────────────────────
|
||||
# scripts/verify-state-keys.sh gate the repo (honours baseline)
|
||||
# scripts/verify-state-keys.sh --strict ignore the baseline: show the debt
|
||||
# scripts/verify-state-keys.sh --verbose also dump every write pattern
|
||||
# scripts/verify-state-keys.sh --root DIR audit a different tree
|
||||
# exit 0 = clean; 1 = finding(s); 2 = the gate itself could not run.
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
STRICT=0
|
||||
PASS_THROUGH=()
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--strict) STRICT=1; shift ;;
|
||||
--root) ROOT="${2:?--root needs a directory}"; shift 2 ;;
|
||||
-h|--help) awk 'NR>1 && /^#/ {print; next} NR>1 {exit}' "${BASH_SOURCE[0]}"; exit 0 ;;
|
||||
*) PASS_THROUGH+=("$1"); shift ;;
|
||||
esac
|
||||
done
|
||||
|
||||
command -v python3 >/dev/null 2>&1 || {
|
||||
echo "[state-keys] CANNOT RUN: python3 not found" >&2; exit 2; }
|
||||
[ -d "$ROOT" ] || { echo "[state-keys] CANNOT RUN: no such tree: $ROOT" >&2; exit 2; }
|
||||
|
||||
AUDIT="$SCRIPT_DIR/state-key-audit.py"
|
||||
[ -f "$AUDIT" ] || { echo "[state-keys] CANNOT RUN: missing $AUDIT" >&2; exit 2; }
|
||||
|
||||
ARGS=("$ROOT" "--external" "$SCRIPT_DIR/state-key-external.txt")
|
||||
[ "$STRICT" -eq 0 ] && ARGS+=("--baseline" "$SCRIPT_DIR/state-key-baseline.txt")
|
||||
[ ${#PASS_THROUGH[@]} -gt 0 ] && ARGS+=("${PASS_THROUGH[@]}")
|
||||
|
||||
python3 "$AUDIT" "${ARGS[@]}"
|
||||
RC=$?
|
||||
if [ "$RC" -gt 1 ]; then
|
||||
echo "[state-keys] CANNOT RUN: the audit itself failed (exit $RC)" >&2
|
||||
exit 2
|
||||
fi
|
||||
exit "$RC"
|
||||
+7
-7
@@ -87,7 +87,7 @@ fn session_create(body: String) -> String {
|
||||
let folder: String = json_get(body, "folder")
|
||||
let content: String = session_make_content(id, title, ts, ts, folder)
|
||||
let tags: String = "[\"session\",\"session:meta\",\"Conversation\"]"
|
||||
let node_id: String = wt_node(
|
||||
let node_id: String = engram_node_full(
|
||||
content, "Conversation", "session:meta",
|
||||
el_from_float(0.7), el_from_float(0.7), el_from_float(0.9),
|
||||
"Episodic", tags
|
||||
@@ -358,7 +358,7 @@ fn session_update_patch(session_id: String, body: String) -> String {
|
||||
let created_int: Int = str_to_int(old_created)
|
||||
let new_content: String = session_make_content(session_id, eff_title, created_int, ts, eff_folder)
|
||||
let tags: String = "[\"session\",\"session:meta\",\"Conversation\"]"
|
||||
let new_node_id: String = wt_node(
|
||||
let new_node_id: String = engram_node_full(
|
||||
new_content, "Conversation", "session:meta",
|
||||
el_from_float(0.7), el_from_float(0.7), el_from_float(0.9),
|
||||
"Episodic", tags
|
||||
@@ -456,7 +456,7 @@ fn session_hist_save(session_id: String, hist: String) -> Void {
|
||||
// TODO(reliability #7): delete-then-insert is not atomic — concurrent saves for the
|
||||
// same session can produce orphan history nodes. State is primary truth; engram fallback.
|
||||
let tags: String = "[\"session\",\"session-history\",\"Conversation\"]"
|
||||
let discard: String = wt_node(
|
||||
let discard: String = engram_node_full(
|
||||
hist, "Conversation", "session:messages:" + session_id,
|
||||
el_from_float(0.6), el_from_float(0.6), el_from_float(0.9),
|
||||
"Episodic", tags
|
||||
@@ -488,7 +488,7 @@ fn session_hist_save(session_id: String, hist: String) -> Void {
|
||||
+ " | ts:" + int_to_str(ts_now)
|
||||
let summary_tags: String = "[\"session-emotional-summary\",\"affective\",\"bell:" + eff_level + "\",\"BellEvent\"]"
|
||||
let summary_sal: String = if str_eq(eff_level, "hard") { el_from_float(0.95) } else { el_from_float(0.85) }
|
||||
let sum_discard: String = wt_node(
|
||||
let sum_discard: String = engram_node_full(
|
||||
summary_content,
|
||||
"BellEvent",
|
||||
"session:emotional-summary",
|
||||
@@ -529,7 +529,7 @@ fn session_hist_save(session_id: String, hist: String) -> Void {
|
||||
if !str_eq(ot_id, "") { engram_forget(ot_id) }
|
||||
let oti = oti + 1
|
||||
}
|
||||
let discard_topic: String = wt_node(
|
||||
let discard_topic: String = engram_node_full(
|
||||
topic_content, "Conversation", topic_label,
|
||||
el_from_float(0.7), el_from_float(0.7), el_from_float(0.9),
|
||||
"Episodic", topic_tags
|
||||
@@ -582,7 +582,7 @@ fn session_update_meta_timestamp(session_id: String) -> Void {
|
||||
let created_int: Int = str_to_int(old_created)
|
||||
let new_content: String = session_make_content(session_id, old_title, created_int, ts, old_folder)
|
||||
let tags: String = "[\"session\",\"session:meta\",\"Conversation\"]"
|
||||
let new_id: String = wt_node(
|
||||
let new_id: String = engram_node_full(
|
||||
new_content, "Conversation", "session:meta",
|
||||
el_from_float(0.7), el_from_float(0.7), el_from_float(0.9),
|
||||
"Episodic", tags
|
||||
@@ -629,7 +629,7 @@ fn session_auto_title(session_id: String, first_message: String) -> Void {
|
||||
let created_int: Int = str_to_int(old_created)
|
||||
let new_content: String = session_make_content(session_id, new_title, created_int, ts, old_folder)
|
||||
let tags: String = "[\"session\",\"session:meta\",\"Conversation\"]"
|
||||
let new_id: String = wt_node(
|
||||
let new_id: String = engram_node_full(
|
||||
new_content, "Conversation", "session:meta",
|
||||
el_from_float(0.7), el_from_float(0.7), el_from_float(0.9),
|
||||
"Episodic", tags
|
||||
|
||||
@@ -657,23 +657,6 @@ if is_genesis && safe_to_seed {
|
||||
}
|
||||
}
|
||||
|
||||
// CRASH RECOVERY (neuron#117). Deltas the previous process staged but could not
|
||||
// hand to the owner are still on disk — the spool is a filesystem queue, not a
|
||||
// memory buffer, precisely so that a soul that died mid-flight does not take its
|
||||
// unpushed writes with it. Drain them before serving, so recovered memories are
|
||||
// durable and recallable from the owner from the first request onward.
|
||||
//
|
||||
// Safe on a clean boot: an empty spool means no HTTP call at all. Safe in file
|
||||
// mode: wt_drain returns immediately when ENGRAM_URL is unset.
|
||||
let wt_recovered: Int = wt_drain()
|
||||
if wt_recovered > 0 {
|
||||
println("[soul] write-through: recovered " + int_to_str(wt_recovered)
|
||||
+ " nodes from a previous process's spool -> persistence owner")
|
||||
}
|
||||
if wt_recovered < 0 {
|
||||
println("[soul] write-through: spool present but the persistence owner is unreachable — queued, will retry on heartbeat")
|
||||
}
|
||||
|
||||
println("[soul] serving on port " + int_to_str(port))
|
||||
http_serve_async(port, "handle_request")
|
||||
println("[soul] awareness loop starting")
|
||||
|
||||
+2
-2
@@ -11,7 +11,7 @@ import "memory.el"
|
||||
fn steward_log_event(kind: String, detail: String) -> Void {
|
||||
let content: String = "STEWARD:" + kind + " | " + detail
|
||||
let tags: String = "[\"stewardship\",\"steward:" + kind + "\"]"
|
||||
let discard: String = wt_node(
|
||||
let discard: String = engram_node_full(
|
||||
content,
|
||||
"StewardshipEvent",
|
||||
"steward:" + kind,
|
||||
@@ -221,7 +221,7 @@ fn steward_fingerprint_session(input: String, session_id: String) -> String {
|
||||
+ " formality=" + fs_str
|
||||
+ " time=" + tb_str
|
||||
let sample_tags: String = "[\"behavior\",\"BehaviorSample\",\"stewardship\"]"
|
||||
let discard: String = wt_node(
|
||||
let discard: String = engram_node_full(
|
||||
sample_content,
|
||||
"BehaviorSample",
|
||||
"behavior:" + session_id,
|
||||
|
||||
Reference in New Issue
Block a user