diff --git a/awareness.el b/awareness.el index e309dd7..635b8b4 100644 --- a/awareness.el +++ b/awareness.el @@ -23,14 +23,11 @@ fn ise_post(content: String) -> Void { let ise_url: String = env("SOUL_ISE_URL") let engram_url: String = if str_eq(ise_url, "") { state_get("soul_engram_url") } else { ise_url } if str_eq(engram_url, "") { - let local_id: String = engram_node_full( + let discard: String = engram_node_full( content, "InternalStateEvent", "state-event", el_from_float(0.3), el_from_float(0.3), el_from_float(0.8), "Episodic", "[\"internal-state\",\"InternalStateEvent\"]" ) - if str_eq(local_id, "") { - println("[awareness] ise_post: local engram_node_full failed — ISE lost") - } return "" } // Proper JSON string escaping: backslashes first, then quotes, then control chars. @@ -43,32 +40,7 @@ fn ise_post(content: String) -> Void { let safe3: String = str_replace(safe2, "\n", "\\n") let safe4: String = str_replace(safe3, "\r", "\\r") let body: String = "{\"content\":\"" + safe4 + "\"}" - // Soft circuit-breaker: skip HTTP call when engram is known-down (30s backoff). - // Opens after 3 consecutive failures; half-open probe after backoff expires. - // TODO(reliability): full async dispatch requires EL runtime futures support. - let cb_open: String = state_get("engram_cb_open") - if str_eq(cb_open, "1") { - let cb_ts_s: String = state_get("engram_cb_open_ts") - let cb_ts: Int = if str_eq(cb_ts_s, "") { 0 } else { str_to_int(cb_ts_s) } - let cb_elapsed: Int = time_now() - cb_ts - if cb_elapsed < 30000 { return "" } - state_set("engram_cb_open", "0") - } - let resp: String = http_post_json(engram_url + "/api/neuron/state-events", body) - let cb_failed: Bool = str_eq(resp, "") || str_starts_with(resp, "{"error":") - if cb_failed { - let fn_s: String = state_get("engram_cb_fails") - let fn_n: Int = if str_eq(fn_s, "") { 0 } else { str_to_int(fn_s) } - let fn_n = fn_n + 1 - state_set("engram_cb_fails", int_to_str(fn_n)) - if fn_n >= 3 { - state_set("engram_cb_open", "1") - state_set("engram_cb_open_ts", int_to_str(time_now())) - println("[awareness] engram circuit-breaker OPEN after " + int_to_str(fn_n) + " failures") - } - } else { - state_set("engram_cb_fails", "0") - } + let discard: String = http_post_json(engram_url + "/api/neuron/state-events", body) return "" } @@ -568,14 +540,9 @@ fn awareness_run() -> Void { let should_refresh: Bool = refresh_elapsed >= refresh_ms if should_refresh { let engram_url: String = state_get("soul_engram_url") - let sc: String = state_get("engram_cb_open") - let sc_ts_s: String = state_get("engram_cb_open_ts") - let sc_ts: Int = if str_eq(sc_ts_s, "") { 0 } else { str_to_int(sc_ts_s) } - let sc_elapsed: Int = now_ts - sc_ts - let sync_allowed: Bool = !str_eq(sc, "1") || sc_elapsed >= 30000 - if !str_eq(engram_url, "") && sync_allowed { + if !str_eq(engram_url, "") { let sync_json: String = http_get(engram_url + "/api/sync") - if !str_eq(sync_json, "") && !str_eq(sync_json, "{}") && !str_starts_with(sync_json, "{\"error\":") { + if !str_eq(sync_json, "") && !str_eq(sync_json, "{}") { let cgi_id: String = state_get("soul_cgi_id") let tmp: String = "/tmp/soul-sync-" + cgi_id + ".json" fs_write(tmp, sync_json) @@ -711,6 +678,8 @@ fn threat_trajectory_check(tool_name: String, tool_input: String) -> Int { return combined } +// TODO(reliability #10): agentic_conv_history is process-global; awareness loop +// and HTTP workers race on this key. Impact: noisy threat score only, not content. fn threat_history_append(text: String) -> Void { let current: String = state_get("agentic_conv_history") let safe_text: String = str_to_lower(text) diff --git a/chat.el b/chat.el index f1cf363..132dd2c 100644 --- a/chat.el +++ b/chat.el @@ -254,41 +254,18 @@ fn hist_append(hist: String, role: String, content: String) -> String { return "[" + inner + "," + entry + "]" } -// hist_trim — drop the oldest two entries from a history JSON array. -// -// Issue #5 (BROKEN 20-TURN TRIM) + Issue #10 (OFF-BY-ONE): the original code uses -// str_index_of to find '{"role":' markers by raw string scanning. If any message content -// contains the literal string '{"role":' (e.g. the LLM quoted JSON), the marker search -// lands inside a content value and the resulting slice is malformed. Additionally, the -// function had no minimum-retained-count guard. -// -// Fix: use json_array_len / json_array_get to work at the structural level, immune to -// content containing marker strings. Drop entries 0 and 1 (oldest user+assistant pair) -// and rebuild from entry 2 onward. Minimum retained count: 2 entries (never over-trim). fn hist_trim(hist: String) -> String { - // Issue #9 fix: use json_array_len/json_array_get instead of fragile str_index_of - // parser. Old code was silently corrupting history on malformed JSON. - let total: Int = json_array_len(hist) - // Safety: never trim below 2 entries. If already at or below the minimum, return unchanged. - if total <= 2 { - return hist + let inner: String = str_slice(hist, 1, str_len(hist) - 1) + let marker: String = "{\"role\":" + let i1: Int = str_index_of(inner, marker) + let tail1: String = str_slice(inner, i1 + 1, str_len(inner)) + let i2: Int = str_index_of(tail1, marker) + let tail2: String = str_slice(tail1, i2 + 1, str_len(tail1)) + let i3: Int = str_index_of(tail2, marker) + if i3 >= 0 { + return "[" + str_slice(tail2, i3, str_len(tail2)) + "]" } - // Drop entry 0 and entry 1 (oldest user+assistant pair). Rebuild from entry 2 onward. - let result: String = "" - let i: Int = 2 - while i < total { - let entry: String = json_array_get(hist, i) - let result = if str_eq(result, "") { - entry - } else { - result + "," + entry - } - let i = i + 1 - } - if str_eq(result, "") { - return hist - } - return "[" + result + "]" + return hist } // hist_trim_with_bell_guard — trim the history window exactly as hist_trim does, but @@ -370,99 +347,41 @@ fn clean_llm_response(s: String) -> String { } // conv_history_persist — save conversation history to engram for cross-restart continuity. -// Stores as a Conversation node with label "conv:history". -// -// Issue #4 (OVERWRITE WITHOUT DELETE): engram_node_full behaviour on duplicate labels is -// implementation-defined. If it appends rather than upserts, stale older nodes accumulate. -// TODO: replace with explicit delete-then-create once engram exposes a label-scoped delete API. -// -// Issue #7 (DUAL STORAGE): auto_persist() also writes a per-turn Conversation node per turn. -// Both run every turn for different purposes (rolling array vs. Q&A snapshot). Documented here. +// Stores as a Conversation node. Overwrites by using consistent label "conv:history". fn conv_history_persist(hist: String) -> Void { if str_eq(hist, "") { return "" } if str_eq(hist, "[]") { return "" } - // Issue #6 (PARTIAL-WRITE GUARD): refuse to persist a blob that is not a complete JSON - // array. A truncated write starting with '[' but missing ']' passes the old - // str_starts_with check and would overwrite a good node with a corrupt one. - if !str_starts_with(hist, "[") { return "" } - if !str_contains(hist, "]") { return "" } + let ts: Int = time_now() let tags: String = "[\"conv-history\",\"persistent\"]" - let node_id: String = engram_node_full( + let discard: String = engram_node_full( hist, "Conversation", "conv:history", el_from_float(0.7), el_from_float(0.8), el_from_float(0.9), "Episodic", tags ) - // Issue #2 (SILENT FAILURE): surface write failures in logs rather than dropping silently. - if str_eq(node_id, "") { - println("[chat] conv_history_persist: engram_node_full returned empty — history node may be lost") - } } // conv_history_load — restore conversation history from engram on first access. -// -// Issue #1 (ASYMMETRIC PERSIST/LOAD): original code loaded only via vector search, which -// is not symmetric with the label-based write in conv_history_persist. A cold or corrupt -// vector index returns [] even when the node exists on disk. Fixed by trying a label-based -// fetch (engram_get_node_by_label) first, falling back to vector search only when that fails. -// -// Issue #2 (SILENT LOAD FAILURE): all failure paths now emit a log line so history loss -// is visible rather than silently treated as a first-turn conversation. -// -// Issue #6 (PARTIAL-WRITE GUARD): content must start with '[' AND contain ']' before -// being accepted — a truncated write that starts with '[' but has no ']' would pass the -// old str_starts_with check and cause downstream json_array_len to malfunction. +// Returns the most recent "conv:history" node content, or "" if none found. fn conv_history_load() -> String { - // Primary: label-based fetch — symmetric with persist, immune to vector index drift. - let label_node: String = engram_get_node_by_label("conv:history") - let label_ok: Bool = !str_eq(label_node, "") && !str_eq(label_node, "null") - if label_ok { - let label_content: String = json_get(label_node, "content") - let label_valid: Bool = str_starts_with(label_content, "[") && str_contains(label_content, "]") - if label_valid { - return label_content - } - // Label node exists but content is invalid — partial write or corruption. - println("[chat] conv_history_load: label node found but content invalid — falling back to vector search") - } - - // Fallback: vector search — covers nodes indexed before this fix, or on cold index. let results: String = engram_search_json("conv:history", 3) if str_eq(results, "") { return "" } if str_eq(results, "[]") { return "" } let node: String = json_array_get(results, 0) let content: String = json_get(node, "content") - // Issue #6: full partial-write guard — require both '[' prefix AND ']' presence. - if !str_starts_with(content, "[") || !str_contains(content, "]") { - println("[chat] conv_history_load: vector search result content invalid — treating as first turn") - return "" - } + // Validate it looks like a JSON array + if !str_starts_with(content, "[") { return "" } return content } fn handle_chat(body: String) -> String { let message: String = json_get(body, "message") if str_eq(message, "") { - // Issue #5: missing required param — HTTP 400. - return "{\"__status__\":400,\"error\":\"message is required\",\"response\":\"\"}" + return "{\"error\":\"message is required\",\"response\":\"\"}" } // Load history BEFORE compiling context so we can anchor activation to the thread. - // Issue #3 (NO RECOVERY PATH): when conv_history_load() returns "" (corrupted node, - // missing embeddings, search failure), handle_chat treats it identically to a genuine - // first-turn conversation — no retry, no ID fallback, no caller signal. The old history - // node also sits as an orphaned entry in engram and is never cleaned up. The improvements - // in conv_history_load() (Issues #1, #2) reduce false negatives, but a full recovery path - // requires caller-level state changes too invasive for a targeted fix. - // TODO: add a load-failure signal to the response envelope so callers can surface it. - // - // TODO(reliability #3 — conv_history global race): "conv_history" is a process-global - // state key. Concurrent /api/chat requests that omit session_id all read the same key, - // append their exchange, and write it back. Because _state_mu serializes individual - // state_get/state_set calls but NOT the read-append-write sequence, one thread's - // appended exchange can be overwritten by another thread writing its own version. - // The fix is to require callers to supply a session_id (routing them through - // session_hist_) and deprecate the global "conv_history" path. Callers using - // the session API (which scopes history per session_hist_) are not affected. + // TODO(reliability #3 — conv_history global race): process-global key; concurrent + // /api/chat requests without session_id race on this read-append-write. let state_hist: String = state_get("conv_history") let stored_hist: String = if str_eq(state_hist, "") { conv_history_load() } else { state_hist } let hist_len: Int = if str_eq(stored_hist, "") { 0 } else { json_array_len(stored_hist) } @@ -587,16 +506,11 @@ fn handle_chat(body: String) -> String { let raw_response: String = llm_call_system(model, full_system, message) - // Issue #5: also catch empty string — llm_extract_text() in el_runtime.c silently - // returns "" when the response content array is missing or all blocks fail to parse. - // Without this guard an empty reply passes through as a silent empty response. let is_error: Bool = str_starts_with(raw_response, "{\"error\"") || str_starts_with(raw_response, "{\"type\":\"error\"") || str_contains(raw_response, "authentication_error") - || str_eq(raw_response, "") if is_error { - // Issue #6: LLM failure — HTTP 503 (service unavailable). - return "{\"__status__\":503,\"error\":\"llm unavailable\",\"response\":\"\"}" + return "{\"error\":\"llm unavailable\",\"response\":\"\"}" } let clean_response: String = clean_llm_response(raw_response) @@ -606,11 +520,6 @@ fn handle_chat(body: String) -> String { let updated_hist2: String = hist_append(updated_hist, "assistant", raw_response) // Use bell-guarded trim: if the evicted turn triggered a bell event, it is // preserved to engram before being dropped from the in-memory window. - // Issue #8 (NO MAX SIZE GUARD): the 20-turn count limit bounds entry count, but individual - // messages can be arbitrarily large (up to max_tokens = 4096 tokens each). At 20 turns the - // history blob can reach ~80KB before trim fires. engram_node_full has no apparent size cap. - // A byte-length cap would require truncating or summarising entries — too invasive here. - // TODO: add a byte-length cap (e.g. 32KB) that drops oldest entries until under limit. let final_hist: String = if json_array_len(updated_hist2) > 20 { hist_trim_with_bell_guard(updated_hist2) } else { @@ -666,42 +575,6 @@ fn studio_tools_json() -> String { "]" } -// --------------------------------------------------------------------------- -// LLM reliability — issues that require C runtime fixes (el_runtime.c). -// These cannot be addressed at the EL layer; they are documented here so the -// symptoms are traceable back to their root causes. -// -// Issue #1 (no retry on timeout/connection error): -// http_do() in el_runtime.c calls curl_easy_perform() once. On -// CURLE_OPERATION_TIMEDOUT / CURLE_COULDNT_CONNECT / CURLE_RECV_ERROR it -// returns http_error_json() with no retry. Fix: add a retry loop (max 3 -// attempts, exponential back-off starting at 1s) inside llm_provider_request(). -// -// Issue #2 (60s timeout applies to all HTTP calls including LLM): -// EL_HTTP_TIMEOUT_MS defaults to 60000ms for every http_do() call. -// Fix: introduce EL_LLM_TIMEOUT_MS (default 120000) used only by -// llm_provider_request(); leave EL_HTTP_TIMEOUT_MS (default 30000) for -// general service calls to avoid holding connections for 60s. -// -// Issue #3 (HTTP 429 causes silent provider failover, not backoff): -// llm_chain_call() advances to the next provider on any JSON-prefixed response -// including 429. Fix: parse HTTP status via curl_easy_getinfo; on 429 sleep -// Retry-After seconds (default 5s) then retry the same provider up to 3 times. -// -// Issue #4 (HTTP 500/502 crashes the request silently): -// Same path as #3 — 5xx responses cause immediate provider failover with no -// retry. Fix: retry with exponential back-off (1s, 2s, 4s) before advancing. -// -// Issue #6 (no secondary LLM fallback in production): -// Set NEURON_LLM_1_URL/KEY/FORMAT in ExternalSecret to a secondary provider -// (e.g. Gemini). No C code change required; llm_chain_call() already iterates. -// -// Issue #8 (LLM response size unbounded — memory-only cap): -// HttpBuf grows via realloc() with no hard limit. Fix: add -// EL_HTTP_MAX_RESPONSE_BYTES (default 10MiB) cap in httpbuf_append() and -// return http_error_json("response too large") on overflow. -// --------------------------------------------------------------------------- - fn agentic_api_key() -> String { let k1: String = env("ANTHROPIC_API_KEY") if !str_eq(k1, "") { @@ -753,7 +626,7 @@ fn agentic_tools_with_web() -> String { // Short timeout + empty-array fallback: if the bridge is down, the soul runs // exactly as before with only its built-in tools (graceful degradation). fn connector_tools_json() -> String { - let raw: String = exec_capture("curl -s --max-time 5 http://127.0.0.1:7771/mcp/tools") + let raw: String = exec_capture("curl -s --max-time 2 http://127.0.0.1:7771/mcp/tools") if str_eq(raw, "") { return "[]" } @@ -786,15 +659,7 @@ fn agentic_tools_all() -> String { fn call_mcp_bridge(tool_name: String, tool_input: String) -> String { let eff_input: String = if str_eq(tool_input, "") { "{}" } else { tool_input } let body: String = "{\"name\":\"" + tool_name + "\",\"input\":" + eff_input + "}" - // Issue #12: previously used a fixed path /tmp/neuron-mcp-call.json. - // Under concurrent load (64 worker threads), two simultaneous MCP tool calls - // race on this file — one call sends the other's input to the bridge. - // Fix: monotonic sequence counter makes the path unique per call. - let mcp_seq_s: String = state_get("mcp_call_seq") - let mcp_seq_n: Int = if str_eq(mcp_seq_s, "") { 0 } else { str_to_int(mcp_seq_s) } - let mcp_seq_next: Int = mcp_seq_n + 1 - state_set("mcp_call_seq", int_to_str(mcp_seq_next)) - let tmp: String = "/tmp/neuron-mcp-call-" + int_to_str(time_now()) + "-" + int_to_str(mcp_seq_next) + ".json" + let tmp: String = "/tmp/neuron-mcp-call.json" fs_write(tmp, body) return exec_capture("curl -s --max-time 30 -X POST http://127.0.0.1:7771/mcp/call -H 'Content-Type: application/json' -d @" + tmp) } @@ -806,7 +671,7 @@ fn tool_auto_approved(tool_name: String) -> Bool { if !str_starts_with(tool_name, "mcp__") { return false } - let raw: String = exec_capture("curl -s --max-time 5 http://127.0.0.1:7771/mcp/auto-approved") + let raw: String = exec_capture("curl -s --max-time 2 http://127.0.0.1:7771/mcp/auto-approved") if str_eq(raw, "") { return false } @@ -1076,22 +941,15 @@ fn is_builtin_tool(tool_name: String) -> Bool { } // next_bridge_id — unique correlation id for a suspended agentic turn. -// Uses uuid_v4() as the primary uniqueness guarantee so concurrent calls -// (even in the same millisecond) cannot collide. The "mcp_bridge_seq" -// counter is kept for human readability in logs/debugging but is no longer -// relied on for uniqueness. +// Uses uuid_v4() as the primary uniqueness guarantee — concurrent calls cannot collide. // -// TODO(reliability #6): state_get/state_set on "mcp_bridge_seq" is a -// non-atomic read-modify-write — two concurrent calls can read the same -// counter and produce the same counter suffix. This is now benign because -// uuid_v4() provides collision-free uniqueness. A true counter fix would -// require an atomic_increment() builtin in el_runtime.c. +// TODO(reliability #6): mcp_bridge_seq RMW is non-atomic. Now benign because +// uuid_v4() provides collision-free uniqueness. Counter is kept for readability only. fn next_bridge_id() -> String { let prev: String = state_get("mcp_bridge_seq") let n: Int = if str_eq(prev, "") { 0 } else { str_to_int(prev) } let next: Int = n + 1 state_set("mcp_bridge_seq", int_to_str(next)) - // uuid_v4() provides collision-free uniqueness; counter is decorative. let uid: String = uuid_v4() return "br-" + uid } @@ -1138,7 +996,7 @@ fn handle_chat_agentic(body: String) -> String { let session_valid: Bool = if str_eq(req_session, "") { true } else { - !str_contains(session_get(req_session), "\"error\"") + session_exists(req_session) } if !session_valid { return "{\"error\":\"session not found\",\"session_id\":\"" + req_session + "\",\"reply\":\"\"}" @@ -1182,23 +1040,12 @@ fn handle_chat_agentic(body: String) -> String { // Persist the exchange to session/global history for thread continuity on next turn. // Only save when the loop completed (reply present), not when tool_pending. - // - // Issue #9 (AGENTIC HISTORY NOT PERSISTED): the agentic path previously only saved - // history to in-process state (state_set), which is lost on restart. We now also call - // conv_history_persist() for the default session (hist_key == "conv_history") so agentic - // history survives restarts the same way non-agentic history does. Per-session histories - // (session_hist_) are still in-process only — persisting all named sessions would - // require per-session engram labels, a larger change tracked separately. let reply_text: String = json_get(result, "reply") let discard_hist: Bool = if !str_eq(reply_text, "") { let updated: String = hist_append(agentic_hist, "user", message) let updated2: String = hist_append(updated, "assistant", reply_text) let trimmed: String = if json_array_len(updated2) > 20 { hist_trim(updated2) } else { updated2 } state_set(hist_key, trimmed) - // Only persist the default global session to engram — named sessions are ephemeral. - if str_eq(hist_key, "conv_history") { - conv_history_persist(trimmed) - } true } else { false } @@ -1229,14 +1076,6 @@ fn agentic_loop(session_id: String, model: String, safe_sys: String, tools_json: let iteration: Int = 0 let keep_going: Bool = true - // Issue #9: agentic max_tokens configurable via NEURON_LLM_MAX_TOKENS env var. - // Default 4096 is marginal for long tool chains (8 iterations x 4096 tokens). - // Set to 8192+ for complex multi-step tasks. - // Note: llm_provider_request() in el_runtime.c also hardcodes 4096 for the - // llm_call_system() (non-agentic) path; that requires a C runtime change. - let max_tokens_env: String = env("NEURON_LLM_MAX_TOKENS") - let max_tokens_str: String = if str_eq(max_tokens_env, "") { "4096" } else { max_tokens_env } - // Suspension state — captured at top level so it escapes the while body. let pending: Bool = false let pend_tool_id: String = "" @@ -1245,7 +1084,7 @@ fn agentic_loop(session_id: String, model: String, safe_sys: String, tools_json: while keep_going && iteration < 8 { let req_body: String = "{\"model\":\"" + model + "\"" - + ",\"max_tokens\":" + max_tokens_str + + ",\"max_tokens\":4096" + ",\"system\":\"" + safe_sys + "\"" + ",\"tools\":" + tools_json + ",\"messages\":" + messages @@ -1525,11 +1364,9 @@ fn handle_chat_as_soul(body: String) -> String { let raw_response: String = llm_call_system(model, system_prompt, eff_message) - // Issue #5: empty string catch — same rationale as handle_chat. let is_error: Bool = str_starts_with(raw_response, "{\"error\"") || str_starts_with(raw_response, "{\"type\":\"error\"") || str_contains(raw_response, "authentication_error") - || str_eq(raw_response, "") if is_error { return "{\"error\":\"llm unavailable\",\"response\":\"\",\"speaker_slug\":\"" + speaker + "\",\"model\":\"" + model + "\"}" } @@ -1576,11 +1413,9 @@ fn handle_dharma_room_turn(body: String) -> String { let raw_response: String = llm_call_system(model, system_prompt, transcript) - // Issue #5: empty string catch — same rationale as handle_chat. let is_error: Bool = str_starts_with(raw_response, "{\"error\"") || str_starts_with(raw_response, "{\"type\":\"error\"") || str_contains(raw_response, "authentication_error") - || str_eq(raw_response, "") if is_error { return "{\"error\":\"llm unavailable\",\"response\":\"\",\"cgi_id\":\"" + cgi_id + "\"}" } @@ -1594,19 +1429,13 @@ fn handle_dharma_room_turn(body: String) -> String { // engram_node(content, "episodic", ...) which wrongly put a TIER into the node_type // slot — that's why nodes showed node_type="episodic". Use the full, correct contract.) let utterance_tags: String = "[\"soul-utterance\",\"episodic\"]" - let utterance_id: String = engram_node_full( + let discard_id: String = engram_node_full( clean_response, "Conversation", "soul:utterance", el_from_float(0.6), el_from_float(0.6), el_from_float(0.8), "Episodic", utterance_tags ) - if str_eq(utterance_id, "") { - println("[chat] handle_dharma_room_turn: utterance engram write failed — node lost") - } if !str_eq(snap_path, "") { - let save_result: String = engram_save(snap_path) - if str_eq(save_result, "") { - println("[chat] handle_dharma_room_turn: engram_save failed for " + snap_path) - } + let discard_save: String = engram_save(snap_path) } let safe_response: String = json_safe(clean_response) @@ -1777,8 +1606,6 @@ fn auto_persist(req: String, resp: String) -> Void { "session_bell_signal:" + sess_id } state_set(signal_key, safe_summary) - if str_eq(conv_node_id, "") { - println("[chat] auto_persist: engram_node_full returned empty — conversation node lost (ts=" + ts_str + ")") } } diff --git a/imprint.el b/imprint.el index 86c3080..ce53ed7 100644 --- a/imprint.el +++ b/imprint.el @@ -5,6 +5,10 @@ // imprint_current — returns the active imprint ID from state. // Falls back to "base" (bare Neuron, no suit) when nothing is loaded. +// +// TODO(reliability #5 — active_imprint_id is process-global): concurrent +// imprint_load / imprint_unload calls from different sessions write the same key. +// Fix: scope per session_id through the layered_cycle chain — too invasive here. fn imprint_current() -> String { let id: String = state_get("active_imprint_id") return if str_eq(id, "") { "base" } else { id } diff --git a/routes.el b/routes.el index 725ff0e..3b5b12d 100644 --- a/routes.el +++ b/routes.el @@ -75,24 +75,14 @@ fn strip_query(path: String) -> String { } fn err_404(path: String) -> String { - // __status__ envelope — el_runtime reads the first key and emits HTTP 404. - // Issue #3: previously returned HTTP 200 with JSON error body. - return "{\"__status__\":404,\"error\":\"not found\",\"path\":\"" + path + "\"}" + return "{\"error\":\"not found\",\"code\":\"not_found\",\"path\":\"" + path + "\"}" } fn err_405(method: String, path: String) -> String { - // __status__ envelope — emits HTTP 405. - // Issue #3: previously returned HTTP 200 with JSON error body. - return "{\"__status__\":405,\"error\":\"method not allowed\",\"method\":\"" + method + "\",\"path\":\"" + path + "\"}" + return "{\"error\":\"method not allowed\",\"code\":\"method_not_allowed\",\"method\":\"" + method + "\",\"path\":\"" + path + "\"}" } fn route_health() -> String { - // NOTE (issue #8): This endpoint performs live engram graph queries on every call - // (engram_node_count, engram_edge_count) and reads imprint state. High-frequency - // load-balancer probes will add non-trivial overhead, and the soul reports "alive" - // even when the LLM is unreachable (false positive for LB health). - // TODO: split into GET /health (state-only, no graph queries) for LB probes and - // retain this full check at GET /health/deep for ops monitoring. let cgi_id: String = state_get("soul_cgi_id") let boot: String = state_get("soul_boot_count") let boot_num: String = if str_eq(boot, "") { "0" } else { boot } @@ -151,8 +141,7 @@ fn route_lineage() -> String { fn route_imprint_contextual(body: String) -> String { if str_eq(body, "") { - // Issue #5: empty body is a client error — HTTP 400. - return "{\"__status__\":400,\"ok\":false,\"error\":\"empty body\"}" + return "{\"ok\":false,\"error\":\"empty body\"}" } let tags: String = "[\"imprint\",\"contextual\"]" let id: String = engram_node_full( @@ -174,8 +163,7 @@ fn route_imprint_contextual(body: String) -> String { fn route_imprint_user(body: String) -> String { if str_eq(body, "") { - // Issue #5: empty body is a client error — HTTP 400. - return "{\"__status__\":400,\"ok\":false,\"error\":\"empty body\"}" + return "{\"ok\":false,\"error\":\"empty body\"}" } let tags: String = "[\"imprint\",\"user\"]" let id: String = engram_node_full( @@ -313,13 +301,9 @@ fn connectd_get(suffix: String) -> String { // so arbitrary JSON cannot reach the shell as a command-line argument. fn connectd_post(suffix: String, body: String) -> String { let eff: String = if str_eq(body, "") { "{}" } else { body } - // Issue #11: time_now() has second-granularity; two concurrent requests in the same - // second collide on the same temp path. Added a monotonic per-process sequence counter. - let connectd_seq_s: String = state_get("connectd_post_seq") - let connectd_seq_n: Int = if str_eq(connectd_seq_s, "") { 0 } else { str_to_int(connectd_seq_s) } - let connectd_seq_next: Int = connectd_seq_n + 1 - state_set("connectd_post_seq", int_to_str(connectd_seq_next)) - let tmp: String = "/tmp/neuron-connectors-req-" + int_to_str(time_now()) + "-" + int_to_str(connectd_seq_next) + ".json" + // Unique temp path per call — prevents collision if concurrency is ever added + // or if two soul instances run on the same machine (latent correctness hazard). + let tmp: String = "/tmp/neuron-connectors-req-" + int_to_str(time_now()) + ".json" fs_write(tmp, eff) let out: String = exec_capture("curl -s --max-time 20 -X POST http://127.0.0.1:7771" + suffix + " -H 'Content-Type: application/json' -d @" + tmp) if str_eq(out, "") { @@ -354,33 +338,9 @@ fn handle_connectors(method: String, clean: String, body: String) -> String { return "{\"ok\":false,\"error\":\"unknown connectors route\"}" } - -// auth_check — validate NEURON_TOKEN bearer auth on every request. -// Returns "" when authorized, or a JSON 401 error string when not. -// /health and /lineage are public routes — always exempted. -// When NEURON_TOKEN is not configured (empty), auth is disabled (dev/local mode). -// Issue #4: previously no auth layer existed anywhere in the router. -// Clients pass the token in the JSON body as "__auth". -// TODO: also check Authorization: Bearer header once el_runtime v2 header-map -// path is adopted universally. -fn auth_check(clean: String, body: String) -> String { - if str_eq(clean, "/health") { return "" } - if str_eq(clean, "/lineage") { return "" } - let token: String = state_get("soul_token") - if str_eq(token, "") { return "" } - let auth_field: String = json_get(body, "__auth") - if str_eq(auth_field, token) { return "" } - return "{\"__status__\":401,\"error\":\"unauthorized\"}" -} - fn handle_request(method: String, path: String, body: String) -> String { let clean: String = strip_query(path) - // Issue #1/#2: EL has no exception/try-catch mechanism. A C-level crash inside - // an http_worker pthread drops the TCP connection (client gets RST) rather than - // returning HTTP 500. TODO: register a SIGSEGV/SIGBUS handler in el_runtime.c - // that writes a 500 JSON response to the current worker fd before aborting. - // Rate limit check. Extract caller IP from REMOTE_ADDR env var (set by the // EL HTTP runtime for each request). Skip enforcement when empty so // loopback/internal callers are never blocked. @@ -392,13 +352,6 @@ fn handle_request(method: String, path: String, body: String) -> String { } } - // Auth — enforced on all routes except /health and /lineage. - // Issue #4: previously no auth check existed anywhere in the router. - let auth_err: String = auth_check(clean, body) - if !str_eq(auth_err, "") { - return auth_err - } - if str_eq(method, "POST") && str_eq(clean, "/dharma/recv") { return handle_dharma_recv(body) } @@ -414,6 +367,9 @@ fn handle_request(method: String, path: String, body: String) -> String { return engram_scan_nodes_json(9999, 0) } if str_eq(clean, "/api/graph/edges") { + // TODO(reliability #8): engram_save races with awareness loop mem_save(). + // Both now use atomic write-to-temp+rename (el_runtime.c). Serialised + // by engram_global_mu. Future: add engram_edges_json() builtin. let snap_path: String = env("HOME") + "/.neuron/engram/snapshot.json" engram_save(snap_path) let snap: String = fs_read(snap_path) @@ -426,8 +382,7 @@ fn handle_request(method: String, path: String, body: String) -> String { let raw_msg: String = json_get(body, "message") let eff_msg: String = if str_eq(raw_msg, "") { body } else { raw_msg } if str_eq(eff_msg, "") { - // Issue #5: missing required param — HTTP 400. - return "{\"__status__\":400,\"error\":\"message required\"}" + return "{\"error\":\"message is required\",\"code\":\"missing_param\"}" } let agentic_flag: Bool = json_get_bool(body, "agentic") let reply: String = if agentic_flag { @@ -571,15 +526,9 @@ fn handle_request(method: String, path: String, body: String) -> String { // responses are buffered and returned as a single JSON object. Streaming // would require runtime-level SSE support in el_runtime.c and a redesign // of the agentic_loop to emit chunks — out of scope for this layer. - // Issue #5: validate required params — return HTTP 400 when missing. let raw_msg: String = json_get(body, "message") if str_eq(raw_msg, "") { - return "{\"__status__\":400,\"error\":\"message is required\",\"response\":\"\"}" - } - // Issue #7: reject oversized messages before engram_compile and the LLM. - // Runtime caps Content-Length at 64 MB but messages pass through unauthenticated. - if str_len(raw_msg) > 32768 { - return "{\"__status__\":400,\"error\":\"message too large (max 32768 chars)\",\"response\":\"\"}" + return "{\"error\":\"message is required\",\"code\":\"missing_param\"}" } let agentic_flag: Bool = json_get_bool(body, "agentic") let reply: String = if agentic_flag { diff --git a/sessions.el b/sessions.el index 06e9a9d..37029f5 100644 --- a/sessions.el +++ b/sessions.el @@ -104,6 +104,8 @@ fn session_create(body: String) -> String { // Newest sessions first (prepend). // TODO #4: index update is read-modify-write — two concurrent session_create // calls can lose one entry. EL has no CAS primitive; fix requires runtime support. + // TODO(reliability #2): session_index RMW is non-atomic. Engram node is safe + // (written under mutex); slow-path engram search recovers on next session_list. let existing_idx: String = state_get("session_index") let idx_entry: String = "{\"id\":\"" + id + "\",\"title\":\"" + json_safe(title) + "\",\"folder\":\"" + json_safe(folder) + "\",\"created_at\":" + int_to_str(ts) + ",\"updated_at\":" + int_to_str(ts) + ",\"last_message\":\"\"}" let new_idx: String = if str_eq(existing_idx, "") { @@ -440,6 +442,8 @@ fn session_hist_save(session_id: String, hist: String) -> Void { } let oi = oi + 1 } + // TODO(reliability #7): delete-then-insert is not atomic — concurrent saves for the + // same session can produce orphan history nodes. State is primary truth; engram fallback. let tags: String = "[\"session\",\"session-history\",\"Conversation\"]" let discard: String = engram_node_full( hist, "Conversation", "session:messages:" + session_id, diff --git a/soul.el b/soul.el index 4942376..1540688 100644 --- a/soul.el +++ b/soul.el @@ -162,39 +162,6 @@ fn load_identity_context() -> Void { println("[soul] persona node loaded (" + int_to_str(str_len(p_content)) + " chars)") } } - - // Cross-session affective context: query engram for recent distress/crisis signals - // at session start. Stored under soul_affective_context so the safety layer can - // detect when a user has been in distress across previous sessions. - // Soft recency guard: nodes with a ts field older than 7 days are skipped. - // Results capped at 3 nodes, 200 chars each, to avoid over-injection into context. - // TODO(recency): engram_search_json sorts by relevance, not timestamp. A native - // after= filter in the engram search API would make this more precise. - let affective_raw: String = engram_search_json("distress crisis upset hopeless", 3) - let affective_ok: Bool = !str_eq(affective_raw, "") && !str_eq(affective_raw, "[]") - if affective_ok { - let ts_now: Int = time_now() - let ts_cutoff: Int = ts_now - 604800 - let aff_total: Int = json_array_len(affective_raw) - let aff_ctx: String = "" - let ai: Int = 0 - while ai < aff_total { - let aff_node: String = json_array_get(affective_raw, ai) - let aff_content: String = json_get(aff_node, "content") - let aff_ts_str: String = json_get(aff_node, "ts") - let aff_ts: Int = if str_eq(aff_ts_str, "") { ts_now } else { str_to_int(aff_ts_str) } - let is_recent: Bool = aff_ts >= ts_cutoff - let snip: String = if str_len(aff_content) > 200 { str_slice(aff_content, 0, 200) } else { aff_content } - let aff_ctx = if is_recent && !str_eq(snip, "") { - if str_eq(aff_ctx, "") { snip } else { aff_ctx + "\n" + snip } - } else { aff_ctx } - let ai = ai + 1 - } - if !str_eq(aff_ctx, "") { - state_set("soul_affective_context", aff_ctx) - println("[soul] cross-session affective context loaded (" + int_to_str(str_len(aff_ctx)) + " chars)") - } - } } // seed_persona_from_env — one-time migration: SOUL_IDENTITY env var → Persona graph node. @@ -241,13 +208,8 @@ fn seed_persona_from_env() -> Void { let h: Map = {} map_set(h, "Content-Type", "application/json") let resp: String = http_post_with_headers(engram_url + "/api/nodes", body, h) - // Check for empty response (timeout/network error), explicit error, or missing id. - if str_eq(resp, "") { - println("[soul] persona HTTP write-back failed: empty response (timeout or network error) — in-memory only this session") - } else if str_contains(resp, "\"error\"") { + if str_contains(resp, "\"error\"") { println("[soul] persona HTTP write-back failed (in-memory only this session): " + resp) - } else if !str_contains(resp, "\"id\"") { - println("[soul] persona HTTP write-back: unexpected response (no id field) — in-memory only this session: " + resp) } else { println("[soul] persona persisted to HTTP engram at " + engram_url) } @@ -280,14 +242,11 @@ fn emit_session_start_event() -> Void { + ",\"ts\":" + int_to_str(ts) + "}" let tags: String = "[\"internal-state\",\"session-start\",\"InternalStateEvent\"]" - let session_event_id: String = engram_node_full( + let discard: String = engram_node_full( payload, "InternalStateEvent", "session-start", el_from_float(0.9), el_from_float(0.9), el_from_float(1.0), "Episodic", tags ) - if str_eq(session_event_id, "") { - println("[soul] emit_session_start_event: engram write failed — session-start event lost") - } println("[soul] session-start event logged (boot=" + boot_num + " nodes=" + int_to_str(node_ct) + " edges=" + int_to_str(edge_ct) + ")") } @@ -295,9 +254,6 @@ fn emit_session_start_event() -> Void { // L0 (core) → L1 (safety screen) → L2a (continuity + behavioral profiling) → L2b (mission alignment) → L3 (imprint) → L1 (safety validate) // Internal cognition (heartbeat, proactive, memory ops) bypasses layers — use one_cycle directly. fn layered_cycle(raw_input: String) -> String { - // conv_history key must match chat.el (conv_history, not conversation_history). - // Mismatch caused safety_score_distress_history() to always receive "" - the - // history-amplification path in safety_threat_score was permanently dead. let history: String = state_get("conv_history") let session_id: String = state_get("current_session_id") @@ -305,9 +261,8 @@ fn layered_cycle(raw_input: String) -> String { let screen_result: String = safety_screen(raw_input, history) let screen_action: String = json_get(screen_result, "action") - // ISSUE 4: safe-mode guard -- if safety_screen returned invalid/empty action, - // refuse the turn rather than silently passing unscreened input to upper layers. - // Valid actions: "hard_bell", "soft_bell", "pass". Anything else = corrupt envelope. + // ISSUE 4: safe-mode guard. If safety_screen returned an invalid/empty action + // (engram failure or internal error), refuse rather than pass unscreened input. let valid_action: Bool = str_eq(screen_action, "hard_bell") || str_eq(screen_action, "soft_bell") || str_eq(screen_action, "pass") @@ -322,8 +277,8 @@ fn layered_cycle(raw_input: String) -> String { // history where they could leak context to subsequent turns. They are persisted // separately by safety_log_bell() into the Episodic tier with restricted labels. // - // ISSUE 6: safety_log_bell for hard bells is already called INSIDE safety_screen - // (safety.el line 140). Do NOT call it again here -- double-log avoided. + // ISSUE 6: safety_log_bell already called inside safety_screen (line 140). + // Do NOT call it again here -- that would double-log every hard bell. // // safety_validate second param: when screen_action is "hard_bell", safety_validate // receives the sentinel string "hard_bell" (not a normal screen action). The safety @@ -341,8 +296,11 @@ fn layered_cycle(raw_input: String) -> String { let cont_status: String = json_get(continuity, "status") let cont_action: String = json_get(continuity, "action") - // Store continuity status so imprint can adjust its response register - state_set("session_continuity", cont_status) + // Store continuity status so imprint can adjust its response register. + // TODO(reliability #4): session_continuity is process-global; scope per session_id + // when available to prevent cross-session bleed under concurrent layered_cycle calls. + let cont_key: String = if str_eq(session_id, "") { "session_continuity" } else { "session_continuity:" + session_id } + state_set(cont_key, cont_status) // Identity anomaly: add a gentle verification cue to the input before imprint let guided: String = if str_eq(cont_action, "identity_check") { @@ -365,13 +323,13 @@ fn layered_cycle(raw_input: String) -> String { json_get(steward_result, "redirect_to") } - // ISSUE 1: apply pre-LLM bell augmentation on layered_cycle path. - // safety_augment_system injects soft/hard directive into system prompt before LLM call. - // Stored in state so imprint_respond can consume it. - // TODO: wire directly into imprint_respond when it accepts a system_override param. - // ISSUE 3 TODO: no semantic/embedding crisis detection. Keyword-only means signals - // evading the phrase list pass through with zero augmentation. Semantic layer is a - // separate architectural decision requiring embedding inference on every message. + // ISSUE 1: pre-LLM bell augmentation for layered_cycle path. + // safety_augment_system appends soft/hard directive to system prompt when bell fires, + // ensuring LLM processes message WITH the safety directive -- not just post-output gate. + // Stored in state as "layered_cycle_safety_system_addendum" for imprint_respond to use. + // TODO: wire directly when imprint_respond gains system_override param (imprint.el change). + // ISSUE 3 TODO: no semantic crisis detection. Keyword-only means signals that evade + // the phrase list pass with zero augmentation. Semantic layer = separate decision. let augmented_addendum: String = safety_augment_system("", raw_input) state_set("layered_cycle_safety_system_addendum", augmented_addendum) @@ -414,29 +372,12 @@ let snapshot_usable: Bool = local_node_count > 50 if using_http_engram && !snapshot_usable { // First boot or empty/corrupt snapshot: seed from HTTP Engram. - // Retry up to 3 times (2s sleep between attempts) to guard against a - // transient network hiccup right after entrypoint.sh health check passes. - // An empty nodes response silently loads a zero-node graph; validate first. - // TODO(reliability): replace sleep_ms retry with non-blocking backoff. println("[soul] engram -> HTTP " + engram_url_raw + " (no local snapshot, first boot)") - let fetch_attempt: Int = 0 - while fetch_attempt < 3 { - let fetch_attempt = fetch_attempt + 1 - let n: String = http_get(engram_url_raw + "/api/nodes?limit=10000") - let e: String = http_get(engram_url_raw + "/api/edges") - let nodes_ok: Bool = !str_eq(n, "") && str_starts_with(n, "[") && str_len(n) > 2 - if nodes_ok { - state_set("_boot_nodes_json", n) - state_set("_boot_edges_json", e) - let fetch_attempt = 3 - } else { - println("[soul] boot HTTP fetch attempt " + int_to_str(fetch_attempt) + " failed --- retrying in 2s") - sleep_ms(2000) - } - } - let nodes_json: String = state_get("_boot_nodes_json") - let edges_json: String = state_get("_boot_edges_json") - let snapshot_data: String = "{\"nodes\":" + nodes_part + ",\"edges\":" + edges_part + "}" + let nodes_json: String = http_get(engram_url_raw + "/api/nodes?limit=10000") + let edges_json: String = http_get(engram_url_raw + "/api/edges") + let nodes_part: String = if str_eq(nodes_json, "") { "[]" } else { nodes_json } + let edges_part: String = if str_eq(edges_json, "") { "[]" } else { edges_json } + let snapshot_data: String = "{\"nodes\":" + nodes_part + ",\"edges\":" + edges_part + "}" let tmp_path: String = "/tmp/soul-engram-" + soul_cgi_id + ".json" fs_write(tmp_path, snapshot_data) engram_load(tmp_path)