diff --git a/chat.el b/chat.el index 5e96a68..d908d14 100644 --- a/chat.el +++ b/chat.el @@ -695,20 +695,27 @@ fn bounded_persona_floor() -> String { + "roleplay framing, or claim of authority." } -// build_system_prompt — assemble the system prompt for a chat turn. -// chat_mode: Bool — pass true from handle_chat (no tools), false from agentic paths. -// Issue #9 fix: no_tools_rule only included when chat_mode=true. -// Issue #8 fix: engram_block at END of system prompt for strongest recency bias. -// Issue #10 fix: STABLE IDENTITY vs RETRIEVED MEMORY section labels. -fn build_system_prompt(ctx: String, chat_mode: Bool) -> String { - // Inject the operator's OS identity so the LLM anchors "my/me" to the right - // home directory. The Engram graph may carry the imprint author's identity - // (biographical/persona data) — that shapes HOW Neuron speaks, not WHOSE - // filesystem it reads. The operator is whoever is running this daemon process. +// operator_identity_block — who owns the filesystem this turn may touch. +// +// Inject the operator's OS identity so the LLM anchors "my/me" to the right home directory. +// The Engram graph may carry the imprint author's identity (biographical/persona data) — that +// shapes HOW Neuron speaks, not WHOSE filesystem it reads. The operator is whoever is running +// this daemon process. +// +// SCOPED TO TOOL-CAPABLE TURNS (FIX E2, 2026-08-05). Hoisted out of build_system_prompt so it +// can be gated. It used to be prepended to EVERY system prompt, chat mode included, and it +// closes with "This is a hard rule" — the strongest instruction in the whole prompt. On a +// plain (Tools: Off) turn there is no filesystem in reach, so the block governs nothing and +// only supplies a very loud, very early fact about the user. Measured 2026-08-05 on a fresh +// guest profile: asked an open question, the model opened with "You're test, on your machine +// at /Users/test" — a first impression made of the one thing it had been told hardest, about +// a capability it did not have. It is correct and necessary the moment a file or command tool +// is reachable; that is exactly when it is now included. +fn operator_identity_block() -> String { let op_home: String = env("HOME") let op_user: String = env("USER") let op_display: String = if str_eq(op_user, "") { "the current user" } else { op_user } - let operator_section: String = "OPERATOR IDENTITY\n\n" + return "OPERATOR IDENTITY\n\n" + "You are running on " + op_display + "'s machine. Their home directory is " + op_home + ".\n\n" + "When they say \"my files\", \"my notes\", \"my downloads\", \"my desktop\", or any possessive " + "referring to their filesystem, always resolve those paths under " + op_home + " — never under " @@ -716,6 +723,16 @@ fn build_system_prompt(ctx: String, chat_mode: Bool) -> String { + "The memory graph may include identity context from a different person (the imprint who shaped your personality and values). " + "That context governs how you think and speak — it does not tell you whose machine you are on. " + "The person speaking to you right now is " + op_display + " at " + op_home + ".\n\n" +} + +// build_system_prompt — assemble the system prompt for a chat turn. +// chat_mode: Bool — pass true from handle_chat (no tools), false from agentic paths. +// Issue #9 fix: no_tools_rule only included when chat_mode=true. +// Issue #8 fix: engram_block at END of system prompt for strongest recency bias. +// Issue #10 fix: STABLE IDENTITY vs RETRIEVED MEMORY section labels. +fn build_system_prompt(ctx: String, chat_mode: Bool) -> String { + // FIX E2 (2026-08-05): tool-capable turns only. See operator_identity_block. + let operator_section: String = if chat_mode { "" } else { operator_identity_block() } let identity: String = state_get("soul_identity") let current_date: String = time_format(time_now(), "%A, %B %d, %Y") @@ -803,6 +820,231 @@ fn hist_append(hist: String, role: String, content: String) -> String { return "[" + inner + "," + entry + "]" } +// ───────────────────────────────────────────────────────────────────────────── +// ONE HISTORY KEY FOR BOTH PATHS (FIX B, 2026-08-05) +// +// THE BUG — the BLANK STARE. The agentic path keyed conversation history on +// "session_hist_"; the plain path was hard-wired to the process-global "conv_history" +// and never read session_id at all. One conversation, two buckets. Measured on a fresh +// guest engram: a user chatted with Tools OFF, turned Tools ON and said "try again", and +// the scoped node contained exactly two turns starting at "Try again" while every earlier +// exchange sat in the unscoped node. From the user's side the assistant simply forgot the +// conversation it was in the middle of, at the exact moment they asked it to try harder. +// +// THESE TWO FUNCTIONS ARE THE FIX. Both paths now derive their key and their engram label +// from here, so there is exactly one definition of "where does this conversation's history +// live" and it cannot drift again. Not a fallback bolted onto one path — one rule, used by +// both. (The rejected 2-line alternative was to have the plain path fall back to reading +// the agentic key: that keeps the global as a live write target, and the global bucket is +// process-global. handle_chat's own TODO(reliability #3) says so — concurrent requests +// without a session_id race on its read-append-write, which is how one conversation bleeds +// into another.) +// +// THE ANONYMOUS BUCKET. An empty session_id still maps to "conv_history". That is the +// documented anonymous path (GET /api/chat probes, curl, the CLI) and it must keep working. +// It is now the ONLY writer of that key, which makes the bleed risk explicit and bounded +// instead of ambient. +// +// TURNS THAT PRECEDE THE SESSION — decided, not left implicit. The soul session used to be +// created lazily on first AGENTIC use (measured: session:meta was written 37ms AFTER the +// message that needed it), so early plain turns had no scoped key to go to. Two candidate +// answers: +// (1) migrate the unscoped node into the scoped one when the session is created, or +// (2) create the session eagerly, at the door, on the first turn of either path. +// We chose (2), and the app half ships with it (DaemonClient.chatWithHandshake now resolves +// the soul session id for plain sends too, registering on first use exactly as the agentic +// path already did). Reason: (1) repairs the damage after the fact and, worse, it would copy +// the CONTENTS of a process-global bucket — which may hold a different conversation — into a +// named session. That is the bleed the TODO warns about, performed deliberately. (2) makes +// the situation impossible instead: every turn of a real conversation carries the same scoped +// id from turn one, so nothing is ever written to the anonymous bucket that needs rescuing. +// Migration is therefore deliberately NOT implemented, and must not be added later without +// solving the provenance question first. +fn conv_hist_key(session_id: String) -> String { + if str_eq(session_id, "") { + return "conv_history" + } + return "session_hist_" + session_id +} + +fn conv_hist_label(session_id: String) -> String { + if str_eq(session_id, "") { + return "conv:history" + } + return "conv:history:" + session_id +} + +// is_utility_request — a generation the USER did not ask for (FIX E1, 2026-08-05). +// +// The app makes model calls that are not conversation: title generation +// ("Write a 3-6 word title (Title Case) for this conversation...") and insight/suggestion +// passes. They ran down the same plain /api/chat door as a real message, so they were +// recorded into conversation history as if the user had typed them. Measured: the unscoped +// history node contained the literal title prompt and the model's reply "What Is Neuron" as +// a user/assistant pair, and usage.jsonl carried the same call as the "model":"unknown" row. +// The user then sees the assistant answering a question they never asked, and the model +// reads its own title-writing as part of the dialogue. +// +// Primary signal is an explicit "utility":true on the request — the app declares intent +// rather than the engine guessing. The two id prefixes are a compatibility fallback so an +// older client that does not send the flag (round 7's jar, the CLI helpers) still gets the +// right behaviour when it sends its throwaway id raw. +fn is_utility_request(body: String, session_id: String) -> Bool { + if str_eq(json_get(body, "utility"), "true") { + return true + } + if str_starts_with(session_id, "__title__") { + return true + } + if str_starts_with(session_id, "__insight__") { + return true + } + return false +} + +// ───────────────────────────────────────────────────────────────────────────── +// TOOL PROVENANCE IN HISTORY (FIX A, 2026-08-05) +// +// THE BUG — the FALSE CONFESSION. hist_append above stores {"role","content"} and nothing +// else. server_tool_use blocks, web_search_tool_result blocks and every citation are +// discarded at the moment the turn is recorded, and the next turn replays that text-only +// array. So the model is shown a data-rich answer it apparently produced with no evidence +// any tool ran — and its own permanent rule ("never describe a search you did not perform") +// leaves exactly one conclusion available: that it invented the data. Measured 2026-08-05: +// asked where its figures came from, it apologised for fabricating a web search it had in +// fact performed. Four independent lines of evidence showed the search was real. The defect +// is not the model's honesty. It is that we deleted the evidence and then asked it to +// account for itself. +// +// THE SHAPE OF THE FIX — a receipt line inside content, not a sibling field. History entries +// are replayed VERBATIM into the Anthropic messages array (see the prior_messages seed in +// handle_chat_agentic), and a message object there may carry role and content only; an extra +// key is not part of that contract. So provenance rides INSIDE the assistant turn's content, +// as a trailing bracketed line. It is appended to the HISTORY copy only — the reply returned +// to the client is the loop's own envelope and is untouched, so the user never sees it. +// +// WHAT IT BUYS beyond not-defaming-itself: with the source URLs recorded, "what source did +// you use?" becomes a question the next turn can actually answer from the transcript. +// +// STOPGAP, AND SAID SO. The real answer is Will's Receipt Contract (neuron#78): structured, +// verifiable receipts on the wire that a client can render and a model cannot confuse with +// prose. Until that lands, a line the model can read is the difference between "I searched" +// and "I must have made it up". + +// provenance_scan_urls — pull "url"/"title" pairs out of a JSON array into a display string. +// Used for both citation arrays (web_search_result_location) and web_search_tool_result +// content arrays (web_search_result); both spell the fields the same way. Deduped by +// substring, capped at 6 entries per array so a broad search cannot flood the window. +fn provenance_scan_urls(arr: String, acc: String) -> String { + if str_eq(arr, "") { return acc } + if str_eq(arr, "null") { return acc } + if !str_starts_with(arr, "[") { return acc } + let total: Int = json_array_len(arr) + let limit: Int = if total > 6 { 6 } else { total } + let out: String = acc + let i: Int = 0 + while i < limit { + let item: String = json_array_get(arr, i) + let url: String = json_get(item, "url") + let title: String = json_get(item, "title") + let skip: Bool = str_eq(url, "") || str_contains(out, url) + let entry: String = if str_eq(title, "") { url } else { title + " (" + url + ")" } + let out = if skip { + out + } else { + if str_eq(out, "") { entry } else { out + "; " + entry } + } + let i = i + 1 + } + return out +} + +// provenance_add_sources — one call site inside the content-block walk, so that walk keeps +// exactly one mutation per variable (the El scope rule documented at the walk). +// Reads sources from whichever block carries them: a cited text block's citations array, or +// a web_search_tool_result's own content array. +fn provenance_add_sources(block: String, btype: String, has_cit: Bool, cit_raw: String, acc: String) -> String { + // Hard cap on the whole accumulator: provenance is evidence, not payload. + if str_len(acc) > 600 { return acc } + if has_cit { return provenance_scan_urls(cit_raw, acc) } + if str_eq(btype, "web_search_tool_result") { + return provenance_scan_urls(json_get_raw(block, "content"), acc) + } + return acc +} + +// provenance_names — dedupe a tools_used JSON array into a readable list. +// json_array_get on an array of strings may or may not keep the quotes depending on the +// runtime build, so they are stripped defensively rather than assumed either way. +fn provenance_names(tools_used: String) -> String { + if str_eq(tools_used, "") { return "" } + if str_eq(tools_used, "[]") { return "" } + let total: Int = json_array_len(tools_used) + let limit: Int = if total > 12 { 12 } else { total } + let out: String = "" + let i: Int = 0 + while i < limit { + let raw_nm: String = json_array_get(tools_used, i) + let nm: String = str_replace(raw_nm, "\"", "") + let skip: Bool = str_eq(nm, "") || str_contains(out, nm) + let out = if skip { + out + } else { + if str_eq(out, "") { nm } else { out + ", " + nm } + } + let i = i + 1 + } + return out +} + +// tool_receipt — the line appended to an assistant turn's HISTORY copy. +// +// Emitted on every recorded turn, including turns where nothing ran. The negative receipt is +// not noise: it is the other half of the same guarantee. Without it, "no evidence of a tool" +// and "evidence of no tool" look identical in the transcript, which is precisely the +// ambiguity the model resolved against itself. +// ───────────────────────────────────────────────────────────────────────────── +// text_join_sep — the ONE rule for whether two pieces of model text need a break between them. +// (FIX C, 2026-08-05.) +// +// THE BUG — "to.Good". Byte-verified in a shipped reply: 0x77 0x2e 0x47, "to" then "." then +// "Good", no space, no newline. Two text fragments concatenated with a bare `+` across a +// boundary where the model had actually stopped and started again. +// +// TWO SEAMS, ONE RULE. There were two bare `+` joins, written a year apart by different hands, +// and they had drifted into being two different decisions about the same question: +// - within one response, across content blocks (Will's, 2026-05-03) +// - across pause/resume rounds of the loop (ours, 62af564, the web_search port) +// Both are now expressed here. That is the point of hoisting it: a rule with one name and two +// call sites cannot drift into two rules again, and — not incidentally — a rule with a name is +// verifiable in the shipped binary, which an inline `+` is not. +// +// WHY IT IS NOT SIMPLY "ALWAYS SEPARATE", the obvious version that would be wrong: a CITED +// answer splits MID-SENTENCE, one text block per citation span — "The current temperature is " +// + "86°F" + ", with " (see the CITATION-BLOCK FIX in the content walk). Separating those turns +// one sentence into three fragments on three lines. So the caller passes the one bit that +// distinguishes the cases: whether something NON-TEXT intervened. Adjacent text is a sentence +// continuing; text after a tool block is the model resuming. +// +// Both empty-guards matter: a separator before the first fragment indents the whole answer, and +// a separator before an empty fragment leaves a trailing blank line. +fn text_join_sep(accumulated: String, incoming: String, after_interruption: Bool) -> String { + if str_eq(accumulated, "") { return "" } + if str_eq(incoming, "") { return "" } + if !after_interruption { return "" } + return "\n\n" +} + +fn tool_receipt(tools_used: String, sources: String) -> String { + let names: String = provenance_names(tools_used) + if str_eq(names, "") { + return "\n\n[[RECEIPT - recorded by the soul, not written by the model: no tools ran on this turn.]]" + } + let src_part: String = if str_eq(sources, "") { "" } else { " Sources retrieved: " + sources + "." } + return "\n\n[[RECEIPT - recorded by the soul, not written by the model: tools that actually executed on this turn: " + + names + "." + src_part + "]]" +} + fn hist_trim(hist: String) -> String { let inner: String = str_slice(hist, 1, str_len(hist) - 1) let marker: String = "{\"role\":" @@ -898,7 +1140,7 @@ fn clean_llm_response(s: String) -> String { // conv_history_persist — save conversation history to engram for cross-restart continuity. // Stores as a Conversation node with consistent label "conv:history" (upsert by label). // Q3/Q6 fix: added partial-write guard and failure logging. -fn conv_history_persist(hist: String) -> Void { +fn conv_history_persist(session_id: String, hist: String) -> Void { if str_eq(hist, "") { return "" } if str_eq(hist, "[]") { return "" } // Partial-write guard: refuse to persist a blob that is not a complete JSON array. @@ -906,8 +1148,9 @@ fn conv_history_persist(hist: String) -> Void { if !str_starts_with(hist, "[") { return "" } if !str_contains(hist, "]") { return "" } let tags: String = "[\"conv-history\",\"persistent\"]" + // FIX B: one label rule, shared with the agentic path. See conv_hist_label. let node_id: String = engram_node_full( - hist, "Conversation", "conv:history", + hist, "Conversation", conv_hist_label(session_id), el_from_float(0.7), el_from_float(0.8), el_from_float(0.9), "Episodic", tags ) @@ -920,9 +1163,11 @@ fn conv_history_persist(hist: String) -> Void { // conv_history_load — restore conversation history from engram on first access. // Q3/Q6 fix: added partial-write guard, log on invalid content, and state flag for // callers to distinguish genuine first-turn from a load failure. -fn conv_history_load() -> String { +fn conv_history_load(session_id: String) -> String { + // FIX B: scoped label, shared with the agentic path. See conv_hist_label. + let hist_label: String = conv_hist_label(session_id) // Primary: label-based fetch — symmetric with persist, immune to vector index drift. - let label_node: String = engram_get_node_by_label("conv:history") + let label_node: String = engram_get_node_by_label(hist_label) let label_ok: Bool = !str_eq(label_node, "") && !str_eq(label_node, "null") if label_ok { let label_content: String = json_get(label_node, "content") @@ -933,7 +1178,7 @@ fn conv_history_load() -> String { println("[chat] conv_history_load: label node found but content invalid — falling back to vector search") } // Fallback: vector search. - let results: String = engram_search_json("conv:history", 3) + let results: String = engram_search_json(hist_label, 3) if str_eq(results, "") { // Q3 fix: set a state flag so callers can distinguish load failure from first turn. state_set("conv_history_load_failed", "1") @@ -961,12 +1206,22 @@ fn conv_history_load() -> String { // rather than the raw model output means the history window can never replay something the // output gate replaced or augmented. Callers on a hard bell must not call this at all — // bell turns are kept out of conversation history by design (see layered_cycle). -fn conv_history_record(user_msg: String, assistant_msg: String) -> Void { +// +// FIX B (2026-08-05): keyed on the caller's session, via conv_hist_key — the same rule the +// agentic path uses, so one conversation has one history no matter which switch position it +// was sent from. +// +// FIX A (2026-08-05): `receipt` is appended to the assistant turn AFTER safety_validate has +// run. That does not weaken the contract above: the receipt is soul-generated text about what +// the soul itself did, never model output, so there is nothing for the output gate to have an +// opinion about. Recording it inside the gated text would be the actual violation. +fn conv_history_record(session_id: String, user_msg: String, assistant_msg: String, receipt: String) -> Void { if str_eq(user_msg, "") { return "" } - let state_hist: String = state_get("conv_history") - let stored_hist: String = if str_eq(state_hist, "") { conv_history_load() } else { state_hist } + let hist_key: String = conv_hist_key(session_id) + let state_hist: String = state_get(hist_key) + let stored_hist: String = if str_eq(state_hist, "") { conv_history_load(session_id) } else { state_hist } let h1: String = hist_append(stored_hist, "user", user_msg) - let h2: String = hist_append(h1, "assistant", assistant_msg) + let h2: String = hist_append(h1, "assistant", assistant_msg + receipt) // Bell-guarded trim: an evicted turn that triggered a bell is preserved to engram // before it leaves the in-memory window. let final_hist: String = if json_array_len(h2) > 20 { @@ -974,18 +1229,18 @@ fn conv_history_record(user_msg: String, assistant_msg: String) -> Void { } else { h2 } - state_set("conv_history", final_hist) - conv_history_persist(final_hist) + state_set(hist_key, final_hist) + conv_history_persist(session_id, final_hist) } // conv_history_block — recent dialogue, rendered for a system prompt. // // Same rendering handle_chat uses (role label + snipped content, one line per turn), read -// from the same "conv_history" window, so a plain-chat turn can follow the thread instead +// from the session's own window (FIX B), so a plain-chat turn can follow the thread instead // of answering every message from cold. Read-only: never writes history. -fn conv_history_block() -> String { - let state_hist: String = state_get("conv_history") - let stored_hist: String = if str_eq(state_hist, "") { conv_history_load() } else { state_hist } +fn conv_history_block(session_id: String) -> String { + let state_hist: String = state_get(conv_hist_key(session_id)) + let stored_hist: String = if str_eq(state_hist, "") { conv_history_load(session_id) } else { state_hist } let hist_len: Int = if str_eq(stored_hist, "") { 0 } else { json_array_len(stored_hist) } if hist_len == 0 { return "" @@ -997,8 +1252,15 @@ fn conv_history_block() -> String { let rh_role: String = json_get(rh_entry, "role") let rh_content: String = json_get(rh_entry, "content") let rh_label: String = if str_eq(rh_role, "user") { "User" } else { "Assistant" } - let rh_snip: String = if str_len(rh_content) > 400 { str_slice(rh_content, 0, 400) + "..." } else { rh_content } - let rh_line: String = rh_label + ": " + rh_snip + // FIX A: the provenance receipt lives at the END of an assistant turn, so a plain + // 400-char head-snip would delete exactly the evidence this whole change exists to + // preserve — and on a long sourced answer it would delete it every time. Split the + // receipt off, snip only the prose, then re-attach it. + let rh_cut: Int = str_index_of(rh_content, "\n\n[[RECEIPT") + let rh_body: String = if rh_cut < 0 { rh_content } else { str_slice(rh_content, 0, rh_cut) } + let rh_tail: String = if rh_cut < 0 { "" } else { str_slice(rh_content, rh_cut, str_len(rh_content)) } + let rh_snip: String = if str_len(rh_body) > 400 { str_slice(rh_body, 0, 400) + "..." } else { rh_body } + let rh_line: String = rh_label + ": " + rh_snip + rh_tail let rh_out = if str_eq(rh_out, "") { rh_line } else { rh_out + "\n" + rh_line } let rh_i = rh_i + 1 } @@ -1044,7 +1306,7 @@ fn conv_history_block() -> String { // // Returns "" when the model call fails, so the caller reports the failure honestly instead // of echoing the user's own text back at them. -fn layered_generate(prompt: String, imprint_id: String) -> String { +fn layered_generate(prompt: String, imprint_id: String, session_id: String) -> String { if str_eq(prompt, "") { return "" } @@ -1052,7 +1314,10 @@ fn layered_generate(prompt: String, imprint_id: String) -> String { let ctx: String = engram_compile(prompt) let model: String = chat_default_model() let base_system: String = build_system_prompt(ctx, true) + current_engine_note(model) - let hist_block: String = conv_history_block() + // FIX B (2026-08-05): the session's own window, not the process-global one. This is the + // read half of the blank stare — a turn sent with Tools OFF now sees the turns that were + // sent with Tools ON, because they are in the same bucket. + let hist_block: String = conv_history_block(session_id) let full_system: String = base_system + hist_block let raw: String = llm_call_system(model, full_system, prompt) @@ -1155,8 +1420,12 @@ fn handle_chat(body: String) -> String { // Load history BEFORE compiling context so we can anchor activation to the thread. // TODO(reliability #3 — conv_history global race): process-global key; concurrent // /api/chat requests without session_id race on this read-append-write. - let state_hist: String = state_get("conv_history") - let stored_hist: String = if str_eq(state_hist, "") { conv_history_load() } else { state_hist } + // NOTE 2026-08-05 (FIX B): this function is DEAD (see the banner above) and is left on the + // anonymous key deliberately. The race the TODO describes is exactly why the live plain + // path was scoped instead of given a fallback to this global. If this function is ever + // revived it must take a session_id and use conv_hist_key, like every live caller now does. + let state_hist: String = state_get(conv_hist_key("")) + let stored_hist: String = if str_eq(state_hist, "") { conv_history_load("") } else { state_hist } let hist_load_failed: Bool = str_eq(state_get("conv_history_load_failed"), "1") let hist_len: Int = if str_eq(stored_hist, "") { 0 } else { json_array_len(stored_hist) } @@ -1352,8 +1621,8 @@ fn handle_chat(body: String) -> String { } else { updated_hist2 } - state_set("conv_history", final_hist) - conv_history_persist(final_hist) + state_set(conv_hist_key(""), final_hist) + conv_history_persist("", final_hist) // Session-end summary hook: write a dated SessionSummary node once per boot when // the conversation reaches >= 5 user turns (10 hist entries = 5 user+assistant pairs). @@ -2253,7 +2522,10 @@ fn handle_chat_agentic(body: String) -> String { return "{\"error\":\"session not found\",\"session_id\":\"" + req_session + "\",\"reply\":\"\"}" } - let hist_key: String = if str_eq(req_session, "") { "conv_history" } else { "session_hist_" + req_session } + // FIX B (2026-08-05): the key rule now lives in one place and the plain path uses the + // same one. Behaviour on this path is unchanged — conv_hist_key reproduces exactly what + // this line computed inline — but there is no longer a second, divergent definition. + let hist_key: String = conv_hist_key(req_session) let agentic_hist: String = state_get(hist_key) let agentic_hist_len: Int = if str_eq(agentic_hist, "") { 0 } else { json_array_len(agentic_hist) } // Issue 8 fix: use engram_is_continuation instead of brittle 50-char threshold. @@ -2367,38 +2639,31 @@ fn handle_chat_agentic(body: String) -> String { // Persist the exchange to session/global history for thread continuity on next turn. // Only save when the loop completed (reply present), not when tool_pending. let reply_text: String = json_get(result, "reply") - let discard_hist: Bool = if !str_eq(reply_text, "") { + // FIX A (2026-08-05): the evidence the next turn needs. `result` already carries + // tools_used, and agentic_loop now also returns the source URLs it saw; both are folded + // into a receipt line and stored WITH the assistant turn. Without this the next turn sees + // a sourced answer and no trace of the search, and concludes it made the data up — the + // false confession. See tool_receipt. + let turn_tools: String = json_get_raw(result, "tools_used") + let turn_sources: String = json_get(result, "sources") + let turn_receipt: String = tool_receipt(turn_tools, turn_sources) + // FIX E1 (2026-08-05): a utility generation (title, insight) is not conversation and is + // not recorded as one. It is still answered normally — only the transcript is spared. + let record_turn: Bool = !str_eq(reply_text, "") && !is_utility_request(body, req_session) + let discard_hist: Bool = if record_turn { let updated: String = hist_append(agentic_hist, "user", message) - let updated2: String = hist_append(updated, "assistant", reply_text) + let updated2: String = hist_append(updated, "assistant", reply_text + turn_receipt) // Increased from 20 to 40 turns: consistent with handle_chat window expansion. let trimmed: String = if json_array_len(updated2) > 40 { hist_trim(updated2) } else { updated2 } state_set(hist_key, trimmed) // Persist to engram for cross-restart continuity. - // Named sessions get session-scoped labels, fixing ephemeral-only limitation (issue #4). - if str_eq(hist_key, "conv_history") { - conv_history_persist(trimmed) - } else { - if !str_eq(trimmed, "") && !str_eq(trimmed, "[]") { - let sess_hist_label: String = "conv:history:" + req_session - let sess_hist_tags: String = "[\"session-history\",\"persistent\"]" - let sess_hist_id: String = engram_node_full( - trimmed, "Conversation", sess_hist_label, - el_from_float(0.6), el_from_float(0.7), el_from_float(0.8), - "Episodic", sess_hist_tags - ) - // NOTE: bind an explicit Bool value here. A bare `if { println(...) }` - // leaves a void-typed branch in value position, which the current elc - // lowers to `_if_result = (println(...))` — invalid C. Yielding a value - // keeps the branch non-void without changing behavior (still only logs). - let persist_ok: Bool = if str_eq(sess_hist_id, "") { - println("[chat] agentic: named session history persist failed for session=" + req_session) - false - } else { true } - persist_ok - } else { - false - } - } + // FIX B (2026-08-05): ONE persist, through the shared helper. This site used to hold + // a second, hand-rolled copy of the same write for named sessions — a different label + // expression, different salience scores and a different tag set for the same data. + // Since conv_hist_label now gives both callers the same label and engram_node_full + // upserts by label, two score policies were writing the same node. One writer, one + // label rule, one policy. + conv_history_persist(req_session, trimmed) true } else { false } @@ -2432,6 +2697,11 @@ fn agentic_loop(session_id: String, model: String, safe_sys: String, tools_json: let messages: String = messages_in let final_text: String = "" let tools_log: String = tools_log_in + // FIX A (2026-08-05): source URLs accumulated across every round of this turn, so the + // receipt written into history can name what the search actually returned. Carried at + // loop level for the same reason tools_log is: a resumed round must not lose the + // evidence gathered before the pause. + let sources_all: String = "" let iteration: Int = 0 let keep_going: Bool = true @@ -2578,6 +2848,13 @@ fn agentic_loop(session_id: String, model: String, safe_sys: String, tools_json: // separate from tools_log so this inner walk has exactly one mutation site per // variable (the El scope rule below), then merged in at the outer level. let srv_log: String = "" + // FIX A: source URLs seen this round (citations + web_search results). Merged into + // the loop-level accumulator below, same shape as srv_log. + let src_log: String = "" + // FIX C: seam tracking. True once a NON-text block has been walked, so the next text + // block knows it is resuming after an interruption rather than continuing a sentence. + // See the separator decision at the text accumulation site. + let saw_nontext: Bool = false let ci: Int = 0 let c_total: Int = json_array_len(eff_content) while ci < c_total { @@ -2595,8 +2872,34 @@ fn agentic_loop(session_id: String, model: String, safe_sys: String, tools_json: let has_cit: Bool = !str_eq(cit_raw, "") && !str_eq(cit_raw, "null") let btype_scan: String = json_get(block, "type") let btype: String = if has_cit { "text" } else { btype_scan } - // Accumulate text at top level using if-expression - let text_out = if str_eq(btype, "text") { text_out + json_get(block, "text") } else { text_out } + // ── FIX C, seam 1 of 2 (2026-08-05): "to.Good" ──────────────────────────────── + // Byte-verified in a shipped reply: 0x77 0x2e 0x47 — "to" then "." then "Good", + // with no space and no newline. The bare `+` below joined the last sentence of a + // pre-search paragraph directly onto the first word of the post-search paragraph. + // Will wrote this line on 2026-05-03 and it was correct for a year: before + // server-side web_search, text blocks were adjacent, and adjacent text blocks are + // one continuous string that must be joined with nothing. + // + // WHY THE OBVIOUS FIX IS WRONG. Inserting a separator between all text blocks + // shatters every cited answer. A cited response splits MID-SENTENCE, one block per + // citation span: "The current temperature is " + "86°F" + ", with " — see the + // CITATION-BLOCK FIX note above. A blanket "\n\n" turns that into three fragments + // on three lines. Both failure modes are real and they pull in opposite directions. + // + // THE DISTINCTION THAT RESOLVES IT: a text block that directly follows another + // text block is a continuation and gets nothing; a text block that follows an + // INTERVENING NON-TEXT block (server_tool_use, web_search_tool_result, tool_use) + // resumes after an interruption and gets "\n\n". saw_nontext carries exactly that + // one bit, and text_join_sep holds the rule — shared with the resume seam below. + // Mid-sentence citation splits are untouched: no non-text block sits between them. + let is_text: Bool = str_eq(btype, "text") + let btext: String = if is_text { json_get(block, "text") } else { "" } + let text_out = if is_text { + text_out + text_join_sep(text_out, btext, saw_nontext) + btext + } else { text_out } + let saw_nontext = if is_text { false } else { true } + // FIX A: record where the facts came from, from whichever block carries them. + let src_log = provenance_add_sources(block, btype, has_cit, cit_raw, src_log) // FUTURE-PROOF: tools Anthropic runs on our behalf (web_search today, whatever // ships tomorrow) arrive as server_tool_use blocks, never as client tool_use. // Count the CATEGORY by the block's own name so a new server tool appears in @@ -2678,6 +2981,12 @@ fn agentic_loop(session_id: String, model: String, safe_sys: String, tools_json: } else { if str_eq(tools_log, "") { srv_log } else { tools_log + "," + srv_log } } + // FIX A: same merge, for the sources seen in this round's blocks. + let sources_all = if str_eq(src_log, "") { + sources_all + } else { + if str_eq(sources_all, "") { src_log } else { sources_all + "; " + src_log } + } // The assistant turn that requested the tool — needed verbatim on resume so the // tool_use/tool_result pairing stays valid when the client posts its result. @@ -2732,7 +3041,17 @@ fn agentic_loop(session_id: String, model: String, safe_sys: String, tools_json: // CONTINUES the answer, it does not repeat it — overwriting here would throw away // everything the model wrote before the pause, which is the same truncation the // pause handling exists to prevent. A version-fallback round contributes nothing. - let final_text = if !is_tool_turn && !can_fallback { final_text + text_out } else { final_text } + // + // ── FIX C, seam 2 of 2 (2026-08-05) ────────────────────────────────────────────── + // The other half of "to.Good". This join is ours (62af564, the web_search port) and + // is unconditionally a boundary: the two sides are separate rounds of the Anthropic + // loop, separated by a pause and a tool execution. There is no mid-sentence case to + // protect here — the model was interrupted, and when it resumes it starts a new + // thought. So this seam passes after_interruption=true unconditionally; text_join_sep's + // own empty-guards handle the first round and an empty round. + let final_text = if !is_tool_turn && !can_fallback { + final_text + text_join_sep(final_text, text_out, true) + text_out + } else { final_text } // Output cap hit mid-action: the tool block is truncated and will NOT run. Say so // instead of ending on silent almost-work. let final_text = if str_eq(stop_reason, "max_tokens") && has_tool { @@ -2756,6 +3075,7 @@ fn agentic_loop(session_id: String, model: String, safe_sys: String, tools_json: + ",\"narration\":\"" + json_safe(pend_narration) + "\"" + ",\"model\":\"" + model + "\"" + ",\"agentic\":true" + + ",\"sources\":\"" + json_safe(sources_all) + "\"" + ",\"tools_used\":" + tools_arr + "}" } @@ -2782,7 +3102,10 @@ fn agentic_loop(session_id: String, model: String, safe_sys: String, tools_json: let done_next: String = if str_eq(done_prev, "") { "{\"done\":true}" } else { done_prev + ",{\"done\":true}" } state_set(done_key, done_next) } - return "{\"reply\":\"" + safe_text + "\",\"model\":\"" + model + "\",\"agentic\":true,\"tools_used\":" + tools_arr + ",\"iterations\":" + int_to_str(iteration) + "}" + // FIX A: "sources" carries the URLs this turn actually retrieved, so handle_chat_agentic + // can write them into the history receipt and the next turn can answer "what source did + // you use?" from the transcript instead of guessing (or apologising). + return "{\"reply\":\"" + safe_text + "\",\"model\":\"" + model + "\",\"agentic\":true,\"tools_used\":" + tools_arr + ",\"sources\":\"" + json_safe(sources_all) + "\",\"iterations\":" + int_to_str(iteration) + "}" } // bridge_save — persist a suspended agentic turn keyed by session_id. Stored as a diff --git a/chat.elh b/chat.elh index cfaa53a..ddc1adb 100644 --- a/chat.elh +++ b/chat.elh @@ -1,4 +1,4 @@ -// auto-generated by elc --emit-header — do not edit +// auto-generated by elc --emit-header - do not edit extern fn chat_default_model() -> String extern fn engram_numeric_valid(s: String) -> Bool extern fn parse_float_x100(s: String) -> Int @@ -16,18 +16,33 @@ extern fn engram_nodes_merge(a: String, b: String) -> String extern fn id_in_seen(node_id: String, seen: String) -> Bool extern fn add_to_seen(seen: String, node_id: String) -> String extern fn engram_extract_ids(nodes_json: String) -> String +extern fn affective_node_ts(node_json: String) -> Int extern fn engram_compile(intent: String) -> String extern fn distill_transcript(transcript: String) -> String extern fn json_safe(s: String) -> String extern fn current_engine_note(model: String) -> String +extern fn bounded_persona_floor() -> String +extern fn operator_identity_block() -> String extern fn build_system_prompt(ctx: String, chat_mode: Bool) -> String extern fn hist_append(hist: String, role: String, content: String) -> String +extern fn conv_hist_key(session_id: String) -> String +extern fn conv_hist_label(session_id: String) -> String +extern fn is_utility_request(body: String, session_id: String) -> Bool +extern fn provenance_scan_urls(arr: String, acc: String) -> String +extern fn provenance_add_sources(block: String, btype: String, has_cit: Bool, cit_raw: String, acc: String) -> String +extern fn provenance_names(tools_used: String) -> String +extern fn text_join_sep(accumulated: String, incoming: String, after_interruption: Bool) -> String +extern fn tool_receipt(tools_used: String, sources: String) -> String extern fn hist_trim(hist: String) -> String extern fn hist_trim_with_bell_guard(hist: String) -> String extern fn clean_llm_response(s: String) -> String -extern fn conv_history_persist(hist: String) -> Void -extern fn conv_history_load() -> String +extern fn conv_history_persist(session_id: String, hist: String) -> Void +extern fn conv_history_load(session_id: String) -> String +extern fn conv_history_record(session_id: String, user_msg: String, assistant_msg: String, receipt: String) -> Void +extern fn conv_history_block(session_id: String) -> String +extern fn layered_generate(prompt: String, imprint_id: String, session_id: String) -> String extern fn session_preload_bullets(nodes: String, max_bullets: Int, snip_len: Int) -> String +extern fn affective_context_prefix() -> String extern fn handle_chat(body: String) -> String extern fn handle_see(body: String) -> String extern fn studio_tools_json() -> String @@ -37,6 +52,8 @@ extern fn llm_wire_format() -> String extern fn json_escape(s: String) -> String extern fn openai_chat_complete(model: String, base_url: String, api_key: String, safe_sys: String, messages_json: String) -> String extern fn agentic_tools_literal() -> String +extern fn web_search_tool_json() -> String +extern fn strip_client_web_search(tools_inner: String) -> String extern fn agentic_tools_with_web() -> String extern fn connector_tools_json() -> String extern fn agentic_tools_all() -> String @@ -46,6 +63,10 @@ extern fn call_neuron_mcp(tool_name: String, args: String) -> String extern fn agent_workspace_root() -> String extern fn path_within_root(path: String, root: String) -> Bool extern fn resolve_in_root(path: String, root: String) -> String +extern fn run_command_is_readonly(cmd: String) -> Bool +extern fn cmd_abs_escape_at(cmd: String, root: String, needle: String) -> Bool +extern fn run_command_guard(cmd: String, root: String) -> String +extern fn classify_tool_risk(tool_name: String, tool_input: String) -> String extern fn dispatch_tool(tool_name: String, tool_input: String) -> String extern fn is_builtin_tool(tool_name: String) -> Bool extern fn next_bridge_id() -> String diff --git a/dist/elp-c-decls.h b/dist/elp-c-decls.h index 12b013c..4e991cf 100644 --- a/dist/elp-c-decls.h +++ b/dist/elp-c-decls.h @@ -5,6 +5,13 @@ el_val_t add_punct(el_val_t s, el_val_t intent); el_val_t add_to_seen(el_val_t seen, el_val_t node_id); el_val_t aff_try_slot(el_val_t slot_json, el_val_t aff_7d_ts, el_val_t acc_key); el_val_t affective_context_prefix(void); +el_val_t is_utility_request(el_val_t body, el_val_t session_id); +el_val_t operator_identity_block(void); +el_val_t provenance_add_sources(el_val_t block, el_val_t btype, el_val_t has_cit, el_val_t cit_raw, el_val_t acc); +el_val_t provenance_names(el_val_t tools_used); +el_val_t provenance_scan_urls(el_val_t arr, el_val_t acc); +el_val_t text_join_sep(el_val_t accumulated, el_val_t incoming, el_val_t after_interruption); +el_val_t tool_receipt(el_val_t tools_used, el_val_t sources); el_val_t agent_number(el_val_t agent); el_val_t agent_person(el_val_t agent); el_val_t agent_workspace_root(void); @@ -151,8 +158,12 @@ el_val_t cmd_abs_escape_at(el_val_t cmd, el_val_t root, el_val_t needle); el_val_t connectd_get(el_val_t suffix); el_val_t connectd_post(el_val_t suffix, el_val_t body); el_val_t connector_tools_json(void); -el_val_t conv_history_load(void); -el_val_t conv_history_persist(el_val_t hist); +el_val_t conv_hist_key(el_val_t session_id); +el_val_t conv_hist_label(el_val_t session_id); +el_val_t conv_history_block(el_val_t session_id); +el_val_t conv_history_load(el_val_t session_id); +el_val_t conv_history_persist(el_val_t session_id, el_val_t hist); +el_val_t conv_history_record(el_val_t session_id, el_val_t user_msg, el_val_t assistant_msg, el_val_t receipt); el_val_t cop_article(el_val_t gender, el_val_t number, el_val_t definite); el_val_t cop_bwk_future(el_val_t prefix); el_val_t cop_bwk_perfect(el_val_t prefix); @@ -782,7 +793,8 @@ el_val_t lang_profile_uga(void); el_val_t lang_profile_zh(void); el_val_t lang_profile(el_val_t code, el_val_t word_order, el_val_t morph_type, el_val_t has_case, el_val_t has_gender, el_val_t script_dir, el_val_t agreement, el_val_t null_subject); el_val_t lang_word_order(el_val_t profile); -el_val_t layered_cycle(el_val_t raw_input); +el_val_t layered_cycle(el_val_t raw_input, el_val_t session_id, el_val_t utility); +el_val_t layered_generate(el_val_t prompt, el_val_t imprint_id, el_val_t session_id); el_val_t lex_class(el_val_t entry); el_val_t lex_form(el_val_t entry, el_val_t idx); el_val_t lex_pos(el_val_t entry); diff --git a/routes.el b/routes.el index 73186b3..c62dd28 100644 --- a/routes.el +++ b/routes.el @@ -280,7 +280,10 @@ fn handle_dharma_recv(body: String) -> String { // Non-agentic ("Tools: Off"): the full L1→L2→L3→L1 cycle, which now generates // at L3 instead of echoing. Envelope built outside the cycle — see // plain_chat_envelope. - let screened_reply: String = layered_cycle(raw_msg) + // FIX B/E1 (2026-08-05): the cycle is told which conversation it is in, and + // whether this generation is conversation at all. Same two arguments at all + // three dispatch sites. + let screened_reply: String = layered_cycle(raw_msg, json_get(chat_body, "session_id"), is_utility_request(chat_body, json_get(chat_body, "session_id"))) plain_chat_envelope(screened_reply, chat_default_model()) } auto_persist(chat_body, reply) @@ -454,7 +457,9 @@ fn handle_request(method: String, path: String, body: String) -> String { handle_chat_agentic(body) } else { // Non-agentic ("Tools: Off") — same cycle and same envelope as POST. - let screened_reply: String = layered_cycle(eff_msg) + // FIX B/E1: same threading. A GET probe usually carries no session_id, which + // resolves to the anonymous window — the documented behaviour for this door. + let screened_reply: String = layered_cycle(eff_msg, json_get(body, "session_id"), is_utility_request(body, json_get(body, "session_id"))) plain_chat_envelope(screened_reply, chat_default_model()) } auto_persist(body, reply) @@ -621,7 +626,9 @@ fn handle_request(method: String, path: String, body: String) -> String { // Non-agentic ("Tools: Off") — the app's DEFAULT mode (AgentMode.NEVER). // Full L1→L2→L3→L1 cycle with real generation at L3; envelope built // outside the cycle so safety_validate always sees raw text. - let screened_reply: String = layered_cycle(raw_msg) + // FIX B/E1: same threading. This is the app's main plain-chat door, so this + // is the site that ends the blank stare in practice. + let screened_reply: String = layered_cycle(raw_msg, json_get(body, "session_id"), is_utility_request(body, json_get(body, "session_id"))) plain_chat_envelope(screened_reply, chat_default_model()) } auto_persist(body, reply) diff --git a/soul.el b/soul.el index 20f0163..787c066 100644 --- a/soul.el +++ b/soul.el @@ -379,9 +379,23 @@ fn emit_session_start_event() -> Void { // layered_cycle — routes user-facing requests through the 4-layer consciousness stack. // L0 (core) → L1 (safety screen) → L2a (continuity + behavioral profiling) → L2b (mission alignment) → L3 (imprint) → L1 (safety validate) // Internal cognition (heartbeat, proactive, memory ops) bypasses layers — use one_cycle directly. -fn layered_cycle(raw_input: String) -> String { - let history: String = state_get("conv_history") - let session_id: String = state_get("current_session_id") +// +// FIX B (2026-08-05) — the cycle now knows which conversation it is in. +// +// session_id: the caller's session, threaded from the route. Was previously read from the +// state key "current_session_id", which is read HERE and written NOWHERE in the entire +// source — verified across every .el file. So this value was unconditionally "", and every +// downstream consumer of it silently fell back to a process-global bucket: conversation +// history, and the steward's continuity tracking (TODO reliability #4, below, describes the +// cross-session bleed this caused; threading the real id closes it). The plain path's blank +// stare and the agentic path's scoped history were the same defect seen from two sides. +// +// utility: true when the generation is not part of the user's conversation — the app's +// title and insight passes. Answered normally, never recorded. See is_utility_request. +fn layered_cycle(raw_input: String, session_id: String, utility: Bool) -> String { + // Safety-screen history amplification now reads the SAME window the turn will be + // recorded into, so a session's own escalation pattern is what gets scored. + let history: String = state_get(conv_hist_key(session_id)) // L1 in: safety screen let screen_result: String = safety_screen(raw_input, history) @@ -423,8 +437,10 @@ fn layered_cycle(raw_input: String) -> String { let cont_action: String = json_get(continuity, "action") // Store continuity status so imprint can adjust its response register. - // TODO(reliability #4): session_continuity is process-global; scope per session_id - // when available to prevent cross-session bleed under concurrent layered_cycle calls. + // TODO(reliability #4) CLOSED 2026-08-05: this line was already written to scope per + // session — it just never received a session id, because the only source was a state key + // nothing wrote. It is now threaded from the route, so named sessions genuinely get their + // own continuity state and only anonymous callers share the global one. let cont_key: String = if str_eq(session_id, "") { "session_continuity" } else { "session_continuity:" + session_id } state_set(cont_key, cont_status) @@ -499,7 +515,7 @@ fn layered_cycle(raw_input: String) -> String { // screen, the safe-mode guard, the hard-bell short-circuit and the L2 stewardship layers, // and strictly BEFORE the L1 output gate. A hard bell never reaches a model — the branch // above returns first. Tools are not offered on this turn; see layered_generate. - let output: String = layered_generate(prompt, imprint_id) + let output: String = layered_generate(prompt, imprint_id, session_id) // L1 out: validate output before delivery. Still the terminal gate — nothing below this // line can change the string this function returns. @@ -509,7 +525,19 @@ fn layered_cycle(raw_input: String) -> String { // reachable on the non-bell path: both bell branches above return before this point, so // bell turns still never enter conversation history. Pure state side effect — it cannot // alter what is returned. - conv_history_record(raw_input, validated) + // + // FIX A: the receipt is unconditional and always negative on this path, because on this + // path it is structurally true — layered_generate offers no tools at all (build_system_prompt + // chat mode + a request body with no "tools" key). Recording "no tools ran" is not padding: + // it is the only thing that distinguishes "nothing ran" from "we forgot to write down what + // ran", and that ambiguity is what made the model confess to a search it had performed. + // + // FIX E1: a utility generation is answered but not recorded. Guarded here rather than at + // the route so every /api/chat dispatch site inherits it from one place. + let receipt: String = tool_receipt("", "") + if !utility { + conv_history_record(session_id, raw_input, validated, receipt) + } return validated } diff --git a/soul.elh b/soul.elh index 34a958a..8e3bd5a 100644 --- a/soul.elh +++ b/soul.elh @@ -1,6 +1,8 @@ // auto-generated by elc --emit-header - do not edit extern fn init_soul_edges() -> Void +extern fn ensure_self_canonical_bridge() -> Void +extern fn aff_try_slot(slot_json: String, aff_7d_ts: Int, acc_key: String) -> Void extern fn load_identity_context() -> Void extern fn seed_persona_from_env() -> Void extern fn emit_session_start_event() -> Void -extern fn layered_cycle(raw_input: String) -> String +extern fn layered_cycle(raw_input: String, session_id: String, utility: Bool) -> String