Compare commits

..

1 Commits

Author SHA1 Message Date
Tim Lingo f47c92a71a feat: vision in the agentic chat path (image content block)
Neuron Soul CI / build (pull_request) Failing after 23m26s
handle_chat_agentic now reads body image + image_media_type and, when present, sends the current
user turn as an Anthropic content-block array [{text},{image}] instead of a plain string — so the
model sees raw pixels alongside memory, history, and tools (parity with the CLI). Additive: no image
=> output byte-identical to before. elc-clean. Pairs with neuron-ui fix/chat-vision-attachments.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-27 12:25:26 -05:00
2 changed files with 17 additions and 29 deletions
+15 -2
View File
@@ -1688,12 +1688,25 @@ fn handle_chat_agentic(body: String) -> String {
let safe_msg: String = json_safe(message) let safe_msg: String = json_safe(message)
let safe_sys: String = json_safe(system) let safe_sys: String = json_safe(system)
// Vision in the agentic brain (2026-06-27): when the client attaches an image
// (base64 in body "image", mime in "image_media_type"), send it as a real Anthropic
// image content block on THIS user turn so the model sees raw pixels WITH memory,
// history, and tools (parity with the CLI). img_b64 == "" => byte-identical to before.
let img_b64: String = json_get(body, "image")
let img_mt_raw: String = json_get(body, "image_media_type")
let img_mt: String = if str_eq(img_mt_raw, "") { "image/png" } else { img_mt_raw }
let cur_user_content: String = if str_eq(img_b64, "") {
"\"" + safe_msg + "\""
} else {
"[{\"type\":\"text\",\"text\":\"" + safe_msg + "\"},{\"type\":\"image\",\"source\":{\"type\":\"base64\",\"media_type\":\"" + img_mt + "\",\"data\":\"" + img_b64 + "\"}}]"
}
// Seed the messages array with recent history if available, so the LLM sees the thread. // Seed the messages array with recent history if available, so the LLM sees the thread.
let prior_messages: String = if agentic_hist_len > 0 { let prior_messages: String = if agentic_hist_len > 0 {
let inner: String = str_slice(agentic_hist, 1, str_len(agentic_hist) - 1) let inner: String = str_slice(agentic_hist, 1, str_len(agentic_hist) - 1)
"[" + inner + ",{\"role\":\"user\",\"content\":\"" + safe_msg + "\"}]" "[" + inner + ",{\"role\":\"user\",\"content\":" + cur_user_content + "}]"
} else { } else {
"[{\"role\":\"user\",\"content\":\"" + safe_msg + "\"}]" "[{\"role\":\"user\",\"content\":" + cur_user_content + "}]"
} }
let messages: String = prior_messages let messages: String = prior_messages
let api_url: String = "https://api.anthropic.com/v1/messages" let api_url: String = "https://api.anthropic.com/v1/messages"
+2 -27
View File
@@ -267,27 +267,6 @@ fn recall_or_list(query: String, limit: Int) -> String {
return http_post_json(neuron_url() + "/recall", body) return http_post_json(neuron_url() + "/recall", body)
} }
// Create a real typed node via /api/neuron/node/create (handle_api_node_create) so it is a proper
// BacklogItem/Artifact/etc. listable by type via /api/neuron/list/<type> instead of a generic
// memory blob. Maps title->label, content/description->content, project/priority->tags.
fn create_node_typed(args: String, node_type: String, tier: String) -> String {
let content: String = pick_content(args)
if str_eq(content, "") {
return mcp_text_result("error: content/title is required for " + node_type)
}
let title: String = json_get_string(args, "title")
let label: String = if str_eq(title, "") { node_type } else { title }
let project: String = json_get_string(args, "project")
let priority: String = json_get_string(args, "priority")
let proj_tag: String = if str_eq(project, "") { "" } else { ",\"project:" + project + "\"" }
let prio_tag: String = if str_eq(priority, "") { "" } else { ",\"priority:" + priority + "\"" }
let tags: String = "[\"" + node_type + "\"" + proj_tag + prio_tag + "]"
let body: String = "{\"node_type\":\"" + node_type + "\",\"content\":\"" + json_escape(content)
+ "\",\"label\":\"" + json_escape(label) + "\",\"tier\":\"" + tier + "\",\"tags\":" + tags + "}"
let resp: String = http_post_json(neuron_url() + "/node/create", body)
return mcp_json_result(resp)
}
fn search_with_query(args: String, default_limit: Int) -> String { fn search_with_query(args: String, default_limit: Int) -> String {
let query: String = json_get_string(args, "query") let query: String = json_get_string(args, "query")
if str_eq(query, "") { let query = pick_content(args) } if str_eq(query, "") { let query = pick_content(args) }
@@ -652,12 +631,8 @@ fn dispatch_tool_call(tool_name: String, args: String) -> String {
} }
// Backlog + work // Backlog + work
// planWork: create a REAL typed BacklogItem via /api/neuron/node/create (the old path fell through if str_eq(tool_name, "planWork") { return create_typed_node(args, "BacklogItem", "0.65") }
// create_typed_node to a generic /memory write, dropping title/project/priority and never making a if str_eq(tool_name, "reviewBacklog") { return search_with_query(args, 50) }
// BacklogItem). reviewBacklog: LIST BacklogItem nodes (was a lexical /recall that never filtered by
// type). Both depend on the /api/neuron/list/<type> slice fix (neuron PR #58) to round-trip.
if str_eq(tool_name, "planWork") { return create_node_typed(args, "BacklogItem", "Working") }
if str_eq(tool_name, "reviewBacklog") { return list_typed("BacklogItem", 50, args) }
if str_eq(tool_name, "trackWork") { return evolve_by_supersede(args, "Memory") } if str_eq(tool_name, "trackWork") { return evolve_by_supersede(args, "Memory") }
if str_eq(tool_name, "listWork") { return list_typed("WorkContext", 50, args) } if str_eq(tool_name, "listWork") { return list_typed("WorkContext", 50, args) }
if str_eq(tool_name, "beginWork") { return create_typed_node(args, "Memory", "0.70") } if str_eq(tool_name, "beginWork") { return create_typed_node(args, "Memory", "0.70") }