Add Ollama provider, portable memory, cultivation digest, refugee importer, GLM-OCR spike
- P0: unified soul binary with engram_node_full fix, read-back-verify, search fix - P0: move API keys from plaintext plists to macOS Keychain - P0: fix MCP backend URL (port 8742 → 7770) - P1.6: memory-export/import scripts (AES-256-CBC, versioned .neuronmem format) - P1.7: nightly cultivation digest with sharpness metric (launchd at 23:55) - P2.10: Ollama provider in agentic loop (SOUL_LLM_PROVIDER=ollama) - P3.12: refugee importer for ChatGPT/Screenpipe/generic formats - P3.13: GLM-OCR spike — SHIP IT (mlx-vlm, 1.59GB, photo-to-memory.sh)
This commit is contained in:
+140
-2
@@ -25258,7 +25258,18 @@ el_val_t tier_canonical(void) {
|
||||
}
|
||||
|
||||
el_val_t mem_store(el_val_t content, el_val_t label, el_val_t tags) {
|
||||
return engram_node_full(content, EL_STR("Memory"), label, el_from_float(el_from_float(0.5)), el_from_float(el_from_float(0.5)), el_from_float(el_from_float(0.8)), EL_STR("Working"), tags);
|
||||
el_val_t id = engram_node_full(content, EL_STR("Memory"), label, el_from_float(el_from_float(0.5)), el_from_float(el_from_float(0.5)), el_from_float(el_from_float(0.8)), EL_STR("Working"), tags);
|
||||
if (str_eq(id, EL_STR(""))) {
|
||||
println(el_str_concat(EL_STR("[memory] write rejected by engram (empty id): label="), label));
|
||||
return EL_STR("");
|
||||
}
|
||||
el_val_t readback = engram_get_node_json(id);
|
||||
if (str_eq(readback, EL_STR("")) || str_eq(readback, EL_STR("{}"))) {
|
||||
println(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("[memory] WRITE VERIFY FAILED: label="), label), EL_STR(" id=")), id), EL_STR(" \xe2\x80\x94 node absent after write")));
|
||||
return EL_STR("");
|
||||
}
|
||||
println(el_str_concat(el_str_concat(EL_STR("[memory] write verified: "), id), EL_STR(" ok")));
|
||||
return id;
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -27028,6 +27039,27 @@ el_val_t next_bridge_id(void) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* === P2.10: Convert Anthropic tools format to OpenAI function-calling format === */
|
||||
el_val_t anthropic_tools_to_openai(el_val_t tools_json) {
|
||||
el_val_t len = json_array_len(tools_json);
|
||||
if (len <= 0) { return EL_STR("[]"); }
|
||||
el_val_t result = EL_STR("[");
|
||||
el_val_t i = 0;
|
||||
while (i < len) {
|
||||
el_val_t tool = json_array_get(tools_json, i);
|
||||
el_val_t tname = json_get(tool, EL_STR("name"));
|
||||
el_val_t tdesc = json_safe(json_get(tool, EL_STR("description")));
|
||||
el_val_t tschema = json_get_raw(tool, EL_STR("input_schema"));
|
||||
if (str_eq(tschema, EL_STR(""))) { tschema = EL_STR("{\"type\":\"object\",\"properties\":{}}"); }
|
||||
el_val_t oai_tool = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"type\":\"function\",\"function\":{\"name\":\""), tname), EL_STR("\",\"description\":\"")), tdesc), EL_STR("\",\"parameters\":")), tschema), EL_STR("}}"));
|
||||
if (i > 0) { result = el_str_concat(result, EL_STR(",")); }
|
||||
result = el_str_concat(result, oai_tool);
|
||||
i = (i + 1);
|
||||
}
|
||||
return el_str_concat(result, EL_STR("]"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
el_val_t agentic_loop(el_val_t session_id, el_val_t model, el_val_t safe_sys, el_val_t tools_json, el_val_t messages_in, el_val_t h, el_val_t tools_log_in) {
|
||||
el_val_t api_url = EL_STR("https://api.anthropic.com/v1/messages");
|
||||
el_val_t messages = messages_in;
|
||||
@@ -27039,6 +27071,87 @@ el_val_t agentic_loop(el_val_t session_id, el_val_t model, el_val_t safe_sys, el
|
||||
el_val_t pend_tool_id = EL_STR("");
|
||||
el_val_t pend_tool_name = EL_STR("");
|
||||
el_val_t pend_tool_input = EL_STR("");
|
||||
/* === P2.10: OLLAMA/OPENAI-COMPAT PROVIDER BRANCH === */
|
||||
{
|
||||
el_val_t _ol_prov = env(EL_STR("SOUL_LLM_PROVIDER"));
|
||||
if (str_eq(_ol_prov, EL_STR("ollama"))) {
|
||||
el_val_t _ol_model = env(EL_STR("SOUL_LLM_MODEL"));
|
||||
if (str_eq(_ol_model, EL_STR(""))) { _ol_model = env(EL_STR("OLLAMA_MODEL")); }
|
||||
if (str_eq(_ol_model, EL_STR(""))) { _ol_model = EL_STR("llama3.1"); }
|
||||
el_val_t _ol_base = env(EL_STR("OLLAMA_API_BASE"));
|
||||
if (str_eq(_ol_base, EL_STR(""))) { _ol_base = EL_STR("http://localhost:11434"); }
|
||||
el_val_t _ol_url = el_str_concat(_ol_base, EL_STR("/v1/chat/completions"));
|
||||
println(el_str_concat(el_str_concat(el_str_concat(EL_STR("[soul] provider: ollama @ "), _ol_base), EL_STR(" (model: ")), el_str_concat(_ol_model, EL_STR(")"))));
|
||||
el_val_t _ol_oai_tools = anthropic_tools_to_openai(tools_json);
|
||||
/* Build initial OpenAI-format messages: prepend system message to existing turns */
|
||||
el_val_t _ol_sys_msg = el_str_concat(el_str_concat(EL_STR("{\"role\":\"system\",\"content\":\""), safe_sys), EL_STR("\"}"));
|
||||
el_val_t _ol_msgs_inner = str_slice(messages_in, 1, (str_len(messages_in) - 1));
|
||||
el_val_t _ol_msgs = el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("["), _ol_sys_msg), EL_STR(",")), _ol_msgs_inner), EL_STR("]"));
|
||||
el_val_t _ol_h = el_map_new(0);
|
||||
map_set(_ol_h, EL_STR("content-type"), EL_STR("application/json"));
|
||||
el_val_t _ol_keep = 1;
|
||||
el_val_t _ol_iter = 0;
|
||||
el_val_t _ol_final = EL_STR("");
|
||||
while (_ol_keep && (_ol_iter < 8)) {
|
||||
el_val_t _ol_req = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"model\":\""), _ol_model), EL_STR("\",\"messages\":")), _ol_msgs), EL_STR(",\"stream\":false,\"tools\":")), _ol_oai_tools), EL_STR("}"));
|
||||
el_val_t _ol_resp = http_post_with_headers(_ol_url, _ol_req, _ol_h);
|
||||
if (str_eq(_ol_resp, EL_STR("")) || str_starts_with(_ol_resp, EL_STR("{\"error\""))) {
|
||||
return EL_STR("{\"error\":\"llm unavailable\",\"reply\":\"\"}");
|
||||
}
|
||||
el_val_t _ol_choices = json_get_raw(_ol_resp, EL_STR("choices"));
|
||||
if (str_eq(_ol_choices, EL_STR("")) || str_eq(_ol_choices, EL_STR("null"))) {
|
||||
return EL_STR("{\"error\":\"no choices in response\",\"reply\":\"\"}");
|
||||
}
|
||||
el_val_t _ol_c0 = json_array_get(_ol_choices, 0);
|
||||
el_val_t _ol_c0_msg = json_get_raw(_ol_c0, EL_STR("message"));
|
||||
el_val_t _ol_content = json_get(_ol_c0_msg, EL_STR("content"));
|
||||
el_val_t _ol_tcs = json_get_raw(_ol_c0_msg, EL_STR("tool_calls"));
|
||||
el_val_t _ol_has_tc = (!str_eq(_ol_tcs, EL_STR("")) && !str_eq(_ol_tcs, EL_STR("null")));
|
||||
el_val_t _ol_text = EL_STR("");
|
||||
if (!str_eq(_ol_content, EL_STR("")) && !str_eq(_ol_content, EL_STR("null"))) { _ol_text = _ol_content; }
|
||||
el_val_t _ol_tname = EL_STR("");
|
||||
el_val_t _ol_tid = EL_STR("");
|
||||
el_val_t _ol_tinput = EL_STR("");
|
||||
if (_ol_has_tc) {
|
||||
el_val_t _ol_tc0 = json_array_get(_ol_tcs, 0);
|
||||
_ol_tid = json_get(_ol_tc0, EL_STR("id"));
|
||||
el_val_t _ol_fn = json_get_raw(_ol_tc0, EL_STR("function"));
|
||||
_ol_tname = json_get(_ol_fn, EL_STR("name"));
|
||||
_ol_tinput = json_get(_ol_fn, EL_STR("arguments"));
|
||||
}
|
||||
el_val_t _ol_is_tool = (_ol_has_tc && !str_eq(_ol_tname, EL_STR("")));
|
||||
el_val_t _ol_result_raw = EL_STR("");
|
||||
if (_ol_is_tool) { _ol_result_raw = dispatch_tool(_ol_tname, _ol_tinput); }
|
||||
el_val_t _ol_result = _ol_result_raw;
|
||||
if (str_len(_ol_result_raw) > 6000) { _ol_result = el_str_concat(str_slice(_ol_result_raw, 0, 6000), EL_STR("...[truncated]")); }
|
||||
if (_ol_has_tc) {
|
||||
el_val_t _ol_tq = el_str_concat(el_str_concat(EL_STR("\""), _ol_tname), EL_STR("\""));
|
||||
if (str_eq(tools_log, EL_STR(""))) { tools_log = _ol_tq; } else { tools_log = el_str_concat(el_str_concat(tools_log, EL_STR(",")), _ol_tq); }
|
||||
}
|
||||
/* arguments must be re-serialized as JSON string for OpenAI assistant message */
|
||||
el_val_t _ol_tinput_escaped = el_str_concat(el_str_concat(EL_STR("\""), json_safe(_ol_tinput)), EL_STR("\""));
|
||||
if (_ol_is_tool) {
|
||||
/* Append assistant tool_call message and tool result to messages */
|
||||
el_val_t _ol_asst_tc = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\""), _ol_tid), EL_STR("\",\"type\":\"function\",\"function\":{\"name\":\"")), _ol_tname), EL_STR("\",\"arguments\":")), _ol_tinput_escaped), EL_STR("}}]}"));
|
||||
el_val_t _ol_tool_msg = el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"role\":\"tool\",\"tool_call_id\":\""), _ol_tid), EL_STR("\",\"content\":\"")), json_safe(_ol_result)), EL_STR("\"}"));
|
||||
el_val_t _ol_cur_inner = str_slice(_ol_msgs, 1, (str_len(_ol_msgs) - 1));
|
||||
_ol_msgs = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("["), _ol_cur_inner), EL_STR(",")), _ol_asst_tc), EL_STR(",")), _ol_tool_msg), EL_STR("]"));
|
||||
} else {
|
||||
_ol_final = _ol_text;
|
||||
_ol_keep = 0;
|
||||
}
|
||||
_ol_iter = (_ol_iter + 1);
|
||||
}
|
||||
if (str_eq(_ol_final, EL_STR(""))) {
|
||||
return EL_STR("{\"error\":\"no response\",\"reply\":\"\"}");
|
||||
}
|
||||
el_val_t _ol_safe_final = json_safe(_ol_final);
|
||||
el_val_t _ol_tools_arr = EL_STR("[]");
|
||||
if (!str_eq(tools_log, EL_STR(""))) { _ol_tools_arr = el_str_concat(el_str_concat(EL_STR("["), tools_log), EL_STR("]")); }
|
||||
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"reply\":\""), _ol_safe_final), EL_STR("\",\"model\":\"")), _ol_model), EL_STR("\",\"agentic\":true,\"tools_used\":")), _ol_tools_arr), EL_STR("}"));
|
||||
}
|
||||
}
|
||||
/* === END OLLAMA BRANCH === */
|
||||
while (keep_going && (iteration < 8)) {
|
||||
el_val_t req_body = el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("{\"model\":\""), model), EL_STR("\"")), EL_STR(",\"max_tokens\":4096")), EL_STR(",\"system\":\"")), safe_sys), EL_STR("\"")), EL_STR(",\"tools\":")), tools_json), EL_STR(",\"messages\":")), messages), EL_STR("}"));
|
||||
el_val_t raw_resp = http_post_with_headers(api_url, req_body, h);
|
||||
@@ -27236,7 +27349,16 @@ el_val_t handle_dharma_room_turn(el_val_t body) {
|
||||
}
|
||||
el_val_t clean_response = clean_llm_response(raw_response);
|
||||
el_val_t snap_path = state_get(EL_STR("soul_snapshot_path"));
|
||||
el_val_t discard_id = engram_node(clean_response, EL_STR("episodic"), el_from_float(el_from_float(0.6)));
|
||||
el_val_t utterance_tags = EL_STR("[\"soul-utterance\",\"episodic\"]");
|
||||
el_val_t discard_id = engram_node_full(clean_response, EL_STR("Conversation"), EL_STR("soul:utterance"), el_from_float(el_from_float(0.6)), el_from_float(el_from_float(0.6)), el_from_float(el_from_float(0.8)), EL_STR("Episodic"), utterance_tags);
|
||||
if (!str_eq(discard_id, EL_STR(""))) {
|
||||
el_val_t utterance_verify = engram_get_node_json(discard_id);
|
||||
if (str_eq(utterance_verify, EL_STR("")) || str_eq(utterance_verify, EL_STR("{}"))) {
|
||||
println(el_str_concat(el_str_concat(EL_STR("[memory] WRITE VERIFY FAILED: soul:utterance id="), discard_id), EL_STR(" \xe2\x80\x94 node absent after write")));
|
||||
} else {
|
||||
println(el_str_concat(el_str_concat(EL_STR("[memory] write verified: "), discard_id), EL_STR(" ok")));
|
||||
}
|
||||
}
|
||||
if (!str_eq(snap_path, EL_STR(""))) {
|
||||
el_val_t discard_save = engram_save(snap_path);
|
||||
}
|
||||
@@ -27750,6 +27872,14 @@ el_val_t handle_api_remember(el_val_t body) {
|
||||
el_val_t base_tags = ({ el_val_t _if_result_308 = 0; if (str_eq(tags_raw, EL_STR(""))) { _if_result_308 = (EL_STR("[\"Memory\"]")); } else { _if_result_308 = (tags_raw); } _if_result_308; });
|
||||
el_val_t final_tags = ({ el_val_t _if_result_309 = 0; if (str_eq(project, EL_STR(""))) { _if_result_309 = (base_tags); } else { el_val_t inner = str_slice(base_tags, 1, (str_len(base_tags) - 1)); _if_result_309 = (el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("["), inner), EL_STR(",\"project:")), project), EL_STR("\"]"))); } _if_result_309; });
|
||||
el_val_t id = engram_node_full(content, EL_STR("Memory"), EL_STR("memory:remembered"), el_from_float(sal), el_from_float(sal), el_from_float(el_from_float(0.9)), EL_STR("Episodic"), final_tags);
|
||||
if (str_eq(id, EL_STR(""))) {
|
||||
return EL_STR("{\"ok\":false,\"error\":\"write_not_persisted\",\"id\":\"\"}");
|
||||
}
|
||||
el_val_t remember_readback = engram_get_node_json(id);
|
||||
if (str_eq(remember_readback, EL_STR("")) || str_eq(remember_readback, EL_STR("{}"))) {
|
||||
println(el_str_concat(el_str_concat(EL_STR("[neuron-api] WRITE VERIFY FAILED remember id="), id), EL_STR(" \xe2\x80\x94 node absent after write")));
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"ok\":false,\"error\":\"write_not_persisted\",\"id\":\""), id), EL_STR("\"}"));
|
||||
}
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"id\":\""), id), EL_STR("\",\"ok\":true}"));
|
||||
return 0;
|
||||
}
|
||||
@@ -27804,6 +27934,14 @@ el_val_t handle_api_capture_knowledge(el_val_t body) {
|
||||
el_val_t full = ({ el_val_t _if_result_317 = 0; if (str_eq(title, EL_STR(""))) { _if_result_317 = (content); } else { _if_result_317 = (el_str_concat(el_str_concat(title, EL_STR(": ")), content)); } _if_result_317; });
|
||||
el_val_t tags = EL_STR("[\"Knowledge\",\"captured\"]");
|
||||
el_val_t id = engram_node_full(full, EL_STR("Knowledge"), EL_STR("knowledge:captured"), el_from_float(el_from_float(0.85)), el_from_float(el_from_float(0.8)), el_from_float(el_from_float(0.9)), EL_STR("Episodic"), tags);
|
||||
if (str_eq(id, EL_STR(""))) {
|
||||
return EL_STR("{\"ok\":false,\"error\":\"write_not_persisted\",\"id\":\"\"}");
|
||||
}
|
||||
el_val_t captured_readback = engram_get_node_json(id);
|
||||
if (str_eq(captured_readback, EL_STR("")) || str_eq(captured_readback, EL_STR("{}"))) {
|
||||
println(el_str_concat(el_str_concat(EL_STR("[neuron-api] WRITE VERIFY FAILED capture id="), id), EL_STR(" \xe2\x80\x94 node absent after write")));
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"ok\":false,\"error\":\"write_not_persisted\",\"id\":\""), id), EL_STR("\"}"));
|
||||
}
|
||||
return el_str_concat(el_str_concat(EL_STR("{\"id\":\""), id), EL_STR("\",\"ok\":true}"));
|
||||
return 0;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user