diff --git a/lang/el-compiler/runtime/el_runtime.c b/lang/el-compiler/runtime/el_runtime.c index 2f4c839..452e836 100644 --- a/lang/el-compiler/runtime/el_runtime.c +++ b/lang/el-compiler/runtime/el_runtime.c @@ -1475,10 +1475,13 @@ static void http_send_response(int fd, const char* body) { } const char* eff_body = is_envelope ? env_body : body; - /* Use the real byte count from fs_read if available (handles binary files - * with embedded null bytes — PNG, WOFF2, etc.). Fall back to strlen for - * normal text/JSON responses where _tl_fs_read_len is 0. */ - size_t blen = (_tl_fs_read_len > 0) ? _tl_fs_read_len : strlen(eff_body); + /* Use max(strlen, fs_read_len). fs_read_len is the real byte count for binary + * files (strlen stops at embedded NULs — PNG, WOFF2). strlen is correct AND larger + * when a handler WRAPS fs_read output in a longer text/JSON response (e.g. + * /api/safety-contact returns {"configured":...,"contact": }); using + * fs_read_len alone truncated those responses to the file's length. */ + size_t _blen_s = strlen(eff_body); + size_t blen = (_tl_fs_read_len > _blen_s) ? _tl_fs_read_len : _blen_s; _tl_fs_read_len = 0; /* consume — one-shot per response */ int head_only = _tl_http_head_only; @@ -1552,7 +1555,8 @@ static void* http_worker(void* arg) { /* Copy response out BEFORE arena teardown. * For binary files, _tl_fs_read_len holds the real byte count — * use memcpy instead of strdup so null bytes are preserved. */ - size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0); + size_t _rlen_s = rs ? strlen(rs) : 0; + size_t rlen = (_tl_fs_read_len > _rlen_s) ? _tl_fs_read_len : _rlen_s; response = malloc(rlen + 1); if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; } else if (response) { response[0] = '\0'; } @@ -1799,7 +1803,8 @@ static void* http_worker_v2(void* arg) { el_val_t hmap = http_build_headers_map(hdr_block ? hdr_block : ""); el_val_t r = h(EL_STR(dispatch_method), EL_STR(path), hmap, EL_STR(body)); const char* rs = EL_CSTR(r); - size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0); + size_t _rlen_s = rs ? strlen(rs) : 0; + size_t rlen = (_tl_fs_read_len > _rlen_s) ? _tl_fs_read_len : _rlen_s; response = malloc(rlen + 1); if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; } else if (response) { response[0] = '\0'; } @@ -1882,83 +1887,6 @@ el_val_t http_serve_v2(el_val_t port, el_val_t handler) { return 0; } -/* ── http_serve_async — non-blocking HTTP server ─────────────────────────── */ -/* Runs the accept loop in a background pthread, returns immediately so the - * calling EL script can continue (e.g. to run an awareness loop). - * - * El signature: http_serve_async(port, handler) -> Void */ - -typedef struct { int sock; } HttpServeAsyncArg; - -static void* _http_serve_async_loop(void* raw) { - HttpServeAsyncArg* a = (HttpServeAsyncArg*)raw; - int sock = a->sock; - free(a); - while (1) { - struct sockaddr_in6 cli; - socklen_t clen = sizeof(cli); - int cfd = accept(sock, (struct sockaddr*)&cli, &clen); - if (cfd < 0) { - if (errno == EINTR) continue; - perror("accept"); break; - } - pthread_mutex_lock(&_http_conn_mu); - while (_http_conn_active >= HTTP_MAX_CONNS) { - pthread_cond_wait(&_http_conn_cv, &_http_conn_mu); - } - _http_conn_active++; - pthread_mutex_unlock(&_http_conn_mu); - HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg)); - if (!arg) { close(cfd); continue; } - arg->fd = cfd; - pthread_t tid; - if (pthread_create(&tid, NULL, http_worker, arg) != 0) { - close(cfd); free(arg); - pthread_mutex_lock(&_http_conn_mu); - _http_conn_active--; - pthread_cond_signal(&_http_conn_cv); - pthread_mutex_unlock(&_http_conn_mu); - continue; - } - pthread_detach(tid); - } - close(sock); - return NULL; -} - -void http_serve_async(el_val_t port, el_val_t handler) { - const char* hname = EL_CSTR(handler); - if (hname && looks_like_string(handler)) { - http_set_handler(handler); - } - int p = (int)port; - if (p <= 0 || p > 65535) { fprintf(stderr, "http_serve_async: invalid port %d\n", p); return; } - int sock = socket(AF_INET6, SOCK_STREAM, 0); - if (sock < 0) { perror("socket"); return; } - int yes = 1; int no = 0; - setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); - setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &no, sizeof(no)); - struct sockaddr_in6 addr; - memset(&addr, 0, sizeof(addr)); - addr.sin6_family = AF_INET6; - addr.sin6_addr = in6addr_any; - addr.sin6_port = htons((uint16_t)p); - if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) { - perror("bind"); close(sock); return; - } - if (listen(sock, 64) < 0) { perror("listen"); close(sock); return; } - fprintf(stderr, "[http] async listening on [::]:%d (dual-stack)\n", p); - HttpServeAsyncArg* a = malloc(sizeof(HttpServeAsyncArg)); - if (!a) { close(sock); return; } - a->sock = sock; - pthread_t tid; - if (pthread_create(&tid, NULL, _http_serve_async_loop, a) != 0) { - perror("pthread_create"); free(a); close(sock); return; - } - pthread_detach(tid); - /* Returns immediately — caller can now run awareness_run() or any loop. */ -} - /* Build the response envelope a 4-arg handler can return. We hand-write * the JSON so the discriminator key always lands first — the runtime's * http_parse_envelope() detects it via prefix match. headers_json must be @@ -6322,7 +6250,9 @@ static void engram_grow_edges(void) { static char* engram_new_id(void) { el_val_t v = uuid_new(); const char* s = EL_CSTR(v); - return el_strdup(s ? s : ""); + /* Persistent: node ids live in the global store; an arena (el_strdup) id is + * freed at el_request_end(), corrupting the node after the creating request. */ + return el_strdup_persist(s ? s : ""); } /* Convert a node into an ElMap of its fields. */ @@ -6417,12 +6347,17 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label, const char* lb = EL_CSTR(label); const char* ti = EL_CSTR(tier); const char* tg = EL_CSTR(tags); - n->content = el_strdup(c ? c : ""); - n->node_type = el_strdup(nt && *nt ? nt : "Memory"); - n->label = el_strdup(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : "")); - n->tier = el_strdup(ti && *ti ? ti : "Working"); - n->tags = el_strdup(tg ? tg : ""); - n->metadata = el_strdup("{}"); + /* Persistent (el_strdup_persist, NOT el_strdup): these strings are owned by the + * persistent global node store. el_strdup tracks into the per-request arena, which + * el_request_end() frees when the creating HTTP request completes — leaving the + * stored node with dangling pointers (corrupted ids, "saved but never listed"). + * This is the root cause of the hallucinated/lost-saves class of bugs. */ + n->content = el_strdup_persist(c ? c : ""); + n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory"); + n->label = el_strdup_persist(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : "")); + n->tier = el_strdup_persist(ti && *ti ? ti : "Working"); + n->tags = el_strdup_persist(tg ? tg : ""); + n->metadata = el_strdup_persist("{}"); n->salience = engram_decode_score(salience); n->importance = engram_decode_score(importance); n->confidence = engram_decode_score(confidence); @@ -7992,257 +7927,6 @@ el_val_t engram_query_range(el_val_t start_ms_v, el_val_t end_ms_v) { return el_wrap_str(b.buf); } -/* engram_load_merge — like engram_load but WITHOUT resetting the store. - * Reads a JSON snapshot from `path` and adds any nodes/edges not already - * present in the in-memory graph. Dedup is by node id (for nodes) and by - * (from_id, to_id, relation) tuple (for edges). - * - * Returns (as an EL int) the count of new nodes added. Embeddings are - * intentionally skipped on merged nodes to avoid Ollama delays at runtime; - * auto_link_semantic will handle them when nodes are next activated. - * - * Does not merge layers — the in-process layer registry is authoritative. */ -el_val_t engram_load_merge(el_val_t path) { - const char* p = EL_CSTR(path); - if (!p || !*p) return 0; - FILE* f = fopen(p, "rb"); - if (!f) return 0; - fseek(f, 0, SEEK_END); - long sz = ftell(f); - rewind(f); - if (sz <= 0) { fclose(f); return 0; } - char* data = malloc((size_t)sz + 1); - if (!data) { fclose(f); return 0; } - size_t got = fread(data, 1, (size_t)sz, f); - fclose(f); - data[got] = '\0'; - - EngramStore* g = engram_get(); - int64_t added_nodes = 0; - - /* Walk nodes array — skip any node whose id already exists */ - const char* nodes_p = json_find_key(data, "nodes"); - if (nodes_p) { - nodes_p = eg_skip_ws(nodes_p); - if (*nodes_p == '[') { - nodes_p++; - nodes_p = eg_skip_ws(nodes_p); - while (*nodes_p && *nodes_p != ']') { - if (*nodes_p != '{') { nodes_p++; continue; } - const char* end = json_skip_value(nodes_p); - size_t n = (size_t)(end - nodes_p); - char* obj = malloc(n + 1); - memcpy(obj, nodes_p, n); obj[n] = '\0'; - char* nid = eg_get_str_field(obj, "id"); - int already = (nid && *nid && engram_find_node(nid) != NULL); - free(nid); - if (!already) { - engram_grow_nodes(); - EngramNode* nn = &g->nodes[g->node_count]; - memset(nn, 0, sizeof(*nn)); - nn->id = eg_get_str_field(obj, "id"); - nn->content = eg_get_str_field(obj, "content"); - nn->node_type = eg_get_str_field(obj, "node_type"); - nn->label = eg_get_str_field(obj, "label"); - nn->tier = eg_get_str_field(obj, "tier"); - nn->tags = eg_get_str_field(obj, "tags"); - nn->metadata = eg_get_str_field(obj, "metadata"); - if (!nn->metadata || !*nn->metadata) { free(nn->metadata); nn->metadata = strdup("{}"); } - nn->salience = eg_get_num_field(obj, "salience"); - nn->importance = eg_get_num_field(obj, "importance"); - nn->confidence = eg_get_num_field(obj, "confidence"); - nn->temporal_decay_rate = eg_get_num_field(obj, "temporal_decay_rate"); - nn->activation_count = eg_get_int_field(obj, "activation_count"); - nn->last_activated = eg_get_int_field(obj, "last_activated"); - nn->created_at = eg_get_int_field(obj, "created_at"); - nn->updated_at = eg_get_int_field(obj, "updated_at"); - nn->background_activation = eg_get_num_field(obj, "background_activation"); - nn->working_memory_weight = eg_get_num_field(obj, "working_memory_weight"); - if (!isfinite(nn->working_memory_weight) || nn->working_memory_weight < 0.0 || nn->working_memory_weight > 1.0) - nn->working_memory_weight = 0.0; /* clamp corrupt snapshot values */ - nn->suppression_count = (int32_t)eg_get_int_field(obj, "suppression_count"); - if (json_find_key(obj, "layer_id")) { - nn->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); - } else { - nn->layer_id = ENGRAM_LAYER_DEFAULT; - } - g->node_count++; - added_nodes++; - } - free(obj); - nodes_p = end; - nodes_p = eg_skip_ws(nodes_p); - if (*nodes_p == ',') { nodes_p++; nodes_p = eg_skip_ws(nodes_p); } - } - } - } - - /* Walk edges array — skip if (from_id, to_id, relation) already present */ - const char* edges_p = json_find_key(data, "edges"); - if (edges_p) { - edges_p = eg_skip_ws(edges_p); - if (*edges_p == '[') { - edges_p++; - edges_p = eg_skip_ws(edges_p); - while (*edges_p && *edges_p != ']') { - if (*edges_p != '{') { edges_p++; continue; } - const char* end = json_skip_value(edges_p); - size_t n = (size_t)(end - edges_p); - char* obj = malloc(n + 1); - memcpy(obj, edges_p, n); obj[n] = '\0'; - char* efrom = eg_get_str_field(obj, "from_id"); - char* eto = eg_get_str_field(obj, "to_id"); - char* erel = eg_get_str_field(obj, "relation"); - /* Check for duplicate by scanning existing edges */ - int dup = 0; - if (efrom && eto && erel) { - for (int64_t ei = 0; ei < g->edge_count; ei++) { - EngramEdge* ex = &g->edges[ei]; - if (ex->from_id && ex->to_id && ex->relation && - strcmp(ex->from_id, efrom) == 0 && - strcmp(ex->to_id, eto) == 0 && - strcmp(ex->relation, erel) == 0) { - dup = 1; break; - } - } - } - if (!dup) { - engram_grow_edges(); - EngramEdge* ee = &g->edges[g->edge_count]; - memset(ee, 0, sizeof(*ee)); - ee->id = eg_get_str_field(obj, "id"); - ee->from_id = efrom ? efrom : strdup(""); - ee->to_id = eto ? eto : strdup(""); - ee->relation = erel ? erel : strdup(""); - ee->metadata = eg_get_str_field(obj, "metadata"); - if (!ee->metadata || !*ee->metadata) { free(ee->metadata); ee->metadata = strdup("{}"); } - ee->weight = eg_get_num_field(obj, "weight"); - ee->confidence = eg_get_num_field(obj, "confidence"); - ee->created_at = eg_get_int_field(obj, "created_at"); - ee->updated_at = eg_get_int_field(obj, "updated_at"); - ee->last_fired = eg_get_int_field(obj, "last_fired"); - ee->inhibitory = (int)eg_get_int_field(obj, "inhibitory"); - if (json_find_key(obj, "layer_id")) { - ee->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); - } else { - ee->layer_id = ENGRAM_LAYER_DEFAULT; - } - g->edge_count++; - /* NOTE: efrom/eto/erel ownership transferred to ee above */ - efrom = NULL; eto = NULL; erel = NULL; - } else { - free(efrom); free(eto); free(erel); - } - free(obj); - edges_p = end; - edges_p = eg_skip_ws(edges_p); - if (*edges_p == ',') { edges_p++; edges_p = eg_skip_ws(edges_p); } - } - } - } - - free(data); - return (el_val_t)added_nodes; -} - -el_val_t engram_wm_count(void) { - EngramStore* g = engram_get(); - int64_t count = 0; - for (int64_t i = 0; i < g->node_count; i++) { - if (g->nodes[i].working_memory_weight > 0.0) count++; - } - return (el_val_t)count; -} - -/* Average working_memory_weight across all promoted nodes (wm > 0). - * Returns the float bit-pattern via el_from_float so EL can use it with - * float_to_str / float_gt. Returns 0.0 when no nodes are promoted. - * Useful in heartbeat ISEs to distinguish "many weak activations" (sparse - * graph, low avg) from "few strong activations" (dense subgraph, high avg). - * Added 2026-06-04 self-review for graph health observability. */ -el_val_t engram_wm_avg_weight(void) { - EngramStore* g = engram_get(); - double sum = 0.0; - int64_t count = 0; - for (int64_t i = 0; i < g->node_count; i++) { - double w = g->nodes[i].working_memory_weight; - /* Defensive guard: skip any corrupt/out-of-range values so a single - * bad snapshot node doesn't produce a garbage average (e.g. 1.77e+234). */ - if (w > 0.0 && w <= 1.0 && isfinite(w)) { sum += w; count++; } - } - double avg = (count > 0) ? (sum / (double)count) : 0.0; - return el_from_float(avg); -} - -/* engram_wm_top_json — return top N working-memory nodes (by wm weight) as a - * compact JSON array for ISE heartbeat reporting. - * - * Each element: {"label":"...","node_type":"...","tier":"...","wm":0.42} - * - * Purpose: the heartbeat ISE reports wm_active (count) and wm_avg_weight but - * gives zero visibility into WM *composition* — which types/tiers are active. - * After long uptime every WM slot is in steady-state decay+re-promotion so - * wm_promotion ISEs never fire (they only fire on 0→>0.1 transitions). - * This function fills the observability gap by snapshotting the current top-N - * WM nodes on every heartbeat. Inserted 2026-06-05 self-review. */ -el_val_t engram_wm_top_json(el_val_t n_v) { - int64_t top_n = (int64_t)n_v; - if (top_n <= 0) top_n = 10; - if (top_n > 50) top_n = 50; - EngramStore* g = engram_get(); - - /* Collect indices of promoted nodes, excluding monitoring noise. - * InternalStateEvent nodes are system-observation artifacts — they reflect - * what the daemon is doing, not what it knows. Including them in wm_top - * buries real knowledge (Memory, Knowledge, Belief nodes) under a wall of - * heartbeat/curiosity ISEs, making the heartbeat ISE useless for diagnosing - * WM composition. Filter them out here so wm_top always shows substantive - * content. (2026-06-07 self-review) */ - int64_t* idx = malloc((size_t)(g->node_count + 1) * sizeof(int64_t)); - if (!idx) return el_wrap_str(el_strdup("[]")); - int64_t mc = 0; - for (int64_t i = 0; i < g->node_count; i++) { - if (g->nodes[i].working_memory_weight > 0.0) { - const char* nt = g->nodes[i].node_type; - if (nt && strcmp(nt, "InternalStateEvent") == 0) continue; - idx[mc++] = i; - } - } - - /* Insertion-sort descending by wm weight (mc is typically small). */ - for (int64_t i = 1; i < mc; i++) { - int64_t key = idx[i]; - double kw = g->nodes[key].working_memory_weight; - int64_t j = i; - while (j > 0 && g->nodes[idx[j-1]].working_memory_weight < kw) { - idx[j] = idx[j-1]; j--; - } - idx[j] = key; - } - - int64_t emit = mc < top_n ? mc : top_n; - JsonBuf b; jb_init(&b); - jb_putc(&b, '['); - for (int64_t k = 0; k < emit; k++) { - EngramNode* n = &g->nodes[idx[k]]; - if (k > 0) jb_putc(&b, ','); - jb_putc(&b, '{'); - jb_puts(&b, "\"label\":"); - jb_emit_escaped(&b, n->label ? n->label : ""); - jb_puts(&b, ",\"node_type\":"); - jb_emit_escaped(&b, n->node_type ? n->node_type : ""); - jb_puts(&b, ",\"tier\":"); - jb_emit_escaped(&b, n->tier ? n->tier : ""); - char tmp[48]; - snprintf(tmp, sizeof(tmp), ",\"wm\":%.3f", n->working_memory_weight); - jb_puts(&b, tmp); - jb_putc(&b, '}'); - } - free(idx); - jb_putc(&b, ']'); - return el_wrap_str(b.buf); -} - #ifdef HAVE_CURL /* ── DHARMA network ───────────────────────────────────────────────────────── * Real implementation. Peers are addressed by `dharma_id` — either bare @@ -8883,7 +8567,7 @@ static el_val_t llm_provider_request(const char* url, const char* key, } } -static el_val_t llm_chain_call(const char* model_pref, const char* system_str, const char* user_str) { +static el_val_t llm_chain_call(const char* system_str, const char* user_str) { char url_key[64], key_key[64], fmt_key[64], model_key[64]; for (int i = 0; i < LLM_MAX_PROVIDERS; i++) { snprintf(url_key, sizeof(url_key), "NEURON_LLM_%d_URL", i); @@ -8896,7 +8580,6 @@ static el_val_t llm_chain_call(const char* model_pref, const char* system_str, c const char* fmt_s = getenv(fmt_key); int fmt = (fmt_s && strcmp(fmt_s, "anthropic") == 0) ? 1 : 0; const char* model = getenv(model_key); - if (!model || !*model) model = model_pref; /* fall back to the caller-requested model */ fprintf(stderr, "[llm] trying provider %d (%s)\n", i, url); el_val_t result = llm_provider_request(url, key, fmt, model, system_str, user_str); const char* t = EL_CSTR(result); @@ -8907,7 +8590,7 @@ static el_val_t llm_chain_call(const char* model_pref, const char* system_str, c const char* api_key = getenv("ANTHROPIC_API_KEY"); if (!api_key || !*api_key) return http_error_json("no LLM providers configured"); fprintf(stderr, "[llm] using legacy ANTHROPIC_API_KEY fallback\n"); - return llm_provider_request(LLM_API_URL, api_key, 1, model_pref, system_str, user_str); + return llm_provider_request(LLM_API_URL, api_key, 1, NULL, system_str, user_str); } /* Legacy llm_request — kept for backward compat with agentic loop internals */ @@ -8971,16 +8654,14 @@ static el_val_t llm_extract_text(el_val_t resp_val) { } el_val_t llm_call(el_val_t model, el_val_t prompt) { - const char* m = EL_CSTR(model); const char* u = EL_CSTR(prompt); if (!u) u = ""; - return llm_chain_call(m, NULL, u); + return llm_chain_call(NULL, u); } el_val_t llm_call_system(el_val_t model, el_val_t system_prompt, el_val_t user_prompt) { - const char* m = EL_CSTR(model); const char* s = EL_CSTR(system_prompt); if (!s) s = ""; const char* u = EL_CSTR(user_prompt); if (!u) u = ""; - return llm_chain_call(m, s, u); + return llm_chain_call(s, u); } /* ── Tool registry for llm_call_agentic ─────────────────────────────────── */ diff --git a/lang/el-compiler/runtime/el_runtime.h b/lang/el-compiler/runtime/el_runtime.h index 4ee01d8..2f9583f 100644 --- a/lang/el-compiler/runtime/el_runtime.h +++ b/lang/el-compiler/runtime/el_runtime.h @@ -176,7 +176,6 @@ el_val_t http_set_handler(el_val_t name); * existing handlers (e.g. products/web/server.el): it dispatches with * (method, path, body), hardcodes 200 OK, and auto-detects content type. */ el_val_t http_serve_v2(el_val_t port, el_val_t handler); -void http_serve_async(el_val_t port, el_val_t handler); el_val_t http_set_handler_v2(el_val_t name); /* Build an HTTP response envelope. `headers_json` should be a JSON object @@ -639,12 +638,6 @@ el_val_t engram_list_layers_json(void); * no nodes promoted to working memory. */ el_val_t engram_compile_layered_json(el_val_t intent, el_val_t depth); -/* ── Working memory ──────────────────────────────────────────────────────────*/ -el_val_t engram_wm_count(void); -el_val_t engram_wm_avg_weight(void); -el_val_t engram_wm_top_json(el_val_t n); -el_val_t engram_load_merge(el_val_t path); - /* ── LLM (Anthropic API client) ───────────────────────────────────────────── * All functions call https://api.anthropic.com/v1/messages with the API key * from env ANTHROPIC_API_KEY. Default model when empty: claude-sonnet-4-5. */