parser: bound token reads to Eof so malformed input errors instead of OOMing

Out-of-range tok_kind/tok_value reads returned runtime null (el_list_get OOB -> 0) rather than the Eof sentinel, so the inner parse loops (parse_block, call-arg, array-literal, match-arm) that terminate only on their close delimiter or k=="Eof" never saw Eof once the cursor ran past the single trailing Eof token. On unclosed-delimiter input the parser then appended AST nodes forever -> unbounded allocation -> ~700GB -> OOM (observed compiling neuron/sessions.el). Fix at the choke point: tok_kind returns "Eof" and tok_value returns "" for out-of-range positions, restoring the parser-wide contract that reads at/after the end yield Eof. expect() no longer steps past the Eof sentinel on mismatch. This terminates every overrun loop simultaneously; a malformed program now surfaces as a normal (best-effort) parse end instead of exhausting memory. Requires a self-hosted bootstrap rebuild of elc to take effect.
runtime: fix the memory leak + write-corruption pair in el_runtime.c
2026-07-14 14:21:39 -05:00 · 2026-07-13 16:22:02 -05:00 · 2026-07-01 16:30:17 +00:00 · 2026-07-01 11:27:54 -05:00 · 2026-07-01 11:25:09 -05:00 · 2026-07-01 11:21:23 -05:00
6 changed files with 857 additions and 227 deletions
@@ -20,11 +20,11 @@ el_val_t route_create_edge(el_val_t method, el_val_t path, el_val_t body);
 el_val_t route_neighbors(el_val_t method, el_val_t path, el_val_t body);
 el_val_t route_strengthen(el_val_t method, el_val_t path, el_val_t body);
 el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body);
+el_val_t route_create_ise(el_val_t method, el_val_t path, el_val_t body);
+el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body);
 el_val_t route_save(el_val_t method, el_val_t path, el_val_t body);
 el_val_t route_load(el_val_t method, el_val_t path, el_val_t body);
 el_val_t route_health(el_val_t method, el_val_t path, el_val_t body);
-el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body);
-el_val_t route_emit_ise(el_val_t method, el_val_t path, el_val_t body);
 el_val_t check_auth_ok(el_val_t method, el_val_t body);
 el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body);

@@ -245,6 +245,34 @@ el_val_t route_forget(el_val_t method, el_val_t path, el_val_t body) {
  return 0;
 }

+el_val_t route_create_ise(el_val_t method, el_val_t path, el_val_t body) {
+  el_val_t content = json_get_string(body, EL_STR("content"));
+  if (str_eq(content, EL_STR(""))) {
+    return err_json(EL_STR("missing content"));
+  }
+  el_val_t sal = el_from_float(0.3);
+  el_val_t imp = el_from_float(0.3);
+  el_val_t conf = el_from_float(0.8);
+  el_val_t id = engram_node_full(content, EL_STR("InternalStateEvent"), EL_STR("state-event"), sal, imp, conf, EL_STR("Episodic"), EL_STR("[\"internal-state\",\"InternalStateEvent\"]"));
+  return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\"}"));
+  return 0;
+}
+
+el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body) {
+  el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR"));
+  if (str_eq(dir, EL_STR(""))) {
+    dir = EL_STR("/tmp/engram");
+  }
+  el_val_t snap_path = el_str_concat(dir, EL_STR("/sync-export.json"));
+  engram_save(snap_path);
+  el_val_t snap = fs_read(snap_path);
+  if (str_eq(snap, EL_STR(""))) {
+    return EL_STR("{\"nodes\":[],\"edges\":[]}");
+  }
+  return snap;
+  return 0;
+}
+
 el_val_t route_save(el_val_t method, el_val_t path, el_val_t body) {
  el_val_t p = json_get_string(body, EL_STR("path"));
  if (str_eq(p, EL_STR(""))) {
@@ -278,34 +306,6 @@ el_val_t route_health(el_val_t method, el_val_t path, el_val_t body) {
  return 0;
 }

-el_val_t route_sync(el_val_t method, el_val_t path, el_val_t body) {
-  el_val_t dir = env(EL_STR("ENGRAM_DATA_DIR"));
-  if (str_eq(dir, EL_STR(""))) {
-    dir = EL_STR("/tmp/engram");
-  }
-  el_val_t snap_path = el_str_concat(dir, EL_STR("/snapshot.json"));
-  engram_save(snap_path);
-  el_val_t snap = fs_read(snap_path);
-  if (str_eq(snap, EL_STR(""))) {
-    return EL_STR("{\"nodes\":[],\"edges\":[]}");
-  }
-  return snap;
-  return 0;
-}
-
-el_val_t route_emit_ise(el_val_t method, el_val_t path, el_val_t body) {
-  el_val_t content = json_get_string(body, EL_STR("content"));
-  if (str_eq(content, EL_STR(""))) {
-    return err_json(EL_STR("missing content"));
-  }
-  el_val_t sal = el_from_float(0.3);
-  el_val_t imp = el_from_float(0.3);
-  el_val_t conf = el_from_float(0.8);
-  el_val_t id = engram_node_full(content, EL_STR("InternalStateEvent"), EL_STR("state-event"), sal, imp, conf, EL_STR("Episodic"), EL_STR("[\"internal-state\",\"InternalStateEvent\"]"));
-  return el_str_concat(el_str_concat(EL_STR("{\"ok\":true,\"id\":\""), id), EL_STR("\"}"));
-  return 0;
-}
-
 el_val_t check_auth_ok(el_val_t method, el_val_t body) {
  el_val_t key = env(EL_STR("ENGRAM_API_KEY"));
  if (str_eq(key, EL_STR(""))) {
@@ -329,8 +329,8 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
      return route_health(method, path, body);
    }
  }
-  if (str_eq(method, EL_STR("POST")) && str_eq(clean, EL_STR("/api/neuron/state-events"))) {
-    return route_emit_ise(method, path, body);
+  if (str_eq(method, EL_STR("POST")) && str_starts_with(clean, EL_STR("/api/neuron/state-events"))) {
+    return route_create_ise(method, path, body);
  }
  if (!check_auth_ok(method, body)) {
    return err_json(EL_STR("unauthorized"));
@@ -374,15 +374,15 @@ el_val_t handle_request(el_val_t method, el_val_t path, el_val_t body) {
  if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/strengthen")) || str_eq(clean, EL_STR("/strengthen")))) {
    return route_strengthen(method, path, body);
  }
+  if (str_eq(method, EL_STR("GET")) && (str_eq(clean, EL_STR("/api/sync")) || str_eq(clean, EL_STR("/sync")))) {
+    return route_sync(method, path, body);
+  }
  if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/save")) || str_eq(clean, EL_STR("/save")))) {
    return route_save(method, path, body);
  }
  if (str_eq(method, EL_STR("POST")) && (str_eq(clean, EL_STR("/api/load")) || str_eq(clean, EL_STR("/load")))) {
    return route_load(method, path, body);
  }
-  if (str_eq(method, EL_STR("GET")) && str_eq(clean, EL_STR("/api/sync"))) {
-    return route_sync(method, path, body);
-  }
  return el_str_concat(el_str_concat(EL_STR("{\"error\":\"not found\",\"path\":\""), clean), EL_STR("\"}"));
  return 0;
 }
@@ -710,6 +710,7 @@ static void jb_init(JsonBuf* b);
 static void jb_putc(JsonBuf* b, char c);
 static void jb_puts(JsonBuf* b, const char* s);
 static void jb_emit_escaped(JsonBuf* b, const char* s);
+static char* jb_finish(JsonBuf* b);
 static int  looks_like_string(el_val_t v);
 static const char* json_find_key(const char* s, const char* key);
 static const char* json_skip_value(const char* p);
@@ -2014,7 +2015,7 @@ el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body) {
    jb_puts(&out, ",\"body\":");
    jb_emit_escaped(&out, b);
    jb_putc(&out, '}');
-    return el_wrap_str(out.buf);
+    return el_wrap_str(jb_finish(&out));
 }

 /* ── Filesystem ──────────────────────────────────────────────────────────── */
@@ -2121,7 +2122,7 @@ el_val_t exec_capture(el_val_t cmdv) {
    char buf[4096];
    while (fgets(buf, sizeof(buf), f)) jb_puts(&b, buf);
    pclose(f);
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 // exec — run a shell command via /bin/sh, capture stdout, return as String.
@@ -2145,7 +2146,7 @@ el_val_t exec(el_val_t cmdv) {
        jb_puts(&b, buf);
    }
    pclose(f);
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 // exec_bg — run a shell command in background, return PID as String.
@@ -3292,6 +3293,23 @@ static void jb_puts(JsonBuf* b, const char* s) {
    b->buf[b->len] = '\0';
 }

+/* jb_finish — call exactly once, at the point a JsonBuf's buffer is handed
+ * off as an el_val_t return value (via el_wrap_str). JsonBuf allocates through
+ * raw malloc/realloc (jb_init/jb_reserve), NOT el_strdup/el_strbuf, so unlike
+ * those helpers it was never registered with the per-request arena — every
+ * JsonBuf-built JSON string leaked unconditionally, whether or not a request
+ * arena was active, because nothing ever freed it. el_arena_track() is a
+ * documented no-op when no arena is active, so this is safe to call from any
+ * context (HTTP request handler, CLI arena-scope, or no arena at all — the
+ * last case still leaks, same as before, until the caller's context is itself
+ * arena-scoped). Do NOT call this on a JsonBuf whose ->buf is consumed by
+ * another builder (jb_puts(&other, x.buf)) and freed manually — only on the
+ * one actually returned via el_wrap_str. */
+static char* jb_finish(JsonBuf* b) {
+    el_arena_track(b->buf);
+    return b->buf;
+}
+
 static void jb_emit_escaped(JsonBuf* b, const char* s) {
    jb_putc(b, '"');
    const unsigned char* p = (const unsigned char*)s;
@@ -3400,7 +3418,7 @@ static void jb_emit_value(JsonBuf* b, el_val_t v) {
 el_val_t json_stringify(el_val_t v) {
    JsonBuf b; jb_init(&b);
    jb_emit_value(&b, v);
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 /* ── JSON substring accessors ────────────────────────────────────────────── */
@@ -3587,7 +3605,7 @@ el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value) {
        jb_putc(&b, ':');
        jb_puts(&b, raw_val);
        jb_putc(&b, '}');
-        return el_wrap_str(b.buf);
+        return el_wrap_str(jb_finish(&b));
    }
    const char* existing = json_find_key(json, k);
    JsonBuf b; jb_init(&b);
@@ -3601,7 +3619,7 @@ el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value) {
        b.buf[b.len] = '\0';
        jb_puts(&b, raw_val);
        jb_puts(&b, end);
-        return el_wrap_str(b.buf);
+        return el_wrap_str(jb_finish(&b));
    }
    /* Insert before closing '}'. Find last '}' */
    size_t jl = strlen(json);
@@ -3633,7 +3651,7 @@ el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value) {
    jb_puts(&b, raw_val);
    /* Append from close_idx onward */
    jb_puts(&b, json + close_idx);
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 el_val_t json_array_len(el_val_t json_str) {
@@ -3771,7 +3789,7 @@ el_val_t json_build_object(el_val_t kvs) {
        jb_putc(&b, '"');
    }
    jb_putc(&b, '}');
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 /* json_build_array — build a JSON array from a list of raw JSON values.
@@ -3789,7 +3807,7 @@ el_val_t json_build_array(el_val_t items) {
        jb_puts(&b, vs);
    }
    jb_putc(&b, ']');
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 /* ── Time ────────────────────────────────────────────────────────────────── */
@@ -5193,7 +5211,7 @@ el_val_t state_keys(void) {
    }
    jb_putc(&b, ']');
    pthread_mutex_unlock(&_state_mu);
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 /* Returns 1 (true) if the key is present in the state store, else 0 (false). */
@@ -5399,7 +5417,7 @@ el_val_t str_format(el_val_t fmt, el_val_t data) {
        jb_putc(&b, *p);
        p++;
    }
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 el_val_t str_lower(el_val_t s) { return str_to_lower(s); }
@@ -5815,7 +5833,7 @@ el_val_t list_join(el_val_t listv, el_val_t sep) {
            jb_puts(&b, tmp);
        }
    }
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 el_val_t list_range(el_val_t start, el_val_t end) {
@@ -6465,12 +6483,12 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) {
    n->id        = engram_new_id();
    const char* c  = EL_CSTR(content);
    const char* nt = EL_CSTR(node_type);
-    n->content   = el_strdup(c ? c : "");
-    n->node_type = el_strdup(nt && *nt ? nt : "Memory");
-    n->label     = engram_first_n_chars(c, 60);
-    n->tier      = el_strdup("Working");
-    n->tags      = el_strdup("");
-    n->metadata  = el_strdup("{}");
+    n->content   = el_strdup_persist(c ? c : "");
+    n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory");
+    { char* _lb = engram_first_n_chars(c, 60); n->label = el_strdup_persist(_lb); } /* persist: stored field must outlive request arena */
+    n->tier      = el_strdup_persist("Working");
+    n->tags      = el_strdup_persist("");
+    n->metadata  = el_strdup_persist("{}");
    n->salience    = engram_decode_score(salience);
    if (n->salience <= 0.0 || n->salience > 1.0) n->salience = 0.5;
    n->importance  = 0.5;
@@ -6597,11 +6615,11 @@ el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t labe
    const char* lb = EL_CSTR(label);
    const char* tg = EL_CSTR(tags);
    const char* st = EL_CSTR(status);
-    n->content   = el_strdup(c ? c : "");
-    n->node_type = el_strdup(nt && *nt ? nt : "Memory");
-    n->label     = el_strdup(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : ""));
-    n->tier      = el_strdup("Working");
-    n->tags      = el_strdup(tg ? tg : "");
+    n->content   = el_strdup_persist(c ? c : "");
+    n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory");
+    n->label     = el_strdup_persist(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : ""));
+    n->tier      = el_strdup_persist("Working");
+    n->tags      = el_strdup_persist(tg ? tg : "");
    if (st && *st) {
        /* Minimal metadata payload: {"status":"..."}. Keep it cheap so
         * callers using `status` don't pay JSON parse cost on every read. */
@@ -6610,7 +6628,7 @@ el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t labe
        snprintf(meta, sl, "{\"status\":\"%s\"}", st);
        n->metadata = meta;
    } else {
-        n->metadata = el_strdup("{}");
+        n->metadata = el_strdup_persist("{}");
    }
    n->salience    = engram_decode_score(salience);
    n->importance  = engram_decode_score(certainty);
@@ -6883,10 +6901,10 @@ void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t
    EngramEdge* e = &g->edges[g->edge_count];
    memset(e, 0, sizeof(*e));
    e->id        = engram_new_id();
-    e->from_id   = el_strdup(f);
-    e->to_id     = el_strdup(t);
-    e->relation  = el_strdup(r && *r ? r : "associate");
-    e->metadata  = el_strdup("{}");
+    e->from_id   = el_strdup_persist(f);
+    e->to_id     = el_strdup_persist(t);
+    e->relation  = el_strdup_persist(r && *r ? r : "associate");
+    e->metadata  = el_strdup_persist("{}");
    e->weight    = engram_decode_score(weight);
    if (e->weight <= 0.0 || e->weight > 1.0) e->weight = 0.5;
    e->confidence = 1.0;
@@ -7519,11 +7537,11 @@ el_val_t engram_save(el_val_t path) {
 /* Helper: extract a string field from a JSON object substring. */
 static char* eg_get_str_field(const char* obj, const char* key) {
    const char* p = json_find_key(obj, key);
-    if (!p) return el_strdup("");
-    if (*p != '"') return el_strdup("");
+    if (!p) return el_strdup_persist("");
+    if (*p != '"') return el_strdup_persist("");
    JsonParser jp = { .p = p, .end = p + strlen(p), .err = 0 };
    char* out = jp_parse_string_raw(&jp);
-    if (jp.err) { free(out); return el_strdup(""); }
+    if (jp.err) { free(out); return el_strdup_persist(""); }
    return out;
 }

@@ -7597,7 +7615,7 @@ el_val_t engram_load(el_val_t path) {
                nn->tier      = eg_get_str_field(obj, "tier");
                nn->tags      = eg_get_str_field(obj, "tags");
                nn->metadata  = eg_get_str_field(obj, "metadata");
-                if (!nn->metadata || !*nn->metadata) { free(nn->metadata); nn->metadata = el_strdup("{}"); }
+                if (!nn->metadata || !*nn->metadata) { free(nn->metadata); nn->metadata = el_strdup_persist("{}"); }
                nn->salience           = eg_get_num_field(obj, "salience");
                nn->importance         = eg_get_num_field(obj, "importance");
                nn->confidence         = eg_get_num_field(obj, "confidence");
@@ -7648,7 +7666,7 @@ el_val_t engram_load(el_val_t path) {
                ee->to_id    = eg_get_str_field(obj, "to_id");
                ee->relation = eg_get_str_field(obj, "relation");
                ee->metadata = eg_get_str_field(obj, "metadata");
-                if (!ee->metadata || !*ee->metadata) { free(ee->metadata); ee->metadata = el_strdup("{}"); }
+                if (!ee->metadata || !*ee->metadata) { free(ee->metadata); ee->metadata = el_strdup_persist("{}"); }
                ee->weight     = eg_get_num_field(obj, "weight");
                ee->confidence = eg_get_num_field(obj, "confidence");
                ee->created_at = eg_get_int_field(obj, "created_at");
@@ -7740,7 +7758,7 @@ el_val_t engram_get_node_json(el_val_t id) {
    if (!n) return el_wrap_str(el_strdup("{}"));
    JsonBuf b; jb_init(&b);
    engram_emit_node_json(&b, n);
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 el_val_t engram_search_json(el_val_t query, el_val_t limit) {
@@ -7768,7 +7786,7 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
        }
    }
    jb_putc(&b, ']');
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset) {
@@ -7777,9 +7795,9 @@ el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset) {
    int64_t off = (int64_t)offset; if (off < 0) off = 0;
    JsonBuf b; jb_init(&b);
    jb_putc(&b, '[');
-    if (g->node_count == 0) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
+    if (g->node_count == 0) { jb_putc(&b, ']'); return el_wrap_str(jb_finish(&b)); }
    int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t));
-    if (!idx) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
+    if (!idx) { jb_putc(&b, ']'); return el_wrap_str(jb_finish(&b)); }
    /* Skip transparent layers — introspection filter, same as engram_scan_nodes. */
    int64_t live = 0;
    for (int64_t i = 0; i < g->node_count; i++) {
@@ -7797,7 +7815,7 @@ el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset) {
    }
    free(idx);
    jb_putc(&b, ']');
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 /* engram_scan_nodes_by_type_json — filter by node_type before paginating.
@@ -7813,9 +7831,9 @@ el_val_t engram_scan_nodes_by_type_json(el_val_t type_v, el_val_t limit, el_val_
    int64_t off = (int64_t)offset; if (off < 0) off = 0;
    JsonBuf b; jb_init(&b);
    jb_putc(&b, '[');
-    if (g->node_count == 0) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
+    if (g->node_count == 0) { jb_putc(&b, ']'); return el_wrap_str(jb_finish(&b)); }
    int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t));
-    if (!idx) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
+    if (!idx) { jb_putc(&b, ']'); return el_wrap_str(jb_finish(&b)); }
    int64_t live = 0;
    for (int64_t i = 0; i < g->node_count; i++) {
        if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue;
@@ -7834,7 +7852,7 @@ el_val_t engram_scan_nodes_by_type_json(el_val_t type_v, el_val_t limit, el_val_
    }
    free(idx);
    jb_putc(&b, ']');
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction) {
@@ -7849,7 +7867,7 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t di
    int allow_in  = (strcmp(dir, "in")  == 0) || (strcmp(dir, "both") == 0);
    JsonBuf b; jb_init(&b);
    jb_putc(&b, '[');
-    if (!sid || !*sid) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
+    if (!sid || !*sid) { jb_putc(&b, ']'); return el_wrap_str(jb_finish(&b)); }

    /* Frontier of (node_id, hops). Cap to a sane size. */
    char** frontier = calloc(1024, sizeof(char*));
@@ -7859,7 +7877,7 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t di
    int64_t vc = 0;
    if (!frontier || !frontier_h || !visited) {
        free(frontier); free(frontier_h); free(visited);
-        jb_putc(&b, ']'); return el_wrap_str(b.buf);
+        jb_putc(&b, ']'); return el_wrap_str(jb_finish(&b));
    }
    /* Use plain strdup (not el_strdup) so arena doesn't track these pointers.
     * The BFS loop manually frees them below — arena would double-free them. */
@@ -7902,7 +7920,7 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t di
    for (int64_t i = 0; i < vc; i++) free(visited[i]);
    free(frontier); free(frontier_h); free(visited);
    jb_putc(&b, ']');
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 el_val_t engram_activate_json(el_val_t query, el_val_t depth) {
@@ -7943,7 +7961,7 @@ el_val_t engram_activate_json(el_val_t query, el_val_t depth) {
        }
    }
    jb_putc(&b, ']');
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 el_val_t engram_stats_json(void) {
@@ -7963,7 +7981,7 @@ el_val_t engram_list_layers_json(void) {
    jb_putc(&b, '[');
    /* Build a sorted index over live layers. */
    size_t* idx = malloc((g->layer_count + 1) * sizeof(size_t));
-    if (!idx) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
+    if (!idx) { jb_putc(&b, ']'); return el_wrap_str(jb_finish(&b)); }
    size_t live = 0;
    for (size_t i = 0; i < g->layer_count; i++) {
        if (g->layers[i].name) idx[live++] = i;
@@ -8000,7 +8018,7 @@ el_val_t engram_list_layers_json(void) {
    }
    free(idx);
    jb_putc(&b, ']');
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 /* engram_compile_layered_json — produce a prompt-ready context block split
@@ -8086,7 +8104,7 @@ el_val_t engram_compile_layered_json(el_val_t intent, el_val_t depth) {
        free(b.buf);
        return el_wrap_str(el_strdup(""));
    }
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 /* engram_query_range — temporal range query.
@@ -8131,7 +8149,7 @@ el_val_t engram_query_range(el_val_t start_ms_v, el_val_t end_ms_v) {
    }
    jb_putc(&b, ']');
    free(idx);
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 /* engram_load_merge — like engram_load but WITHOUT resetting the store.
@@ -8382,7 +8400,7 @@ el_val_t engram_wm_top_json(el_val_t n_v) {
    }
    free(idx);
    jb_putc(&b, ']');
-    return el_wrap_str(b.buf);
+    return el_wrap_str(jb_finish(&b));
 }

 #ifdef HAVE_CURL
@@ -8778,13 +8796,13 @@ static int64_t dharma_find_or_create_relation_edge(const char* peer_base, int cr
        engram_grow_nodes();
        EngramNode* n = &g->nodes[g->node_count];
        memset(n, 0, sizeof(*n));
-        n->id        = el_strdup(self_id);
-        n->content   = el_strdup(_el_cgi_dharma_id ? _el_cgi_dharma_id : "(self)");
-        n->node_type = el_strdup("DharmaSelf");
-        n->label     = el_strdup("dharma:self");
-        n->tier      = el_strdup("Working");
-        n->tags      = el_strdup("dharma");
-        n->metadata  = el_strdup("{}");
+        n->id        = el_strdup_persist(self_id);
+        n->content   = el_strdup_persist(_el_cgi_dharma_id ? _el_cgi_dharma_id : "(self)");
+        n->node_type = el_strdup_persist("DharmaSelf");
+        n->label     = el_strdup_persist("dharma:self");
+        n->tier      = el_strdup_persist("Working");
+        n->tags      = el_strdup_persist("dharma");
+        n->metadata  = el_strdup_persist("{}");
        n->salience  = 1.0; n->importance = 1.0; n->confidence = 1.0;
        int64_t now  = engram_now_ms();
        n->created_at = now; n->updated_at = now; n->last_activated = now;
@@ -8795,13 +8813,13 @@ static int64_t dharma_find_or_create_relation_edge(const char* peer_base, int cr
        engram_grow_nodes();
        EngramNode* n = &g->nodes[g->node_count];
        memset(n, 0, sizeof(*n));
-        n->id        = el_strdup(peer_node);
-        n->content   = el_strdup(peer_base);
-        n->node_type = el_strdup("DharmaPeer");
-        n->label     = el_strdup(peer_node);
-        n->tier      = el_strdup("Working");
-        n->tags      = el_strdup("dharma");
-        n->metadata  = el_strdup("{}");
+        n->id        = el_strdup_persist(peer_node);
+        n->content   = el_strdup_persist(peer_base);
+        n->node_type = el_strdup_persist("DharmaPeer");
+        n->label     = el_strdup_persist(peer_node);
+        n->tier      = el_strdup_persist("Working");
+        n->tags      = el_strdup_persist("dharma");
+        n->metadata  = el_strdup_persist("{}");
        n->salience  = 0.5; n->importance = 0.5; n->confidence = 1.0;
        int64_t now  = engram_now_ms();
        n->created_at = now; n->updated_at = now; n->last_activated = now;
@@ -8813,10 +8831,10 @@ static int64_t dharma_find_or_create_relation_edge(const char* peer_base, int cr
    EngramEdge* e = &g->edges[g->edge_count];
    memset(e, 0, sizeof(*e));
    e->id        = engram_new_id();
-    e->from_id   = el_strdup(self_id);
-    e->to_id     = el_strdup(peer_node);
-    e->relation  = el_strdup("dharma-relation");
-    e->metadata  = el_strdup("{}");
+    e->from_id   = el_strdup_persist(self_id);
+    e->to_id     = el_strdup_persist(peer_node);
+    e->relation  = el_strdup_persist("dharma-relation");
+    e->metadata  = el_strdup_persist("{}");
    e->weight    = 0.0;
    e->confidence = 1.0;
    int64_t now  = engram_now_ms();
@@ -9109,7 +9127,7 @@ static el_val_t llm_extract_text(el_val_t resp_val) {
        free(obj);
        p = end;
    }
-    return el_wrap_str(out.buf);
+    return el_wrap_str(jb_finish(&out));
 }

 el_val_t llm_call(el_val_t model, el_val_t prompt) {
@@ -23,10 +23,29 @@ fn tok_at(tokens: [Any], pos: Int) -> Map<String, Any> {
 }

 fn tok_kind(tokens: [Any], pos: Int) -> String {
+    // Out-of-range reads must report the Eof sentinel so every `== "Eof"`
+    // termination guard in the parser fires. Without this, reading past the
+    // single trailing Eof token returns runtime null (el_list_get OOB -> 0),
+    // which matches no delimiter, letting inner parse loops append AST nodes
+    // forever on malformed input -> unbounded allocation -> OOM.
+    let n: Int = native_list_len(tokens) / 2
+    if pos < 0 {
+        return "Eof"
+    }
+    if pos >= n {
+        return "Eof"
+    }
    native_list_get(tokens, pos * 2)
 }

 fn tok_value(tokens: [Any], pos: Int) -> String {
+    let n: Int = native_list_len(tokens) / 2
+    if pos < 0 {
+        return ""
+    }
+    if pos >= n {
+        return ""
+    }
    native_list_get(tokens, pos * 2 + 1)
 }

@@ -35,7 +54,12 @@ fn expect(tokens: [Any], pos: Int, kind: String) -> Int {
    if k == kind {
        return pos + 1
    }
-    // On mismatch just advance; error recovery is best-effort
+    // On mismatch, error recovery is best-effort. But never step PAST the Eof
+    // sentinel: once at Eof a mismatch means the input ended early, and
+    // advancing would run the cursor off the token list.
+    if k == "Eof" {
+        return pos
+    }
    pos + 1
 }

@@ -601,6 +601,13 @@ el_val_t  engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t d
 el_val_t  engram_activate_json(el_val_t query, el_val_t depth);
 el_val_t  engram_stats_json(void);
 el_val_t  engram_list_layers_json(void);
+/* Working memory introspection — count, mean weight, and top-N snapshot.
+ * Ported from el-compiler/runtime on 2026-06-30 self-review. */
+el_val_t  engram_wm_count(void);
+el_val_t  engram_wm_avg_weight(void);
+el_val_t  engram_wm_top_json(el_val_t n);
+/* Merge-load: add nodes/edges from a snapshot without resetting the store. */
+el_val_t  engram_load_merge(el_val_t path);
 /* engram_compile_layered_json — produce a prompt-ready text block split
 * into "[LAYER 0 — STRUCTURAL]" (non-suppressible layers, sacred fire)
 * and "[ENGRAM CONTEXT]" (standard suppressible layers). Returns "" if
Author	SHA1	Message	Date
will.anderson	0a0a2bcb44	parser: bound token reads to Eof so malformed input errors instead of OOMing El SDK Release / build-and-release (pull_request) Failing after 16s Details Out-of-range tok_kind/tok_value reads returned runtime null (el_list_get OOB -> 0) rather than the Eof sentinel, so the inner parse loops (parse_block, call-arg, array-literal, match-arm) that terminate only on their close delimiter or k=="Eof" never saw Eof once the cursor ran past the single trailing Eof token. On unclosed-delimiter input the parser then appended AST nodes forever -> unbounded allocation -> ~700GB -> OOM (observed compiling neuron/sessions.el). Fix at the choke point: tok_kind returns "Eof" and tok_value returns "" for out-of-range positions, restoring the parser-wide contract that reads at/after the end yield Eof. expect() no longer steps past the Eof sentinel on mismatch. This terminates every overrun loop simultaneously; a malformed program now surfaces as a normal (best-effort) parse end instead of exhausting memory. Requires a self-hosted bootstrap rebuild of elc to take effect.	2026-07-14 14:21:39 -05:00
will.anderson	f78da81aa4	runtime: fix the memory leak + write-corruption pair in el_runtime.c El SDK Release / build-and-release (pull_request) Failing after 11m58s Details Two independent investigations, one runtime, complementary halves: 1. Leak (Jul 2, this machine): JsonBuf buffers returned via el_wrap_str were raw malloc, never arena-tracked — every engram_*_json call leaked its output unconditionally. Added jb_finish() arena-tracking across all ~30 return sites. Plus el_arena_push/pop per-tick bracketing support for the soul's awareness loop (the loop ran outside any request arena, so even correctly-tracked allocations were permanent — 7.5GB RSS in under a minute at 1s tick). 2. Corruption (Tim's container soak, docs findings/container-migration): stored engram node/edge fields (content, node_type, label, tier, tags, metadata, from/to ids) were arena el_strdup — freed at request end, leaving dangling pointers that read back as recycled request-buffer bytes one request later. This is the June corruption root cause and the mechanism that grew snapshot.json to 18GB of empty-type junk (21.6M nodes, 3,335 real). 39 sites switched to el_strdup_persist, plus a latent double-free fix in engram_load metadata fixup. Interaction note: fix 1's per-tick arena reclamation makes fix 2 mandatory — more aggressive arena recycling widens the use-after-free window if stored fields still live in the arena. Apply as a pair, never separately. Verified live: soul + engram rebuilt from this runtime, booted against the recovered real snapshot (3,335 nodes/40,146 edges), 5h stable at <100MB RSS, write-then-next-request field-integrity test passes (the June corruption fingerprint does not reproduce). engram/dist/engram binary updated from this build. Investigation credit: leak diagnosis this machine Jul 2-6; corruption diagnosis + persist-fix patch by Tim's instance (docs PR #4).	2026-07-13 16:22:02 -05:00
will.anderson	2597a092bb	Merge pull request 'chore: integrate local main commits' (#63 ) from integrate/local-main-commits into main El SDK Release / build-and-release (push) Successful in 10m58s Details	2026-07-01 16:30:17 +00:00
will.anderson	226b798407	Merge branch 'fix/windows-rusage-guard' (PR #61 ): UTF-8 guard, engram sync route, native platform backends, UI vessels El SDK Release / build-and-release (pull_request) Failing after 13m57s Details	2026-07-01 11:27:54 -05:00
will.anderson	cfe8cb1c80	fix(release-snapshot): fflush stdout in println and update Knowledge threshold El SDK Release / build-and-release (pull_request) Failing after 20s Details	2026-07-01 11:25:09 -05:00
will.anderson	688b8508fb	feat(runtime): native platform backends and UI vessels onto main	2026-07-01 11:21:23 -05:00
will.anderson	59cea116c5	build(engram): rebuild binary with engram_load_merge runtime (`deb0520`) El SDK Release / build-and-release (pull_request) Failing after 19s Details Runtime now includes engram_load_merge — soul daemon awareness.el calls this function during its periodic sync refresh cycle. Binary rebuilt from server.el (unchanged source) + updated el_runtime.c.	2026-06-30 08:59:01 -05:00
will.anderson	deb0520551	feat(runtime): port engram_load_merge to released runtime + add missing WM headers engram_load_merge was added to el-compiler/runtime in `35c1897` but never ported to the released runtime used by Engram and the soul daemon. awareness.el calls engram_load_merge in its sync refresh cycle; without this function in lang/releases/v1.0.0-20260501/el_runtime.c the soul daemon fails to compile. Also adds header declarations for engram_wm_count, engram_wm_avg_weight, engram_wm_top_json, and engram_load_merge — all four were added as implementations (`da116b2` / `35c1897`) but their prototypes were missing from el_runtime.h, causing implicit-function-declaration warnings and potential ABI breakage on stricter compilers. Identified during self-review 2026-06-30.	2026-06-30 08:57:22 -05:00
will.anderson	da116b2884	self-review 2026-06-30: WM cap, breakthrough floor, ISE exclusion + route Port critical WM fixes from self-review 2026-06-26 branch (`f7bd99a`) that were never merged to HEAD. Running binary had these fixes; source did not — rebuild would have silently regressed all three improvements. 1. ENGRAM_BREAKTHROUGH_WEIGHT 0.25→0.10 With 0.25, naturally-promoted nodes (threshold ≥0.15) decayed below the breakthrough floor within one activation call and lost their WM slot to fresh breakthrough candidates. All 524/525 WM nodes were at floor = useless. Invariant: BREAKTHROUGH_WEIGHT < min(type_thresholds = 0.15 Canonical). 2. ENGRAM_WM_CAP=24 with Pass 4 (per-call) + Pass 5 (global) enforcement Without cap, broad curiosity seeds promote 500+ nodes simultaneously. wm_avg_weight collapses, goal-bias differentiation is lost. Verified: "knowledge" query now promotes exactly 24 nodes (was 525). Cowan (2001) cognitive basis: WM capacity ~4 chunks; 24 allows rich multi-topic context. 3. ISE exclusion from WM (Pass 2 guard) InternalStateEvent JSON content ("knowledge", "memory", etc.) triggered lexical seeding → suppression accumulation → breakthrough at floor. ISEs are observability-only and must never surface in context compilation. suppression_count cleared so ISEs never build toward breakthrough. 4. route_create_ise importance fix (0.5→0.3) Corrects mismatch between HTTP route and awareness.el in-process fallback. Also adds body comment clarifying auth-exempt rationale. SYNAPSE (arXiv 2601.02744) validates WM cap design and ISE exclusion principle. Next priority: cosine similarity seeding to complement lexical BFS.	2026-06-30 08:48:19 -05:00