/* * el_runtime.c — El language C runtime implementation * * All functions use el_val_t (= int64_t) as the universal value type. * Strings are transported as their pointer address cast to int64_t. * On any 64-bit system sizeof(pointer) <= sizeof(int64_t), so this is safe. * * Compile with: * cc -std=c11 -I -lcurl -lpthread -o .c el_runtime.c * * Link requirements: -lcurl (HTTP client + LLM), -lpthread (HTTP server). */ /* Feature-test macros must be set before any standard headers. _GNU_SOURCE * exposes clock_gettime/CLOCK_REALTIME, strcasecmp, and the dlfcn extensions * (RTLD_DEFAULT) — all of which macOS hands us without asking but glibc on * Debian gates behind an explicit opt-in. */ #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include "el_runtime.h" #include #include /* strcasecmp */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* dlsym for http_set_handler fallback */ #include #include #include #include #include #include /* ── Internal allocators ─────────────────────────────────────────────────── */ /* * Per-request string arena * * Every El string allocated via el_strbuf / el_strdup during an HTTP request * is registered in a thread-local arena. When el_request_end() is called at * the end of the worker thread, every arena entry is freed — recovering all * the intermediate strings from el_str_concat chains (build_system_prompt, * engram_compile, etc.) that are otherwise leaked forever. * * Long-lived allocations (state_set values, engram internal storage) call * el_strdup_persist() / el_strbuf_persist() which bypass the arena entirely. */ #define EL_ARENA_INITIAL 512 typedef struct { char** ptrs; size_t count; size_t cap; } ElArena; static _Thread_local ElArena _tl_arena = {NULL, 0, 0}; static _Thread_local int _tl_arena_active = 0; /* Binary-safe fs_read length — set by fs_read, consumed by http_send_response. * Allows serving PNGs and other binary files without strlen truncation. */ static _Thread_local size_t _tl_fs_read_len = 0; static void el_arena_track(char* p) { if (!_tl_arena_active || !p) return; if (_tl_arena.count >= _tl_arena.cap) { size_t nc = _tl_arena.cap == 0 ? EL_ARENA_INITIAL : _tl_arena.cap * 2; char** grown = realloc(_tl_arena.ptrs, nc * sizeof(char*)); if (!grown) return; /* can't track — will leak this one ptr, but don't crash */ _tl_arena.ptrs = grown; _tl_arena.cap = nc; } _tl_arena.ptrs[_tl_arena.count++] = p; } /* Called by http_worker before dispatching the El handler. */ void el_request_start(void) { _tl_arena.count = 0; _tl_arena_active = 1; } /* Called by http_worker after the El handler returns and the response is sent. * Frees every intermediate string allocated during the request. */ void el_request_end(void) { _tl_arena_active = 0; for (size_t i = 0; i < _tl_arena.count; i++) { free(_tl_arena.ptrs[i]); } _tl_arena.count = 0; } /* ── Scoped arena for CLI use ─────────────────────────────────────────────── * * CLI programs never call el_request_start/end, so all strdup allocations are * permanent. el_arena_push/pop let the compiler free intermediate strings * after each compilation unit. Ported verbatim from el-compiler/runtime on * 2026-07-17: the soul daemon's awareness loop arena-scopes each tick with * these, and they were present only in the dev runtime copy. * * el_arena_push() — activates the arena if not already active, saves the * current arena count as a mark, and returns it as an el_val_t Int. * el_arena_pop(mark) — frees all strings allocated since the push mark and * resets the count. If count reaches 0, deactivates the arena. */ #define EL_ARENA_SCOPE_DEPTH 32 static _Thread_local size_t _tl_arena_scope[EL_ARENA_SCOPE_DEPTH]; static _Thread_local int _tl_arena_scope_depth = 0; el_val_t el_arena_push(void) { if (!_tl_arena_active) { _tl_arena_active = 1; } if (_tl_arena_scope_depth < EL_ARENA_SCOPE_DEPTH) { _tl_arena_scope[_tl_arena_scope_depth++] = _tl_arena.count; } return (el_val_t)(int64_t)_tl_arena.count; } el_val_t el_arena_pop(el_val_t mark) { size_t save = (size_t)(int64_t)mark; if (save > _tl_arena.count) save = 0; for (size_t i = save; i < _tl_arena.count; i++) { if (_tl_arena.ptrs[i]) { free(_tl_arena.ptrs[i]); _tl_arena.ptrs[i] = NULL; } } _tl_arena.count = save; if (_tl_arena_scope_depth > 0) _tl_arena_scope_depth--; if (save == 0) _tl_arena_active = 0; return 0; } /* Persistent allocation — bypasses the arena (state_set, engram internals). */ static char* el_strdup_persist(const char* s) { if (!s) return strdup(""); return strdup(s); } static char* el_strbuf_persist(size_t n) { char* p = malloc(n + 1); if (!p) { fputs("el_runtime: out of memory\n", stderr); exit(1); } p[0] = '\0'; return p; } static char* el_strdup(const char* s) { if (!s) { char* p = strdup(""); el_arena_track(p); return p; } char* p = strdup(s); el_arena_track(p); return p; } static char* el_strbuf(size_t n) { char* p = malloc(n + 1); if (!p) { fputs("el_runtime: out of memory\n", stderr); exit(1); } p[0] = '\0'; el_arena_track(p); return p; } /* Wrap an allocated C string as el_val_t */ static el_val_t el_wrap_str(char* s) { return EL_STR(s); } /* ── I/O ──────────────────────────────────────────────────────────────────── */ void println(el_val_t s) { const char* str = EL_CSTR(s); if (str) puts(str); else puts(""); } void print(el_val_t s) { const char* str = EL_CSTR(s); if (str) fputs(str, stdout); } el_val_t readline(void) { char buf[4096]; if (!fgets(buf, sizeof(buf), stdin)) return el_wrap_str(el_strdup("")); size_t len = strlen(buf); if (len > 0 && buf[len - 1] == '\n') buf[len - 1] = '\0'; return el_wrap_str(el_strdup(buf)); } /* ── String builtins ─────────────────────────────────────────────────────── */ el_val_t el_str_concat(el_val_t av, el_val_t bv) { const char* a = EL_CSTR(av); const char* b = EL_CSTR(bv); if (!a) a = ""; if (!b) b = ""; size_t la = strlen(a); size_t lb = strlen(b); char* out = el_strbuf(la + lb); memcpy(out, a, la); memcpy(out + la, b, lb); out[la + lb] = '\0'; return el_wrap_str(out); } el_val_t str_eq(el_val_t av, el_val_t bv) { const char* a = EL_CSTR(av); const char* b = EL_CSTR(bv); if (!a || !b) return (el_val_t)(a == b); return (el_val_t)(strcmp(a, b) == 0); } el_val_t str_starts_with(el_val_t sv, el_val_t prefv) { const char* s = EL_CSTR(sv); const char* prefix = EL_CSTR(prefv); if (!s || !prefix) return 0; size_t lp = strlen(prefix); return (el_val_t)(strncmp(s, prefix, lp) == 0); } el_val_t str_ends_with(el_val_t sv, el_val_t sufv) { const char* s = EL_CSTR(sv); const char* suffix = EL_CSTR(sufv); if (!s || !suffix) return 0; size_t ls = strlen(s); size_t lsuf = strlen(suffix); if (lsuf > ls) return 0; return (el_val_t)(strcmp(s + ls - lsuf, suffix) == 0); } el_val_t str_len(el_val_t sv) { const char* s = EL_CSTR(sv); if (!s) return 0; return (el_val_t)strlen(s); } el_val_t str_concat(el_val_t a, el_val_t b) { return el_str_concat(a, b); } el_val_t int_to_str(el_val_t n) { char buf[32]; snprintf(buf, sizeof(buf), "%lld", (long long)n); return el_wrap_str(el_strdup(buf)); } el_val_t str_to_int(el_val_t sv) { const char* s = EL_CSTR(sv); if (!s) return 0; return (el_val_t)atoll(s); } el_val_t str_slice(el_val_t sv, el_val_t start, el_val_t end) { const char* s = EL_CSTR(sv); if (!s) return el_wrap_str(el_strdup("")); int64_t len = (int64_t)strlen(s); if (start < 0) start = 0; if (end > len) end = len; if (start >= end) return el_wrap_str(el_strdup("")); int64_t sz = end - start; char* out = el_strbuf((size_t)sz); memcpy(out, s + start, (size_t)sz); out[sz] = '\0'; return el_wrap_str(out); } el_val_t str_contains(el_val_t sv, el_val_t subv) { const char* s = EL_CSTR(sv); const char* sub = EL_CSTR(subv); if (!s || !sub) return 0; return (el_val_t)(strstr(s, sub) != NULL); } el_val_t str_replace(el_val_t sv, el_val_t fromv, el_val_t tov) { const char* s = EL_CSTR(sv); const char* from = EL_CSTR(fromv); const char* to = EL_CSTR(tov); if (!s || !from || !to) return el_wrap_str(el_strdup(s ? s : "")); size_t ls = strlen(s); size_t lf = strlen(from); size_t lt = strlen(to); if (lf == 0) return el_wrap_str(el_strdup(s)); size_t count = 0; const char* p = s; while ((p = strstr(p, from)) != NULL) { count++; p += lf; } size_t out_sz = ls + count * lt + 1; char* out = el_strbuf(out_sz); char* dst = out; p = s; const char* found; while ((found = strstr(p, from)) != NULL) { size_t chunk = (size_t)(found - p); memcpy(dst, p, chunk); dst += chunk; memcpy(dst, to, lt); dst += lt; p = found + lf; } strcpy(dst, p); return el_wrap_str(out); } el_val_t str_to_upper(el_val_t sv) { const char* s = EL_CSTR(sv); if (!s) return el_wrap_str(el_strdup("")); size_t n = strlen(s); char* out = el_strbuf(n); for (size_t i = 0; i < n; i++) out[i] = (char)toupper((unsigned char)s[i]); out[n] = '\0'; return el_wrap_str(out); } el_val_t str_to_lower(el_val_t sv) { const char* s = EL_CSTR(sv); if (!s) return el_wrap_str(el_strdup("")); size_t n = strlen(s); char* out = el_strbuf(n); for (size_t i = 0; i < n; i++) out[i] = (char)tolower((unsigned char)s[i]); out[n] = '\0'; return el_wrap_str(out); } el_val_t str_trim(el_val_t sv) { const char* s = EL_CSTR(sv); if (!s) return el_wrap_str(el_strdup("")); while (*s && isspace((unsigned char)*s)) s++; size_t n = strlen(s); while (n > 0 && isspace((unsigned char)s[n - 1])) n--; char* out = el_strbuf(n); memcpy(out, s, n); out[n] = '\0'; return el_wrap_str(out); } /* ── Math ────────────────────────────────────────────────────────────────── */ el_val_t el_abs(el_val_t n) { return n < 0 ? -n : n; } el_val_t el_max(el_val_t a, el_val_t b) { return a > b ? a : b; } el_val_t el_min(el_val_t a, el_val_t b) { return a < b ? a : b; } /* ── Refcounted heap objects ────────────────────────────────────────────────── * * ElList and ElMap carry a magic-tagged header at offset 0: * { uint32_t magic; uint32_t refcount; ... payload ... } * * The magic tag distinguishes refcounted objects from raw C strings (whose * first byte is printable ASCII < 0x80) and from small integers (which can't * be dereferenced). el_retain / el_release sniff the magic and act only on * matching values; everything else is a safe no-op. * * Both ElList and ElMap use INDIRECTION: the header is fixed-size and never * moves. The payload arrays (elems, keys, values) live in separate heap * allocations, so realloc-grow on append never invalidates the caller's * pointer to the header. This is what lets us mutate-in-place safely when * the refcount is 1 and copy-on-write when it's higher. * * Memory model in practice: * Single-owner accumulator (the cg_stmts pattern) — refcount stays at 1, * appends amortize to O(1), total memory O(N) for an N-element list. * Multi-owner branching (the cg_if_stmt pattern) — refcount > 1, each * append on a shared list copies, so the original is preserved for the * else-branch. Persistent semantics where they're needed; mutation where * they're not. */ #define EL_MAGIC_LIST 0xE15710A1u /* >= 0x80 in MSB so 'looks_like_string' rejects */ #define EL_MAGIC_MAP 0xE19A704Bu typedef struct { uint32_t magic; uint32_t refcount; } ElHeader; /* ── List ────────────────────────────────────────────────────────────────── */ typedef struct { ElHeader hdr; int64_t length; int64_t capacity; el_val_t* elems; } ElList; static ElList* list_alloc(int64_t cap) { if (cap < 4) cap = 4; ElList* lst = malloc(sizeof(ElList)); if (!lst) { fputs("el_runtime: out of memory\n", stderr); exit(1); } lst->hdr.magic = EL_MAGIC_LIST; lst->hdr.refcount = 1; lst->length = 0; lst->capacity = cap; lst->elems = malloc((size_t)cap * sizeof(el_val_t)); if (!lst->elems) { fputs("el_runtime: out of memory\n", stderr); exit(1); } return lst; } el_val_t el_list_empty(void) { return EL_STR(list_alloc(4)); } el_val_t el_list_new(el_val_t count, ...) { ElList* lst = list_alloc(count > 0 ? count : 4); va_list ap; va_start(ap, count); for (int64_t i = 0; i < count; i++) { lst->elems[i] = va_arg(ap, el_val_t); } va_end(ap); lst->length = count; return EL_STR(lst); } el_val_t el_list_len(el_val_t listv) { ElList* lst = (ElList*)(uintptr_t)listv; if (!lst) return 0; return lst->length; } el_val_t el_list_get(el_val_t listv, el_val_t index) { ElList* lst = (ElList*)(uintptr_t)listv; if (!lst) return 0; if (index < 0 || index >= lst->length) return 0; return lst->elems[index]; } el_val_t el_list_append(el_val_t listv, el_val_t elem) { ElList* old = (ElList*)(uintptr_t)listv; if (!old) { ElList* fresh = list_alloc(4); fresh->elems[0] = elem; fresh->length = 1; return EL_STR(fresh); } /* Uniquely owned: grow the elems buffer in place. The header pointer the * caller holds doesn't move (we only realloc the inner array). This is * the common case in compiler accumulators, and it's amortized O(1). */ if (old->hdr.refcount <= 1) { if (old->length >= old->capacity) { int64_t new_cap = old->capacity > 0 ? old->capacity * 2 : 4; el_val_t* grown = realloc(old->elems, (size_t)new_cap * sizeof(el_val_t)); if (!grown) { fputs("el_runtime: out of memory\n", stderr); exit(1); } old->elems = grown; old->capacity = new_cap; } old->elems[old->length++] = elem; return listv; } /* Shared: copy-on-write. The original is preserved for its other owners. */ int64_t new_cap = old->length + 1; if (new_cap < 4) new_cap = 4; ElList* fresh = malloc(sizeof(ElList)); if (!fresh) { fputs("el_runtime: out of memory\n", stderr); exit(1); } fresh->hdr.magic = EL_MAGIC_LIST; fresh->hdr.refcount = 1; fresh->length = old->length + 1; fresh->capacity = new_cap; fresh->elems = malloc((size_t)new_cap * sizeof(el_val_t)); if (!fresh->elems) { fputs("el_runtime: out of memory\n", stderr); exit(1); } if (old->length > 0) { memcpy(fresh->elems, old->elems, (size_t)old->length * sizeof(el_val_t)); } fresh->elems[old->length] = elem; return EL_STR(fresh); } el_val_t el_list_clone(el_val_t listv) { /* Shallow copy: the new ElList owns its own header and elems buffer, but * the elements themselves are shared (which is what callers want for the * cg_if_stmt 'declared' pattern — cloning the spine, not its contents). * Used by codegen at scope branch points where two child scopes need to * see the same starting set of declared names without each other's * mutations. */ ElList* old = (ElList*)(uintptr_t)listv; if (!old) return el_list_empty(); int64_t cap = old->capacity > 0 ? old->capacity : 4; if (cap < old->length) cap = old->length; if (cap < 4) cap = 4; ElList* fresh = malloc(sizeof(ElList)); if (!fresh) { fputs("el_runtime: out of memory\n", stderr); exit(1); } fresh->hdr.magic = EL_MAGIC_LIST; fresh->hdr.refcount = 1; fresh->length = old->length; fresh->capacity = cap; fresh->elems = malloc((size_t)cap * sizeof(el_val_t)); if (!fresh->elems) { fputs("el_runtime: out of memory\n", stderr); exit(1); } if (old->length > 0) { memcpy(fresh->elems, old->elems, (size_t)old->length * sizeof(el_val_t)); } return EL_STR(fresh); } /* ── Map ─────────────────────────────────────────────────────────────────── */ typedef struct { ElHeader hdr; int64_t count; int64_t capacity; el_val_t* keys; el_val_t* values; } ElMap; static ElMap* map_alloc(int64_t cap) { if (cap < 4) cap = 4; ElMap* m = malloc(sizeof(ElMap)); if (!m) { fputs("el_runtime: out of memory\n", stderr); exit(1); } m->hdr.magic = EL_MAGIC_MAP; m->hdr.refcount = 1; m->count = 0; m->capacity = cap; m->keys = malloc((size_t)cap * sizeof(el_val_t)); m->values = malloc((size_t)cap * sizeof(el_val_t)); if (!m->keys || !m->values) { fputs("el_runtime: out of memory\n", stderr); exit(1); } return m; } el_val_t el_map_new(el_val_t pair_count, ...) { ElMap* m = map_alloc(pair_count > 0 ? pair_count : 4); va_list ap; va_start(ap, pair_count); for (int64_t i = 0; i < pair_count; i++) { m->keys[i] = va_arg(ap, el_val_t); m->values[i] = va_arg(ap, el_val_t); } va_end(ap); m->count = pair_count; return EL_STR(m); } static ElMap* as_map(el_val_t v) { return (ElMap*)(uintptr_t)v; } el_val_t el_map_get(el_val_t mapv, el_val_t keyv) { ElMap* m = as_map(mapv); const char* key = EL_CSTR(keyv); if (!m || !key) return 0; for (int64_t i = 0; i < m->count; i++) { const char* k = EL_CSTR(m->keys[i]); if (k && strcmp(k, key) == 0) return m->values[i]; } return 0; } el_val_t el_get_field(el_val_t mapv, el_val_t keyv) { return el_map_get(mapv, keyv); } /* Internal: in-place set on a uniquely-owned map. */ static el_val_t map_set_in_place(ElMap* m, el_val_t keyv, el_val_t value) { const char* key = EL_CSTR(keyv); if (key) { for (int64_t i = 0; i < m->count; i++) { const char* k = EL_CSTR(m->keys[i]); if (k && strcmp(k, key) == 0) { m->values[i] = value; return EL_STR(m); } } } if (m->count >= m->capacity) { int64_t new_cap = m->capacity > 0 ? m->capacity * 2 : 4; el_val_t* gk = realloc(m->keys, (size_t)new_cap * sizeof(el_val_t)); el_val_t* gv = realloc(m->values, (size_t)new_cap * sizeof(el_val_t)); if (!gk || !gv) { fputs("el_runtime: out of memory\n", stderr); exit(1); } m->keys = gk; m->values = gv; m->capacity = new_cap; } m->keys[m->count] = keyv; m->values[m->count] = value; m->count++; return EL_STR(m); } el_val_t el_map_set(el_val_t mapv, el_val_t keyv, el_val_t value) { ElMap* m = as_map(mapv); if (!m) return 0; if (m->hdr.refcount <= 1) { return map_set_in_place(m, keyv, value); } /* Shared: copy then set. The original is preserved for its other owners. */ int64_t new_cap = m->count + 1; if (new_cap < 4) new_cap = 4; ElMap* fresh = malloc(sizeof(ElMap)); if (!fresh) { fputs("el_runtime: out of memory\n", stderr); exit(1); } fresh->hdr.magic = EL_MAGIC_MAP; fresh->hdr.refcount = 1; fresh->count = m->count; fresh->capacity = new_cap; fresh->keys = malloc((size_t)new_cap * sizeof(el_val_t)); fresh->values = malloc((size_t)new_cap * sizeof(el_val_t)); if (!fresh->keys || !fresh->values) { fputs("el_runtime: out of memory\n", stderr); exit(1); } if (m->count > 0) { memcpy(fresh->keys, m->keys, (size_t)m->count * sizeof(el_val_t)); memcpy(fresh->values, m->values, (size_t)m->count * sizeof(el_val_t)); } return map_set_in_place(fresh, keyv, value); } /* ── Refcount ops ─────────────────────────────────────────────────────────── */ /* * Both retain and release sniff the magic header to decide whether a value * is a refcounted heap object. For small integers, raw C strings, and any * value whose magic word doesn't match, both functions are no-ops. This lets * codegen emit them on every let-binding without having to track types. * * Safety: we filter out obvious non-pointers (small magnitudes, misaligned * addresses) before dereferencing. For any value that passes the filter and * lives in a mapped page, reading the first 4 bytes is safe — strings start * with printable ASCII (< 0x80), so their magic word will never collide with * EL_MAGIC_LIST (0xE1...) or EL_MAGIC_MAP (0xE1...). Random integers that * happen to look like aligned heap pointers are exceedingly unlikely to land * on a page whose first 4 bytes match either magic. */ static int looks_like_heap_obj(el_val_t v) { if (v == 0) return 0; int64_t s = (int64_t)v; if (s > -0x10000 && s < 0x10000) return 0; /* small ints */ uintptr_t p = (uintptr_t)v; if (p < 0x10000) return 0; /* low addresses */ if (p & 0x7) return 0; /* malloc returns 8-aligned */ return 1; } void el_retain(el_val_t v) { if (!looks_like_heap_obj(v)) return; ElHeader* h = (ElHeader*)(uintptr_t)v; if (h->magic == EL_MAGIC_LIST || h->magic == EL_MAGIC_MAP) { h->refcount++; } } void el_release(el_val_t v) { if (!looks_like_heap_obj(v)) return; ElHeader* h = (ElHeader*)(uintptr_t)v; if (h->magic == EL_MAGIC_LIST) { if (h->refcount > 0 && --h->refcount == 0) { ElList* l = (ElList*)h; free(l->elems); l->hdr.magic = 0; /* poison so use-after-free is detected */ free(l); } } else if (h->magic == EL_MAGIC_MAP) { if (h->refcount > 0 && --h->refcount == 0) { ElMap* m = (ElMap*)h; free(m->keys); free(m->values); m->hdr.magic = 0; free(m); } } } /* ── Batch 2/3 forward decls (defined later in JSON section) ────────────── */ typedef struct JsonBuf JsonBuf; typedef struct JsonParser JsonParser; static void jb_init(JsonBuf* b); static void jb_putc(JsonBuf* b, char c); static void jb_puts(JsonBuf* b, const char* s); static void jb_emit_escaped(JsonBuf* b, const char* s); static int looks_like_string(el_val_t v); static const char* json_find_key(const char* s, const char* key); static const char* json_skip_value(const char* p); static char* jp_parse_string_raw(JsonParser* jp); /* Struct definitions are visible here because batch 2/3 helpers above use * them by value; the bodies (jb_init, etc.) appear in the JSON section. */ struct JsonBuf { char* buf; size_t len; size_t cap; }; struct JsonParser { const char* p; const char* end; int err; }; /* ── Batch 2: Real HTTP (libcurl client + POSIX-socket server) ───────────── */ /* * Client: blocking libcurl easy-handle calls. Errors are returned as a JSON * fragment {"error":"..."} so callers can detect via str_starts_with("{") / * json_get_string("error", ...). * * Server: bind/listen/accept loop on a TCP socket. Each accepted connection * is handled in its own pthread (detached). A semaphore-style counter caps * concurrent in-flight connections at HTTP_MAX_CONNS (64). When the cap is * reached, accept() blocks until a worker exits. This prevents runaway * thread creation under high load. * * Handler dispatch: El does not expose first-class function references at * the runtime layer, so the second argument to http_serve(port, handler) is * treated as a string name (or any el_val_t — the runtime ignores its * value and uses the registry). Callers register a C-level handler via * * extern void el_runtime_register_handler(const char* name, * el_val_t (*fn)(el_val_t, * el_val_t, * el_val_t)); * * and select the active handler by calling http_set_handler("name") from * El, or by setting it directly through the C registry. If no handler is * registered, the server replies with a 200 carrying a default message so * the loop is observable. */ /* ── HTTP client write-callback buffer ───────────────────────────────────── */ typedef struct { char* data; size_t len; size_t cap; } HttpBuf; static void httpbuf_init(HttpBuf* b) { b->cap = 1024; b->len = 0; b->data = malloc(b->cap); if (!b->data) { fputs("el_runtime: out of memory\n", stderr); exit(1); } b->data[0] = '\0'; } static void httpbuf_append(HttpBuf* b, const void* src, size_t n) { if (b->len + n + 1 > b->cap) { while (b->len + n + 1 > b->cap) b->cap *= 2; b->data = realloc(b->data, b->cap); if (!b->data) { fputs("el_runtime: out of memory\n", stderr); exit(1); } } memcpy(b->data + b->len, src, n); b->len += n; b->data[b->len] = '\0'; } static size_t http_write_cb(char* ptr, size_t size, size_t nmemb, void* ud) { size_t n = size * nmemb; httpbuf_append((HttpBuf*)ud, ptr, n); return n; } /* JSON-escape an arbitrary C string into an allocated buffer. */ static char* json_escape_alloc(const char* s) { if (!s) return el_strdup(""); JsonBuf b; jb_init(&b); for (const char* p = s; *p; p++) { unsigned char c = (unsigned char)*p; switch (c) { case '"': jb_puts(&b, "\\\""); break; case '\\': jb_puts(&b, "\\\\"); break; case '\n': jb_puts(&b, "\\n"); break; case '\r': jb_puts(&b, "\\r"); break; case '\t': jb_puts(&b, "\\t"); break; default: if (c < 0x20) { char tmp[8]; snprintf(tmp, sizeof(tmp), "\\u%04x", c); jb_puts(&b, tmp); } else jb_putc(&b, (char)c); } } return b.buf; } static el_val_t http_error_json(const char* msg) { char* esc = json_escape_alloc(msg ? msg : "unknown error"); char* buf = el_strbuf(strlen(esc) + 16); sprintf(buf, "{\"error\":\"%s\"}", esc); free(esc); return el_wrap_str(buf); } /* HTTP timeout (ms) — read once from EL_HTTP_TIMEOUT_MS, default 60000. * Applied via CURLOPT_TIMEOUT_MS on every libcurl request. */ static long _el_http_timeout_ms = -1; static long el_http_timeout_ms(void) { long v = __atomic_load_n(&_el_http_timeout_ms, __ATOMIC_ACQUIRE); if (v >= 0) return v; const char* s = getenv("EL_HTTP_TIMEOUT_MS"); long parsed = 60000L; if (s && *s) { char* end = NULL; long n = strtol(s, &end, 10); if (end != s && n > 0) parsed = n; } __atomic_store_n(&_el_http_timeout_ms, parsed, __ATOMIC_RELEASE); return parsed; } /* Internal: do a libcurl request; takes optional body/headers, optional method * override, and an optional timeout override (0 = use EL_HTTP_TIMEOUT_MS). * The override exists for the embedding path: activation must never wait the * full 60s default on a wedged Ollama. (2026-07-24 self-review) */ static el_val_t http_do_t(const char* method, const char* url, const char* body, struct curl_slist* extra_headers, long timeout_ms) { if (!url || !*url) return http_error_json("empty url"); CURL* c = curl_easy_init(); if (!c) return http_error_json("curl_easy_init failed"); HttpBuf rb; httpbuf_init(&rb); char errbuf[CURL_ERROR_SIZE]; errbuf[0] = '\0'; curl_easy_setopt(c, CURLOPT_URL, url); curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_write_cb); curl_easy_setopt(c, CURLOPT_WRITEDATA, &rb); curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, timeout_ms > 0 ? timeout_ms : el_http_timeout_ms()); curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf); curl_easy_setopt(c, CURLOPT_USERAGENT, "el-runtime/1.0"); if (extra_headers) curl_easy_setopt(c, CURLOPT_HTTPHEADER, extra_headers); if (method && strcmp(method, "POST") == 0) { curl_easy_setopt(c, CURLOPT_POST, 1L); curl_easy_setopt(c, CURLOPT_POSTFIELDS, body ? body : ""); curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)(body ? strlen(body) : 0)); } else if (method && strcmp(method, "DELETE") == 0) { curl_easy_setopt(c, CURLOPT_CUSTOMREQUEST, "DELETE"); /* DELETE with a body (2026-07-24): the engram server authenticates * mutating requests via an "_auth" field in the JSON body, so EL * code must be able to send DELETE + body. Absent body → unchanged. */ if (body && *body) { curl_easy_setopt(c, CURLOPT_POSTFIELDS, body); curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)strlen(body)); } } CURLcode rc = curl_easy_perform(c); curl_easy_cleanup(c); if (rc != CURLE_OK) { free(rb.data); const char* m = errbuf[0] ? errbuf : curl_easy_strerror(rc); return http_error_json(m); } return el_wrap_str(rb.data); } /* Legacy entry point: default timeout. */ static el_val_t http_do(const char* method, const char* url, const char* body, struct curl_slist* extra_headers) { return http_do_t(method, url, body, extra_headers, 0); } el_val_t http_get(el_val_t url) { return http_do("GET", EL_CSTR(url), NULL, NULL); } el_val_t http_post(el_val_t url, el_val_t body) { return http_do("POST", EL_CSTR(url), EL_CSTR(body), NULL); } el_val_t http_post_json(el_val_t url, el_val_t json_body) { struct curl_slist* h = NULL; h = curl_slist_append(h, "Content-Type: application/json"); el_val_t r = http_do("POST", EL_CSTR(url), EL_CSTR(json_body), h); curl_slist_free_all(h); return r; } /* Build a curl_slist from an ElMap of name -> value strings. */ static struct curl_slist* headers_from_map(el_val_t headers_map) { struct curl_slist* h = NULL; ElMap* m = as_map(headers_map); if (!m) return NULL; for (int64_t i = 0; i < m->count; i++) { const char* k = EL_CSTR(m->keys[i]); const char* v = EL_CSTR(m->values[i]); if (!k || !v) continue; size_t n = strlen(k) + strlen(v) + 4; char* line = malloc(n); if (!line) continue; snprintf(line, n, "%s: %s", k, v); h = curl_slist_append(h, line); free(line); } return h; } el_val_t http_get_with_headers(el_val_t url, el_val_t headers_map) { struct curl_slist* h = headers_from_map(headers_map); el_val_t r = http_do("GET", EL_CSTR(url), NULL, h); if (h) curl_slist_free_all(h); return r; } el_val_t http_post_with_headers(el_val_t url, el_val_t body, el_val_t headers_map) { struct curl_slist* h = headers_from_map(headers_map); el_val_t r = http_do("POST", EL_CSTR(url), EL_CSTR(body), h); if (h) curl_slist_free_all(h); return r; } el_val_t http_post_form_auth(el_val_t url, el_val_t form_body, el_val_t auth_header) { struct curl_slist* h = NULL; h = curl_slist_append(h, "Content-Type: application/x-www-form-urlencoded"); const char* a = EL_CSTR(auth_header); if (a && *a) { size_t n = strlen(a) + 32; char* line = malloc(n); snprintf(line, n, "Authorization: %s", a); h = curl_slist_append(h, line); free(line); } el_val_t r = http_do("POST", EL_CSTR(url), EL_CSTR(form_body), h); curl_slist_free_all(h); return r; } /* HTTP DELETE — mirrors http_post but with CURLOPT_CUSTOMREQUEST=DELETE. * Returns response body on success; on transport failure returns an error * JSON fragment (same convention as http_get/http_post). Callers that * expect "" on failure should check for a leading '{' and an "error" key. */ el_val_t http_delete(el_val_t url) { return http_do("DELETE", EL_CSTR(url), NULL, NULL); } /* DELETE with a JSON body — required by the engram server's body-based * "_auth" scheme for mutating requests. (2026-07-24 self-review) */ el_val_t http_delete_json(el_val_t url, el_val_t json_body) { struct curl_slist* h = NULL; h = curl_slist_append(h, "Content-Type: application/json"); el_val_t r = http_do("DELETE", EL_CSTR(url), EL_CSTR(json_body), h); curl_slist_free_all(h); return r; } /* ── HTTP → file streaming ──────────────────────────────────────────────── * * Why this exists: el_val_t strings are NUL-terminated by convention, so * accumulating an HTTP response into an httpbuf and then wrapping its * `.data` pointer with el_wrap_str() loses the byte length. Any consumer * that does strlen() on the wrapped pointer truncates the body at the * first embedded NUL. Audio (MP3, WAV, OGG), images (PNG, JPEG), and any * other binary payload hits this. The vessels that download such bodies * (e.g. ElevenLabs TTS → MP3) get silently corrupted files. * * The fix: wire libcurl's CURLOPT_WRITEFUNCTION directly to fwrite() * against a fopen()-ed FILE*. The bytes never pass through an el_val_t * string, so embedded NULs are preserved verbatim. Caller's contract is * just "a file at this path with the response body in it". */ static size_t http_file_write_cb(char* ptr, size_t size, size_t nmemb, void* ud) { FILE* f = (FILE*)ud; return fwrite(ptr, size, nmemb, f); } /* Internal: stream body to file. method is "GET" or "POST". body may be NULL * (GET) or NUL-terminated (POST). headers may be NULL. Returns 1/0. */ static el_val_t http_do_to_file(const char* method, const char* url, const char* body, struct curl_slist* extra_headers, const char* output_path) { if (!url || !*url) return 0; if (!output_path || !*output_path) return 0; FILE* f = fopen(output_path, "wb"); if (!f) return 0; CURL* c = curl_easy_init(); if (!c) { fclose(f); remove(output_path); return 0; } char errbuf[CURL_ERROR_SIZE]; errbuf[0] = '\0'; curl_easy_setopt(c, CURLOPT_URL, url); curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_file_write_cb); curl_easy_setopt(c, CURLOPT_WRITEDATA, f); curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, el_http_timeout_ms()); curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf); curl_easy_setopt(c, CURLOPT_USERAGENT, "el-runtime/1.0"); curl_easy_setopt(c, CURLOPT_FAILONERROR, 1L); /* 4xx/5xx → CURLE_HTTP_RETURNED_ERROR */ if (extra_headers) curl_easy_setopt(c, CURLOPT_HTTPHEADER, extra_headers); if (method && strcmp(method, "POST") == 0) { curl_easy_setopt(c, CURLOPT_POST, 1L); curl_easy_setopt(c, CURLOPT_POSTFIELDS, body ? body : ""); /* For the request body we still rely on strlen — POST bodies are * caller-controlled and JSON/text in every known El use case. * If a future caller needs a binary POST body, add a *_bytes * variant that takes an explicit length, mirroring fs_write_bytes. */ curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)(body ? strlen(body) : 0)); } CURLcode rc = curl_easy_perform(c); curl_easy_cleanup(c); /* Flush + close before signalling success, so the file is fully on disk * by the time the caller reads back. */ int flush_ok = (fflush(f) == 0); int close_ok = (fclose(f) == 0); if (rc != CURLE_OK || !flush_ok || !close_ok) { remove(output_path); return 0; } return 1; } el_val_t http_get_to_file(el_val_t url, el_val_t headers_map, el_val_t output_path) { struct curl_slist* h = headers_from_map(headers_map); el_val_t r = http_do_to_file("GET", EL_CSTR(url), NULL, h, EL_CSTR(output_path)); if (h) curl_slist_free_all(h); return r; } el_val_t http_post_to_file(el_val_t url, el_val_t body, el_val_t headers_map, el_val_t output_path) { struct curl_slist* h = headers_from_map(headers_map); el_val_t r = http_do_to_file("POST", EL_CSTR(url), EL_CSTR(body), h, EL_CSTR(output_path)); if (h) curl_slist_free_all(h); return r; } /* ── HTTP server (POSIX sockets + pthreads) ──────────────────────────────── */ #define HTTP_MAX_CONNS 64 typedef el_val_t (*http_handler_fn)(el_val_t method, el_val_t path, el_val_t body); typedef struct { char* name; http_handler_fn fn; } HttpHandlerEntry; static HttpHandlerEntry _http_handlers[32]; static size_t _http_handler_count = 0; static char* _http_active_handler = NULL; static pthread_mutex_t _http_handler_mu = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t _http_conn_mu = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t _http_conn_cv = PTHREAD_COND_INITIALIZER; static int _http_conn_active = 0; /* Public C-level API: register a handler by name. Programs that want El * `http_serve` to dispatch into their handler call this from main() before * http_serve. Not declared in the header to keep the public API minimal — * extern lookup works since C symbols are global. */ void el_runtime_register_handler(const char* name, http_handler_fn fn); void el_runtime_register_handler(const char* name, http_handler_fn fn) { if (!name || !fn) return; pthread_mutex_lock(&_http_handler_mu); for (size_t i = 0; i < _http_handler_count; i++) { if (strcmp(_http_handlers[i].name, name) == 0) { _http_handlers[i].fn = fn; pthread_mutex_unlock(&_http_handler_mu); return; } } if (_http_handler_count < sizeof(_http_handlers) / sizeof(_http_handlers[0])) { _http_handlers[_http_handler_count].name = el_strdup(name); _http_handlers[_http_handler_count].fn = fn; _http_handler_count++; } pthread_mutex_unlock(&_http_handler_mu); } void http_set_handler(el_val_t name) { const char* n = EL_CSTR(name); pthread_mutex_lock(&_http_handler_mu); free(_http_active_handler); _http_active_handler = el_strdup(n ? n : ""); /* If the name is not yet in the registry, try dlsym lookup against * the running binary's symbol table. Every El `fn name(...)` compiles * to a global C symbol with that exact name, so El programs can self- * register their own handlers just by calling http_set_handler("name"). */ if (n && *n) { int found = 0; for (size_t i = 0; i < _http_handler_count; i++) { if (strcmp(_http_handlers[i].name, n) == 0) { found = 1; break; } } if (!found) { void* sym = dlsym(RTLD_DEFAULT, n); if (sym && _http_handler_count < sizeof(_http_handlers) / sizeof(_http_handlers[0])) { _http_handlers[_http_handler_count].name = el_strdup(n); _http_handlers[_http_handler_count].fn = (http_handler_fn)sym; _http_handler_count++; } } } pthread_mutex_unlock(&_http_handler_mu); } static http_handler_fn http_lookup_active(void) { http_handler_fn out = NULL; pthread_mutex_lock(&_http_handler_mu); if (_http_active_handler) { for (size_t i = 0; i < _http_handler_count; i++) { if (strcmp(_http_handlers[i].name, _http_active_handler) == 0) { out = _http_handlers[i].fn; break; } } } pthread_mutex_unlock(&_http_handler_mu); return out; } /* Auto-detect Content-Type from response body. */ static const char* http_detect_content_type(const char* body) { if (!body) return "text/plain; charset=utf-8"; const char* p = body; /* Binary magic bytes — check before stripping whitespace */ if ((unsigned char)p[0] == 0x89 && p[1]=='P' && p[2]=='N' && p[3]=='G') return "image/png"; if ((unsigned char)p[0] == 0xFF && (unsigned char)p[1] == 0xD8) return "image/jpeg"; if (strncmp(p, "GIF8", 4) == 0) return "image/gif"; if (strncmp(p, "RIFF", 4) == 0) return "image/webp"; if (strncmp(p, "wOFF", 4) == 0) return "font/woff"; if (strncmp(p, "wOF2", 4) == 0) return "font/woff2"; while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; if (strncasecmp(p, "= cap) { if (cap >= 1024 * 1024) { free(buf); return -1; } cap *= 2; buf = realloc(buf, cap); if (!buf) return -1; } ssize_t n = recv(fd, buf + len, cap - len - 1, 0); if (n <= 0) { free(buf); return -1; } len += (size_t)n; buf[len] = '\0'; if (strstr(buf, "\r\n\r\n")) break; } /* Parse request line */ char* sp1 = strchr(buf, ' '); if (!sp1) { free(buf); return -1; } *sp1 = '\0'; *out_method = el_strdup(buf); char* path_start = sp1 + 1; char* sp2 = strchr(path_start, ' '); if (!sp2) { free(*out_method); *out_method = NULL; free(buf); return -1; } *sp2 = '\0'; *out_path = el_strdup(path_start); char* hdr_end = strstr(sp2 + 1, "\r\n\r\n"); /* Capture the raw header block (after the request line's CRLF, up to * but not including the terminating \r\n\r\n) for callers that asked * for it. The legacy 3-arg path passes NULL and skips this. */ if (out_headers_block) { char* hdr_start = strstr(sp2 + 1, "\r\n"); if (hdr_start && hdr_start < hdr_end) { hdr_start += 2; size_t hb_len = (size_t)(hdr_end - hdr_start); char* hb = malloc(hb_len + 1); if (hb) { memcpy(hb, hdr_start, hb_len); hb[hb_len] = '\0'; *out_headers_block = hb; } } else { *out_headers_block = el_strdup(""); } } /* Find Content-Length */ long content_length = 0; char* hp = sp2 + 1; while (hp < hdr_end) { char* line_end = strstr(hp, "\r\n"); /* line_end == hdr_end means we're on the LAST header line — its * trailing \r\n is the same \r\n that begins the \r\n\r\n header * terminator. Process this line; only stop when line_end is past * hdr_end (which means the parser walked off the end of the * header block). The previous condition (line_end >= hdr_end) * silently dropped any Content-Length that appeared as the last * header — exactly what real curl/clients tend to emit. */ if (!line_end || line_end > hdr_end) break; if (strncasecmp(hp, "Content-Length:", 15) == 0) { content_length = strtol(hp + 15, NULL, 10); if (content_length < 0) content_length = 0; if (content_length > 64 * 1024 * 1024) content_length = 64 * 1024 * 1024; } hp = line_end + 2; } /* Body: any bytes already read past hdr_end, plus more recv */ char* body_start = hdr_end + 4; size_t body_have = (buf + len) - body_start; char* body = malloc((size_t)content_length + 1); if (!body) { free(*out_method); free(*out_path); *out_method=NULL; *out_path=NULL; free(buf); return -1; } if ((long)body_have > content_length) body_have = (size_t)content_length; if (body_have > 0) memcpy(body, body_start, body_have); while ((long)body_have < content_length) { ssize_t n = recv(fd, body + body_have, (size_t)content_length - body_have, 0); if (n <= 0) break; body_have += (size_t)n; } body[body_have] = '\0'; *out_body = body; free(buf); return 0; } /* Reason phrase for common HTTP statuses. Falls back to "Status" for the * long tail — clients only care about the numeric code. */ static const char* http_reason_phrase(int status) { switch (status) { case 200: return "OK"; case 201: return "Created"; case 202: return "Accepted"; case 204: return "No Content"; case 301: return "Moved Permanently"; case 302: return "Found"; case 303: return "See Other"; case 304: return "Not Modified"; case 307: return "Temporary Redirect"; case 308: return "Permanent Redirect"; case 400: return "Bad Request"; case 401: return "Unauthorized"; case 403: return "Forbidden"; case 404: return "Not Found"; case 405: return "Method Not Allowed"; case 409: return "Conflict"; case 410: return "Gone"; case 422: return "Unprocessable Entity"; case 429: return "Too Many Requests"; case 500: return "Internal Server Error"; case 501: return "Not Implemented"; case 502: return "Bad Gateway"; case 503: return "Service Unavailable"; case 504: return "Gateway Timeout"; default: return "Status"; } } /* Best-effort send with retry on partial writes. */ static int http_send_all(int fd, const char* p, size_t left) { while (left > 0) { ssize_t w = send(fd, p, left, 0); if (w <= 0) return -1; p += w; left -= (size_t)w; } return 0; } /* Discriminator that http_response() embeds at the start of its envelope. * A handler returning a string starting with this exact prefix is treated * as a structured response; anything else is treated as a raw body. */ #define EL_HTTP_RESPONSE_TAG "{\"el_http_response\":1" /* Keys that conflict with runtime-managed headers are silently dropped to * avoid double-emission — the runtime always emits its own Content-Length * and Connection: close. Content-Type from the envelope IS allowed and * overrides auto-detection. */ static int http_header_is_managed(const char* k) { return strcasecmp(k, "Content-Length") == 0 || strcasecmp(k, "Connection") == 0; } /* Walk an ElMap of header pairs and emit each as `K: V\r\n` into JsonBuf b. * Sets *out_saw_content_type to 1 if the map contained an explicit * Content-Type so the caller can skip auto-detection. */ static void http_emit_headers_from_map(JsonBuf* b, el_val_t headers_map, int* out_saw_content_type) { *out_saw_content_type = 0; if (headers_map == 0) return; ElMap* m = (ElMap*)(uintptr_t)headers_map; if (!m || m->hdr.magic != EL_MAGIC_MAP) return; for (int64_t i = 0; i < m->count; i++) { const char* k = EL_CSTR(m->keys[i]); const char* v = EL_CSTR(m->values[i]); if (!k || !v) continue; if (http_header_is_managed(k)) continue; if (strcasecmp(k, "Content-Type") == 0) *out_saw_content_type = 1; jb_puts(b, k); jb_puts(b, ": "); jb_puts(b, v); jb_puts(b, "\r\n"); } } /* Parse the envelope produced by http_response(). On success returns 1 and * populates *out_status, *out_headers_map (an ElMap el_val_t — caller must * el_release), and *out_body (allocated). On failure returns 0. * * Implementation: feeds the entire envelope through the recursive-descent * JSON parser (which builds proper ElMap/ElList values), then pulls the * three top-level fields by name. Avoids re-stringifying the headers map * since json_stringify() does not support nested objects. */ static int http_parse_envelope(const char* s, int* out_status, el_val_t* out_headers_map, char** out_body, el_val_t* out_parsed_root) { if (!s) return 0; if (strncmp(s, EL_HTTP_RESPONSE_TAG, sizeof(EL_HTTP_RESPONSE_TAG) - 1) != 0) return 0; el_val_t parsed = json_parse(EL_STR(s)); if (parsed == EL_NULL) return 0; int status = 200; el_val_t hmap = 0; char* body = NULL; el_val_t sv = el_map_get(parsed, EL_STR("status")); if (sv != 0) { /* status comes back as an integer — el_val_t holds it directly. */ long sc = (long)sv; if (sc >= 100 && sc <= 599) status = (int)sc; } el_val_t hv = el_map_get(parsed, EL_STR("headers")); if (hv != 0) { ElMap* hm = (ElMap*)(uintptr_t)hv; if (hm && hm->hdr.magic == EL_MAGIC_MAP) hmap = hv; } el_val_t bv = el_map_get(parsed, EL_STR("body")); if (bv != 0) { const char* bs = EL_CSTR(bv); if (bs) body = el_strdup(bs); } if (!body) body = el_strdup(""); *out_status = status; *out_headers_map = hmap; *out_body = body; *out_parsed_root = parsed; /* caller releases to free hmap + entries */ return 1; } /* Lightweight `__status__` envelope: if the body's first key is `__status__` * and its value is a numeric literal, lift the status to the HTTP layer and * strip the marker from the body before sending. This is the common case for * El handlers that want to return 4xx/5xx without going through * http_response() — they just prepend `{"__status__":,...}` to the JSON * they were already returning. * * We deliberately recognise ONLY the first-key form so the contract is cheap * to detect and unambiguous: `{"__status__":401,"error":"unauthorized"}` is * an envelope, but `{"error":"...","__status__":401}` is not. Product code * controls placement. * * On success returns 1 with *out_status set and *out_body_alloc populated * with a freshly malloc'd body (caller frees). On failure returns 0 and * leaves outputs untouched. */ static int http_parse_status_envelope(const char* s, int* out_status, char** out_body_alloc) { if (!s) return 0; const char* p = s; while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; if (*p != '{') return 0; p++; while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; static const char marker[] = "\"__status__\""; size_t mlen = sizeof(marker) - 1; if (strncmp(p, marker, mlen) != 0) return 0; p += mlen; while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; if (*p != ':') return 0; p++; while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; if (*p < '0' || *p > '9') return 0; /* non-numeric -> not an envelope */ int status = 0; while (*p >= '0' && *p <= '9') { status = status * 10 + (*p - '0'); p++; } if (status < 100 || status > 599) return 0; while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; /* Two trailing shapes accepted: * ,"k":v,...} -> body becomes {"k":v,...} * } -> body becomes {} * Anything else (e.g. `:` re-appearing, garbage) drops the envelope so * we don't strip what we shouldn't. */ if (*p == '}') { *out_status = status; *out_body_alloc = el_strdup("{}"); return 1; } if (*p != ',') return 0; p++; /* skip the comma; the rest of the object follows */ while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; /* Build the trimmed body: '{' + remainder. */ size_t rest_len = strlen(p); char* out = (char*)malloc(rest_len + 2); if (!out) return 0; out[0] = '{'; memcpy(out + 1, p, rest_len); out[rest_len + 1] = '\0'; *out_status = status; *out_body_alloc = out; return 1; } /* Send a fully-built HTTP response. If `body` starts with the envelope tag, * unpack status/headers/body. Otherwise emit the historical 200-OK with * auto-detected Content-Type. */ /* Thread-local flag: if 1, http_send_response writes status + headers but * NO body (HEAD method behaviour). Set by http_worker before calling * http_send_response, cleared after. */ static __thread int _tl_http_head_only = 0; static void http_send_response(int fd, const char* body) { if (!body) body = ""; int status = 200; el_val_t env_headers_map = 0; char* env_body = NULL; el_val_t env_parsed_root = 0; int is_envelope = http_parse_envelope(body, &status, &env_headers_map, &env_body, &env_parsed_root); /* If the rich http_response() envelope didn't claim this body, try the * lightweight `__status__` form. This second envelope is malloc-backed so * we route it through env_body and let the existing cleanup path free it * — same lifetime contract, no special case at the bottom of the * function. */ if (!is_envelope) { char* trimmed = NULL; if (http_parse_status_envelope(body, &status, &trimmed)) { env_body = trimmed; is_envelope = 1; } } const char* eff_body = is_envelope ? env_body : body; /* Use the real byte count from fs_read if available (handles binary files * with embedded null bytes — PNG, WOFF2, etc.). Fall back to strlen for * normal text/JSON responses where _tl_fs_read_len is 0. */ size_t blen = (_tl_fs_read_len > 0) ? _tl_fs_read_len : strlen(eff_body); _tl_fs_read_len = 0; /* consume — one-shot per response */ int head_only = _tl_http_head_only; JsonBuf hdrs; jb_init(&hdrs); int saw_content_type = 0; if (is_envelope) { http_emit_headers_from_map(&hdrs, env_headers_map, &saw_content_type); } if (!saw_content_type) { jb_puts(&hdrs, "Content-Type: "); jb_puts(&hdrs, http_detect_content_type(eff_body)); jb_puts(&hdrs, "\r\n"); } char status_line[64]; int sl = snprintf(status_line, sizeof(status_line), "HTTP/1.1 %d %s\r\n", status, http_reason_phrase(status)); if (sl < 0) { if (env_parsed_root) el_release(env_parsed_root); free(env_body); free(hdrs.buf); return; } char tail[128]; int tl = snprintf(tail, sizeof(tail), "Content-Length: %zu\r\n" "Connection: close\r\n" "\r\n", blen); if (tl < 0) { if (env_parsed_root) el_release(env_parsed_root); free(env_body); free(hdrs.buf); return; } if (http_send_all(fd, status_line, (size_t)sl) == 0 && http_send_all(fd, hdrs.buf, hdrs.len) == 0 && http_send_all(fd, tail, (size_t)tl) == 0 && (head_only /* HEAD requests echo headers + Content-Length but no body. */ ? 1 : http_send_all(fd, eff_body, blen) == 0)) { /* sent successfully */ } if (env_parsed_root) el_release(env_parsed_root); free(env_body); free(hdrs.buf); } typedef struct { int fd; } HttpWorkerArg; static void* http_worker(void* arg) { HttpWorkerArg* a = (HttpWorkerArg*)arg; int fd = a->fd; free(a); char *method = NULL, *path = NULL, *body = NULL; if (http_read_request(fd, &method, &path, &body, NULL) == 0) { http_handler_fn h = http_lookup_active(); char* response = NULL; /* HEAD: dispatch as GET so existing handlers respond with the same * body, but flag the response writer to emit headers only. RFC 9110 * requires HEAD to mirror GET headers + Content-Length without body. */ int head_only = (method && strcmp(method, "HEAD") == 0); const char* dispatch_method = head_only ? "GET" : method; el_request_start(); /* begin per-request arena */ if (h) { el_val_t r = h(EL_STR(dispatch_method), EL_STR(path), EL_STR(body)); const char* rs = EL_CSTR(r); /* Copy response out BEFORE arena teardown. * For binary files, _tl_fs_read_len holds the real byte count — * use memcpy instead of strdup so null bytes are preserved. */ size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0); response = malloc(rlen + 1); if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; } else if (response) { response[0] = '\0'; } } else { response = el_strdup_persist("el-runtime: no http handler registered"); } el_request_end(); /* free all intermediate strings */ _tl_http_head_only = head_only; http_send_response(fd, response); _tl_http_head_only = 0; free(response); } free(method); free(path); free(body); close(fd); /* release a slot */ pthread_mutex_lock(&_http_conn_mu); _http_conn_active--; pthread_cond_signal(&_http_conn_cv); pthread_mutex_unlock(&_http_conn_mu); return NULL; } void http_serve(el_val_t port, el_val_t handler) { /* If `handler` looks like a string name, register it as the active handler. */ const char* hname = EL_CSTR(handler); if (hname && looks_like_string(handler)) { http_set_handler(handler); } int p = (int)port; if (p <= 0 || p > 65535) { fprintf(stderr, "http_serve: invalid port %d\n", p); return; } /* Dual-stack: AF_INET6 with IPV6_V6ONLY=0 accepts both IPv4 and IPv6. * This makes `localhost` work in browsers that resolve it to ::1 first. */ int sock = socket(AF_INET6, SOCK_STREAM, 0); if (sock < 0) { perror("socket"); return; } int yes = 1; int no = 0; setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &no, sizeof(no)); struct sockaddr_in6 addr; memset(&addr, 0, sizeof(addr)); addr.sin6_family = AF_INET6; addr.sin6_addr = in6addr_any; addr.sin6_port = htons((uint16_t)p); if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) { perror("bind"); close(sock); return; } if (listen(sock, 64) < 0) { perror("listen"); close(sock); return; } fprintf(stderr, "[http] listening on [::]:%d (dual-stack)\n", p); while (1) { struct sockaddr_in6 cli; socklen_t clen = sizeof(cli); int cfd = accept(sock, (struct sockaddr*)&cli, &clen); if (cfd < 0) { if (errno == EINTR) continue; perror("accept"); break; } pthread_mutex_lock(&_http_conn_mu); while (_http_conn_active >= HTTP_MAX_CONNS) { pthread_cond_wait(&_http_conn_cv, &_http_conn_mu); } _http_conn_active++; pthread_mutex_unlock(&_http_conn_mu); HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg)); if (!arg) { close(cfd); continue; } arg->fd = cfd; pthread_t tid; if (pthread_create(&tid, NULL, http_worker, arg) != 0) { close(cfd); free(arg); pthread_mutex_lock(&_http_conn_mu); _http_conn_active--; pthread_cond_signal(&_http_conn_cv); pthread_mutex_unlock(&_http_conn_mu); continue; } pthread_detach(tid); } close(sock); } /* ── http_serve_async — non-blocking HTTP server ─────────────────────────── */ /* Runs the accept loop in a background pthread, returns immediately so the * calling EL script can continue (e.g. to run an awareness loop). * Ported verbatim from el-compiler/runtime on 2026-07-17: the soul daemon * (soul.el) builds against this release runtime and calls http_serve_async, * which was present only in the dev runtime copy. * * El signature: http_serve_async(port, handler) -> Void */ typedef struct { int sock; } HttpServeAsyncArg; static void* _http_serve_async_loop(void* raw) { HttpServeAsyncArg* a = (HttpServeAsyncArg*)raw; int sock = a->sock; free(a); while (1) { struct sockaddr_in6 cli; socklen_t clen = sizeof(cli); int cfd = accept(sock, (struct sockaddr*)&cli, &clen); if (cfd < 0) { if (errno == EINTR) continue; perror("accept"); break; } pthread_mutex_lock(&_http_conn_mu); while (_http_conn_active >= HTTP_MAX_CONNS) { pthread_cond_wait(&_http_conn_cv, &_http_conn_mu); } _http_conn_active++; pthread_mutex_unlock(&_http_conn_mu); HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg)); if (!arg) { close(cfd); continue; } arg->fd = cfd; pthread_t tid; if (pthread_create(&tid, NULL, http_worker, arg) != 0) { close(cfd); free(arg); pthread_mutex_lock(&_http_conn_mu); _http_conn_active--; pthread_cond_signal(&_http_conn_cv); pthread_mutex_unlock(&_http_conn_mu); continue; } pthread_detach(tid); } close(sock); return NULL; } void http_serve_async(el_val_t port, el_val_t handler) { const char* hname = EL_CSTR(handler); if (hname && looks_like_string(handler)) { http_set_handler(handler); } int p = (int)port; if (p <= 0 || p > 65535) { fprintf(stderr, "http_serve_async: invalid port %d\n", p); return; } int sock = socket(AF_INET6, SOCK_STREAM, 0); if (sock < 0) { perror("socket"); return; } int yes = 1; int no = 0; setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &no, sizeof(no)); struct sockaddr_in6 addr; memset(&addr, 0, sizeof(addr)); addr.sin6_family = AF_INET6; addr.sin6_addr = in6addr_any; addr.sin6_port = htons((uint16_t)p); if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) { perror("bind"); close(sock); return; } if (listen(sock, 64) < 0) { perror("listen"); close(sock); return; } fprintf(stderr, "[http] async listening on [::]:%d (dual-stack)\n", p); HttpServeAsyncArg* a = malloc(sizeof(HttpServeAsyncArg)); if (!a) { close(sock); return; } a->sock = sock; pthread_t tid; if (pthread_create(&tid, NULL, _http_serve_async_loop, a) != 0) { perror("pthread_create"); free(a); close(sock); return; } pthread_detach(tid); /* Returns immediately — caller can now run awareness_run() or any loop. */ } /* ── HTTP server v2 — request headers + structured response ──────────────── */ /* * v2 widens the handler signature from * (method, path, body) -> body_string * to * (method, path, headers_map, body) -> body_string_or_envelope * * The response envelope is detected uniformly inside http_send_response — so * 4-arg handlers can return either a plain body or http_response(...). The * 3-arg path stays untouched in spirit (its handlers still build plain * bodies; the envelope tag, being `{"el_http_response":1`, will never * collide with normal JSON the legacy server.el routes return). * * Registry is parallel to the 3-arg handler registry: separate name table, * separate active-handler slot, separate dlsym fallback. Mixing v1 and v2 * handlers in the same process is fine — they don't share the active slot. */ typedef el_val_t (*http_handler4_fn)(el_val_t method, el_val_t path, el_val_t headers_map, el_val_t body); typedef struct { char* name; http_handler4_fn fn; } HttpHandler4Entry; static HttpHandler4Entry _http_handlers4[32]; static size_t _http_handler4_count = 0; static char* _http_active_handler4 = NULL; void el_runtime_register_handler_v2(const char* name, http_handler4_fn fn); void el_runtime_register_handler_v2(const char* name, http_handler4_fn fn) { if (!name || !fn) return; pthread_mutex_lock(&_http_handler_mu); for (size_t i = 0; i < _http_handler4_count; i++) { if (strcmp(_http_handlers4[i].name, name) == 0) { _http_handlers4[i].fn = fn; pthread_mutex_unlock(&_http_handler_mu); return; } } if (_http_handler4_count < sizeof(_http_handlers4) / sizeof(_http_handlers4[0])) { _http_handlers4[_http_handler4_count].name = el_strdup(name); _http_handlers4[_http_handler4_count].fn = fn; _http_handler4_count++; } pthread_mutex_unlock(&_http_handler_mu); } void http_set_handler_v2(el_val_t name) { const char* n = EL_CSTR(name); pthread_mutex_lock(&_http_handler_mu); free(_http_active_handler4); _http_active_handler4 = el_strdup(n ? n : ""); if (n && *n) { int found = 0; for (size_t i = 0; i < _http_handler4_count; i++) { if (strcmp(_http_handlers4[i].name, n) == 0) { found = 1; break; } } if (!found) { void* sym = dlsym(RTLD_DEFAULT, n); if (sym && _http_handler4_count < sizeof(_http_handlers4) / sizeof(_http_handlers4[0])) { _http_handlers4[_http_handler4_count].name = el_strdup(n); _http_handlers4[_http_handler4_count].fn = (http_handler4_fn)sym; _http_handler4_count++; } } } pthread_mutex_unlock(&_http_handler_mu); } static http_handler4_fn http_lookup_active_v2(void) { http_handler4_fn out = NULL; pthread_mutex_lock(&_http_handler_mu); if (_http_active_handler4) { for (size_t i = 0; i < _http_handler4_count; i++) { if (strcmp(_http_handlers4[i].name, _http_active_handler4) == 0) { out = _http_handlers4[i].fn; break; } } } pthread_mutex_unlock(&_http_handler_mu); return out; } /* Build an ElMap from the raw header block produced by http_read_request. * Keys are lowercased (RFC 7230 — case-insensitive); values have leading * whitespace trimmed. Repeated headers with the same name are joined with * ", " in arrival order, matching standard library behaviour elsewhere. */ static el_val_t http_build_headers_map(const char* hdr_block) { el_val_t m = el_map_new(0); if (!hdr_block || !*hdr_block) return m; const char* p = hdr_block; while (*p) { const char* line_end = strstr(p, "\r\n"); const char* end = line_end ? line_end : p + strlen(p); const char* colon = NULL; for (const char* c = p; c < end; c++) { if (*c == ':') { colon = c; break; } } if (colon && colon > p) { size_t klen = (size_t)(colon - p); char* key = malloc(klen + 1); if (key) { for (size_t i = 0; i < klen; i++) { unsigned char ch = (unsigned char)p[i]; key[i] = (char)tolower(ch); } key[klen] = '\0'; const char* vstart = colon + 1; while (vstart < end && (*vstart == ' ' || *vstart == '\t')) vstart++; size_t vlen = (size_t)(end - vstart); /* Strip trailing OWS just in case. */ while (vlen > 0 && (vstart[vlen - 1] == ' ' || vstart[vlen - 1] == '\t')) vlen--; /* Coalesce repeats: if key already present, append ", value". */ el_val_t existing = el_map_get(m, EL_STR(key)); if (existing != 0 && looks_like_string(existing)) { const char* old = EL_CSTR(existing); size_t olen = strlen(old); char* combined = malloc(olen + 2 + vlen + 1); if (combined) { memcpy(combined, old, olen); memcpy(combined + olen, ", ", 2); memcpy(combined + olen + 2, vstart, vlen); combined[olen + 2 + vlen] = '\0'; m = el_map_set(m, EL_STR(key), EL_STR(combined)); } free(key); } else { char* val = malloc(vlen + 1); if (val) { memcpy(val, vstart, vlen); val[vlen] = '\0'; m = el_map_set(m, EL_STR(key), EL_STR(val)); } else { free(key); } } } } if (!line_end) break; p = line_end + 2; } return m; } static void* http_worker_v2(void* arg) { HttpWorkerArg* a = (HttpWorkerArg*)arg; int fd = a->fd; free(a); char *method = NULL, *path = NULL, *body = NULL, *hdr_block = NULL; if (http_read_request(fd, &method, &path, &body, &hdr_block) == 0) { http_handler4_fn h = http_lookup_active_v2(); char* response = NULL; int head_only = (method && strcmp(method, "HEAD") == 0); const char* dispatch_method = head_only ? "GET" : method; el_request_start(); /* begin per-request arena */ if (h) { el_val_t hmap = http_build_headers_map(hdr_block ? hdr_block : ""); el_val_t r = h(EL_STR(dispatch_method), EL_STR(path), hmap, EL_STR(body)); const char* rs = EL_CSTR(r); size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0); response = malloc(rlen + 1); if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; } else if (response) { response[0] = '\0'; } el_release(hmap); } else { response = el_strdup_persist( "el-runtime: no v2 http handler registered " "(call http_set_handler_v2)"); } el_request_end(); /* free all intermediate strings */ _tl_http_head_only = head_only; http_send_response(fd, response); _tl_http_head_only = 0; free(response); } free(method); free(path); free(body); free(hdr_block); close(fd); pthread_mutex_lock(&_http_conn_mu); _http_conn_active--; pthread_cond_signal(&_http_conn_cv); pthread_mutex_unlock(&_http_conn_mu); return NULL; } void http_serve_v2(el_val_t port, el_val_t handler) { const char* hname = EL_CSTR(handler); if (hname && looks_like_string(handler)) { http_set_handler_v2(handler); } int p = (int)port; if (p <= 0 || p > 65535) { fprintf(stderr, "http_serve_v2: invalid port %d\n", p); return; } /* Dual-stack: same as http_serve - AF_INET6 + IPV6_V6ONLY=0. */ int sock = socket(AF_INET6, SOCK_STREAM, 0); if (sock < 0) { perror("socket"); return; } int yes = 1; int no = 0; setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &no, sizeof(no)); struct sockaddr_in6 addr; memset(&addr, 0, sizeof(addr)); addr.sin6_family = AF_INET6; addr.sin6_addr = in6addr_any; addr.sin6_port = htons((uint16_t)p); if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) { perror("bind"); close(sock); return; } if (listen(sock, 64) < 0) { perror("listen"); close(sock); return; } fprintf(stderr, "[http v2] listening on [::]:%d (dual-stack)\n", p); while (1) { struct sockaddr_in6 cli; socklen_t clen = sizeof(cli); int cfd = accept(sock, (struct sockaddr*)&cli, &clen); if (cfd < 0) { if (errno == EINTR) continue; perror("accept"); break; } pthread_mutex_lock(&_http_conn_mu); while (_http_conn_active >= HTTP_MAX_CONNS) { pthread_cond_wait(&_http_conn_cv, &_http_conn_mu); } _http_conn_active++; pthread_mutex_unlock(&_http_conn_mu); HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg)); if (!arg) { close(cfd); continue; } arg->fd = cfd; pthread_t tid; if (pthread_create(&tid, NULL, http_worker_v2, arg) != 0) { close(cfd); free(arg); pthread_mutex_lock(&_http_conn_mu); _http_conn_active--; pthread_cond_signal(&_http_conn_cv); pthread_mutex_unlock(&_http_conn_mu); continue; } pthread_detach(tid); } close(sock); } /* Build the response envelope a 4-arg handler can return. We hand-write * the JSON so the discriminator key always lands first — the runtime's * http_parse_envelope() detects it via prefix match. headers_json must be * either "" (empty), "{}" (empty object), or a well-formed JSON object * literal; anything else will produce a malformed envelope and the runtime * will treat the whole string as a plain body (no envelope detected). */ el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body) { long sc = (long)status; if (sc < 100 || sc > 599) sc = 200; const char* hj = EL_CSTR(headers_json); if (!hj || !*hj) hj = "{}"; /* Light validation: must start with '{' and end with '}'. */ size_t hlen = strlen(hj); int hj_ok = (hlen >= 2 && hj[0] == '{' && hj[hlen - 1] == '}'); if (!hj_ok) hj = "{}"; const char* b = EL_CSTR(body); if (!b) b = ""; JsonBuf out; jb_init(&out); jb_puts(&out, EL_HTTP_RESPONSE_TAG); /* {"el_http_response":1 */ jb_puts(&out, ",\"status\":"); char num[32]; snprintf(num, sizeof(num), "%ld", sc); jb_puts(&out, num); jb_puts(&out, ",\"headers\":"); jb_puts(&out, hj); jb_puts(&out, ",\"body\":"); jb_emit_escaped(&out, b); jb_putc(&out, '}'); return el_wrap_str(out.buf); } /* ── Filesystem ──────────────────────────────────────────────────────────── */ el_val_t fs_read(el_val_t pathv) { const char* path = EL_CSTR(pathv); _tl_fs_read_len = 0; if (!path) return el_wrap_str(el_strdup("")); FILE* f = fopen(path, "rb"); if (!f) return el_wrap_str(el_strdup("")); fseek(f, 0, SEEK_END); long sz = ftell(f); rewind(f); if (sz < 0) { fclose(f); return el_wrap_str(el_strdup("")); } /* pipe/special file */ char* buf = el_strbuf((size_t)sz); size_t got = fread(buf, 1, (size_t)sz, f); buf[got] = '\0'; _tl_fs_read_len = got; /* store real byte count for binary-safe send */ fclose(f); return el_wrap_str(buf); } el_val_t fs_write(el_val_t pathv, el_val_t contentv) { const char* path = EL_CSTR(pathv); const char* content = EL_CSTR(contentv); if (!path || !content) return 0; FILE* f = fopen(path, "wb"); if (!f) return 0; size_t n = strlen(content); size_t written = fwrite(content, 1, n, f); fclose(f); return written == n ? 1 : 0; } /* fs_write_bytes — explicit-length binary write. Bypasses strlen so embedded * NULs survive. Caller must know the byte count (e.g. from base64_decode, * or the fixed 32-byte sha256_bytes/hmac_sha256_bytes outputs). * * If `length` is negative, treats as failure. If `length` is 0, creates an * empty file (still useful as a "touch with content" primitive). */ el_val_t fs_write_bytes(el_val_t pathv, el_val_t bytesv, el_val_t lengthv) { const char* path = EL_CSTR(pathv); const char* bytes = EL_CSTR(bytesv); int64_t n = (int64_t)lengthv; if (!path || !bytes) return 0; if (n < 0) return 0; FILE* f = fopen(path, "wb"); if (!f) return 0; size_t written = (n > 0) ? fwrite(bytes, 1, (size_t)n, f) : 0; int flush_ok = (fflush(f) == 0); int close_ok = (fclose(f) == 0); if (!flush_ok || !close_ok || written != (size_t)n) { remove(path); return 0; } return 1; } // exec_command — run a shell command, return exit code (0 = success). // Used by elb and other El tooling to invoke subprocesses. el_val_t exec_command(el_val_t cmdv) { const char* cmd = EL_CSTR(cmdv); if (!cmd) return (el_val_t)(int64_t)-1; int ret = system(cmd); return (el_val_t)(int64_t)ret; } // exec_capture — run a shell command, capture stdout, return as String. // Returns "" on failure. el_val_t exec_capture(el_val_t cmdv) { const char* cmd = EL_CSTR(cmdv); if (!cmd) return el_wrap_str(el_strdup("")); FILE* f = popen(cmd, "r"); if (!f) return el_wrap_str(el_strdup("")); JsonBuf b; jb_init(&b); char buf[4096]; while (fgets(buf, sizeof(buf), f)) jb_puts(&b, buf); pclose(f); return el_wrap_str(b.buf); } // exec — run a shell command via /bin/sh, capture stdout, return as String. // Times out after 30 seconds. Returns "" on any error. // El name: exec(cmd) -> String el_val_t exec(el_val_t cmdv) { const char* cmd = EL_CSTR(cmdv); if (!cmd || !*cmd) return el_wrap_str(el_strdup("")); /* Build a time-limited command: wrap with timeout(1) if available, * otherwise rely on the 30s read loop guard below. We use the simple * popen approach with a deadline measured by wall clock so the caller * is never blocked indefinitely. */ FILE* f = popen(cmd, "r"); if (!f) return el_wrap_str(el_strdup("")); JsonBuf b; jb_init(&b); char buf[4096]; /* 30-second wall-clock deadline */ time_t deadline = time(NULL) + 30; while (time(NULL) < deadline) { if (fgets(buf, sizeof(buf), f) == NULL) break; jb_puts(&b, buf); } pclose(f); return el_wrap_str(b.buf); } // exec_bg — run a shell command in background, return PID as String. // The child process runs independently; the caller is not blocked. // Returns "" on fork failure. // El name: exec_bg(cmd) -> String el_val_t exec_bg(el_val_t cmdv) { const char* cmd = EL_CSTR(cmdv); if (!cmd || !*cmd) return el_wrap_str(el_strdup("")); pid_t pid = fork(); if (pid < 0) { /* fork failed */ return el_wrap_str(el_strdup("")); } if (pid == 0) { /* child: detach from parent's stdio, exec via shell */ setsid(); int devnull = open("/dev/null", O_RDWR); if (devnull >= 0) { dup2(devnull, STDIN_FILENO); dup2(devnull, STDOUT_FILENO); dup2(devnull, STDERR_FILENO); close(devnull); } execl("/bin/sh", "sh", "-c", cmd, (char*)NULL); _exit(127); } /* parent: convert pid to string and return immediately */ char pidbuf[32]; snprintf(pidbuf, sizeof(pidbuf), "%d", (int)pid); return el_wrap_str(el_strdup(pidbuf)); } el_val_t fs_list(el_val_t pathv) { const char* path = EL_CSTR(pathv); el_val_t lst = el_list_empty(); if (!path) return lst; DIR* d = opendir(path); if (!d) return lst; struct dirent* e; while ((e = readdir(d)) != NULL) { if (strcmp(e->d_name, ".") == 0 || strcmp(e->d_name, "..") == 0) continue; lst = el_list_append(lst, el_wrap_str(el_strdup(e->d_name))); } closedir(d); return lst; } /* fs_exists — true iff stat(path) succeeds. Symlinks are followed. */ el_val_t fs_exists(el_val_t pathv) { const char* path = EL_CSTR(pathv); if (!path || !*path) return 0; struct stat st; return (el_val_t)(stat(path, &st) == 0 ? 1 : 0); } /* fs_mkdir — create directory at path with mode 0755, mkdir -p semantics. * Returns 1 if path exists or was created (incl. all parents); 0 on failure. * Walks the path component-by-component so missing intermediate dirs are * also created. An existing leaf is not an error. */ el_val_t fs_mkdir(el_val_t pathv) { const char* path = EL_CSTR(pathv); if (!path || !*path) return 0; size_t n = strlen(path); char* buf = malloc(n + 1); if (!buf) return 0; memcpy(buf, path, n + 1); /* Walk components; create each prefix in turn. */ for (size_t i = 1; i <= n; i++) { if (buf[i] == '/' || buf[i] == '\0') { char saved = buf[i]; buf[i] = '\0'; if (buf[0] != '\0') { if (mkdir(buf, 0755) != 0 && errno != EEXIST) { /* Tolerate the case where this prefix exists as a non-dir * only when stat says it's a directory. */ struct stat st; if (stat(buf, &st) != 0 || !S_ISDIR(st.st_mode)) { free(buf); return 0; } } } buf[i] = saved; } } free(buf); return 1; } /* ── URL encoding ─────────────────────────────────────────────────────────── */ /* RFC 3986 percent-encoding for URL components (form bodies, query strings). * Unreserved set: A-Z a-z 0-9 - _ . ~ — passed through verbatim. * Everything else (including space) becomes %XX hex. */ el_val_t url_encode(el_val_t sv) { const char* s = EL_CSTR(sv); if (!s) return el_wrap_str(el_strdup("")); static const char hex[] = "0123456789ABCDEF"; size_t n = strlen(s); char* out = el_strbuf(n * 3); size_t o = 0; for (size_t i = 0; i < n; i++) { unsigned char c = (unsigned char)s[i]; if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '.' || c == '~') { out[o++] = (char)c; } else { out[o++] = '%'; out[o++] = hex[(c >> 4) & 0xF]; out[o++] = hex[c & 0xF]; } } out[o] = '\0'; return el_wrap_str(out); } /* Decode percent-encoded URL component. '+' becomes space (form-encoded); * malformed %-escapes are emitted verbatim. */ el_val_t url_decode(el_val_t sv) { const char* s = EL_CSTR(sv); if (!s) return el_wrap_str(el_strdup("")); size_t n = strlen(s); char* out = el_strbuf(n); size_t o = 0; for (size_t i = 0; i < n; i++) { char c = s[i]; if (c == '+') { out[o++] = ' '; } else if (c == '%' && i + 2 < n) { char h1 = s[i + 1], h2 = s[i + 2]; int v1 = (h1 >= '0' && h1 <= '9') ? h1 - '0' : (h1 >= 'a' && h1 <= 'f') ? h1 - 'a' + 10 : (h1 >= 'A' && h1 <= 'F') ? h1 - 'A' + 10 : -1; int v2 = (h2 >= '0' && h2 <= '9') ? h2 - '0' : (h2 >= 'a' && h2 <= 'f') ? h2 - 'a' + 10 : (h2 >= 'A' && h2 <= 'F') ? h2 - 'A' + 10 : -1; if (v1 >= 0 && v2 >= 0) { out[o++] = (char)((v1 << 4) | v2); i += 2; } else { out[o++] = c; } } else { out[o++] = c; } } out[o] = '\0'; return el_wrap_str(out); } /* ── HTML allowlist sanitizer ──────────────────────────────────────────────── * el_html_sanitize(input, allowlist_json) * * Strict allowlist HTML cleaner. Replaces the older denylist patterns * (str_replace cascades that wrapped dangerous tags in HTML comments and * renamed `on*` attributes). The denylist approach is fragile: comment- * wrapping can be re-broken by a literal `-->` inside an attacker-supplied * attribute value, and every new attack vector requires a code change. * * Design: * - Single-pass byte-level state machine. * - Tag and attribute names are matched case-insensitively against the * allowlist. Unknown tags are dropped entirely (the open and close * markers are stripped; their inner text content survives, escaped). * - A small set of "dangerous container" tags (script, style, iframe, * object, embed, form, plus a few rarer ones) drop themselves AND * their full subtree — text between `` is * CDATA-like and must not be re-emitted as escaped text either. * - Comments (), doctype (), CDATA (), * and processing instructions () are dropped entirely. * - Text content outside dropped subtrees is HTML-escaped (&, <, >, ", '). * - Attribute values are unquoted/dequoted, then re-emitted with double * quotes around the cleanly-escaped value. * - For `` and any `src` attribute, the URL scheme is validated: * only http:, https:, mailto:, fragment-only `#anchor`, or relative * paths are allowed. Anything else (javascript:, data:, vbscript:, * about:, file:, etc.) drops the attribute. * - Self-closing void tags (br, hr, img, etc.) emit without a close tag. * - Malformed input (unclosed tag at EOF, bad attribute syntax) drops * the pending tag and continues. Pre-encoded entities (<, &, * etc.) are passed through verbatim — the browser will decode them * safely on render. * * Allowlist format (JSON string): * {"p":[],"a":["href","title"],"strong":[],...} * - Key = lowercase tag name. * - Value = JSON array of allowed attribute names (lowercase). * - Empty array means tag allowed but no attributes survive. * * Output is a freshly-allocated arena-tracked el_val_t string. */ /* Internal byte buffer with realloc-doubling. Used during sanitization; * the final result is copied into an arena-tracked el_strbuf so the caller * sees standard runtime memory semantics. */ typedef struct { char* data; size_t len; size_t cap; } html_buf_t; static void html_buf_init(html_buf_t* b) { b->cap = 256; b->data = malloc(b->cap); if (!b->data) { fputs("el_runtime: out of memory\n", stderr); exit(1); } b->len = 0; } static void html_buf_grow(html_buf_t* b, size_t need) { if (b->len + need + 1 <= b->cap) return; size_t nc = b->cap; while (b->len + need + 1 > nc) nc *= 2; char* nd = realloc(b->data, nc); if (!nd) { fputs("el_runtime: out of memory\n", stderr); exit(1); } b->data = nd; b->cap = nc; } static void html_buf_putc(html_buf_t* b, char c) { html_buf_grow(b, 1); b->data[b->len++] = c; } static void html_buf_puts(html_buf_t* b, const char* s) { if (!s) return; size_t n = strlen(s); html_buf_grow(b, n); memcpy(b->data + b->len, s, n); b->len += n; } static void html_buf_free(html_buf_t* b) { free(b->data); b->data = NULL; b->len = b->cap = 0; } /* ASCII tolower, locale-independent. */ static int html_tolower(int c) { return (c >= 'A' && c <= 'Z') ? c + 32 : c; } /* Case-insensitive ASCII compare of [a, a+n) against c-string `s`. * Returns 1 iff lengths match and bytes are equal under tolower. */ static int html_ieq_n(const char* a, size_t n, const char* s) { if (!a || !s) return 0; if (strlen(s) != n) return 0; for (size_t i = 0; i < n; i++) { if (html_tolower((unsigned char)a[i]) != html_tolower((unsigned char)s[i])) return 0; } return 1; } /* Case-insensitive ASCII compare of two byte slices. */ static int html_iemem(const char* a, const char* b, size_t n) { for (size_t i = 0; i < n; i++) { if (html_tolower((unsigned char)a[i]) != html_tolower((unsigned char)b[i])) return 0; } return 1; } /* Walk a JSON allowlist object and find the value (an array) for a given * tag key, comparing case-insensitively. On hit returns a pointer to the * opening `[` of the array and writes the byte length of the array span * (including the brackets) to *out_len. On miss returns NULL. * * The parser is intentionally tiny: it does not handle escapes inside * keys (allowlist authors do not need them), and it relies on balanced * brackets/quotes within the value array. */ static const char* html_allowlist_find(const char* allow, const char* tag, size_t tag_len, size_t* out_len) { if (!allow) return NULL; const char* p = allow; while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; if (*p != '{') return NULL; p++; while (*p) { while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r' || *p == ',') p++; if (*p == '}' || *p == 0) return NULL; if (*p != '"') return NULL; p++; const char* k = p; while (*p && *p != '"') p++; if (*p != '"') return NULL; size_t klen = (size_t)(p - k); p++; while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; if (*p != ':') return NULL; p++; while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; if (*p != '[') return NULL; const char* arr_start = p; int depth = 0; int in_str = 0; while (*p) { char c = *p; if (in_str) { if (c == '\\' && p[1]) { p += 2; continue; } if (c == '"') in_str = 0; } else { if (c == '"') in_str = 1; else if (c == '[') depth++; else if (c == ']') { depth--; if (depth == 0) { p++; break; } } } p++; } size_t alen = (size_t)(p - arr_start); int match = (klen == tag_len) && html_iemem(k, tag, klen); if (match) { if (out_len) *out_len = alen; return arr_start; } } return NULL; } /* Returns 1 iff `attr` (length attr_len) appears as a string element * in the JSON array slice [arr, arr+arr_len). Comparison is case- * insensitive. */ static int html_attr_in_array(const char* arr, size_t arr_len, const char* attr, size_t attr_len) { if (!arr || arr_len < 2) return 0; const char* p = arr + 1; const char* end = arr + arr_len - 1; while (p < end) { while (p < end && (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r' || *p == ',')) p++; if (p >= end) return 0; if (*p != '"') return 0; p++; const char* s = p; while (p < end && *p != '"') { if (*p == '\\' && p + 1 < end) p++; p++; } if (p >= end) return 0; size_t slen = (size_t)(p - s); p++; if (slen == attr_len && html_iemem(s, attr, slen)) return 1; } return 0; } /* Hard-coded set of tags whose content is ALSO dropped (entire subtree). */ static int html_is_dangerous_container(const char* tag, size_t tag_len) { static const char* names[] = { "script", "style", "iframe", "object", "embed", "form", "noscript", "noembed", "template", "svg", "math", "frame", "frameset", "applet", "audio", "video", "source", "track", NULL }; for (int i = 0; names[i]; i++) { if (html_ieq_n(tag, tag_len, names[i])) return 1; } return 0; } /* HTML void elements — emit without a close tag. */ static int html_is_void(const char* tag, size_t tag_len) { static const char* names[] = { "area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr", NULL }; for (int i = 0; names[i]; i++) { if (html_ieq_n(tag, tag_len, names[i])) return 1; } return 0; } /* Append a single byte HTML-escaped into the output buffer. */ static void html_escape_byte(html_buf_t* out, unsigned char c) { switch (c) { case '<': html_buf_puts(out, "<"); break; case '>': html_buf_puts(out, ">"); break; case '"': html_buf_puts(out, """); break; case '\'': html_buf_puts(out, "'"); break; default: html_buf_putc(out, (char)c); break; } } /* Validate a URL value against the allowlist of safe schemes for hrefs. * Returns 1 iff the URL is safe to emit. Acceptable forms: * - http:// or https:// (case-insensitive) * - mailto: * - fragment-only `#anchor` * - relative path that does not contain a colon before the first * slash/?/# (so `foo/bar`, `/foo`, `?x=1` are OK; `javascript:x` is * not — its colon precedes any path/hash/query separator). * * URL leading whitespace and embedded ASCII control bytes (TAB, LF, CR) * are stripped before the scheme test, mirroring how browsers normalise * URLs (these bytes are otherwise a known XSS bypass: `java\tscript:`). */ static int html_url_is_safe(const char* url, size_t len) { if (!url || len == 0) return 1; /* empty href is harmless */ size_t i = 0; while (i < len) { unsigned char c = (unsigned char)url[i]; if (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == 0x0B || c == 0x0C) { i++; continue; } break; } if (i >= len) return 1; /* whitespace only */ if (url[i] == '#') return 1; /* fragment only */ if (url[i] == '/' || url[i] == '?') return 1; /* relative */ /* Find the first scheme-terminating character. */ size_t scheme_end = (size_t)-1; for (size_t j = i; j < len; j++) { char c = url[j]; if (c == ':') { scheme_end = j; break; } if (c == '/' || c == '?' || c == '#') break; } if (scheme_end == (size_t)-1) return 1; /* no colon → relative path */ /* Lowercase the scheme, stripping embedded control bytes. */ char scheme[32]; size_t sl = 0; for (size_t j = i; j < scheme_end && sl < sizeof(scheme) - 1; j++) { unsigned char c = (unsigned char)url[j]; if (c == '\t' || c == '\n' || c == '\r' || c == 0x0B || c == 0x0C) continue; scheme[sl++] = (char)html_tolower(c); } scheme[sl] = '\0'; if (strcmp(scheme, "http") == 0) return 1; if (strcmp(scheme, "https") == 0) return 1; if (strcmp(scheme, "mailto") == 0) return 1; return 0; } el_val_t el_html_sanitize(el_val_t input_v, el_val_t allowlist_v) { const char* input = EL_CSTR(input_v); const char* allow = EL_CSTR(allowlist_v); if (!input) return el_wrap_str(el_strdup("")); if (!allow) allow = "{}"; size_t in_len = strlen(input); html_buf_t out; html_buf_init(&out); size_t i = 0; while (i < in_len) { unsigned char c = (unsigned char)input[i]; if (c != '<') { /* Plain text — escape and emit. We pass `&` through verbatim * to preserve pre-encoded entities (`<`, `&`, `&#x...;`) * which the browser will decode safely. */ if (c == '&') html_buf_putc(&out, '&'); else html_escape_byte(&out, c); i++; continue; } /* `<` — try to parse a tag. */ if (i + 1 >= in_len) { html_buf_puts(&out, "<"); i++; continue; } /* Comments, doctype, CDATA, processing instructions — drop entirely. */ if (input[i + 1] == '!') { if (i + 3 < in_len && input[i + 2] == '-' && input[i + 3] == '-') { size_t j = i + 4; while (j + 2 < in_len && !(input[j] == '-' && input[j + 1] == '-' && input[j + 2] == '>')) j++; if (j + 2 < in_len) i = j + 3; else i = in_len; continue; } size_t j = i + 2; while (j < in_len && input[j] != '>') j++; i = (j < in_len) ? j + 1 : in_len; continue; } if (input[i + 1] == '?') { size_t j = i + 2; while (j < in_len && input[j] != '>') j++; i = (j < in_len) ? j + 1 : in_len; continue; } int is_close = 0; size_t name_start = i + 1; if (input[i + 1] == '/') { is_close = 1; name_start = i + 2; } if (name_start >= in_len) { html_buf_puts(&out, "<"); i++; continue; } unsigned char nc = (unsigned char)input[name_start]; if (!((nc >= 'a' && nc <= 'z') || (nc >= 'A' && nc <= 'Z'))) { /* `<` followed by non-letter — emit as escaped text. */ html_buf_puts(&out, "<"); i++; continue; } size_t name_end = name_start; while (name_end < in_len) { unsigned char x = (unsigned char)input[name_end]; if ((x >= 'a' && x <= 'z') || (x >= 'A' && x <= 'Z') || (x >= '0' && x <= '9') || x == '-' || x == '_' || x == ':') { name_end++; } else { break; } } const char* tag = input + name_start; size_t tag_len = name_end - name_start; /* Find the `>` that closes this tag, respecting quoted attrs. */ size_t cur = name_end; int self_close = 0; while (cur < in_len) { unsigned char x = (unsigned char)input[cur]; if (x == '"' || x == '\'') { unsigned char q = x; cur++; while (cur < in_len && (unsigned char)input[cur] != q) cur++; if (cur < in_len) cur++; /* skip closing quote */ continue; } if (x == '/' && cur + 1 < in_len && input[cur + 1] == '>') { self_close = 1; break; } if (x == '>') break; cur++; } if (cur >= in_len) { /* Malformed: unclosed tag at EOF. Drop the rest of the input. */ i = in_len; continue; } size_t tag_end = self_close ? cur + 2 : cur + 1; /* one past `>` */ /* Dangerous container — drop the whole subtree. */ if (!is_close && html_is_dangerous_container(tag, tag_len)) { if (self_close || html_is_void(tag, tag_len)) { i = tag_end; continue; } size_t scan = tag_end; int found_close = 0; while (scan < in_len) { if (input[scan] != '<') { scan++; continue; } if (scan + 1 < in_len && input[scan + 1] == '/') { size_t cn_start = scan + 2; size_t cn_end = cn_start; while (cn_end < in_len) { unsigned char x = (unsigned char)input[cn_end]; if ((x >= 'a' && x <= 'z') || (x >= 'A' && x <= 'Z') || (x >= '0' && x <= '9') || x == '-' || x == '_' || x == ':') { cn_end++; } else break; } if (cn_end - cn_start == tag_len && html_iemem(input + cn_start, tag, tag_len)) { size_t end_close = cn_end; while (end_close < in_len && input[end_close] != '>') end_close++; i = (end_close < in_len) ? end_close + 1 : in_len; found_close = 1; break; } } scan++; } if (!found_close) { /* No matching close — drop everything from here on. */ i = in_len; } continue; } /* Look up the tag in the allowlist. */ size_t arr_len = 0; const char* arr = html_allowlist_find(allow, tag, tag_len, &arr_len); if (!arr) { /* Tag not allowed. Drop the open/close marker; inner text is * processed by the outer loop and re-emitted as escaped text. */ i = tag_end; continue; } if (is_close) { if (!html_is_void(tag, tag_len)) { html_buf_putc(&out, '<'); html_buf_putc(&out, '/'); for (size_t k = 0; k < tag_len; k++) { html_buf_putc(&out, (char)html_tolower((unsigned char)tag[k])); } html_buf_putc(&out, '>'); } i = tag_end; continue; } /* Allowed open tag. Emit ``. */ html_buf_putc(&out, '<'); for (size_t k = 0; k < tag_len; k++) { html_buf_putc(&out, (char)html_tolower((unsigned char)tag[k])); } size_t a = name_end; while (a < cur) { unsigned char x = (unsigned char)input[a]; if (x == ' ' || x == '\t' || x == '\n' || x == '\r' || x == '/') { a++; continue; } size_t an_start = a; while (a < cur) { unsigned char y = (unsigned char)input[a]; if (y == '=' || y == ' ' || y == '\t' || y == '\n' || y == '\r' || y == '/' || y == '>') break; a++; } size_t an_len = a - an_start; if (an_len == 0) { a++; continue; } size_t av_start = 0; size_t av_len = 0; int has_value = 0; size_t b = a; while (b < cur && (input[b] == ' ' || input[b] == '\t' || input[b] == '\n' || input[b] == '\r')) b++; if (b < cur && input[b] == '=') { has_value = 1; b++; while (b < cur && (input[b] == ' ' || input[b] == '\t' || input[b] == '\n' || input[b] == '\r')) b++; if (b < cur && (input[b] == '"' || input[b] == '\'')) { unsigned char q = (unsigned char)input[b]; b++; av_start = b; while (b < cur && (unsigned char)input[b] != q) b++; av_len = b - av_start; if (b < cur) b++; } else { av_start = b; while (b < cur) { unsigned char y = (unsigned char)input[b]; if (y == ' ' || y == '\t' || y == '\n' || y == '\r' || y == '>') break; b++; } av_len = b - av_start; } a = b; } if (!html_attr_in_array(arr, arr_len, input + an_start, an_len)) continue; int is_href = (an_len == 4 && html_iemem(input + an_start, "href", 4)); int is_src = (an_len == 3 && html_iemem(input + an_start, "src", 3)); if ((is_href || is_src) && has_value) { if (!html_url_is_safe(input + av_start, av_len)) continue; } html_buf_putc(&out, ' '); for (size_t k = 0; k < an_len; k++) { html_buf_putc(&out, (char)html_tolower((unsigned char)input[an_start + k])); } if (has_value) { html_buf_puts(&out, "=\""); for (size_t k = 0; k < av_len; k++) { unsigned char y = (unsigned char)input[av_start + k]; /* Re-escape so the emitted attribute is well-formed * double-quoted HTML. `&` passes through to preserve * pre-encoded entities. */ if (y == '"') html_buf_puts(&out, """); else if (y == '<') html_buf_puts(&out, "<"); else if (y == '>') html_buf_puts(&out, ">"); else html_buf_putc(&out, (char)y); } html_buf_putc(&out, '"'); } } html_buf_putc(&out, '>'); i = tag_end; } /* Copy into arena-tracked buffer so the standard runtime memory model * applies to the returned string. */ char* result = el_strbuf(out.len); memcpy(result, out.data, out.len); result[out.len] = '\0'; html_buf_free(&out); return el_wrap_str(result); } /* ── JSON ────────────────────────────────────────────────────────────────── */ /* True iff the segment is non-empty and every byte is an ASCII digit. We treat * such segments as numeric array indices when walking a dot-path; mixed names * like "0a" remain object-key lookups, so a key named "0" still wins over an * index when the surrounding container is an object. */ static int json_path_seg_is_index(const char* seg, size_t n) { if (n == 0) return 0; for (size_t i = 0; i < n; i++) { if (seg[i] < '0' || seg[i] > '9') return 0; } return 1; } /* Skip JSON whitespace. */ static const char* json_skip_ws(const char* p) { while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; return p; } /* Descend one segment into the JSON cursor `p`. * - If `p` points at an array `[...]` and the segment is all digits, * advance to that element (zero-based). * - Otherwise treat the segment as an object key and use json_find_key * scoped to a one-level slice of the current container. * Returns NULL if the descent fails (segment not found, container mismatch). * * `seg` is a pointer into the original path string and `seg_len` is its * byte length — this avoids an extra alloc per segment. */ static const char* json_path_descend(const char* p, const char* seg, size_t seg_len) { if (!p || !seg) return NULL; p = json_skip_ws(p); if (*p == '[' && json_path_seg_is_index(seg, seg_len)) { long idx = 0; for (size_t i = 0; i < seg_len; i++) idx = idx * 10 + (seg[i] - '0'); p++; /* step past '[' */ p = json_skip_ws(p); long cur = 0; while (*p && *p != ']') { if (cur == idx) return p; const char* end = json_skip_value(p); if (!end || end == p) return NULL; p = json_skip_ws(end); if (*p == ',') { p++; p = json_skip_ws(p); cur++; continue; } /* No comma after this element — only acceptable at the closing ']', * which means we ran out of elements. */ break; } return NULL; } /* Object lookup. json_find_key walks at depth 1 of whatever container it * receives, so we slice from `p` onwards. Caller already positioned us at * the opening '{' (or at whitespace before it). */ if (*p != '{') return NULL; /* Build a NUL-terminated copy of the key segment for the lookup. We only * pay this cost when the segment isn't a numeric index. */ char stack_key[256]; char* k = stack_key; if (seg_len + 1 > sizeof(stack_key)) { k = malloc(seg_len + 1); if (!k) return NULL; } memcpy(k, seg, seg_len); k[seg_len] = '\0'; const char* found = json_find_key(p, k); if (k != stack_key) free(k); return found; } /* Read the JSON value at `p` into a freshly-allocated, arena-owned el_val_t. * - String -> unescaped, wrapped el_val_t string * - Anything else -> raw JSON slice as a string (matches the historical * json_get behaviour: numbers/bools/null come back stringified). */ static el_val_t json_read_value(const char* p) { p = json_skip_ws(p); if (*p == '"') { p++; size_t cap = strlen(p) + 1; char* out = el_strbuf(cap); char* w = out; while (*p && *p != '"') { if (*p == '\\' && *(p+1)) { p++; switch (*p) { case '"': *w++ = '"'; break; case '\\': *w++ = '\\'; break; case '/': *w++ = '/'; break; case 'n': *w++ = '\n'; break; case 'r': *w++ = '\r'; break; case 't': *w++ = '\t'; break; default: *w++ = *p; break; } } else { *w++ = *p; } p++; } *w = '\0'; return el_wrap_str(out); } /* Object/array/number/bool/null — return the raw slice up to the value's * end. json_skip_value tracks brace/bracket/string state so nested objects * round-trip cleanly. */ const char* end = json_skip_value(p); if (!end) end = p; size_t n = (size_t)(end - p); /* Strip trailing whitespace from scalar values so callers don't see * `123 ` when they parsed a pretty-printed number. */ while (n > 0 && (p[n-1] == ' ' || p[n-1] == '\t' || p[n-1] == '\n' || p[n-1] == '\r')) { n--; } char* out = el_strbuf(n); memcpy(out, p, n); out[n] = '\0'; return el_wrap_str(out); } el_val_t json_get(el_val_t jsonv, el_val_t keyv) { const char* json = EL_CSTR(jsonv); const char* key = EL_CSTR(keyv); if (!json || !key) return el_wrap_str(el_strdup("")); /* Fast path: key contains no '.' — keep the historical single-segment * substring search so existing callers retain their O(strlen) cost * profile. The dot-path walker is only paid for when needed. */ if (!strchr(key, '.')) { size_t klen = strlen(key); char stack_pat[512]; char* pattern; if (klen + 5 <= sizeof(stack_pat)) { pattern = stack_pat; } else { pattern = malloc(klen + 5); if (!pattern) return el_wrap_str(el_strdup("")); } snprintf(pattern, klen + 5, "\"%s\":", key); const char* p = strstr(json, pattern); if (pattern != stack_pat) free(pattern); if (!p) return el_wrap_str(el_strdup("")); p += strlen(key) + 3; /* skip "key": */ return json_read_value(p); } /* Dot-path traversal. Walk segments left to right; at each step, descend * into the current container by either array index (all-digit segment on * an array cursor) or object key. */ const char* cursor = json_skip_ws(json); const char* seg_start = key; const char* k = key; while (1) { if (*k == '.' || *k == '\0') { size_t seg_len = (size_t)(k - seg_start); cursor = json_path_descend(cursor, seg_start, seg_len); if (!cursor) return el_wrap_str(el_strdup("")); if (*k == '\0') break; k++; seg_start = k; continue; } k++; } return json_read_value(cursor); } /* ── Float bit-cast helpers ──────────────────────────────────────────────── */ /* `el_to_float` and `el_from_float` are exposed in el_runtime.h as static * inlines so generated programs (which #include the header) can call them * for Float literals. No definitions are needed here. */ /* ── JSON parser (recursive descent) ─────────────────────────────────────── */ /* * Parsed JSON representation: * - object -> ElMap (keys & values are el_val_t) * - array -> ElList * - string -> EL_STR-wrapped char* (allocated) * - number -> int (el_val_t) if integer, otherwise el_from_float(double) * - true -> 1 * - false -> 0 * - null -> EL_NULL (0) * * Note: there is no runtime type tag — parsed numbers cannot be * distinguished from booleans by the runtime alone. The codegen tracks * types separately. This matches the rest of el_val_t's type-erased model. */ /* JsonParser struct is forward-declared near the HTTP/Engram section. */ static void jp_skip_ws(JsonParser* jp) { while (jp->p < jp->end) { char c = *jp->p; if (c == ' ' || c == '\t' || c == '\n' || c == '\r') jp->p++; else break; } } static el_val_t jp_parse_value(JsonParser* jp); /* Parse a JSON string literal (the opening " has NOT yet been consumed). */ static char* jp_parse_string_raw(JsonParser* jp) { if (jp->p >= jp->end || *jp->p != '"') { jp->err = 1; return el_strdup(""); } jp->p++; size_t cap = 32, len = 0; char* out = malloc(cap); if (!out) { fputs("el_runtime: out of memory\n", stderr); exit(1); } while (jp->p < jp->end && *jp->p != '"') { char c = *jp->p++; if (c == '\\' && jp->p < jp->end) { char esc = *jp->p++; switch (esc) { case '"': c = '"'; break; case '\\': c = '\\'; break; case '/': c = '/'; break; case 'b': c = '\b'; break; case 'f': c = '\f'; break; case 'n': c = '\n'; break; case 'r': c = '\r'; break; case 't': c = '\t'; break; case 'u': { /* Decode \uXXXX (with surrogate pairs) to UTF-8. * * (2026-08-08 self-review) This used to skip the 4 hex * digits and emit a literal '?'. That is a LOSSY, silent, * irreversible transform on every JSON string entering the * runtime — and JSON writers escape non-ASCII by default * (Python json.dumps ships ensure_ascii=True; most MCP * clients do the same). So every em dash, curly quote, * accented letter, and emoji arriving over MCP or HTTP was * replaced by one question mark on the way in, with no * error and no counter. * * Measured on the live store before the fix: 3,119 of * 4,081 non-telemetry nodes (76%) carried the damage, * including the self traversal root and all 13 values * nodes ("Value ? Constraints as Freedom"). Node contents * split cleanly into fully-clean or fully-mangled with * zero overlap — the tell that this was one write path, * not gradual rot. The corruption is unrecoverable in * place (3 bytes collapse to 1), so the only real fix is * to stop producing it; historical repair has to come from * each node's upstream source. * * Malformed escapes keep the old '?' behaviour rather than * failing the parse: a truncated body should not take down * a route that previously tolerated it. */ unsigned cp = 0; int ok = 1; for (int i = 0; i < 4; i++) { if (jp->p >= jp->end) { ok = 0; break; } char h = *jp->p++; unsigned d; if (h >= '0' && h <= '9') d = (unsigned)(h - '0'); else if (h >= 'a' && h <= 'f') d = (unsigned)(h - 'a' + 10); else if (h >= 'A' && h <= 'F') d = (unsigned)(h - 'A' + 10); else { ok = 0; break; } cp = (cp << 4) | d; } if (!ok) { c = '?'; break; } /* High surrogate: pair it with the following low surrogate * so astral-plane codepoints (emoji) survive. If the next * token is not a valid low surrogate, rewind so it is * parsed on its own terms rather than swallowed. */ if (cp >= 0xD800 && cp <= 0xDBFF && (size_t)(jp->end - jp->p) >= 6 && jp->p[0] == '\\' && jp->p[1] == 'u') { const char* save = jp->p; unsigned lo = 0; int ok2 = 1; jp->p += 2; for (int i = 0; i < 4; i++) { char h = *jp->p++; unsigned d; if (h >= '0' && h <= '9') d = (unsigned)(h - '0'); else if (h >= 'a' && h <= 'f') d = (unsigned)(h - 'a' + 10); else if (h >= 'A' && h <= 'F') d = (unsigned)(h - 'A' + 10); else { ok2 = 0; break; } lo = (lo << 4) | d; } if (ok2 && lo >= 0xDC00 && lo <= 0xDFFF) cp = 0x10000u + ((cp - 0xD800u) << 10) + (lo - 0xDC00u); else jp->p = save; } /* Lone surrogate → U+FFFD (WHATWG / serde_json behaviour): * emitting a raw surrogate would produce invalid UTF-8. */ if (cp >= 0xD800 && cp <= 0xDFFF) cp = 0xFFFD; char ub[4]; int un; if (cp < 0x80) { ub[0] = (char)cp; un = 1; } else if (cp < 0x800) { ub[0] = (char)(0xC0 | (cp >> 6)); ub[1] = (char)(0x80 | (cp & 0x3F)); un = 2; } else if (cp < 0x10000) { ub[0] = (char)(0xE0 | (cp >> 12)); ub[1] = (char)(0x80 | ((cp >> 6) & 0x3F)); ub[2] = (char)(0x80 | (cp & 0x3F)); un = 3; } else { ub[0] = (char)(0xF0 | (cp >> 18)); ub[1] = (char)(0x80 | ((cp >> 12) & 0x3F)); ub[2] = (char)(0x80 | ((cp >> 6) & 0x3F)); ub[3] = (char)(0x80 | (cp & 0x3F)); un = 4; } while (len + (size_t)un >= cap) { cap *= 2; out = realloc(out, cap); if (!out) { fputs("el_runtime: out of memory\n", stderr); exit(1); } } for (int i = 0; i < un; i++) out[len++] = ub[i]; continue; /* bytes already appended */ } default: c = esc; break; } } if (len + 1 >= cap) { cap *= 2; out = realloc(out, cap); if (!out) { fputs("el_runtime: out of memory\n", stderr); exit(1); } } out[len++] = c; } if (jp->p < jp->end && *jp->p == '"') jp->p++; else jp->err = 1; out[len] = '\0'; return out; } static el_val_t jp_parse_number(JsonParser* jp) { const char* start = jp->p; int is_float = 0; if (jp->p < jp->end && (*jp->p == '-' || *jp->p == '+')) jp->p++; while (jp->p < jp->end && isdigit((unsigned char)*jp->p)) jp->p++; if (jp->p < jp->end && *jp->p == '.') { is_float = 1; jp->p++; while (jp->p < jp->end && isdigit((unsigned char)*jp->p)) jp->p++; } if (jp->p < jp->end && (*jp->p == 'e' || *jp->p == 'E')) { is_float = 1; jp->p++; if (jp->p < jp->end && (*jp->p == '+' || *jp->p == '-')) jp->p++; while (jp->p < jp->end && isdigit((unsigned char)*jp->p)) jp->p++; } size_t n = (size_t)(jp->p - start); char buf[64]; if (n >= sizeof(buf)) n = sizeof(buf) - 1; memcpy(buf, start, n); buf[n] = '\0'; if (is_float) return el_from_float(strtod(buf, NULL)); return (el_val_t)strtoll(buf, NULL, 10); } static el_val_t jp_parse_array(JsonParser* jp) { if (jp->p < jp->end && *jp->p == '[') jp->p++; el_val_t lst = el_list_empty(); jp_skip_ws(jp); if (jp->p < jp->end && *jp->p == ']') { jp->p++; return lst; } while (jp->p < jp->end) { jp_skip_ws(jp); el_val_t v = jp_parse_value(jp); lst = el_list_append(lst, v); jp_skip_ws(jp); if (jp->p < jp->end && *jp->p == ',') { jp->p++; continue; } if (jp->p < jp->end && *jp->p == ']') { jp->p++; break; } jp->err = 1; break; } return lst; } static el_val_t jp_parse_object(JsonParser* jp) { if (jp->p < jp->end && *jp->p == '{') jp->p++; el_val_t m = el_map_new(0); jp_skip_ws(jp); if (jp->p < jp->end && *jp->p == '}') { jp->p++; return m; } while (jp->p < jp->end) { jp_skip_ws(jp); char* key = jp_parse_string_raw(jp); jp_skip_ws(jp); if (jp->p < jp->end && *jp->p == ':') jp->p++; else { jp->err = 1; free(key); break; } jp_skip_ws(jp); el_val_t v = jp_parse_value(jp); m = el_map_set(m, EL_STR(key), v); jp_skip_ws(jp); if (jp->p < jp->end && *jp->p == ',') { jp->p++; continue; } if (jp->p < jp->end && *jp->p == '}') { jp->p++; break; } jp->err = 1; break; } return m; } static el_val_t jp_parse_value(JsonParser* jp) { jp_skip_ws(jp); if (jp->p >= jp->end) { jp->err = 1; return EL_NULL; } char c = *jp->p; if (c == '"') return el_wrap_str(jp_parse_string_raw(jp)); if (c == '{') return jp_parse_object(jp); if (c == '[') return jp_parse_array(jp); if (c == '-' || isdigit((unsigned char)c)) return jp_parse_number(jp); if (c == 't' && jp->p + 4 <= jp->end && strncmp(jp->p, "true", 4) == 0) { jp->p += 4; return 1; } if (c == 'f' && jp->p + 5 <= jp->end && strncmp(jp->p, "false", 5) == 0) { jp->p += 5; return 0; } if (c == 'n' && jp->p + 4 <= jp->end && strncmp(jp->p, "null", 4) == 0) { jp->p += 4; return EL_NULL; } jp->err = 1; return EL_NULL; } el_val_t json_parse(el_val_t sv) { const char* s = EL_CSTR(sv); if (!s) return EL_NULL; JsonParser jp = { .p = s, .end = s + strlen(s), .err = 0 }; el_val_t v = jp_parse_value(&jp); if (jp.err) return EL_NULL; return v; } /* ── JSON stringify ──────────────────────────────────────────────────────── */ /* * Stringify policy: el_val_t is type-erased, so we cannot perfectly * round-trip arbitrary values. We use these heuristics: * - If value is an ElList pointer (in the heap range), serialize as array. * - If value is an ElMap pointer, serialize as object. * - If value looks like a printable string pointer, serialize as string. * - Otherwise serialize as integer. * This is best-effort. Programs that need exact control should build the * string directly. A pointer test is the cheapest way to disambiguate * from small integers without a separate type tag. */ /* JsonBuf struct is forward-declared near the HTTP section so HTTP helpers * can use it. Its definition appears there. */ static void jb_init(JsonBuf* b) { b->cap = 64; b->len = 0; b->buf = malloc(b->cap); if (!b->buf) { fputs("el_runtime: out of memory\n", stderr); exit(1); } b->buf[0] = '\0'; } static void jb_reserve(JsonBuf* b, size_t add) { if (b->len + add + 1 > b->cap) { while (b->len + add + 1 > b->cap) b->cap *= 2; b->buf = realloc(b->buf, b->cap); if (!b->buf) { fputs("el_runtime: out of memory\n", stderr); exit(1); } } } static void jb_putc(JsonBuf* b, char c) { jb_reserve(b, 1); b->buf[b->len++] = c; b->buf[b->len] = '\0'; } static void jb_puts(JsonBuf* b, const char* s) { size_t n = strlen(s); jb_reserve(b, n); memcpy(b->buf + b->len, s, n); b->len += n; b->buf[b->len] = '\0'; } static void jb_emit_escaped(JsonBuf* b, const char* s) { jb_putc(b, '"'); for (; *s; s++) { unsigned char c = (unsigned char)*s; switch (c) { case '"': jb_puts(b, "\\\""); break; case '\\': jb_puts(b, "\\\\"); break; case '\b': jb_puts(b, "\\b"); break; case '\f': jb_puts(b, "\\f"); break; case '\n': jb_puts(b, "\\n"); break; case '\r': jb_puts(b, "\\r"); break; case '\t': jb_puts(b, "\\t"); break; default: if (c < 0x20) { char tmp[8]; snprintf(tmp, sizeof(tmp), "\\u%04x", c); jb_puts(b, tmp); } else { jb_putc(b, (char)c); } break; } } jb_putc(b, '"'); } /* Heuristic: is this el_val_t likely a pointer to an ElList? * We can't fully verify, but pointers are large addresses, integers small. * Treat values whose magnitude exceeds 2^32 as potential pointers and * sniff by reading the header conservatively. * * Simpler heuristic: if the value reads as a printable string, treat as * string; otherwise as integer. Lists/Maps are encoded as struct pointers, * which have leading binary bytes — so they won't look like strings. */ static int looks_like_string(el_val_t v) { if (v == 0) return 0; /* Treat plausible heap addresses as candidates. * Threshold: 4 GiB (0x100000000). On 64-bit systems heap addresses from * malloc/mmap start well above 4 GiB (ASLR pushes them to ~0x7f...). * El integer values (counters, unix timestamps up to ~2106) all fit below * 0x100000000 (4294967296). The old threshold of 1,000,000 caused unix * timestamps (~1.7e9) to be misidentified as string pointers — a segfault * risk in json_stringify and jb_emit_value. */ uintptr_t p = (uintptr_t)v; if (p < 0x100000000ULL) return 0; /* integers, timestamps, counters */ if (p < 0x1000) return 0; /* Sniff first bytes for printable */ const unsigned char* s = (const unsigned char*)p; for (int i = 0; i < 16; i++) { unsigned char c = s[i]; if (c == '\0') return 1; /* terminated string (empty string is still a valid string) */ /* Reject C0 control chars (non-whitespace), allow UTF-8 high bytes. * 0x09-0x0d = tab/newline/cr/vt/ff (whitespace, OK) * 0x20-0x7e = printable ASCII (OK) * 0x7f = DEL (reject) * 0x80-0xff = UTF-8 continuation/lead bytes (OK for multi-byte chars) */ if (c < 0x09 || (c > 0x0d && c < 0x20) || c == 0x7f) return 0; } return 1; /* 16+ printable bytes — call it a string */ } static void jb_emit_value(JsonBuf* b, el_val_t v); static void jb_emit_int(JsonBuf* b, int64_t n) { char tmp[32]; snprintf(tmp, sizeof(tmp), "%lld", (long long)n); jb_puts(b, tmp); } static void jb_emit_value(JsonBuf* b, el_val_t v) { if (v == EL_NULL) { jb_puts(b, "null"); return; } if (looks_like_string(v)) { jb_emit_escaped(b, EL_CSTR(v)); return; } jb_emit_int(b, (int64_t)v); } el_val_t json_stringify(el_val_t v) { JsonBuf b; jb_init(&b); jb_emit_value(&b, v); return el_wrap_str(b.buf); } /* ── JSON substring accessors ────────────────────────────────────────────── */ /* * These walk the raw JSON string looking for "key": at the top level (depth 1) * of an object. They handle escaped quotes, nested objects/arrays, and * whitespace around the colon. */ /* Find "key": at object-depth == 1 inside the JSON object string `s`. * Returns pointer to the first byte of the value, or NULL. */ static const char* json_find_key(const char* s, const char* key) { if (!s || !key) return NULL; size_t klen = strlen(key); int depth = 0; int in_str = 0; int escape = 0; const char* p = s; while (*p) { char c = *p; if (in_str) { if (escape) { escape = 0; } else if (c == '\\') { escape = 1; } else if (c == '"') { /* End of string. If we're at depth 1, check if this was a key. */ p++; if (depth == 1) { /* The string just ended at p-1. Check if it matches key * and is followed by a colon. We need to backtrack to find * the start of this string and compare. */ } in_str = 0; continue; } p++; continue; } if (c == '"') { /* Start of a string literal */ const char* str_start = p + 1; const char* q = str_start; int e = 0; while (*q) { if (e) { e = 0; q++; continue; } if (*q == '\\') { e = 1; q++; continue; } if (*q == '"') break; q++; } size_t slen = (size_t)(q - str_start); const char* after = (*q == '"') ? q + 1 : q; /* If at depth 1 and matches key and followed by ':' -> got it */ if (depth == 1 && slen == klen && strncmp(str_start, key, klen) == 0) { const char* r = after; while (*r == ' ' || *r == '\t' || *r == '\n' || *r == '\r') r++; if (*r == ':') { r++; while (*r == ' ' || *r == '\t' || *r == '\n' || *r == '\r') r++; return r; } } p = after; continue; } if (c == '{' || c == '[') depth++; else if (c == '}' || c == ']') depth--; p++; } return NULL; } /* Skip a JSON value starting at p; return pointer past the value end. */ static const char* json_skip_value(const char* p) { if (!p || !*p) return p; while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; if (*p == '"') { p++; int e = 0; while (*p) { if (e) { e = 0; p++; continue; } if (*p == '\\') { e = 1; p++; continue; } if (*p == '"') { p++; break; } p++; } return p; } if (*p == '{' || *p == '[') { char open = *p; char close = (open == '{') ? '}' : ']'; int depth = 0; int in_str = 0; int e = 0; while (*p) { char c = *p; if (in_str) { if (e) { e = 0; } else if (c == '\\') { e = 1; } else if (c == '"') in_str = 0; p++; continue; } if (c == '"') { in_str = 1; p++; continue; } if (c == open) depth++; else if (c == close) { depth--; p++; if (depth == 0) return p; continue; } p++; } return p; } /* scalar: number, true/false/null */ while (*p && *p != ',' && *p != '}' && *p != ']' && *p != ' ' && *p != '\t' && *p != '\n' && *p != '\r') p++; return p; } el_val_t json_get_string(el_val_t json_str, el_val_t key) { const char* json = EL_CSTR(json_str); const char* k = EL_CSTR(key); const char* p = json_find_key(json, k); if (!p || *p != '"') return el_wrap_str(el_strdup("")); p++; JsonParser jp = { .p = p - 1, .end = json + (json ? strlen(json) : 0), .err = 0 }; char* parsed = jp_parse_string_raw(&jp); if (jp.err) { free(parsed); return el_wrap_str(el_strdup("")); } return el_wrap_str(parsed); } el_val_t json_get_int(el_val_t json_str, el_val_t key) { const char* json = EL_CSTR(json_str); const char* k = EL_CSTR(key); const char* p = json_find_key(json, k); if (!p) return 0; if (*p == '"' || *p == '{' || *p == '[') return 0; return (el_val_t)strtoll(p, NULL, 10); } el_val_t json_get_float(el_val_t json_str, el_val_t key) { const char* json = EL_CSTR(json_str); const char* k = EL_CSTR(key); const char* p = json_find_key(json, k); if (!p) return 0; if (*p == '"' || *p == '{' || *p == '[') return 0; return el_from_float(strtod(p, NULL)); } el_val_t json_get_bool(el_val_t json_str, el_val_t key) { const char* json = EL_CSTR(json_str); const char* k = EL_CSTR(key); const char* p = json_find_key(json, k); if (!p) return 0; if (strncmp(p, "true", 4) == 0) return 1; return 0; } el_val_t json_get_raw(el_val_t json_str, el_val_t key) { const char* json = EL_CSTR(json_str); const char* k = EL_CSTR(key); const char* p = json_find_key(json, k); /* Clear fs_read binary-length hint — result is a fresh null-terminated * string, not the raw file bytes, so Content-Length must use strlen. */ _tl_fs_read_len = 0; if (!p) return el_wrap_str(el_strdup("")); const char* end = json_skip_value(p); size_t n = (size_t)(end - p); char* out = el_strbuf(n); memcpy(out, p, n); out[n] = '\0'; return el_wrap_str(out); } el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value) { const char* json = EL_CSTR(json_str); const char* k = EL_CSTR(key); if (!k) k = ""; if (!json || !*json) { /* Build a fresh object */ JsonBuf b; jb_init(&b); jb_putc(&b, '{'); jb_emit_escaped(&b, k); jb_putc(&b, ':'); jb_emit_value(&b, value); jb_putc(&b, '}'); return el_wrap_str(b.buf); } const char* existing = json_find_key(json, k); JsonBuf b; jb_init(&b); if (existing) { const char* end = json_skip_value(existing); /* Copy [json .. existing) */ size_t prefix = (size_t)(existing - json); jb_reserve(&b, prefix); memcpy(b.buf + b.len, json, prefix); b.len += prefix; b.buf[b.len] = '\0'; jb_emit_value(&b, value); jb_puts(&b, end); return el_wrap_str(b.buf); } /* Insert before closing '}'. Find last '}' */ size_t jl = strlen(json); if (jl == 0) { free(b.buf); return el_wrap_str(el_strdup("{}")); } /* Find last '}' from the end */ ssize_t close_idx = -1; for (ssize_t i = (ssize_t)jl - 1; i >= 0; i--) { if (json[i] == '}') { close_idx = i; break; } } if (close_idx < 0) { free(b.buf); return el_wrap_str(el_strdup(json)); } /* Determine if object is empty: scan between last '{' and '}' for non-ws */ int empty = 1; for (ssize_t i = close_idx - 1; i >= 0; i--) { char c = json[i]; if (c == '{') break; if (c != ' ' && c != '\t' && c != '\n' && c != '\r') { empty = 0; break; } } /* Copy json[0..close_idx) */ jb_reserve(&b, (size_t)close_idx); memcpy(b.buf + b.len, json, (size_t)close_idx); b.len += (size_t)close_idx; b.buf[b.len] = '\0'; if (!empty) jb_putc(&b, ','); jb_emit_escaped(&b, k); jb_putc(&b, ':'); jb_emit_value(&b, value); /* Append from close_idx onward */ jb_puts(&b, json + close_idx); return el_wrap_str(b.buf); } el_val_t json_array_len(el_val_t json_str) { const char* s = EL_CSTR(json_str); if (!s) return 0; while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; if (*s != '[') return 0; s++; while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; if (*s == ']') return 0; int64_t count = 0; while (*s) { const char* end = json_skip_value(s); if (end == s) break; count++; s = end; while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; if (*s == ',') { s++; continue; } if (*s == ']' || *s == '\0') break; } return (el_val_t)count; } /* json_array_get — return the i-th element of a JSON array as a JSON * fragment string. Nested objects and arrays are returned verbatim * (json_skip_value tracks brace/bracket depth so nested structures are * preserved intact). Out-of-range index → "". */ el_val_t json_array_get(el_val_t json_str, el_val_t index) { const char* s = EL_CSTR(json_str); int64_t idx = (int64_t)index; if (!s || idx < 0) return el_wrap_str(el_strdup("")); while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; if (*s != '[') return el_wrap_str(el_strdup("")); s++; while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; if (*s == ']') return el_wrap_str(el_strdup("")); int64_t i = 0; while (*s) { const char* start = s; const char* end = json_skip_value(s); if (end == s) break; if (i == idx) { size_t n = (size_t)(end - start); char* out = el_strbuf(n); memcpy(out, start, n); out[n] = '\0'; return el_wrap_str(out); } i++; s = end; while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; if (*s == ',') { s++; while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; continue; } if (*s == ']' || *s == '\0') break; } return el_wrap_str(el_strdup("")); } /* json_array_get_string — same as json_array_get, but assume the element * is a JSON string and return the unquoted/unescaped value. Non-string * elements yield "". */ el_val_t json_array_get_string(el_val_t json_str, el_val_t index) { el_val_t raw = json_array_get(json_str, index); const char* s = EL_CSTR(raw); if (!s || *s != '"') return el_wrap_str(el_strdup("")); JsonParser jp = { .p = s, .end = s + strlen(s), .err = 0, }; char* parsed = jp_parse_string_raw(&jp); if (jp.err) { free(parsed); return el_wrap_str(el_strdup("")); } return el_wrap_str(parsed); } /* ── Time ────────────────────────────────────────────────────────────────── */ el_val_t time_now(void) { struct timeval tv; gettimeofday(&tv, NULL); int64_t ms = (int64_t)tv.tv_sec * 1000LL + (int64_t)tv.tv_usec / 1000LL; return (el_val_t)ms; } el_val_t time_now_utc(void) { return time_now(); } el_val_t time_format(el_val_t ts, el_val_t fmt) { int64_t ms = (int64_t)ts; time_t s = (time_t)(ms / 1000); int msec = (int)(ms % 1000); if (msec < 0) { msec += 1000; s -= 1; } struct tm tm; gmtime_r(&s, &tm); const char* fmt_str = EL_CSTR(fmt); if (!fmt_str || strcmp(fmt_str, "ISO") == 0) { char buf[64]; snprintf(buf, sizeof(buf), "%04d-%02d-%02dT%02d:%02d:%02d.%03dZ", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, msec); return el_wrap_str(el_strdup(buf)); } char buf[256]; if (strftime(buf, sizeof(buf), fmt_str, &tm) == 0) buf[0] = '\0'; return el_wrap_str(el_strdup(buf)); } el_val_t time_to_parts(el_val_t ts) { int64_t ms = (int64_t)ts; time_t s = (time_t)(ms / 1000); int msec = (int)(ms % 1000); if (msec < 0) { msec += 1000; s -= 1; } struct tm tm; gmtime_r(&s, &tm); el_val_t m = el_map_new(0); m = el_map_set(m, EL_STR(el_strdup("year")), (el_val_t)(tm.tm_year + 1900)); m = el_map_set(m, EL_STR(el_strdup("month")), (el_val_t)(tm.tm_mon + 1)); m = el_map_set(m, EL_STR(el_strdup("day")), (el_val_t)tm.tm_mday); m = el_map_set(m, EL_STR(el_strdup("hour")), (el_val_t)tm.tm_hour); m = el_map_set(m, EL_STR(el_strdup("minute")), (el_val_t)tm.tm_min); m = el_map_set(m, EL_STR(el_strdup("second")), (el_val_t)tm.tm_sec); m = el_map_set(m, EL_STR(el_strdup("ms")), (el_val_t)msec); return m; } el_val_t time_from_parts(el_val_t secs, el_val_t ns, el_val_t tz) { (void)tz; int64_t s = (int64_t)secs; int64_t n = (int64_t)ns; int64_t ms = s * 1000LL + n / 1000000LL; return (el_val_t)ms; } el_val_t time_add(el_val_t ts, el_val_t n, el_val_t unit) { const char* u = EL_CSTR(unit); int64_t cur = (int64_t)ts; int64_t d = (int64_t)n; int64_t add_ms = d; if (u) { if (strcmp(u, "ms") == 0) add_ms = d; else if (strcmp(u, "sec") == 0) add_ms = d * 1000LL; else if (strcmp(u, "min") == 0) add_ms = d * 60000LL; else if (strcmp(u, "hour") == 0) add_ms = d * 3600000LL; else if (strcmp(u, "day") == 0) add_ms = d * 86400000LL; } return (el_val_t)(cur + add_ms); } el_val_t time_diff(el_val_t ts1, el_val_t ts2, el_val_t unit) { int64_t d = (int64_t)ts2 - (int64_t)ts1; const char* u = EL_CSTR(unit); if (!u || strcmp(u, "ms") == 0) return (el_val_t)d; if (strcmp(u, "sec") == 0) return (el_val_t)(d / 1000LL); if (strcmp(u, "min") == 0) return (el_val_t)(d / 60000LL); if (strcmp(u, "hour") == 0) return (el_val_t)(d / 3600000LL); if (strcmp(u, "day") == 0) return (el_val_t)(d / 86400000LL); return (el_val_t)d; } /* Block the calling thread for `secs` seconds. Negative values are clamped * to 0. Used by El programs that poll external resources (e.g. RunPod * /status, Engram readiness probes). */ el_val_t sleep_secs(el_val_t secs) { int64_t s = (int64_t)secs; if (s < 0) s = 0; struct timespec ts; ts.tv_sec = (time_t)s; ts.tv_nsec = 0; nanosleep(&ts, NULL); return 0; } el_val_t sleep_ms(el_val_t ms) { int64_t m = (int64_t)ms; if (m < 0) m = 0; struct timespec ts; ts.tv_sec = (time_t)(m / 1000LL); ts.tv_nsec = (long)((m % 1000LL) * 1000000LL); nanosleep(&ts, NULL); return 0; } /* ── Instant + Duration: first-class temporal types ────────────────────────── * El's substrate (Neuron) is a temporal cognition system. Memory salience * decay, the six-tier pacemaker, TTL caches, and supersession are all * temporal. Treating time as a raw Int (now() returning ms-since-epoch and * arithmetic done with mixed unit literals) lets bugs through the type * system: `(now - cached_at) < 60` cannot tell ms from sec, and `sleep(30)` * is ambiguous. This block introduces two dedicated representations. * * Representation: * Instant — int64 nanoseconds since the Unix epoch * Duration — int64 nanoseconds (signed; negative durations are legal, * e.g. when a deadline has passed) * * Both share the el_val_t (int64) slot the rest of the runtime uses, so no * boxing / arena allocation is needed. Type discipline is enforced at the * codegen layer: `let x: Duration = ...` registers `x` in __duration_names, * and BinOp dispatches through typed wrappers (el_duration_add, etc.) that * make intent explicit in the generated C. Mismatched ops (Instant+Instant, * Duration+Int) are surfaced via #error directives at codegen time so the * downstream cc step fails with a clear El-source-level message. * * Nanosecond precision matches POSIX clock_gettime / nanosleep granularity. * 2^63 nanos covers ~292 years from epoch — comfortably past 2200, plenty * for a memory-system runtime that never schedules outside a human lifespan. */ /* now() — current Instant. Wraps clock_gettime(CLOCK_REALTIME) for nanosecond * precision. Falls back to gettimeofday on systems where clock_gettime is * unavailable (defensive — every supported platform has it). */ el_val_t el_now_instant(void) { struct timespec ts; if (clock_gettime(CLOCK_REALTIME, &ts) == 0) { int64_t ns = (int64_t)ts.tv_sec * 1000000000LL + (int64_t)ts.tv_nsec; return (el_val_t)ns; } struct timeval tv; gettimeofday(&tv, NULL); int64_t ns = (int64_t)tv.tv_sec * 1000000000LL + (int64_t)tv.tv_usec * 1000LL; return (el_val_t)ns; } el_val_t now(void) { return el_now_instant(); } /* unix_seconds(n) — Instant from a Unix-epoch second count. * unix_millis(n) — Instant from a Unix-epoch millisecond count. */ el_val_t unix_seconds(el_val_t n) { int64_t s = (int64_t)n; return (el_val_t)(s * 1000000000LL); } el_val_t unix_millis(el_val_t n) { int64_t m = (int64_t)n; return (el_val_t)(m * 1000000LL); } /* instant_from_iso8601 — parse a strict subset: * YYYY-MM-DDTHH:MM:SS[.fff]Z * Returns 0 (the Unix-epoch sentinel) on parse failure. Callers that need to * distinguish epoch-zero from a parse error should use a wider sentinel * representation; the current zero-on-failure choice matches existing El * runtime conventions for parse builtins (str_to_int, parse_int). */ el_val_t instant_from_iso8601(el_val_t s) { const char* str = EL_CSTR(s); if (!str) return (el_val_t)0; int Y, M, D, h, m, sec, frac = 0; int n = sscanf(str, "%d-%d-%dT%d:%d:%d.%3d", &Y, &M, &D, &h, &m, &sec, &frac); if (n < 6) { n = sscanf(str, "%d-%d-%dT%d:%d:%dZ", &Y, &M, &D, &h, &m, &sec); if (n < 6) return (el_val_t)0; } struct tm tm; memset(&tm, 0, sizeof(tm)); tm.tm_year = Y - 1900; tm.tm_mon = M - 1; tm.tm_mday = D; tm.tm_hour = h; tm.tm_min = m; tm.tm_sec = sec; /* timegm — UTC. POSIX-Y but available on macOS and glibc. */ time_t t = timegm(&tm); if (t == (time_t)-1) return (el_val_t)0; int64_t ns = (int64_t)t * 1000000000LL + (int64_t)frac * 1000000LL; return (el_val_t)ns; } /* Duration constructors. The El-side postfix literals (30.seconds, 1.hour) * are lowered by the codegen directly into a literal int64 of nanoseconds — * these constructors are for runtime values where the count is dynamic. */ el_val_t el_duration_from_nanos(el_val_t ns) { return (el_val_t)(int64_t)ns; } el_val_t duration_seconds(el_val_t n) { int64_t s = (int64_t)n; return (el_val_t)(s * 1000000000LL); } el_val_t duration_millis(el_val_t n) { int64_t m = (int64_t)n; return (el_val_t)(m * 1000000LL); } el_val_t duration_nanos(el_val_t n) { return (el_val_t)(int64_t)n; } /* Arithmetic — typed wrappers. At the C level these are no-op casts, but * the codegen routes Instant/Duration BinOps through them so the generated * C says `el_instant_add_dur(start, dur)` rather than `start + dur`. The * intent is explicit, the operand order is documented, and a future change * to the underlying representation (saturating arithmetic, overflow guards) * has a single chokepoint. */ el_val_t el_instant_add_dur(el_val_t inst, el_val_t dur) { return (el_val_t)((int64_t)inst + (int64_t)dur); } el_val_t el_instant_sub_dur(el_val_t inst, el_val_t dur) { return (el_val_t)((int64_t)inst - (int64_t)dur); } el_val_t el_instant_diff(el_val_t a, el_val_t b) { /* a - b — yields a Duration (negative if b is later than a). */ return (el_val_t)((int64_t)a - (int64_t)b); } el_val_t el_duration_add(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a + (int64_t)b); } el_val_t el_duration_sub(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a - (int64_t)b); } el_val_t el_duration_scale(el_val_t dur, el_val_t scalar) { return (el_val_t)((int64_t)dur * (int64_t)scalar); } el_val_t el_duration_div(el_val_t dur, el_val_t scalar) { int64_t s = (int64_t)scalar; if (s == 0) return (el_val_t)0; return (el_val_t)((int64_t)dur / s); } /* Comparisons. Return 1/0 in el_val_t convention. */ el_val_t el_instant_lt(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a < (int64_t)b ? 1 : 0); } el_val_t el_instant_le(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a <= (int64_t)b ? 1 : 0); } el_val_t el_instant_gt(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a > (int64_t)b ? 1 : 0); } el_val_t el_instant_ge(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a >= (int64_t)b ? 1 : 0); } el_val_t el_instant_eq(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a == (int64_t)b ? 1 : 0); } el_val_t el_instant_ne(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a != (int64_t)b ? 1 : 0); } el_val_t el_duration_lt(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a < (int64_t)b ? 1 : 0); } el_val_t el_duration_le(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a <= (int64_t)b ? 1 : 0); } el_val_t el_duration_gt(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a > (int64_t)b ? 1 : 0); } el_val_t el_duration_ge(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a >= (int64_t)b ? 1 : 0); } el_val_t el_duration_eq(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a == (int64_t)b ? 1 : 0); } el_val_t el_duration_ne(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a != (int64_t)b ? 1 : 0); } /* Conversions. */ el_val_t instant_to_unix_seconds(el_val_t i) { return (el_val_t)((int64_t)i / 1000000000LL); } el_val_t instant_to_unix_millis(el_val_t i) { return (el_val_t)((int64_t)i / 1000000LL); } el_val_t instant_to_iso8601(el_val_t i) { int64_t ns = (int64_t)i; time_t s = (time_t)(ns / 1000000000LL); int msec = (int)((ns / 1000000LL) % 1000LL); if (msec < 0) { msec += 1000; s -= 1; } struct tm tm; gmtime_r(&s, &tm); char buf[64]; snprintf(buf, sizeof(buf), "%04d-%02d-%02dT%02d:%02d:%02d.%03dZ", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, msec); return el_wrap_str(el_strdup(buf)); } el_val_t duration_to_seconds(el_val_t d) { return (el_val_t)((int64_t)d / 1000000000LL); } el_val_t duration_to_millis(el_val_t d) { return (el_val_t)((int64_t)d / 1000000LL); } el_val_t duration_to_nanos(el_val_t d) { return (el_val_t)(int64_t)d; } /* sleep(Duration) — Phase 1 replacement for ambiguous sleep(Int). The runtime * still exposes sleep_secs/sleep_ms for legacy call sites; codegen lowers * sleep(Duration) to el_sleep_duration(d). Negative durations clamp to 0 so a * stale deadline doesn't block forever. */ el_val_t el_sleep_duration(el_val_t dur) { int64_t ns = (int64_t)dur; if (ns < 0) ns = 0; struct timespec ts; ts.tv_sec = (time_t)(ns / 1000000000LL); ts.tv_nsec = (long)(ns % 1000000000LL); nanosleep(&ts, NULL); return (el_val_t)0; } /* unix_timestamp() — back-compat. Existing El callers expect an Int seconds * value; this stays an Int returner so the type system isn't disturbed for * legacy code. New code should call now() and convert when needed. */ el_val_t unix_timestamp(void) { return instant_to_unix_seconds(el_now_instant()); } /* TTL cache helpers. Backed by the existing process-wide K/V (state_set/get) * with a sibling __ttl_set_at_ entry recording the Instant of the last * write. ttl_cache_get returns "" if the entry is missing or stale, so call * sites can branch on `if v == "" { miss } else { hit }` — the same shape * existing get-with-default code uses. No more (now - cached_at) < 60. */ el_val_t ttl_cache_set(el_val_t key, el_val_t value) { const char* k = EL_CSTR(key); if (!k) return (el_val_t)0; /* Store the value at the user's key. */ state_set(key, value); /* Stamp set_at — opaque schema, namespaced under __ttl: prefix so user * keys can't collide with stamps. */ size_t klen = strlen(k); char* stamp_key = (char*)malloc(klen + 16); if (!stamp_key) return (el_val_t)0; snprintf(stamp_key, klen + 16, "__ttl_at:%s", k); int64_t now_ns = (int64_t)el_now_instant(); char buf[32]; snprintf(buf, sizeof(buf), "%lld", (long long)now_ns); state_set(EL_STR(stamp_key), EL_STR(buf)); free(stamp_key); return (el_val_t)1; } el_val_t ttl_cache_get(el_val_t key, el_val_t max_age) { const char* k = EL_CSTR(key); if (!k) return el_wrap_str(el_strdup("")); /* Look up stamp. */ size_t klen = strlen(k); char* stamp_key = (char*)malloc(klen + 16); if (!stamp_key) return el_wrap_str(el_strdup("")); snprintf(stamp_key, klen + 16, "__ttl_at:%s", k); el_val_t stamp = state_get(EL_STR(stamp_key)); free(stamp_key); const char* sv = EL_CSTR(stamp); if (!sv || !*sv) return el_wrap_str(el_strdup("")); int64_t set_at = (int64_t)atoll(sv); int64_t now_ns = (int64_t)el_now_instant(); int64_t age = now_ns - set_at; int64_t max_ns = (int64_t)max_age; if (age < 0) return el_wrap_str(el_strdup("")); /* clock skew — treat as miss */ if (age > max_ns) return el_wrap_str(el_strdup("")); /* expired */ return state_get(key); } el_val_t ttl_cache_age(el_val_t key) { const char* k = EL_CSTR(key); if (!k) return (el_val_t)INT64_MAX; size_t klen = strlen(k); char* stamp_key = (char*)malloc(klen + 16); if (!stamp_key) return (el_val_t)INT64_MAX; snprintf(stamp_key, klen + 16, "__ttl_at:%s", k); el_val_t stamp = state_get(EL_STR(stamp_key)); free(stamp_key); const char* sv = EL_CSTR(stamp); if (!sv || !*sv) return (el_val_t)INT64_MAX; int64_t set_at = (int64_t)atoll(sv); int64_t now_ns = (int64_t)el_now_instant(); return (el_val_t)(now_ns - set_at); } /* ── Calendar + CalendarTime + Rhythm + LocalDate/Time/DateTime ────────────── * Phase 1.5. Calendar is pluggable: EarthCalendar (IANA zones + Gregorian + * DST), MarsCalendar (sols, MTC), CycleCalendar(period), NoCycleCalendar, * RelativeCalendar(epoch). Phase 1 zone wrapping folds INTO EarthCalendar; * UTC and IANA zones are themselves Earth-parochial and cannot live at the * lowest type layer. * * A Rhythm is a small AST that asks the Calendar for cycle phase, weekday, * etc. Most rhythm logic is calendar-agnostic at runtime: rhythm_cycle_phase * means "midpoint of cycle" whether the cycle is 24h on Earth or 30h on a * station or 300y on a long-cycle world. */ /* Magic headers — used by the runtime to recognize boxed temporal values * arriving through el_val_t. Distinct constants so accidental misuse fails * loudly rather than silently. */ #define EL_CAL_MAGIC 0xE1CA1EDDU #define EL_CALTIME_MAGIC 0xE1CA1747U #define EL_RHYTHM_MAGIC 0xE1287A11U #define EL_LDATE_MAGIC 0xE1DA7E00U #define EL_LDT_MAGIC 0xE1DA7E1DU #define EL_ZONE_MAGIC 0xE12017E0U typedef enum { EL_CALENDAR_EARTH = 1, EL_CALENDAR_MARS = 2, EL_CALENDAR_CYCLE = 3, EL_CALENDAR_NO_CYCLE = 4, EL_CALENDAR_RELATIVE = 5 } el_calendar_kind_t; typedef struct { uint32_t magic; char* id; /* IANA name or "+HH:MM" / "-HH:MM" */ int fixed; /* 1 for fixed offset, 0 for IANA */ int64_t offset_ns; /* fixed offset in nanos (only when fixed) */ } el_zone_t; typedef struct { uint32_t magic; el_calendar_kind_t kind; el_zone_t* zone; /* EarthCalendar; MarsCalendar uses MTC */ int64_t cycle_period_ns;/* CycleCalendar; computed for Earth (86400 s) and Mars (88775.244 s) */ int64_t epoch_ns; /* RelativeCalendar; Unix-epoch zero otherwise */ } el_calendar_t; typedef struct { uint32_t magic; int64_t instant_ns; el_calendar_t* cal; } el_caltime_t; /* Rhythm AST. */ typedef enum { EL_RHYTHM_CYCLE_START = 1, EL_RHYTHM_CYCLE_PHASE = 2, EL_RHYTHM_DURATION = 3, EL_RHYTHM_SESSION_START = 4, EL_RHYTHM_EVENT = 5, EL_RHYTHM_AND = 6, EL_RHYTHM_OR = 7, EL_RHYTHM_WEEKDAY = 8, EL_RHYTHM_WEEKLY_AT = 9 } el_rhythm_kind_t; typedef struct el_rhythm_s { uint32_t magic; el_rhythm_kind_t kind; double phase; /* CYCLE_PHASE */ int64_t period_ns; /* DURATION */ int weekday; /* 1..7 Mon..Sun */ int hour; int minute; char* event_name; /* EVENT */ struct el_rhythm_s* a; /* AND/OR */ struct el_rhythm_s* b; } el_rhythm_t; typedef struct { uint32_t magic; int year; int month; int day; } el_localdate_t; typedef struct { uint32_t magic; el_localdate_t* date; int64_t time_ns; /* nanos since midnight */ } el_localdt_t; /* Magic-tag check helpers — peek the first 4 bytes of an el_val_t pointer * and compare against the expected magic. Strings are NUL-terminated and * never start with our magic byte sequence, so this is safe. */ static int el_is_magic(el_val_t v, uint32_t want) { if (v == 0) return 0; /* Defensive: only follow pointers in plausible address space. * On 64-bit unix processes pointers are above 0x10000. */ if ((uint64_t)v < 0x10000ULL) return 0; uint32_t got = *(volatile uint32_t*)(uintptr_t)v; return got == want; } /* Sol length on Mars in nanoseconds: 88775.244 seconds. */ #define EL_MARS_SOL_NS ((int64_t)88775244000000LL) /* Earth solar day in nanoseconds: 86400 seconds. */ #define EL_EARTH_DAY_NS ((int64_t)86400000000000LL) /* ── Zone construction ────────────────────────────────────────────────────── * Zones intern by id string so equality comparisons are pointer-compares. */ #define EL_ZONE_TABLE_CAP 64 static el_zone_t* _el_zone_table[EL_ZONE_TABLE_CAP]; static int _el_zone_count = 0; static el_zone_t* _el_zone_intern(const char* id, int fixed, int64_t offset_ns) { for (int i = 0; i < _el_zone_count; i++) { el_zone_t* z = _el_zone_table[i]; if (z->fixed == fixed && z->offset_ns == offset_ns && strcmp(z->id ? z->id : "", id ? id : "") == 0) { return z; } } if (_el_zone_count >= EL_ZONE_TABLE_CAP) { /* Out of slots: build a non-interned zone. Equality will fail across * such zones but the program still runs. */ el_zone_t* z = (el_zone_t*)malloc(sizeof(el_zone_t)); z->magic = EL_ZONE_MAGIC; z->id = el_strdup_persist(id ? id : ""); z->fixed = fixed; z->offset_ns = offset_ns; return z; } el_zone_t* z = (el_zone_t*)malloc(sizeof(el_zone_t)); z->magic = EL_ZONE_MAGIC; z->id = el_strdup_persist(id ? id : ""); z->fixed = fixed; z->offset_ns = offset_ns; _el_zone_table[_el_zone_count++] = z; return z; } el_val_t zone(el_val_t id) { const char* s = EL_CSTR(id); if (!s || !*s) return (el_val_t)(uintptr_t)_el_zone_intern("UTC", 0, 0); /* Fixed-offset shortcut: "+HH:MM" or "-HH:MM". */ if ((s[0] == '+' || s[0] == '-') && strlen(s) >= 6 && s[3] == ':') { int sign = (s[0] == '-') ? -1 : 1; int hh = (s[1] - '0') * 10 + (s[2] - '0'); int mm = (s[4] - '0') * 10 + (s[5] - '0'); int64_t off = (int64_t)sign * ((int64_t)hh * 3600LL + (int64_t)mm * 60LL) * 1000000000LL; return (el_val_t)(uintptr_t)_el_zone_intern(s, 1, off); } return (el_val_t)(uintptr_t)_el_zone_intern(s, 0, 0); } el_val_t zone_utc(void) { return (el_val_t)(uintptr_t)_el_zone_intern("UTC", 1, 0); } el_val_t zone_local(void) { /* Resolve the local zone via TZ env or system default. tzset() picks * up TZ if set; otherwise the C library reads /etc/localtime. We store * the zone id as "LOCAL" so subsequent equality holds; resolution is * lazy at use time. */ return (el_val_t)(uintptr_t)_el_zone_intern("LOCAL", 0, 0); } el_val_t zone_offset(el_val_t hours, el_val_t minutes) { int hh = (int)(int64_t)hours; int mm = (int)(int64_t)minutes; int sign = (hh < 0 || mm < 0) ? -1 : 1; if (hh < 0) hh = -hh; if (mm < 0) mm = -mm; int64_t off = (int64_t)sign * ((int64_t)hh * 3600LL + (int64_t)mm * 60LL) * 1000000000LL; char buf[16]; snprintf(buf, sizeof(buf), "%c%02d:%02d", sign < 0 ? '-' : '+', hh, mm); return (el_val_t)(uintptr_t)_el_zone_intern(buf, 1, off); } /* ── Calendar interning ──────────────────────────────────────────────────── */ #define EL_CAL_TABLE_CAP 64 static el_calendar_t* _el_cal_table[EL_CAL_TABLE_CAP]; static int _el_cal_count = 0; static el_calendar_t* _el_cal_intern(el_calendar_kind_t kind, el_zone_t* z, int64_t period_ns, int64_t epoch_ns) { for (int i = 0; i < _el_cal_count; i++) { el_calendar_t* c = _el_cal_table[i]; if (c->kind == kind && c->zone == z && c->cycle_period_ns == period_ns && c->epoch_ns == epoch_ns) { return c; } } el_calendar_t* c = (el_calendar_t*)malloc(sizeof(el_calendar_t)); c->magic = EL_CAL_MAGIC; c->kind = kind; c->zone = z; c->cycle_period_ns = period_ns; c->epoch_ns = epoch_ns; if (_el_cal_count < EL_CAL_TABLE_CAP) _el_cal_table[_el_cal_count++] = c; return c; } el_val_t earth_calendar(el_val_t z_val) { el_zone_t* z = NULL; if (z_val != 0 && el_is_magic(z_val, EL_ZONE_MAGIC)) { z = (el_zone_t*)(uintptr_t)z_val; } else { z = (el_zone_t*)(uintptr_t)zone_local(); } return (el_val_t)(uintptr_t)_el_cal_intern(EL_CALENDAR_EARTH, z, EL_EARTH_DAY_NS, 0); } el_val_t earth_calendar_default(void) { return earth_calendar(zone_local()); } el_val_t mars_calendar(void) { el_zone_t* z = (el_zone_t*)(uintptr_t)_el_zone_intern("MTC", 1, 0); return (el_val_t)(uintptr_t)_el_cal_intern(EL_CALENDAR_MARS, z, EL_MARS_SOL_NS, 0); } el_val_t cycle_calendar(el_val_t period_dur) { int64_t period = (int64_t)period_dur; if (period <= 0) period = 1; return (el_val_t)(uintptr_t)_el_cal_intern(EL_CALENDAR_CYCLE, NULL, period, 0); } el_val_t no_cycle_calendar(void) { return (el_val_t)(uintptr_t)_el_cal_intern(EL_CALENDAR_NO_CYCLE, NULL, 0, 0); } el_val_t relative_calendar(el_val_t epoch_inst) { int64_t ep = (int64_t)epoch_inst; return (el_val_t)(uintptr_t)_el_cal_intern(EL_CALENDAR_RELATIVE, NULL, 0, ep); } /* ── CalendarTime ───────────────────────────────────────────────────────── */ static el_caltime_t* _el_caltime_alloc(int64_t inst, el_calendar_t* c) { el_caltime_t* ct = (el_caltime_t*)malloc(sizeof(el_caltime_t)); ct->magic = EL_CALTIME_MAGIC; ct->instant_ns = inst; ct->cal = c; return ct; } static el_calendar_t* _el_resolve_cal(el_val_t cal_val) { if (cal_val == 0 || !el_is_magic(cal_val, EL_CAL_MAGIC)) { return (el_calendar_t*)(uintptr_t)earth_calendar_default(); } return (el_calendar_t*)(uintptr_t)cal_val; } el_val_t now_in(el_val_t cal_val) { el_calendar_t* c = _el_resolve_cal(cal_val); int64_t ns = (int64_t)el_now_instant(); return (el_val_t)(uintptr_t)_el_caltime_alloc(ns, c); } el_val_t in_calendar(el_val_t inst, el_val_t cal_val) { el_calendar_t* c = _el_resolve_cal(cal_val); return (el_val_t)(uintptr_t)_el_caltime_alloc((int64_t)inst, c); } el_val_t cal_to_instant(el_val_t ct_val) { if (!el_is_magic(ct_val, EL_CALTIME_MAGIC)) return (el_val_t)0; el_caltime_t* ct = (el_caltime_t*)(uintptr_t)ct_val; return (el_val_t)ct->instant_ns; } el_val_t cal_in(el_val_t ct_val, el_val_t cal_val) { if (!el_is_magic(ct_val, EL_CALTIME_MAGIC)) return (el_val_t)0; el_caltime_t* ct = (el_caltime_t*)(uintptr_t)ct_val; el_calendar_t* c = _el_resolve_cal(cal_val); return (el_val_t)(uintptr_t)_el_caltime_alloc(ct->instant_ns, c); } el_val_t cal_cycle_phase(el_val_t ct_val) { if (!el_is_magic(ct_val, EL_CALTIME_MAGIC)) return el_from_float(0.0); el_caltime_t* ct = (el_caltime_t*)(uintptr_t)ct_val; el_calendar_t* c = ct->cal; if (c->kind == EL_CALENDAR_NO_CYCLE) { return el_from_float(0.0/0.0); /* NaN sentinel */ } int64_t period = c->cycle_period_ns; if (period <= 0) return el_from_float(0.0); int64_t base = ct->instant_ns - c->epoch_ns; int64_t phase_ns = base % period; if (phase_ns < 0) phase_ns += period; double phase = (double)phase_ns / (double)period; return el_from_float(phase); } /* ── Earth zone resolution: TZ-based offset lookup ────────────────────────── * For an EarthCalendar(zone), we want to convert an instant_ns into local * y/m/d/h/m/s, including DST. Approach: setenv("TZ", id), tzset(), use * localtime_r, then restore. This is not thread-safe by design — El's * runtime is single-threaded for the request handler path. Cache the * computed (instant -> tm) to avoid the syscall churn on repeat formats. */ static void _el_apply_zone(el_zone_t* z) { if (!z) { unsetenv("TZ"); tzset(); return; } if (z->fixed && strcmp(z->id, "UTC") == 0) { setenv("TZ", "UTC0", 1); tzset(); return; } if (z->fixed) { /* Fixed offset: POSIX TZ uses inverted sign (sign convention of * "hours WEST of UTC" rather than east). Build the spec accordingly. */ char buf[32]; int neg_secs = (int)(-z->offset_ns / 1000000000LL); int sign = neg_secs < 0 ? -1 : 1; int abs_secs = neg_secs < 0 ? -neg_secs : neg_secs; int hh = abs_secs / 3600; int mm = (abs_secs % 3600) / 60; snprintf(buf, sizeof(buf), "FIX%c%d:%02d", sign < 0 ? '-' : '+', hh, mm); setenv("TZ", buf, 1); tzset(); return; } if (strcmp(z->id, "LOCAL") == 0) { unsetenv("TZ"); tzset(); return; } setenv("TZ", z->id, 1); tzset(); } static int _el_decompose_earth(el_caltime_t* ct, struct tm* tm_out, int* abbr_len, char* abbr_buf, size_t abbr_cap) { el_calendar_t* c = ct->cal; el_zone_t* z = c->zone; _el_apply_zone(z); time_t s = (time_t)(ct->instant_ns / 1000000000LL); struct tm tm; localtime_r(&s, &tm); *tm_out = tm; if (abbr_buf && abbr_cap > 0) { const char* z_str = tm.tm_zone ? tm.tm_zone : ""; size_t n = strlen(z_str); if (n >= abbr_cap) n = abbr_cap - 1; memcpy(abbr_buf, z_str, n); abbr_buf[n] = '\0'; if (abbr_len) *abbr_len = (int)n; } return 0; } /* Format an Earth CalendarTime under a Java-DateTimeFormatter-ish pattern. * We support a useful core: yyyy MM dd HH mm ss z EEE MMM d h a — enough for * the acceptance tests. Single quotes denote literal text. */ static const char* _el_weekday_short[] = {"Sun","Mon","Tue","Wed","Thu","Fri","Sat"}; static const char* _el_month_short[] = {"Jan","Feb","Mar","Apr","May","Jun", "Jul","Aug","Sep","Oct","Nov","Dec"}; static char* _el_format_earth(el_caltime_t* ct, const char* pattern) { struct tm tm; char abbr[16] = {0}; int abbr_len = 0; _el_decompose_earth(ct, &tm, &abbr_len, abbr, sizeof(abbr)); size_t cap = strlen(pattern) * 4 + 64; char* out = (char*)malloc(cap); size_t pos = 0; size_t i = 0; size_t plen = strlen(pattern); while (i < plen) { char ch = pattern[i]; /* Quoted literal */ if (ch == '\'') { i++; while (i < plen && pattern[i] != '\'') { if (pos + 1 >= cap) { cap *= 2; out = realloc(out, cap); } out[pos++] = pattern[i++]; } if (i < plen) i++; continue; } /* Count run of same letter */ size_t run = 1; while (i + run < plen && pattern[i + run] == ch) run++; char tmp[64]; tmp[0] = '\0'; if (ch == 'y') { if (run >= 4) snprintf(tmp, sizeof(tmp), "%04d", tm.tm_year + 1900); else snprintf(tmp, sizeof(tmp), "%02d", (tm.tm_year + 1900) % 100); } else if (ch == 'M') { if (run >= 3) snprintf(tmp, sizeof(tmp), "%s", _el_month_short[tm.tm_mon]); else if (run == 2) snprintf(tmp, sizeof(tmp), "%02d", tm.tm_mon + 1); else snprintf(tmp, sizeof(tmp), "%d", tm.tm_mon + 1); } else if (ch == 'd') { if (run >= 2) snprintf(tmp, sizeof(tmp), "%02d", tm.tm_mday); else snprintf(tmp, sizeof(tmp), "%d", tm.tm_mday); } else if (ch == 'H') { if (run >= 2) snprintf(tmp, sizeof(tmp), "%02d", tm.tm_hour); else snprintf(tmp, sizeof(tmp), "%d", tm.tm_hour); } else if (ch == 'h') { int h12 = tm.tm_hour % 12; if (h12 == 0) h12 = 12; if (run >= 2) snprintf(tmp, sizeof(tmp), "%02d", h12); else snprintf(tmp, sizeof(tmp), "%d", h12); } else if (ch == 'm') { if (run >= 2) snprintf(tmp, sizeof(tmp), "%02d", tm.tm_min); else snprintf(tmp, sizeof(tmp), "%d", tm.tm_min); } else if (ch == 's') { if (run >= 2) snprintf(tmp, sizeof(tmp), "%02d", tm.tm_sec); else snprintf(tmp, sizeof(tmp), "%d", tm.tm_sec); } else if (ch == 'a') { snprintf(tmp, sizeof(tmp), "%s", tm.tm_hour < 12 ? "AM" : "PM"); } else if (ch == 'E') { snprintf(tmp, sizeof(tmp), "%s", _el_weekday_short[tm.tm_wday]); } else if (ch == 'z') { snprintf(tmp, sizeof(tmp), "%s", abbr); } else { for (size_t k = 0; k < run; k++) { if (pos + 1 >= cap) { cap *= 2; out = realloc(out, cap); } out[pos++] = ch; } i += run; continue; } size_t tl = strlen(tmp); if (pos + tl + 1 >= cap) { cap = (cap + tl) * 2; out = realloc(out, cap); } memcpy(out + pos, tmp, tl); pos += tl; i += run; } out[pos] = '\0'; char* result = el_strdup(out); free(out); return result; } /* Format a Mars CalendarTime: %sol prints the integer sol number since * mission epoch (Unix epoch fallback), %phase prints cycle_phase as a * 0..1 decimal. Other %-specifiers fall through. */ static char* _el_format_mars(el_caltime_t* ct, const char* pattern) { el_calendar_t* c = ct->cal; int64_t period = c->cycle_period_ns > 0 ? c->cycle_period_ns : EL_MARS_SOL_NS; int64_t base = ct->instant_ns - c->epoch_ns; int64_t sol = base / period; int64_t phase_ns = base % period; if (phase_ns < 0) { phase_ns += period; sol -= 1; } double phase = (double)phase_ns / (double)period; size_t cap = strlen(pattern) * 4 + 64; char* out = (char*)malloc(cap); size_t pos = 0; for (size_t i = 0; pattern[i]; i++) { if (pattern[i] == '%' && pattern[i+1]) { char tmp[64]; tmp[0] = '\0'; if (strncmp(pattern + i + 1, "sol", 3) == 0) { snprintf(tmp, sizeof(tmp), "%lld", (long long)sol); i += 3; } else if (strncmp(pattern + i + 1, "phase", 5) == 0) { snprintf(tmp, sizeof(tmp), "%.4f", phase); i += 5; } else if (pattern[i+1] == 'd') { snprintf(tmp, sizeof(tmp), "%lld", (long long)sol); i += 1; } else { tmp[0] = pattern[i+1]; tmp[1] = '\0'; i += 1; } size_t tl = strlen(tmp); if (pos + tl + 1 >= cap) { cap = (cap + tl) * 2; out = realloc(out, cap); } memcpy(out + pos, tmp, tl); pos += tl; } else { if (pos + 1 >= cap) { cap *= 2; out = realloc(out, cap); } out[pos++] = pattern[i]; } } out[pos] = '\0'; char* result = el_strdup(out); free(out); return result; } /* Format a CycleCalendar CalendarTime: %cycle and %phase. */ static char* _el_format_cycle(el_caltime_t* ct, const char* pattern) { el_calendar_t* c = ct->cal; int64_t period = c->cycle_period_ns > 0 ? c->cycle_period_ns : 1; int64_t base = ct->instant_ns - c->epoch_ns; int64_t cycle = base / period; int64_t phase_ns = base % period; if (phase_ns < 0) { phase_ns += period; cycle -= 1; } double phase = (double)phase_ns / (double)period; size_t cap = strlen(pattern) * 4 + 64; char* out = (char*)malloc(cap); size_t pos = 0; for (size_t i = 0; pattern[i]; i++) { if (pattern[i] == '%' && pattern[i+1]) { char tmp[64]; tmp[0] = '\0'; if (strncmp(pattern + i + 1, "cycle", 5) == 0) { snprintf(tmp, sizeof(tmp), "%lld", (long long)cycle); i += 5; } else if (strncmp(pattern + i + 1, "phase", 5) == 0) { snprintf(tmp, sizeof(tmp), "%.4f", phase); i += 5; } else if (pattern[i+1] == 'd') { snprintf(tmp, sizeof(tmp), "%lld", (long long)cycle); i += 1; } else if (pattern[i+1] == 'f') { snprintf(tmp, sizeof(tmp), "%.2f", phase); i += 1; } else { /* Pass through unknown specifier */ tmp[0] = '%'; tmp[1] = pattern[i+1]; tmp[2] = '\0'; i += 1; } size_t tl = strlen(tmp); if (pos + tl + 1 >= cap) { cap = (cap + tl) * 2; out = realloc(out, cap); } memcpy(out + pos, tmp, tl); pos += tl; } else { if (pos + 1 >= cap) { cap *= 2; out = realloc(out, cap); } out[pos++] = pattern[i]; } } out[pos] = '\0'; char* result = el_strdup(out); free(out); return result; } el_val_t cal_format(el_val_t ct_val, el_val_t pattern_val) { if (!el_is_magic(ct_val, EL_CALTIME_MAGIC)) return el_wrap_str(el_strdup("")); el_caltime_t* ct = (el_caltime_t*)(uintptr_t)ct_val; const char* pat = EL_CSTR(pattern_val); if (!pat) pat = ""; char* result = NULL; switch (ct->cal->kind) { case EL_CALENDAR_EARTH: result = _el_format_earth(ct, pat); break; case EL_CALENDAR_MARS: result = _el_format_mars(ct, pat); break; case EL_CALENDAR_CYCLE: result = _el_format_cycle(ct, pat); break; case EL_CALENDAR_RELATIVE: result = _el_format_cycle(ct, pat); break; case EL_CALENDAR_NO_CYCLE: { char buf[64]; snprintf(buf, sizeof(buf), "instant:%lld", (long long)ct->instant_ns); result = el_strdup(buf); break; } default: result = el_strdup(""); } return el_wrap_str(result); } /* ── LocalDate / LocalTime / LocalDateTime ──────────────────────────────── */ static int _el_days_in_month(int y, int m) { static const int dim[12] = {31,28,31,30,31,30,31,31,30,31,30,31}; if (m == 2) { int leap = ((y % 4 == 0) && (y % 100 != 0)) || (y % 400 == 0); return 28 + (leap ? 1 : 0); } if (m < 1 || m > 12) return 30; return dim[m - 1]; } el_val_t local_date(el_val_t y, el_val_t m, el_val_t d) { el_localdate_t* ld = (el_localdate_t*)malloc(sizeof(el_localdate_t)); ld->magic = EL_LDATE_MAGIC; ld->year = (int)(int64_t)y; ld->month = (int)(int64_t)m; ld->day = (int)(int64_t)d; return (el_val_t)(uintptr_t)ld; } el_val_t local_time(el_val_t h, el_val_t m, el_val_t s, el_val_t ns) { int64_t hh = (int64_t)h; int64_t mm = (int64_t)m; int64_t ss = (int64_t)s; int64_t nn = (int64_t)ns; int64_t total = hh * 3600000000000LL + mm * 60000000000LL + ss * 1000000000LL + nn; return (el_val_t)total; } el_val_t local_datetime(el_val_t date_val, el_val_t time_val) { if (!el_is_magic(date_val, EL_LDATE_MAGIC)) return (el_val_t)0; el_localdt_t* ldt = (el_localdt_t*)malloc(sizeof(el_localdt_t)); ldt->magic = EL_LDT_MAGIC; ldt->date = (el_localdate_t*)(uintptr_t)date_val; ldt->time_ns = (int64_t)time_val; return (el_val_t)(uintptr_t)ldt; } el_val_t zoned(el_val_t date_val, el_val_t time_val, el_val_t cal_val) { if (!el_is_magic(date_val, EL_LDATE_MAGIC)) return (el_val_t)0; el_localdate_t* ld = (el_localdate_t*)(uintptr_t)date_val; el_calendar_t* c = _el_resolve_cal(cal_val); int64_t time_ns = (int64_t)time_val; /* Convert (LocalDate, LocalTime, EarthCalendar) -> Instant. * For non-Earth calendars we use day-anchored conversion: treat the * LocalDate's (y,m,d) as a Gregorian projection, convert to seconds via * mktime under the calendar's zone, then add nanos-since-midnight. */ if (c->kind == EL_CALENDAR_EARTH) { _el_apply_zone(c->zone); struct tm tm; memset(&tm, 0, sizeof(tm)); tm.tm_year = ld->year - 1900; tm.tm_mon = ld->month - 1; tm.tm_mday = ld->day; tm.tm_hour = (int)(time_ns / 3600000000000LL); tm.tm_min = (int)((time_ns / 60000000000LL) % 60); tm.tm_sec = (int)((time_ns / 1000000000LL) % 60); tm.tm_isdst = -1; time_t t = mktime(&tm); if (t == (time_t)-1) return (el_val_t)0; int64_t ns = (int64_t)t * 1000000000LL + (time_ns % 1000000000LL); return (el_val_t)(uintptr_t)_el_caltime_alloc(ns, c); } /* Non-Earth fallback: project as if Earth UTC then attach calendar. */ struct tm tm; memset(&tm, 0, sizeof(tm)); tm.tm_year = ld->year - 1900; tm.tm_mon = ld->month - 1; tm.tm_mday = ld->day; tm.tm_hour = (int)(time_ns / 3600000000000LL); tm.tm_min = (int)((time_ns / 60000000000LL) % 60); tm.tm_sec = (int)((time_ns / 1000000000LL) % 60); time_t t = timegm(&tm); if (t == (time_t)-1) return (el_val_t)0; int64_t ns = (int64_t)t * 1000000000LL + (time_ns % 1000000000LL); return (el_val_t)(uintptr_t)_el_caltime_alloc(ns, c); } el_val_t local_date_year(el_val_t v) { if (!el_is_magic(v, EL_LDATE_MAGIC)) return (el_val_t)0; return (el_val_t)((el_localdate_t*)(uintptr_t)v)->year; } el_val_t local_date_month(el_val_t v) { if (!el_is_magic(v, EL_LDATE_MAGIC)) return (el_val_t)0; return (el_val_t)((el_localdate_t*)(uintptr_t)v)->month; } el_val_t local_date_day(el_val_t v) { if (!el_is_magic(v, EL_LDATE_MAGIC)) return (el_val_t)0; return (el_val_t)((el_localdate_t*)(uintptr_t)v)->day; } el_val_t local_time_hour(el_val_t v) { int64_t t = (int64_t)v; return (el_val_t)(t / 3600000000000LL); } el_val_t local_time_minute(el_val_t v) { int64_t t = (int64_t)v; return (el_val_t)((t / 60000000000LL) % 60); } el_val_t local_time_second(el_val_t v) { int64_t t = (int64_t)v; return (el_val_t)((t / 1000000000LL) % 60); } el_val_t local_time_nanos(el_val_t v) { int64_t t = (int64_t)v; return (el_val_t)(t % 1000000000LL); } el_val_t el_local_date_add_dur(el_val_t ld_val, el_val_t dur_val) { if (!el_is_magic(ld_val, EL_LDATE_MAGIC)) return ld_val; el_localdate_t* ld = (el_localdate_t*)(uintptr_t)ld_val; int64_t dur_ns = (int64_t)dur_val; int64_t days = dur_ns / EL_EARTH_DAY_NS; int y = ld->year, m = ld->month, d = ld->day; /* Walk days forward/backward in canonical Gregorian. */ while (days > 0) { int dim = _el_days_in_month(y, m); if (d + days <= dim) { d += (int)days; days = 0; break; } days -= (dim - d + 1); d = 1; m++; if (m > 12) { m = 1; y++; } } while (days < 0) { if (d + days >= 1) { d += (int)days; days = 0; break; } days += d; m--; if (m < 1) { m = 12; y--; } d = _el_days_in_month(y, m); } return local_date((el_val_t)y, (el_val_t)m, (el_val_t)d); } el_val_t el_local_time_add_dur(el_val_t lt_val, el_val_t dur_val) { int64_t t = (int64_t)lt_val + (int64_t)dur_val; /* Wrap mod 24h on Earth-default. CycleCalendar wrapping requires the * caller to use cal_in / cal_format for the right modulus. */ int64_t day = EL_EARTH_DAY_NS; int64_t r = t % day; if (r < 0) r += day; return (el_val_t)r; } el_val_t el_local_date_lt(el_val_t a_val, el_val_t b_val) { if (!el_is_magic(a_val, EL_LDATE_MAGIC) || !el_is_magic(b_val, EL_LDATE_MAGIC)) return (el_val_t)0; el_localdate_t* a = (el_localdate_t*)(uintptr_t)a_val; el_localdate_t* b = (el_localdate_t*)(uintptr_t)b_val; if (a->year != b->year) return (el_val_t)(a->year < b->year ? 1 : 0); if (a->month != b->month) return (el_val_t)(a->month < b->month ? 1 : 0); return (el_val_t)(a->day < b->day ? 1 : 0); } el_val_t el_local_date_eq(el_val_t a_val, el_val_t b_val) { if (!el_is_magic(a_val, EL_LDATE_MAGIC) || !el_is_magic(b_val, EL_LDATE_MAGIC)) return (el_val_t)0; el_localdate_t* a = (el_localdate_t*)(uintptr_t)a_val; el_localdate_t* b = (el_localdate_t*)(uintptr_t)b_val; return (el_val_t)((a->year == b->year && a->month == b->month && a->day == b->day) ? 1 : 0); } /* ── Rhythm ──────────────────────────────────────────────────────────────── */ static el_rhythm_t* _el_rhythm_alloc(el_rhythm_kind_t k) { el_rhythm_t* r = (el_rhythm_t*)calloc(1, sizeof(el_rhythm_t)); r->magic = EL_RHYTHM_MAGIC; r->kind = k; return r; } el_val_t rhythm_cycle_start(void) { return (el_val_t)(uintptr_t)_el_rhythm_alloc(EL_RHYTHM_CYCLE_START); } el_val_t rhythm_cycle_phase(el_val_t phase_val) { el_rhythm_t* r = _el_rhythm_alloc(EL_RHYTHM_CYCLE_PHASE); r->phase = el_to_float(phase_val); return (el_val_t)(uintptr_t)r; } el_val_t rhythm_duration(el_val_t d_val) { el_rhythm_t* r = _el_rhythm_alloc(EL_RHYTHM_DURATION); r->period_ns = (int64_t)d_val; return (el_val_t)(uintptr_t)r; } el_val_t rhythm_session_start(void) { return (el_val_t)(uintptr_t)_el_rhythm_alloc(EL_RHYTHM_SESSION_START); } el_val_t rhythm_event(el_val_t name_val) { el_rhythm_t* r = _el_rhythm_alloc(EL_RHYTHM_EVENT); const char* n = EL_CSTR(name_val); r->event_name = el_strdup_persist(n ? n : ""); return (el_val_t)(uintptr_t)r; } el_val_t rhythm_and(el_val_t a_val, el_val_t b_val) { el_rhythm_t* r = _el_rhythm_alloc(EL_RHYTHM_AND); r->a = el_is_magic(a_val, EL_RHYTHM_MAGIC) ? (el_rhythm_t*)(uintptr_t)a_val : NULL; r->b = el_is_magic(b_val, EL_RHYTHM_MAGIC) ? (el_rhythm_t*)(uintptr_t)b_val : NULL; return (el_val_t)(uintptr_t)r; } el_val_t rhythm_or(el_val_t a_val, el_val_t b_val) { el_rhythm_t* r = _el_rhythm_alloc(EL_RHYTHM_OR); r->a = el_is_magic(a_val, EL_RHYTHM_MAGIC) ? (el_rhythm_t*)(uintptr_t)a_val : NULL; r->b = el_is_magic(b_val, EL_RHYTHM_MAGIC) ? (el_rhythm_t*)(uintptr_t)b_val : NULL; return (el_val_t)(uintptr_t)r; } el_val_t rhythm_weekday(el_val_t day) { el_rhythm_t* r = _el_rhythm_alloc(EL_RHYTHM_WEEKDAY); r->weekday = (int)(int64_t)day; return (el_val_t)(uintptr_t)r; } el_val_t rhythm_weekly_at(el_val_t day, el_val_t hour, el_val_t minute) { el_rhythm_t* r = _el_rhythm_alloc(EL_RHYTHM_WEEKLY_AT); r->weekday = (int)(int64_t)day; r->hour = (int)(int64_t)hour; r->minute = (int)(int64_t)minute; return (el_val_t)(uintptr_t)r; } /* Compute the next instant on or after `after` when rhythm `r` matches, * under calendar `cal`. */ static int64_t _el_next_after(el_rhythm_t* r, int64_t after_ns, el_calendar_t* cal) { if (!r) return after_ns; int64_t period = cal->cycle_period_ns > 0 ? cal->cycle_period_ns : EL_EARTH_DAY_NS; switch (r->kind) { case EL_RHYTHM_CYCLE_START: { int64_t base = after_ns - cal->epoch_ns; int64_t cyc = (base / period) + 1; return cal->epoch_ns + cyc * period; } case EL_RHYTHM_CYCLE_PHASE: { int64_t base = after_ns - cal->epoch_ns; int64_t cyc_ns = (int64_t)(r->phase * (double)period); int64_t cur_cyc = base / period; int64_t candidate = cal->epoch_ns + cur_cyc * period + cyc_ns; if (candidate <= after_ns) candidate += period; return candidate; } case EL_RHYTHM_DURATION: { return after_ns + (r->period_ns > 0 ? r->period_ns : 1); } case EL_RHYTHM_WEEKDAY: case EL_RHYTHM_WEEKLY_AT: { if (cal->kind != EL_CALENDAR_EARTH) { /* Non-Earth calendars: fall back to cycle math, treating * weekday as a 7-cycle-per-period proxy. */ return after_ns + period; } _el_apply_zone(cal->zone); time_t s = (time_t)(after_ns / 1000000000LL); struct tm tm; localtime_r(&s, &tm); /* tm_wday: 0=Sun..6=Sat. We use 1=Mon..7=Sun. */ int target = r->weekday >= 1 && r->weekday <= 7 ? r->weekday : 1; int target_wday = target == 7 ? 0 : target; /* 7→Sun=0, 1→Mon=1 */ int days_ahead = (target_wday - tm.tm_wday + 7) % 7; int hour = (r->kind == EL_RHYTHM_WEEKLY_AT) ? r->hour : 0; int minute = (r->kind == EL_RHYTHM_WEEKLY_AT) ? r->minute : 0; struct tm cand = tm; cand.tm_mday += days_ahead; cand.tm_hour = hour; cand.tm_min = minute; cand.tm_sec = 0; cand.tm_isdst = -1; time_t cand_t = mktime(&cand); int64_t cand_ns = (int64_t)cand_t * 1000000000LL; if (cand_ns <= after_ns) { cand.tm_mday += 7; cand.tm_isdst = -1; cand_t = mktime(&cand); cand_ns = (int64_t)cand_t * 1000000000LL; } return cand_ns; } case EL_RHYTHM_AND: { int64_t a = _el_next_after(r->a, after_ns, cal); int64_t b = _el_next_after(r->b, after_ns, cal); return a > b ? a : b; } case EL_RHYTHM_OR: { int64_t a = _el_next_after(r->a, after_ns, cal); int64_t b = _el_next_after(r->b, after_ns, cal); return a < b ? a : b; } case EL_RHYTHM_SESSION_START: case EL_RHYTHM_EVENT: default: return after_ns; } } el_val_t rhythm_next_after(el_val_t r_val, el_val_t after_val, el_val_t cal_val) { if (!el_is_magic(r_val, EL_RHYTHM_MAGIC)) return after_val; el_rhythm_t* r = (el_rhythm_t*)(uintptr_t)r_val; el_calendar_t* c = _el_resolve_cal(cal_val); int64_t out = _el_next_after(r, (int64_t)after_val, c); return (el_val_t)out; } el_val_t rhythm_matches(el_val_t r_val, el_val_t ct_val) { if (!el_is_magic(r_val, EL_RHYTHM_MAGIC)) return (el_val_t)0; if (!el_is_magic(ct_val, EL_CALTIME_MAGIC)) return (el_val_t)0; el_rhythm_t* r = (el_rhythm_t*)(uintptr_t)r_val; el_caltime_t* ct = (el_caltime_t*)(uintptr_t)ct_val; int64_t period = ct->cal->cycle_period_ns > 0 ? ct->cal->cycle_period_ns : EL_EARTH_DAY_NS; int64_t base = ct->instant_ns - ct->cal->epoch_ns; int64_t phase_ns = base % period; if (phase_ns < 0) phase_ns += period; double phase = (double)phase_ns / (double)period; switch (r->kind) { case EL_RHYTHM_CYCLE_START: return (el_val_t)(phase_ns == 0 ? 1 : 0); case EL_RHYTHM_CYCLE_PHASE: { double diff = phase - r->phase; if (diff < 0) diff = -diff; return (el_val_t)(diff < 0.001 ? 1 : 0); } default: return (el_val_t)0; } } /* ── UUID v4 ─────────────────────────────────────────────────────────────── */ static int _el_uuid_seeded = 0; static void _el_uuid_seed(void) { if (!_el_uuid_seeded) { srand((unsigned)time(NULL) ^ (unsigned)(uintptr_t)&_el_uuid_seeded); _el_uuid_seeded = 1; } } el_val_t uuid_new(void) { _el_uuid_seed(); unsigned char b[16]; for (int i = 0; i < 16; i++) b[i] = (unsigned char)(rand() & 0xff); /* Version 4 */ b[6] = (b[6] & 0x0f) | 0x40; /* RFC 4122 variant */ b[8] = (b[8] & 0x3f) | 0x80; char buf[37]; snprintf(buf, sizeof(buf), "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15]); return el_wrap_str(el_strdup(buf)); } el_val_t uuid_v4(void) { return uuid_new(); } /* ── Environment ─────────────────────────────────────────────────────────── */ el_val_t env(el_val_t key) { const char* k = EL_CSTR(key); if (!k) return el_wrap_str(el_strdup("")); const char* v = getenv(k); return el_wrap_str(el_strdup(v ? v : "")); } /* ── In-process state K/V ────────────────────────────────────────────────── */ typedef struct { char* key; char* value; } StateEntry; static StateEntry* _state_entries = NULL; static size_t _state_count = 0; static size_t _state_cap = 0; /* Mutex protecting all _state_entries access. state_set/state_get are called * concurrently from 64 HTTP worker threads — without this lock, realloc and * free race, producing corruption, double-free, and segfaults. */ static pthread_mutex_t _state_mu = PTHREAD_MUTEX_INITIALIZER; static StateEntry* state_find(const char* key) { for (size_t i = 0; i < _state_count; i++) { if (strcmp(_state_entries[i].key, key) == 0) return &_state_entries[i]; } return NULL; } el_val_t state_set(el_val_t key, el_val_t value) { const char* k = EL_CSTR(key); const char* v = EL_CSTR(value); if (!k) return 0; if (!v) v = ""; pthread_mutex_lock(&_state_mu); StateEntry* e = state_find(k); if (e) { free(e->value); e->value = el_strdup_persist(v); pthread_mutex_unlock(&_state_mu); return 1; } if (_state_count >= _state_cap) { size_t nc = _state_cap == 0 ? 16 : _state_cap * 2; StateEntry* grown = realloc(_state_entries, nc * sizeof(StateEntry)); if (!grown) { pthread_mutex_unlock(&_state_mu); fputs("el_runtime: out of memory\n", stderr); exit(1); } _state_entries = grown; _state_cap = nc; } _state_entries[_state_count].key = el_strdup_persist(k); _state_entries[_state_count].value = el_strdup_persist(v); _state_count++; pthread_mutex_unlock(&_state_mu); return 1; } el_val_t state_get(el_val_t key) { const char* k = EL_CSTR(key); if (!k) return el_wrap_str(el_strdup("")); pthread_mutex_lock(&_state_mu); StateEntry* e = state_find(k); char* result = el_strdup_persist(e ? e->value : ""); pthread_mutex_unlock(&_state_mu); /* wrap in arena-tracked copy for the caller's request lifetime */ char* copy = el_strdup(result); return el_wrap_str(copy); } el_val_t state_del(el_val_t key) { const char* k = EL_CSTR(key); if (!k) return 0; pthread_mutex_lock(&_state_mu); for (size_t i = 0; i < _state_count; i++) { if (strcmp(_state_entries[i].key, k) == 0) { free(_state_entries[i].key); free(_state_entries[i].value); for (size_t j = i + 1; j < _state_count; j++) { _state_entries[j - 1] = _state_entries[j]; } _state_count--; pthread_mutex_unlock(&_state_mu); return 1; } } pthread_mutex_unlock(&_state_mu); return 1; } el_val_t state_keys(void) { pthread_mutex_lock(&_state_mu); el_val_t lst = el_list_empty(); for (size_t i = 0; i < _state_count; i++) { lst = el_list_append(lst, el_wrap_str(el_strdup(_state_entries[i].key))); } pthread_mutex_unlock(&_state_mu); return lst; } /* ── Float formatting ────────────────────────────────────────────────────── */ el_val_t float_to_str(el_val_t f) { char buf[64]; snprintf(buf, sizeof(buf), "%g", el_to_float(f)); return el_wrap_str(el_strdup(buf)); } el_val_t int_to_float(el_val_t n) { return el_from_float((double)(int64_t)n); } el_val_t float_to_int(el_val_t f) { return (el_val_t)(int64_t)el_to_float(f); } el_val_t format_float(el_val_t f, el_val_t decimals) { int d = (int)(int64_t)decimals; if (d < 0) d = 0; if (d > 30) d = 30; char buf[128]; snprintf(buf, sizeof(buf), "%.*f", d, el_to_float(f)); return el_wrap_str(el_strdup(buf)); } el_val_t decimal_round(el_val_t f, el_val_t decimals) { int d = (int)(int64_t)decimals; if (d < 0) d = 0; if (d > 15) d = 15; double mul = pow(10.0, (double)d); double v = el_to_float(f); double r = (v >= 0.0 ? floor(v * mul + 0.5) : -floor(-v * mul + 0.5)) / mul; return el_from_float(r); } el_val_t str_to_float(el_val_t s) { const char* str = EL_CSTR(s); if (!str) return el_from_float(0.0); return el_from_float(strtod(str, NULL)); } /* ── Math (Float-aware) ──────────────────────────────────────────────────── */ el_val_t math_sqrt(el_val_t f) { return el_from_float(sqrt(el_to_float(f))); } el_val_t math_log(el_val_t f) { return el_from_float(log(el_to_float(f))); } el_val_t math_ln(el_val_t f) { return el_from_float(log(el_to_float(f))); } el_val_t math_sin(el_val_t f) { return el_from_float(sin(el_to_float(f))); } el_val_t math_cos(el_val_t f) { return el_from_float(cos(el_to_float(f))); } el_val_t math_pi(void) { return el_from_float(3.141592653589793238462643383279502884); } /* ── String additions ────────────────────────────────────────────────────── */ el_val_t str_index_of(el_val_t s, el_val_t sub) { const char* str = EL_CSTR(s); const char* sb = EL_CSTR(sub); if (!str || !sb) return -1; const char* hit = strstr(str, sb); if (!hit) return -1; return (el_val_t)(int64_t)(hit - str); } el_val_t str_split(el_val_t s, el_val_t sep) { const char* str = EL_CSTR(s); const char* sp = EL_CSTR(sep); el_val_t lst = el_list_empty(); if (!str) return lst; if (!sp || !*sp) { lst = el_list_append(lst, el_wrap_str(el_strdup(str))); return lst; } size_t lp = strlen(sp); const char* p = str; const char* hit; while ((hit = strstr(p, sp)) != NULL) { size_t n = (size_t)(hit - p); char* out = el_strbuf(n); memcpy(out, p, n); out[n] = '\0'; lst = el_list_append(lst, el_wrap_str(out)); p = hit + lp; } lst = el_list_append(lst, el_wrap_str(el_strdup(p))); return lst; } el_val_t str_char_at(el_val_t s, el_val_t i) { const char* str = EL_CSTR(s); int64_t idx = (int64_t)i; if (!str) return el_wrap_str(el_strdup("")); int64_t n = (int64_t)strlen(str); if (idx < 0 || idx >= n) return el_wrap_str(el_strdup("")); char buf[2]; buf[0] = str[idx]; buf[1] = '\0'; return el_wrap_str(el_strdup(buf)); } el_val_t str_char_code(el_val_t s, el_val_t i) { const char* str = EL_CSTR(s); int64_t idx = (int64_t)i; if (!str) return 0; int64_t n = (int64_t)strlen(str); if (idx < 0 || idx >= n) return 0; return (el_val_t)(unsigned char)str[idx]; } static el_val_t str_pad(const char* s, int64_t width, const char* pad, int left) { if (!s) s = ""; if (!pad || !*pad) pad = " "; int64_t lp = (int64_t)strlen(pad); int64_t ls = (int64_t)strlen(s); if (ls >= width) return el_wrap_str(el_strdup(s)); int64_t need = width - ls; char* out = el_strbuf((size_t)width); if (left) { for (int64_t i = 0; i < need; i++) out[i] = pad[i % lp]; memcpy(out + need, s, (size_t)ls); } else { memcpy(out, s, (size_t)ls); for (int64_t i = 0; i < need; i++) out[ls + i] = pad[i % lp]; } out[width] = '\0'; return el_wrap_str(out); } el_val_t str_pad_left(el_val_t s, el_val_t width, el_val_t pad) { return str_pad(EL_CSTR(s), (int64_t)width, EL_CSTR(pad), 1); } el_val_t str_pad_right(el_val_t s, el_val_t width, el_val_t pad) { return str_pad(EL_CSTR(s), (int64_t)width, EL_CSTR(pad), 0); } el_val_t str_format(el_val_t fmt, el_val_t data) { const char* tpl = EL_CSTR(fmt); if (!tpl) return el_wrap_str(el_strdup("")); JsonBuf b; jb_init(&b); const char* p = tpl; while (*p) { if (*p == '{') { const char* q = p + 1; while (*q && *q != '}') q++; if (*q == '}') { size_t klen = (size_t)(q - p - 1); char keybuf[256]; if (klen < sizeof(keybuf)) { memcpy(keybuf, p + 1, klen); keybuf[klen] = '\0'; el_val_t v = el_map_get(data, EL_STR(keybuf)); if (v != 0 && looks_like_string(v)) { jb_puts(&b, EL_CSTR(v)); p = q + 1; continue; } else if (v != 0) { jb_emit_int(&b, (int64_t)v); p = q + 1; continue; } } /* Unknown key — leave {key} verbatim */ jb_reserve(&b, klen + 2); memcpy(b.buf + b.len, p, klen + 2); b.len += klen + 2; b.buf[b.len] = '\0'; p = q + 1; continue; } } jb_putc(&b, *p); p++; } return el_wrap_str(b.buf); } el_val_t str_lower(el_val_t s) { return str_to_lower(s); } el_val_t str_upper(el_val_t s) { return str_to_upper(s); } /* ── Text-processing primitives (Phase 1: byte/codepoint, ASCII char classes) * * Phase 1 covers the operations every text-handling caller used to roll by * hand on top of str_index_of + str_slice. The character-class predicates * (is_letter / is_digit / ...) are ASCII only — Unicode-grapheme awareness, * NFC/NFD normalization, and regex are Phase 2. Single-char input checks the * first byte; multi-char input requires ALL bytes to match (false otherwise). * * Counting: * str_count non-overlapping occurrences of sub in s * str_count_chars codepoint count (UTF-8 leading-byte count) * str_count_bytes explicit byte length (alias of str_len) * str_count_lines \n-delimited line count (\r\n folded to \n) * str_count_words whitespace-delimited tokens, non-empty only * str_count_letters ASCII [A-Za-z] * str_count_digits ASCII [0-9] * * Find / position: * str_index_of_all all byte offsets of sub, [] if none * str_last_index_of last byte offset of sub, -1 if not found * str_find_chars first index of any char in any_of, -1 if none * * Transform: * str_repeat s * n (non-negative) * str_reverse codepoint-reversed (NOT grapheme-aware) * str_strip_prefix s without prefix if present, else s * str_strip_suffix s without suffix if present, else s * str_strip_chars strip leading+trailing chars matching any in chars * str_lstrip strip leading whitespace * str_rstrip strip trailing whitespace * * Char classification (Bool): * is_letter, is_digit, is_alphanumeric, is_whitespace, * is_punctuation, is_uppercase, is_lowercase * * Splitting: * str_split_lines \n-delimited (\r\n folded). Trailing empty dropped. * str_split_chars alias of native_string_chars in str_ namespace * str_split_n split into at most n parts (last part keeps the * rest verbatim, including any further separators) * * Joining: * str_join [String] -> String, sep between elements */ /* Count non-overlapping occurrences of sub in s. Empty sub returns 0. */ el_val_t str_count(el_val_t sv, el_val_t subv) { const char* s = EL_CSTR(sv); const char* sub = EL_CSTR(subv); if (!s || !sub || !*sub) return 0; size_t lp = strlen(sub); int64_t count = 0; const char* p = s; while ((p = strstr(p, sub)) != NULL) { count++; p += lp; /* non-overlapping advance */ } return (el_val_t)count; } /* Codepoint count: walk bytes, count those NOT matching 10xxxxxx. */ el_val_t str_count_chars(el_val_t sv) { const char* s = EL_CSTR(sv); if (!s) return 0; int64_t count = 0; for (const unsigned char* p = (const unsigned char*)s; *p; p++) { if ((*p & 0xC0) != 0x80) count++; } return (el_val_t)count; } el_val_t str_count_bytes(el_val_t sv) { return str_len(sv); } el_val_t str_count_lines(el_val_t sv) { const char* s = EL_CSTR(sv); if (!s || !*s) return 0; int64_t count = 0; int has_content = 0; for (const char* p = s; *p; p++) { has_content = 1; if (*p == '\n') { count++; has_content = 0; /* the \n closed the line */ } } if (has_content) count++; /* trailing line with no terminator */ return (el_val_t)count; } el_val_t str_count_words(el_val_t sv) { const char* s = EL_CSTR(sv); if (!s) return 0; int64_t count = 0; int in_word = 0; for (const unsigned char* p = (const unsigned char*)s; *p; p++) { if (isspace(*p)) { in_word = 0; } else if (!in_word) { in_word = 1; count++; } } return (el_val_t)count; } el_val_t str_count_letters(el_val_t sv) { const char* s = EL_CSTR(sv); if (!s) return 0; int64_t count = 0; for (const unsigned char* p = (const unsigned char*)s; *p; p++) { if ((*p >= 'A' && *p <= 'Z') || (*p >= 'a' && *p <= 'z')) count++; } return (el_val_t)count; } el_val_t str_count_digits(el_val_t sv) { const char* s = EL_CSTR(sv); if (!s) return 0; int64_t count = 0; for (const unsigned char* p = (const unsigned char*)s; *p; p++) { if (*p >= '0' && *p <= '9') count++; } return (el_val_t)count; } el_val_t str_index_of_all(el_val_t sv, el_val_t subv) { const char* s = EL_CSTR(sv); const char* sub = EL_CSTR(subv); el_val_t lst = el_list_empty(); if (!s || !sub || !*sub) return lst; size_t lp = strlen(sub); const char* p = s; const char* hit; while ((hit = strstr(p, sub)) != NULL) { lst = el_list_append(lst, (el_val_t)(int64_t)(hit - s)); p = hit + lp; } return lst; } el_val_t str_last_index_of(el_val_t sv, el_val_t subv) { const char* s = EL_CSTR(sv); const char* sub = EL_CSTR(subv); if (!s || !sub || !*sub) return -1; size_t lp = strlen(sub); int64_t last = -1; const char* p = s; const char* hit; while ((hit = strstr(p, sub)) != NULL) { last = (int64_t)(hit - s); p = hit + lp; } return (el_val_t)last; } el_val_t str_find_chars(el_val_t sv, el_val_t any_of_v) { const char* s = EL_CSTR(sv); const char* any = EL_CSTR(any_of_v); if (!s || !any || !*any) return -1; for (const char* p = s; *p; p++) { if (strchr(any, *p)) return (el_val_t)(int64_t)(p - s); } return -1; } el_val_t str_repeat(el_val_t sv, el_val_t nv) { const char* s = EL_CSTR(sv); int64_t n = (int64_t)nv; if (!s || n <= 0) return el_wrap_str(el_strdup("")); size_t ls = strlen(s); if (ls == 0) return el_wrap_str(el_strdup("")); size_t total = ls * (size_t)n; char* out = el_strbuf(total); for (int64_t i = 0; i < n; i++) { memcpy(out + i * ls, s, ls); } out[total] = '\0'; return el_wrap_str(out); } /* Reverse by codepoint: walk codepoints, copy each backwards into the output. * NOT grapheme-aware (Phase 2). Combining marks attached to a base codepoint * will detach. ASCII strings are byte-reverse equivalent. */ el_val_t str_reverse(el_val_t sv) { const char* s = EL_CSTR(sv); if (!s) return el_wrap_str(el_strdup("")); size_t n = strlen(s); char* out = el_strbuf(n); /* Walk forward, find each codepoint's byte length, then copy from the end. */ size_t out_pos = n; const unsigned char* p = (const unsigned char*)s; while (*p) { int cp_len; if ((*p & 0x80) == 0x00) cp_len = 1; else if ((*p & 0xE0) == 0xC0) cp_len = 2; else if ((*p & 0xF0) == 0xE0) cp_len = 3; else if ((*p & 0xF8) == 0xF0) cp_len = 4; else cp_len = 1; /* invalid byte: passthrough */ out_pos -= cp_len; memcpy(out + out_pos, p, cp_len); p += cp_len; } out[n] = '\0'; return el_wrap_str(out); } el_val_t str_strip_prefix(el_val_t sv, el_val_t prefv) { const char* s = EL_CSTR(sv); const char* pref = EL_CSTR(prefv); if (!s) return el_wrap_str(el_strdup("")); if (!pref || !*pref) return el_wrap_str(el_strdup(s)); size_t lp = strlen(pref); size_t ls = strlen(s); if (lp <= ls && strncmp(s, pref, lp) == 0) { char* out = el_strbuf(ls - lp); memcpy(out, s + lp, ls - lp); out[ls - lp] = '\0'; return el_wrap_str(out); } return el_wrap_str(el_strdup(s)); } el_val_t str_strip_suffix(el_val_t sv, el_val_t sufv) { const char* s = EL_CSTR(sv); const char* suf = EL_CSTR(sufv); if (!s) return el_wrap_str(el_strdup("")); if (!suf || !*suf) return el_wrap_str(el_strdup(s)); size_t ls = strlen(s); size_t lsuf = strlen(suf); if (lsuf <= ls && strcmp(s + ls - lsuf, suf) == 0) { char* out = el_strbuf(ls - lsuf); memcpy(out, s, ls - lsuf); out[ls - lsuf] = '\0'; return el_wrap_str(out); } return el_wrap_str(el_strdup(s)); } el_val_t str_strip_chars(el_val_t sv, el_val_t charsv) { const char* s = EL_CSTR(sv); const char* chars = EL_CSTR(charsv); if (!s) return el_wrap_str(el_strdup("")); if (!chars || !*chars) return el_wrap_str(el_strdup(s)); const char* start = s; while (*start && strchr(chars, *start)) start++; size_t n = strlen(start); while (n > 0 && strchr(chars, start[n - 1])) n--; char* out = el_strbuf(n); memcpy(out, start, n); out[n] = '\0'; return el_wrap_str(out); } el_val_t str_lstrip(el_val_t sv) { const char* s = EL_CSTR(sv); if (!s) return el_wrap_str(el_strdup("")); while (*s && isspace((unsigned char)*s)) s++; return el_wrap_str(el_strdup(s)); } el_val_t str_rstrip(el_val_t sv) { const char* s = EL_CSTR(sv); if (!s) return el_wrap_str(el_strdup("")); size_t n = strlen(s); while (n > 0 && isspace((unsigned char)s[n - 1])) n--; char* out = el_strbuf(n); memcpy(out, s, n); out[n] = '\0'; return el_wrap_str(out); } /* Character classification. * Empty input returns false. Multi-char input requires ALL bytes to match. * ASCII range only; Phase 2 will widen to Unicode. */ static int s_all_match(el_val_t sv, int (*pred)(unsigned char)) { const char* s = EL_CSTR(sv); if (!s || !*s) return 0; for (const unsigned char* p = (const unsigned char*)s; *p; p++) { if (!pred(*p)) return 0; } return 1; } static int p_letter(unsigned char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); } static int p_digit(unsigned char c) { return c >= '0' && c <= '9'; } static int p_alnum(unsigned char c) { return p_letter(c) || p_digit(c); } static int p_white(unsigned char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; } static int p_punct(unsigned char c) { return ispunct(c) ? 1 : 0; } static int p_upper(unsigned char c) { return c >= 'A' && c <= 'Z'; } static int p_lower(unsigned char c) { return c >= 'a' && c <= 'z'; } el_val_t is_letter(el_val_t s) { return (el_val_t)s_all_match(s, p_letter); } el_val_t is_digit(el_val_t s) { return (el_val_t)s_all_match(s, p_digit); } el_val_t is_alphanumeric(el_val_t s) { return (el_val_t)s_all_match(s, p_alnum); } el_val_t is_whitespace(el_val_t s) { return (el_val_t)s_all_match(s, p_white); } el_val_t is_punctuation(el_val_t s) { return (el_val_t)s_all_match(s, p_punct); } el_val_t is_uppercase(el_val_t s) { return (el_val_t)s_all_match(s, p_upper); } el_val_t is_lowercase(el_val_t s) { return (el_val_t)s_all_match(s, p_lower); } /* Split on \n. \r\n is folded to \n first. Trailing empty after final \n * is dropped (so "a\nb\n" -> ["a", "b"], not ["a", "b", ""]). */ el_val_t str_split_lines(el_val_t sv) { const char* s = EL_CSTR(sv); el_val_t lst = el_list_empty(); if (!s) return lst; size_t n = strlen(s); /* Pre-scan: build into a normalized buffer with \r\n folded. */ const char* line_start = s; for (size_t i = 0; i <= n; i++) { if (s[i] == '\n' || s[i] == '\0') { size_t len = (size_t)(s + i - line_start); /* Drop trailing \r if this was \r\n. */ if (len > 0 && line_start[len - 1] == '\r') len--; /* Drop final trailing-empty-after-newline. */ if (s[i] == '\0' && len == 0 && i > 0 && s[i - 1] == '\n') break; char* out = el_strbuf(len); memcpy(out, line_start, len); out[len] = '\0'; lst = el_list_append(lst, el_wrap_str(out)); if (s[i] == '\0') break; line_start = s + i + 1; } } return lst; } el_val_t str_split_chars(el_val_t s) { return native_string_chars(s); } /* Split into at most n parts. The (n-1)th split point is the LAST split; * after it, the remainder is appended verbatim including any further * separators. n <= 0 returns an empty list. n == 1 returns [s]. */ el_val_t str_split_n(el_val_t sv, el_val_t sepv, el_val_t nv) { const char* s = EL_CSTR(sv); const char* sep = EL_CSTR(sepv); int64_t n = (int64_t)nv; el_val_t lst = el_list_empty(); if (!s) return lst; if (n <= 0) return lst; if (n == 1 || !sep || !*sep) { lst = el_list_append(lst, el_wrap_str(el_strdup(s))); return lst; } size_t lp = strlen(sep); const char* p = s; int64_t parts = 0; const char* hit; while (parts < n - 1 && (hit = strstr(p, sep)) != NULL) { size_t len = (size_t)(hit - p); char* out = el_strbuf(len); memcpy(out, p, len); out[len] = '\0'; lst = el_list_append(lst, el_wrap_str(out)); p = hit + lp; parts++; } /* Remainder verbatim. */ lst = el_list_append(lst, el_wrap_str(el_strdup(p))); return lst; } /* Join a [String] with a separator. Empty list -> "". Single-element -> * that element. Non-string elements are stringified via int_to_str. */ el_val_t str_join(el_val_t listv, el_val_t sepv) { return list_join(listv, sepv); } /* ── List additions ──────────────────────────────────────────────────────── */ el_val_t list_push(el_val_t list, el_val_t elem) { return el_list_append(list, elem); } el_val_t list_push_front(el_val_t listv, el_val_t elem) { ElList* lst = (ElList*)(uintptr_t)listv; if (!lst) { el_val_t nl = el_list_empty(); return el_list_append(nl, elem); } /* Append to grow capacity, then shift right */ listv = el_list_append(listv, elem); lst = (ElList*)(uintptr_t)listv; for (int64_t i = lst->length - 1; i > 0; i--) { lst->elems[i] = lst->elems[i - 1]; } lst->elems[0] = elem; return EL_STR(lst); } el_val_t list_join(el_val_t listv, el_val_t sep) { ElList* lst = (ElList*)(uintptr_t)listv; const char* sp = EL_CSTR(sep); if (!sp) sp = ""; if (!lst || lst->length == 0) return el_wrap_str(el_strdup("")); JsonBuf b; jb_init(&b); for (int64_t i = 0; i < lst->length; i++) { if (i > 0) jb_puts(&b, sp); el_val_t v = lst->elems[i]; if (v == 0) continue; if (looks_like_string(v)) { jb_puts(&b, EL_CSTR(v)); } else { char tmp[32]; snprintf(tmp, sizeof(tmp), "%lld", (long long)v); jb_puts(&b, tmp); } } return el_wrap_str(b.buf); } el_val_t list_range(el_val_t start, el_val_t end) { int64_t a = (int64_t)start; int64_t b = (int64_t)end; el_val_t lst = el_list_empty(); for (int64_t i = a; i < b; i++) lst = el_list_append(lst, (el_val_t)i); return lst; } /* ── Bool helpers ────────────────────────────────────────────────────────── */ el_val_t bool_to_str(el_val_t b) { return el_wrap_str(el_strdup(b ? "true" : "false")); } /* ── Numeric parsing ─────────────────────────────────────────────────────── */ /* parse_int — strtoll with a default. str_to_int already exists but does not * distinguish "0" from a parse failure, so callers that need a sentinel use * this. Skips leading whitespace; accepts an optional leading +/-; returns * default_val on empty input or no consumed digits. Trailing junk is ignored * (atoi-style). */ el_val_t parse_int(el_val_t sv, el_val_t default_val) { const char* s = EL_CSTR(sv); if (!s) return default_val; while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; if (*s == '\0') return default_val; char* end = NULL; long long n = strtoll(s, &end, 10); if (end == s) return default_val; return (el_val_t)n; } /* ── Process ─────────────────────────────────────────────────────────────── */ void exit_program(el_val_t code) { exit((int)code); } /* getpid_now — current process id. Named with the _now suffix to avoid * colliding with the libc `getpid` declaration that the runtime already * sees via (calling it `getpid` would fight the prototype). */ el_val_t getpid_now(void) { return (el_val_t)getpid(); } /* ── args() — command-line argument access ────────────────────────────────── * Compiled El programs call args() to get a list of CLI arguments. * Call el_runtime_init_args(argc, argv) at the start of C main() to populate. * The args list excludes argv[0] (the program name). */ static el_val_t _el_args_list = 0; void el_runtime_init_args(int argc, char** argv) { _el_args_list = el_list_empty(); for (int i = 1; i < argc; i++) { _el_args_list = el_list_append(_el_args_list, EL_STR(argv[i])); } } el_val_t args(void) { if (!_el_args_list) _el_args_list = el_list_empty(); return _el_args_list; } /* ── CGI identity ──────────────────────────────────────────────────────────── * Called once at program start by the generated main() of a cgi {} program. * Stores CGI identity so dharma_* builtins can reference it. */ static const char* _el_cgi_name = NULL; static const char* _el_cgi_dharma_id = NULL; static const char* _el_cgi_principal = NULL; static const char* _el_cgi_network = NULL; static const char* _el_cgi_engram = NULL; void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal, el_val_t network, el_val_t engram) { _el_cgi_name = EL_CSTR(name); _el_cgi_dharma_id = EL_CSTR(dharma_id); _el_cgi_principal = EL_CSTR(principal); _el_cgi_network = EL_CSTR(network) ? EL_CSTR(network) : "dharma-mainnet"; _el_cgi_engram = EL_CSTR(engram) ? EL_CSTR(engram) : "http://localhost:8742"; printf("[cgi] identity: name=%s dharma_id=%s principal=%s network=%s engram=%s\n", _el_cgi_name ? _el_cgi_name : "(unset)", _el_cgi_dharma_id ? _el_cgi_dharma_id : "(unset)", _el_cgi_principal ? _el_cgi_principal : "(unset)", _el_cgi_network, _el_cgi_engram); } /* ── Batch 3: Engram in-process graph store ──────────────────────────────── */ /* * Single global EngramStore allocated lazily on first call. All node and * edge content strings are owned (strdup'd) by the store. Linear arrays * with doubling capacity for both nodes and edges. * * Two-layer activation algorithm (engram_activate): * * LAYER 1 — Broad fan-out (background activation): * 1. Find seed nodes whose content/label/tags contain query (case-insens). * 2. BFS up to `depth` hops along ALL edges (excitatory and inhibitory). * Every reachable node fires — nothing is filtered at this layer. * 3. bg_act = seed.salience * temporal_decay * dampening * propagated as: new_bg = parent_bg * edge_weight * 0.7 * (1 + tbonus) * where tbonus ∈ {0, 0.10, 0.20} for co-temporal nodes. * 4. If reached by multiple paths, take max background_activation. * 5. Persist background_activation to EngramNode.background_activation. * * LAYER 2 — Executive filter (working memory promotion): * 6. For each inhibitory edge where source has background_activation > 0: * inhibition[target] = max(bg[source] * e->weight) * 7. For each background-activated node: * raw_wm = bg * goal_bias(node, query) * confidence * * (1 - (1 - INHIBITION_FACTOR) * inhibition) * 8. Per-type threshold gate: raw_wm >= type_threshold → promoted. * Safety/DharmaSelf: 0.05 Canonical: 0.15 Lesson: 0.25 * Belief/Entity: 0.30 Note/Memory/Working: 0.40 * 9. If not promoted: suppression_count++. After * ENGRAM_SUPPRESSION_BREAKTHROUGH suppressions → force breakthrough * at ENGRAM_BREAKTHROUGH_WEIGHT (latent tension surfacing). * 10. Persist working_memory_weight to EngramNode.working_memory_weight. * 11. Sort: promoted nodes (wm > 0) first by wm desc, then background- * only by bg desc. Context compilation uses ONLY promoted nodes. * * Temporal decay: * decay_factor = exp(-lambda * age_hours / T_half) * T_half = 168.0 h (one week), lambda = ln(2) * * Activation dampening: * dampen = 1.0 / (1.0 + log(1 + activation_count)) * * engram_query_range(start_ms, end_ms): * Returns nodes whose created_at OR last_activated falls within * [start_ms, end_ms], sorted by created_at ascending. */ /* Temporal decay constants. * T_HALF_HOURS: half-life in hours — one week. After one week of no * activation a node retains 50% of its base salience contribution. * DECAY_LAMBDA: ln(2) ≈ 0.693147 */ #define ENGRAM_T_HALF_HOURS 168.0 #define ENGRAM_DECAY_LAMBDA 0.693147 /* Two-layer activation constants. * ENGRAM_WM_THRESHOLD: SUPERSEDED — defined here for legacy reference only. * The actual per-call threshold is computed by engram_type_threshold() which * returns per-node-type values (0.05 Safety/DharmaSelf, 0.15 Canonical, * 0.25 Lesson, 0.30 Belief/Entity, 0.40 Note/Memory/Working). This constant * is NOT used in engram_activate(); it matches the Canonical tier value only * by coincidence. (2026-07-01 self-review: clarified stale doc) * ENGRAM_WM_DECAY: SUPERSEDED (2026-07-22 self-review, bl-b17facdd) — the * per-turn multiplicative carry-over decay was replaced by the ACT-R/ * Petrov base-level scheme (see ENGRAM_BLL_* below). Kept for reference; * no longer used in engram_activate. * ENGRAM_SUPPRESSION_BREAKTHROUGH: after this many consecutive suppressions * a latent node forces itself into working memory at reduced weight, * modelling the brain's "intrusive thought" / unresolved-tension surfacing. * ENGRAM_BREAKTHROUGH_WEIGHT: the reduced working_memory_weight assigned * when a suppressed node breaks through. * ENGRAM_INHIBITION_FACTOR: multiplier applied to working_memory_weight when * an inhibitory edge fires against a node (0 = full suppress; current value * 0.1 = near-full suppression — comment previously said 0.3, which drifted * from the actual constant below). */ #define ENGRAM_WM_THRESHOLD 0.15 #define ENGRAM_WM_DECAY 0.7 #define ENGRAM_SUPPRESSION_BREAKTHROUGH 5 /* ENGRAM_BREAKTHROUGH_WEIGHT: lowered 0.25→0.10 (2026-06-30 self-review, porting * fix from self-review 2026-06-26 branch). With 0.25, Knowledge nodes (threshold * 0.15) promoted at ~0.21 decay in one call to ~0.147, fall below the 0.25 floor, * and immediately lose their WM slot to fresh breakthrough candidates at 0.25. * Natural promotion was invisible: live data showed 524/525 WM nodes at 0.25 * breakthrough floor. With 0.10, all per-type thresholds (minimum 0.15 Canonical) * exceed the floor, so naturally-promoted nodes survive multiple decay cycles. * Invariant maintained: BREAKTHROUGH_WEIGHT < min(type_thresholds). */ #define ENGRAM_BREAKTHROUGH_WEIGHT 0.10 /* ENGRAM_BREAKTHROUGH_BUDGET / ENGRAM_BREAKTHROUGH_COOLDOWN (2026-08-02 * self-review): the breakthrough path was an unbounded, self-resetting loop. * Every reached node failing its type threshold incremented suppression_count; * on the 5th failure it was force-promoted at exactly 0.10 AND had its counter * reset to 0 — so it re-entered the identical cycle immediately. Because * BREAKTHROUGH_WEIGHT (0.10) > WM_FLOOR (0.05), all of them cleared the * absolute floor and entered the rank contest tied at 0.10, where all but a * handful were evicted by ENGRAM_WM_CAP. Evicted nodes get no access_ts * record (Pass 6 skips wm_weights<=0), so the short-term inhibition-of-return * damper at ENGRAM_STI_TS never applied to them and they were re-suppressed * completely unmarked. Steady state: N_suppressed/5 breakthroughs per call, * essentially all of them evicted the same call. * * Live telemetry 2026-08-02 (boot 19, uptime 23h48m): breakthroughs_delta * 661–903 and wm_evicted_delta 485–717 PER 60s heartbeat against wm_active * pinned at 22–24. At ~4 activate calls/minute that is ~165–225 breakthroughs * per call — ~825–1125 nodes cycling in 5-call lockstep. Working memory was * not remembering; it was thrashing, and the churn drowned genuine promotion. * * Two bounds, both required: * BUDGET — at most WM_CAP/4 (6) intrusive thoughts may surface per call. * Breakthrough is meant to be an occasional intrusive thought, * not a stampede; it can never again exceed a quarter of WM. * COOLDOWN — on breakthrough, suppression_count is set NEGATIVE rather than * 0, so the node needs COOLDOWN+SUPPRESSION_BREAKTHROUGH further * suppressions before it may surface again (~60 calls ≈ 15 min at * the current cadence, vs 5 calls ≈ 75s before). This is * inhibition-of-return applied to the breakthrough path, matching * the STI damper already applied to the natural path. Stored in * the existing int32_t field — serialized as %d and parsed via * eg_get_int_field, so negative values round-trip through * snapshots without a struct or format change. * When the budget or cooldown blocks a breakthrough, suppression_count is NOT * reset — it saturates, so a starved node surfaces on a later call rather than * restarting its climb. */ #define ENGRAM_BREAKTHROUGH_BUDGET (ENGRAM_WM_CAP / 4) #define ENGRAM_BREAKTHROUGH_COOLDOWN 55 /* ENGRAM_WM_CAP: hard limit on concurrent working-memory nodes (2026-06-30 * self-review, porting fix from self-review 2026-06-26 branch). Without this, * broad curiosity seeds like "knowledge" promote 500+ nodes simultaneously — * wm_avg_weight collapses to the breakthrough floor, goal-bias differentiation * is lost, and heartbeat ISEs show useless WM composition data. Cognitive * basis: WM capacity is ~4 chunks (Cowan 2001); 24 allows richer multi-topic * context while preventing flooding. Enforced in Pass 4 (per-call) and Pass 5 * (global across prior-promoted nodes). */ #define ENGRAM_WM_CAP 24 /* ENGRAM_WM_FLOOR: absolute admission floor for a working-memory slot * (2026-07-30 self-review). Before this, Pass 4/Pass 5/load-cap only ever * trimmed the WM population down TO the cap and never below it — rank-based * eviction guarantees the cap is filled whenever ≥24 nodes hold any nonzero * weight, so wm_active was pinned at 24/24 and wm_saturated:1 was * definitionally true on every heartbeat (carried no information). Soar's WM * forgetting (Derbinsky & Laird, ICCM 2012) removes elements by comparing * activation to an ABSOLUTE threshold θ, independent of how many other * elements exist — fill below capacity is a reachable, meaningful state * ("low cognitive load"). This floor is the weight-domain analogue of that θ: * any slot whose weight sinks below it is dropped even when WM is under cap. * Value 0.05 = the lowest per-type promotion threshold (Safety/DharmaSelf in * engram_type_threshold) and the existing boot-time launder floor, and sits * below ENGRAM_BREAKTHROUGH_WEIGHT (0.10) so intrusive-thought breakthroughs * still surface. Applied: Pass 4, carry-over, Pass 5, load-cap. */ #define ENGRAM_WM_FLOOR 0.05 #define ENGRAM_INHIBITION_FACTOR 0.1 /* ── ACT-R / Petrov hybrid base-level learning (2026-07-22 self-review) ────── * Replaces the per-turn multiplicative WM carry-over decay (weight *= 0.7 per * engram_activate call) with wall-clock power-law decay over actual access * history. The old scheme was call-rate-dependent: 10 curiosity scans in 10 * seconds decayed a carried node as much as 10 scans across an hour, and a * scalar weight loses access FREQUENCY entirely — a node touched 50 times * decayed identically to one touched once. * * Petrov (2006) hybrid: keep the K most recent access timestamps exactly, * approximate the older tail in closed form: * * B = ln( Σ_{j=1..k} t_j^(-d) * + (n-k) · (t_n^(1-d) - t_k^(1-d)) / ((1-d) · (t_n - t_k)) ) * * d = 0.5 (canonical ACT-R decay), t_j = seconds since j-th recent access, * t_k = seconds since oldest RETAINED access, t_n = seconds since first * presentation (node creation), n = total presentations. * * This is the scheme Soar ships for working-memory forgetting (Derbinsky & * Laird 2012). Calibration: a single-touch node has B(t) = -0.5·ln(t); with * τ = -3.0 it falls below threshold at t = e^6 ≈ 403 s. Frequently-touched * nodes accumulate Σ t^-0.5 mass and persist proportionally longer — recency * AND frequency in one formula, which a decayed scalar cannot represent. * * Carry-over weight above threshold is shaped by the ACT-R retrieval- * probability logistic P = σ((B − τ)/s), s = 0.4, applied to the weight the * node held at promotion (wm_anchor) — NOT re-multiplied per call, so the * carried weight is a pure function of wall-clock time regardless of how * often engram_activate runs. * Sources: alexpetrov.com/pub/iccm06 · Soar cli/cmd_wm defaults · * arxiv.org/html/2412.05112v1 (2026-07-21 integration brief, bl-b17facdd). */ #define ENGRAM_BLL_K 10 #define ENGRAM_BLL_D 0.5 #define ENGRAM_BLL_TAU (-3.0) #define ENGRAM_BLL_S 0.4 /* Short-term inhibition-of-return (2026-07-25 self-review). * Lebiere & Best 2009 ("Balancing Long-Term Reinforcement and Short-Term * Inhibition", CogSci): subtract ln(1 + (t_n/t_s)^-d_s) from activation, * where t_n = time since the MOST RECENT access. With their best-fit * d_s = 1.0 the exp of the subtraction reduces to the clean multiplier * m(t_n) = t_n / (t_n + t_s) * applied to raw_wm in Pass 2. Immediately after a promotion the node's * score is crushed (m → 0), then self-heals as a power law — producing an * emergent round-robin over WM candidates instead of winner-take-all. * This replaces the non-decaying suppression_count as the damping * mechanism (that counter never entered the score at all — it only ever * pushed nodes TOWARD surfacing via breakthrough; kept for that role). * t_s = 4× the ~30 s curiosity-scan interval per the paper's guidance * (t_s ≈ peak re-occurrence lag). A node promoted every scan holds * m ≈ 0.2 until it loses its slot; after ~2 min unretrieved, m ≥ 0.5. * Source: act-r.psy.cmu.edu/.../894Cogsci09-Lebiere-Best.pdf */ #define ENGRAM_STI_TS 120.0 /* Carry-over occupancy inhibition (2026-07-26 self-review). * The STI multiplier above only runs in the REACHED branch of Pass 2. * A node carried over WITHOUT being reached (Pass 4½ below) kept * w = wm_anchor * keep, and for a node whose base-level was inflated * during the pre-07-25 unconditional-reinforcement era, keep ≈ 1.0 for * days — the anchor weight is re-emitted verbatim forever. Observed * live: wm_top0_streak = 1407 heartbeats (~23 h), one node frozen at * its anchor 0.589 while every reached candidate rotated at the 0.10 * breakthrough floor. Deterministic argmax over a quasi-static score * fixates regardless of any inhibition applied only to the reached set * (Morita et al. 2021, citing Lebiere & Best 2009). * Fix: key inhibition on OCCUPANCY, not retrieval recency — multiply * the carried weight by the same d_s=1 closed form over hold time * m(t_h) = t_c / (t_c + t_h), t_h = seconds since last_activated * (carry-over nodes are deliberately never reinforced, so * last_activated marks when the node last EARNED its slot). Power-law * self-healing: a node re-reached by any future activation is * re-scored fresh in Pass 2 and re-anchors. t_c = 3600 s → a carried * node keeps ~92% after 5 min, 50% after 1 h, ~4% after 23 h. Unlike * m(t_n), this cannot saturate to no-op while the node camps. */ #define ENGRAM_CARRY_TC 3600.0 /* qsort comparator — descending double, used by WM cap enforcement. */ static int engram_cmp_double_desc(const void* a, const void* b) { double da = *(const double*)a; double db = *(const double*)b; if (da > db) return -1; if (da < db) return 1; return 0; } /* ── Layered consciousness architecture ────────────────────────────────────── * * The engram graph is stratified into LAYERS that gate which suppressions * apply during the executive filter pass. Layers are ordered shallow-to-deep * by `activation_priority`; the deepest layer (priority 0, conventionally * "safety") is the structural floor of the soul: nodes here cannot be * silenced by inhibitory edges from any other layer. Higher layers * (core-identity, domain-knowledge, imprint, suit) are normally * suppressible — they participate in attentional inhibition and goal * focus the way the prior single-graph implementation did. * * The five canonical layers (see engram_init_layers): * 0. safety — structural, transparent, non-injectable, non-suppressible * 1. core-identity — default for legacy nodes; suppressible * 2. domain-knowledge— suppressible * 3. imprint — runtime-injectable (an Imprint package can add/remove) * 4. suit — runtime-injectable (a Suit overlays domain skill) * * Three-pass activation (engram_activate): * Pass 1 — Background fan-out: BFS spreads activation across ALL layers * (existing behavior preserved). Inhibitory edges propagate at * this layer too; no filtering happens here. * Pass 2 — Working memory promotion: type-threshold gate, goal bias, * confidence weighting, inhibitory suppression. Inhibitory edges * ONLY apply against nodes whose layer is `suppressible == 1`. * Nodes in non-suppressible layers (Layer 0) ignore inhibition. * Pass 3 — Layer 0 override: every node in a non-suppressible layer that * received background activation has its working_memory_weight * forced to >= ENGRAM_LAYER0_OVERRIDE_WEIGHT. The sacred fire — * safety nodes that touched any seed unconditionally surface, * even when the executive filter would have silenced them. * * Layer fields: * suppressible : 0 → inhibitory edges are ignored against nodes in this * layer during pass 2. Pass 3 also force-promotes them. * 1 → standard behavior (most layers). * transparent : 1 → emitted into the prompt context so its content shapes * output, but filtered out of "what do you know about * yourself?" introspection queries (engram_search and * friends do not return transparent-layer nodes by * default). 0 → fully visible to introspection. * injectable : 1 → can be added/removed at runtime via engram_add_layer * and engram_remove_layer (imprints, suits). * 0 → built-in, fixed at engram_get() initialization. * * Backward compatibility: * Nodes and edges loaded from snapshots without a `layer_id` field default * to layer 1 (core-identity). The five canonical layers are always present. */ #define ENGRAM_LAYER_SAFETY 0u #define ENGRAM_LAYER_CORE_IDENTITY 1u #define ENGRAM_LAYER_DOMAIN 2u #define ENGRAM_LAYER_IMPRINT 3u #define ENGRAM_LAYER_SUIT 4u #define ENGRAM_LAYER_DEFAULT ENGRAM_LAYER_CORE_IDENTITY /* Pass 3 override floor. Layer 0 nodes that received any background * activation are force-promoted to AT LEAST this working_memory_weight, * regardless of inhibitory suppression in pass 2. */ #define ENGRAM_LAYER0_OVERRIDE_WEIGHT 1.0 /* Per-node-type activation thresholds. * Lower tier / safety-critical nodes fire more readily. */ static double engram_type_threshold(const char* node_type, const char* tier) { if (node_type) { if (strcmp(node_type, "DharmaSelf") == 0) return 0.05; if (strcmp(node_type, "Safety") == 0) return 0.05; } if (tier) { if (strcmp(tier, "Canonical") == 0) return 0.15; if (strcmp(tier, "Lesson") == 0) return 0.25; } if (node_type) { if (strcmp(node_type, "Belief") == 0) return 0.30; if (strcmp(node_type, "Entity") == 0) return 0.30; /* Knowledge nodes (captureKnowledge, world-ingestor) at non-Canonical/ * non-Lesson tiers (Semantic/Episodic/Procedural) previously fell * through to the 0.40 note default — same bar as ephemeral notes — * so curated knowledge mostly entered WM via breakthrough suppression * (visible as wm weights pinned near the breakthrough floor) instead * of natural promotion. Placed AFTER the tier checks so Canonical * (0.15) and Lesson (0.25) still win. Ported from the dev-line fix * (2026-06-13 self-review). (2026-07-19 self-review) */ if (strcmp(node_type, "Knowledge") == 0) return 0.20; } return 0.40; /* Note / Memory / Working (most nodes) */ } typedef struct EngramNode { char* id; char* content; char* node_type; char* label; char* tier; char* tags; char* metadata; double salience; double importance; double confidence; double temporal_decay_rate; /* per-node override for lambda; 0 = use default */ int64_t activation_count; int64_t last_activated; int64_t created_at; int64_t updated_at; /* Two-layer activation fields ───────────────────────────────────────── * background_activation: Layer 1. Set by BFS fan-out on every query. * Every reachable node fires here — nothing is filtered at this stage. * Models the brain's massive parallel sub-threshold activation of all * associated content in response to a stimulus. * working_memory_weight: Layer 2. Executive filter output. Only nodes * that survive goal-state / attentional-bias scoring receive a * non-zero weight here. Context compilation ONLY uses this field. * Background-activated nodes with working_memory_weight == 0 remain * latent — real, available, but silent. * suppression_count: Consecutive turn count where this node was * background-activated but NOT promoted to working memory. High * values signal the node "wants to surface." After * ENGRAM_SUPPRESSION_BREAKTHROUGH consecutive suppressions the node * is force-promoted at a reduced weight (breakthrough activation). */ double background_activation; double working_memory_weight; int32_t suppression_count; /* Layered consciousness — see ENGRAM_LAYER_* macros and engram_init_layers. * Defaults to ENGRAM_LAYER_DEFAULT (1, core-identity) for legacy nodes * created via engram_node / engram_node_full and for snapshots that * predate the layered schema. */ uint32_t layer_id; /* ACT-R base-level learning state (2026-07-22 self-review, ENGRAM_BLL_*). * access_ts: ring buffer of the most recent access timestamps (ms). * access_head: next write slot. access_filled: valid entries (≤ K). * wm_anchor: the WM weight the node held when last promoted; carry-over * decay is computed from this anchor as a pure function of wall-clock * time. All fields zero via memset in creation/load paths; snapshots * without them degrade gracefully to the optimized-form approximation * in engram_bll_base_level. */ int64_t access_ts[ENGRAM_BLL_K]; int32_t access_head; int32_t access_filled; double wm_anchor; /* Semantic embedding (2026-07-24 self-review, bl-b2d1c944). * emb: malloc'd float vector (nomic-embed-text, 768-dim) or NULL. * emb_dim: vector length; 0 = not embedded. Populated lazily by the * backfill pass in engram_activate — NOT on the node-create hot path, * so bulk sync imports never block on Ollama. Freed in engram_forget / * engram_prune_telemetry; shift-copies move the pointer intact. */ float* emb; int32_t emb_dim; /* Hebbian eligibility trace (2026-08-06 self-review, ENGRAM_HEBB_TRACE_*). * hebb_elig: trace amplitude, set to 1.0 whenever the node holds a WM slot. * hebb_elig_ts: wall-clock ms at which that happened; the trace is read as * hebb_elig·exp(−Δt/TC) rather than stored decayed, so it is a pure * function of time and independent of how often activate is called. * * DELIBERATELY NOT SERIALIZED. A trace is a behavioral-timescale quantity * (minutes); persisting it across a restart would resurrect coincidences * from an arbitrarily distant past as if they had just happened. Zero via * the calloc at store init and the memset in engram_grow_nodes, so a cold * boot simply has no eligible pairs until WM starts turning over. */ double hebb_elig; int64_t hebb_elig_ts; } EngramNode; /* Record an access (ACT-R "presentation") into the base-level ring buffer. */ static void engram_bll_record_access(EngramNode* n, int64_t now_ms) { n->access_ts[n->access_head] = now_ms; n->access_head = (n->access_head + 1) % ENGRAM_BLL_K; if (n->access_filled < ENGRAM_BLL_K) n->access_filled++; } /* Petrov hybrid base-level activation B in nats. Exact sum over the retained * ring, closed-form tail for older presentations. Falls back to the ACT-R * optimized-learning approximation B = ln(n/(1−d)) − d·ln(L) for nodes with * no retained access history (legacy snapshots). */ static double engram_bll_base_level(const EngramNode* n, int64_t now_ms) { /* Total presentations: creation counts as the first, retrievals add. */ double n_total = (double)n->activation_count + 1.0; double t_life = (double)(now_ms - n->created_at) / 1000.0; if (t_life < 1.0) t_life = 1.0; if (n->access_filled == 0) { return log(n_total / (1.0 - ENGRAM_BLL_D)) - ENGRAM_BLL_D * log(t_life); } double sum = 0.0; double t_oldest_ring = 0.0; /* seconds since oldest retained access */ for (int32_t j = 0; j < n->access_filled; j++) { double t = (double)(now_ms - n->access_ts[j]) / 1000.0; if (t < 1.0) t = 1.0; sum += pow(t, -ENGRAM_BLL_D); if (t > t_oldest_ring) t_oldest_ring = t; } /* Closed-form tail for the (n − k) presentations older than the ring. * With d = 0.5 this reduces to 2·(n−k)/(√t_n + √t_k). */ double n_older = n_total - (double)n->access_filled; if (n_older > 0.0 && t_life > t_oldest_ring) { sum += n_older * (pow(t_life, 1.0 - ENGRAM_BLL_D) - pow(t_oldest_ring, 1.0 - ENGRAM_BLL_D)) / ((1.0 - ENGRAM_BLL_D) * (t_life - t_oldest_ring)); } if (sum <= 0.0) return -99.0; return log(sum); } /* Parse a persisted "access_ts" comma list (chronological order) into the * ring buffer. Tolerates absence, empty strings, and stray whitespace. */ static void engram_bll_parse_access(EngramNode* nn, const char* s) { if (!s) return; const char* p = s; while (*p) { char* endp = NULL; long long v = strtoll(p, &endp, 10); if (endp == p) break; if (v > 0) engram_bll_record_access(nn, (int64_t)v); p = endp; while (*p == ',' || *p == ' ') p++; } } /* ── Embedding-based activation (2026-07-24 self-review, bl-b2d1c944) ────── * Closes the gap named in every heartbeat since 2026-06-30: "no embedding * call is made during activation. The seed-finding loop uses istr_contains * only." Design per the 2026-07-21 integration brief: * - Lazy backfill: engram_activate embeds up to BACKFILL_PER_CALL * un-embedded, non-telemetry nodes per call, newest first. No latency * on node-create paths; a fresh node is embedded within ~1 scan cycle. * - Semantic seeding (HippoRAG pattern, use similarity twice): the query * is embedded, the top-K nodes by cosine ≥ SEED_MIN join the seed set * with initial activation = the similarity itself. * - Additive WM term: raw_wm += W * relu((cos − S0)/(1 − S0)). Shift-and- * floor at S0 because nomic-embed scores unrelated pairs 0.4–0.5; raw * cosine in a weighted sum is a constant bias, not a signal. * - Circuit breaker: 3 consecutive Ollama failures → stop trying for 5 * minutes. Activation NEVER blocks on a dead embedder beyond timeout. */ #define ENGRAM_EMBED_S0 0.45 #define ENGRAM_EMBED_WM_WEIGHT 0.20 #define ENGRAM_EMBED_SEED_K 8 #define ENGRAM_EMBED_SEED_MIN 0.60 #define ENGRAM_EMBED_BACKFILL_PER_CALL 8 /* ENGRAM_QGATE_FLOOR: minimum propagation multiplier for an EMBEDDED target * node with zero/negative query similarity. Query-aware spreading gate * (arXiv:2606.30133) adapted for partial embedding coverage — see the * propagation loop in engram_activate. 0.25 damps semantically unrelated * branches ~4x without severing them. Unembedded targets are ungated. */ #define ENGRAM_QGATE_FLOOR 0.25 #define ENGRAM_EMBED_MAX_CHARS 2000 #define ENGRAM_EMBED_TIMEOUT_MS 4000L #define ENGRAM_EMBED_BREAKER_LIMIT 3 #define ENGRAM_EMBED_BREAKER_COOLDOWN_MS 300000 /* ── Context centroid (2026-07-29 self-review) ─────────────────────────────── * Closes the last gap from the 2026-07-21 decay/embedding brief: cosine was * computed against the per-call query embedding ONLY, so every curiosity scan * was semantically memoryless — the 4 rotating seed phrases fully determined * what ignited, with zero continuity from what the system actually touched. * * Mechanism (brief spec): a running EMA centroid over touch embeddings, * c ← normalize(μ·c + (1−μ)·e_touch), μ = ENGRAM_CTX_MU * where a "touch" is (a) the query embedding each activate call and (b) the * embeddings of up to ENGRAM_CTX_TOUCH_MAX top WM-promoted survivors of that * call — promotion is the retrieval event (same rule as BLL reinforcement). * * Feedback-loop guard: scoring does NOT use the raw centroid. The lit * failure mode of centroid memories (EMA of your own outputs → runaway * attractor; cf. the wm_top0_streak=1407 freeze this store already hit) is * bounded by scoring against a query-dominant blend: * e_eff = normalize(α·e_q + (1−α)·c), α = ENGRAM_CTX_QALPHA * so the exogenous rotating seeds always contribute the majority of the * scoring direction; the centroid is a context tint, not the signal. * * Observability: _eg_act_ctx_cos = cos(e_q, c) BEFORE the query is blended * in. ~1.0 → centroid aligned with current query; low → context and query * have diverged (expected at domain-rotation boundaries); -2.0 → no centroid * yet / embedder down. Exposed via engram_act_stats_json → heartbeat ISE so * drift is diagnosable from telemetry. In-memory only: context is * short-term by definition, a restart legitimately starts cold. */ #define ENGRAM_CTX_MU 0.90 #define ENGRAM_CTX_QALPHA 0.65 #define ENGRAM_CTX_TOUCH_MAX 8 /* ── Hebbian co-activation potentiation (2026-08-04 self-review) ───────────── * THE GAP: every learning mechanism in this runtime operated on NODES — * salience, base-level learning (ACT-R), activation_count, temporal decay, * WM promotion. Edge weights were written once at engram_connect() and never * changed again. `last_fired` was declared on EngramEdge, persisted, and * emitted in JSON, but the ONLY writer in the entire 12.5k-line runtime was * dharma_strengthen() — an unrelated CGI-relationship path. Activation read * e->weight and never wrote it. So the graph's TOPOLOGY was frozen: an edge * authored at the default 0.5 that proved itself useful on ten thousand * consecutive retrievals stayed at exactly 0.5, indistinguishable from one * that had never carried a useful signal. The nodes learned; the wiring * between them did not. "The system must get smarter over time" was true of * memories and false of the associations among them. * * THE RULE (HeLa-Mem, arXiv:2604.16839): w ← λ·w + η·1[both nodes co-retrieved]. * * ADAPTATION 1 — learned strength is a SEPARATE field, not a mutation of the * authored weight. HeLa-Mem updates the association weight in place because * all of its edges ARE learned associations. Most edges here are authored * structure: `contains` from the self root, `identity`, `supersedes`, `tagged`. * Decaying those would erode identity silently and irreversibly — precisely * the failure the immutable-engram principle exists to prevent. So `weight` * stays exactly as authored (audit trail intact, behavior exactly restorable * by ignoring the field) and `hebb` accumulates alongside it. Propagation uses * eg_edge_eff_weight() = weight × (1 + GAIN·hebb), clamped to 1.0. A cold * graph has hebb == 0 everywhere and therefore behaves bit-identically to the * pre-change runtime — this change cannot regress a fresh deploy. * * ADAPTATION 2 — η is set to exactly (1 − λ), which turns the update into an * exponentially-weighted moving average. hebb then converges to a quantity * with a plain reading: THE FRACTION OF RECENT ACTIVATION CALLS IN WHICH BOTH * ENDPOINTS WERE SIMULTANEOUSLY IN WORKING MEMORY. Not an arbitrary strength * unit — a probability. That makes the homeostatic budget below interpretable * in the same units, and makes the telemetry readable without a decoder ring. * * ADAPTATION 3 — homeostatic scaling, which HeLa-Mem does not have. Their * ablation shows removing adaptive forgetting costs almost nothing (34.74 → * 34.28 F1) because their benchmark runs ~300 turns; this store has 41k edges * and runs continuously for weeks. Pure potentiation lets a high-degree hub * accumulate strength on ALL its edges at once and become a superhighway that * relays activation everywhere — the exact hub-flooding pathology the * query-aware propagation gate was added to fix in the 2026-07-27 review. * The consolidation literature is unanimous that potentiation requires a * compensating normalization (surviving connections are collectively scaled * down to hold firing-rate homeostasis — PNAS 2422602122, two-factor synaptic * consolidation). So: per node, the summed hebb across incident edges is * capped at ENGRAM_HEBB_NODE_BUDGET and scaled down proportionally when * exceeded. A node can hold ~4 strong associations, or many weak ones, but * not unbounded total associative mass. Potentiation is competitive, not free. * * TIMESCALE: decay is per activation CALL, not per wall-clock second. That is * deliberate — associative strength should track cognitive events, not the * clock, so an idle daemon does not forget what it learned while working. At * the current curiosity-scan rate (~2 calls / 30 s) the 0.9999 factor gives a * half-life of ~6,900 calls ≈ 1.2 days: associations form over hours and fade * over days of genuine disuse. */ #define ENGRAM_HEBB_DECAY 0.9999 #define ENGRAM_HEBB_ETA 0.0001 /* == 1 - DECAY ⇒ hebb is an EWMA */ #define ENGRAM_HEBB_GAIN 0.5 /* max +50% effective propagation */ #define ENGRAM_HEBB_NODE_BUDGET 4.0 /* homeostatic cap on per-node Σ hebb */ /* Snap-to-zero floor. MUST stay far below ENGRAM_HEBB_ETA. Set to 0.001 * initially — above the 0.0001 per-step increment — and live telemetry caught * it immediately: hebb_cand_max pinned at exactly 0.0001 across 50 calls while * 15 pairs co-activated every single time. A pair claimed a slot at ETA, the * next call's decay pass saw 0.0001 < 0.001 and cleared it, and the same pair * re-claimed at ETA forever. Nothing could ever cross a threshold 1,500 steps * away when it was being reset every step — the rule was structurally * incapable of learning anything, for edges as well as candidates. At 1e-6 an * association touched once survives ~8 days of pure disuse before cleanup, * which is what a decay floor is actually for. The lesson is the one this * system keeps relearning: a mechanism that is not instrumented is a mechanism * you are guessing about. */ #define ENGRAM_HEBB_MIN 1e-6 /* ── Associative link FORMATION (2026-08-04 self-review, same session) ─────── * MEASURED, NOT ASSUMED: after wiring the potentiation rule above I drove 30 * activations and found zero potentiated edges. Instrumenting the reason gave * the finding that actually matters: * * edges between working-memory members: 0 * * Working memory is populated by semantic seeding (cosine top-K) and by * multi-hop spreading. Both routinely land on nodes that are semantically * close and structurally distant. So the pairs that fire together are, in this * graph, almost never already wired together — and a rule that only reweights * EXISTING edges is a no-op. HeLa-Mem does not hit this because it maintains a * dense association matrix over a small memory set; this is a sparse 41k-edge * graph in which EVERY edge was authored by an explicit tool call. Nothing in * the runtime has ever created an associative edge from experience. The system * could strengthen what it was told; it could not notice anything on its own. * * So take Hebb literally rather than as HeLa-Mem specializes him: cells that * fire together WIRE together — if the wire is absent, grow it. * * Consolidation is deliberately slow and heavily bounded, because unlike a * weight tweak this permanently mutates the persisted graph: * - A pair must sustain co-activation as an EWMA past ENGRAM_HEBB_LINK_MIN * (~15% of recent calls ≈ 1,100 co-activations ≈ 5 h of continuous * association) before any edge is created. One-off coincidences never * consolidate; that is the whole point of the threshold. * - At most ENGRAM_HEBB_LINK_PER_CALL edges are born per activation. * - Candidates live in a fixed 8,192-slot table, in memory only. A restart * discards them, which is a feature, not a limitation: only associations * sustained across one continuous run earn permanence, and the table can * never grow without bound. * - New edges carry relation "hebbian-associate" and start at a deliberately * weak ENGRAM_HEBB_LINK_W0, so they must keep proving themselves through * the potentiation rule to gain any real influence. They are tagged * precisely so every self-formed association stays auditable and the whole * set is removable with one query if this turns out to be wrong. */ /* ── Eligibility traces: why co-activation had to stop meaning "same call" ──── * (2026-08-06 self-review. Measured, not theorized.) * * CENSUS. Across the live graph — 41,213 edges, 13,091 nodes, 23h44m uptime, * boot 23 — the strongest Hebbian association in the entire store measured * hebb = 0.000799, and `hebbian-associate` edges formed since the mechanism * shipped: ZERO. 0.000799 / ENGRAM_HEBB_ETA ≈ 8. The best pair of nodes in the * graph had co-activated eight times, net of decay, ever. ENGRAM_HEBB_LINK_MIN * is 0.15 — 1875× further away. The effective propagation bonus the mechanism * was delivering was GAIN·hebb = 0.5 × 0.0008 = +0.04%. * * Since the awareness loop calls engram_connect nowhere, Hebbian consolidation * is the ONLY path by which this system grows its own structure. It was inert. * Every edge in the graph was authored or imported; none was learned. * * WHY RAISING ETA IS THE WRONG FIX. hebb is an EWMA: h ← DECAY·h + ETA·[event], * with ETA = 1 − DECAY. Its fixed point is P(event) — the learning rate sets how * fast it converges there and has NO effect on where it converges. If the * measured ceiling is 0.0008, then P(event) ≈ 0.0008, and ETA could be raised a * thousandfold without moving the plateau a single decimal. The defect is not * the rate. It is the event. * * WHAT THE EVENT WAS. "Both endpoints hold a WM slot in the same activate call." * With ENGRAM_WM_CAP = 24 against a 13k-node store, and measured turnover of * ~142 evictions per 60s heartbeat, two related nodes essentially never occupy * the same 24 twice. The rule demanded exact simultaneity from a working memory * engineered — by ENGRAM_STI_TS inhibition-of-return, by breakthrough rotation, * by the global cap — to never let anything sit still. The three mechanisms * that make WM healthy are precisely the ones that made this measurement empty. * * THE FIX (three-factor / TD(λ) eligibility traces; Sutton & Barto ch. 7, * Gerstner et al. 2018 on neoHebbian eligibility, PLOS Comp Biol 2018 on * differential Hebbian learning with leaky-integrator traces). Coincidence * detection at behavioral timescales does not require simultaneity. A synapse * that fires sets a decaying flag; potentiation occurs if the partner fires * while the flag is still up. So: every node entering WM sets a trace to 1.0, * the trace decays exponentially in WALL-CLOCK time, and the potentiation * increment becomes ETA · trace(a) · trace(b) instead of ETA · [both in WM now]. * * This is a strict generalization, which is what makes it safe to ship: when * both endpoints are in WM at this instant, both traces read exactly 1.0 and * the increment is exactly ETA — bit-identical to the previous rule. The change * is purely additive on near-coincidences the old rule discarded outright. * * TC = 300s. Chosen against measured cadence, not taste: curiosity scans run * every ~31s, so 300s spans ~10 activation cycles (one cycle back reads 0.90, * three back 0.73, ten back 0.36). Long enough to bridge WM rotation; short * enough that two nodes surfacing an hour apart (exp(−12) ≈ 6e-6) are correctly * treated as unrelated. It sits deliberately between ENGRAM_STI_TS (120s, the * rotation it must survive) and ENGRAM_CARRY_TC (3600s, conversational span). * * TRACE_MIN snaps sub-threshold traces to zero so the warm set stays small and * a node cannot linger as a faint associate of everything for hours. * * The homeostatic ENGRAM_HEBB_NODE_BUDGET cap is what keeps this from running * away: more pairs now potentiate per call, and per-node associative mass is * still hard-bounded and scaled down proportionally. The safety net predates * this change and is exactly why loosening the event is not reckless. */ #define ENGRAM_HEBB_TRACE_TC 300.0 #define ENGRAM_HEBB_TRACE_MIN 0.05 /* Bound on non-WM warm nodes considered for candidate pairing in one call. * At 24 slots × ~10 cycles per trace window the warm set tops out near 240, but * most are re-promoted incumbents already in WM; 96 is non-binding in practice * and caps the pairing loop at 24×96 slot probes. */ #define ENGRAM_HEBB_WARM_MAX 96 #define ENGRAM_HEBB_CAND_SLOTS 8192 #define ENGRAM_HEBB_LINK_MIN 0.15 #define ENGRAM_HEBB_LINK_PER_CALL 2 #define ENGRAM_HEBB_LINK_W0 0.15 /* Hard ceiling on self-formed edges as a fraction of the authored graph. * ENGRAM_HEBB_LINK_PER_CALL alone bounds the RATE (≤2/call) but not the TOTAL: * at the production scan rate that ceiling is ~11k edges/day, which would * swamp a 41k-edge graph inside a week in the worst case. Potentiation decays, * but a link whose hebb has decayed back to zero still persists as a weak * edge — there is currently no pruning path, so growth is one-way. Until there * is one, self-formed structure is capped at 5% of the store: enough room to * learn real associations, not enough to drown what was authored. */ #define ENGRAM_HEBB_LINK_MAX_FRAC 0.05 /* ── Systems consolidation: the durable write-back queue (2026-08-07) ──────── * * THE DEFECT THIS CLOSES. Yesterday's eligibility-trace fix made Hebbian * learning numerically real: hebb_max went 0.000799 → 0.4725 and 1,198 * `hebbian-associate` edges formed in 23h48m. Today's census found where they * went: nowhere. Measured on the live system — * * soul daemon (pid 3269, in-process graph): 42,426 edges, 1,198 hebbian * engram server (:8742, the persistent store): 41,213 edges, 49 hebbian * * Two processes, two graphs, one direction of travel. The soul pulls from the * server every 10 min via GET /api/sync and merges. It never pushes. And it * cannot fall back on saving its own copy: `soul_snapshot_path` is set only * inside `if is_genesis && safe_to_seed` in soul.el, and safe_to_seed is * unconditionally false whenever ENGRAM_URL is set (it is, in the launchd * plist) — because the HTTP server owns persistence and a soul that wrote * snapshot.json would clobber it. That guard is correct. The consequence was * not: mem_save() has never once executed, so every association the soul * learns lives in RAM until the process dies. * * The soul is the ONLY process that runs idle cognition — curiosity scans * every ~8 min, around the clock. It is where essentially all co-activation * happens. So the system's entire capacity to grow its own structure was * pointed at a volatile store. 1,198 associations/day, discarded at restart, * every day, silently. The mechanism worked and the learning still evaporated. * * WHY A QUEUE AND NOT A SAVE. The fix is not to let the soul write the * snapshot — that reintroduces the clobber the guard exists to prevent. It is * to make consolidation a MESSAGE, not a file: the fast volatile store hands * each newly-formed association to the slow durable store, one edge at a time, * over the API the server already exposes (POST /api/edges). This is the * hippocampal→neocortical split the rest of this file is already modeled on. * Fast store learns online and forgets; slow store receives what survived the * threshold and keeps it. Only edges that already cleared ENGRAM_HEBB_LINK_MIN * are enqueued, so what crosses the process boundary is what earned it. * * SHAPE. Fixed 512-slot ring, overwrite-oldest. 512 is ~18x the observed * formation rate per drain interval (14 links per 8-min heartbeat), so the * queue only saturates when the writer is down — and when it is, keeping the * freshest associations is the right loss. Drops are counted, not silent: * a consolidation path that quietly discards is the failure mode this whole * entry exists to correct. Enqueue is strdup'd because g->edges may realloc * and node ids may be freed by later pruning; the queue owns its copies. */ #define ENGRAM_HEBB_WB_SLOTS 512 typedef struct { char* a; char* b; double w; double hebb; } EgHebbWB; static EgHebbWB _eg_hebb_wb[ENGRAM_HEBB_WB_SLOTS]; static int _eg_hebb_wb_head = 0; /* index of the oldest live entry */ static int _eg_hebb_wb_len = 0; static int64_t _eg_hebb_wb_dropped = 0; /* lost to a full queue, cumulative */ static int64_t _eg_hebb_wb_drained = 0; /* handed to the durable store, cum. */ static void eg_hebb_wb_push(const char* a, const char* b, double w, double h) { if (!a || !b) return; int slot; if (_eg_hebb_wb_len >= ENGRAM_HEBB_WB_SLOTS) { slot = _eg_hebb_wb_head; free(_eg_hebb_wb[slot].a); free(_eg_hebb_wb[slot].b); _eg_hebb_wb_head = (_eg_hebb_wb_head + 1) % ENGRAM_HEBB_WB_SLOTS; _eg_hebb_wb_dropped++; } else { slot = (_eg_hebb_wb_head + _eg_hebb_wb_len) % ENGRAM_HEBB_WB_SLOTS; _eg_hebb_wb_len++; } _eg_hebb_wb[slot].a = strdup(a); _eg_hebb_wb[slot].b = strdup(b); _eg_hebb_wb[slot].w = w; _eg_hebb_wb[slot].hebb = h; /* strdup failure leaves a NULL id; the drain skips those rather than * emitting a malformed edge. */ } typedef struct { char* a; char* b; double score; } EgHebbCand; static EgHebbCand _eg_hebb_cand[ENGRAM_HEBB_CAND_SLOTS]; static int64_t _eg_hebb_links_formed = 0; /* Observability for the eligibility-trace rule (2026-08-06). hebb_warm is the * size of the last call's warm set — nodes eligible but not co-resident, i.e. * exactly the population the old simultaneity rule discarded. If this reads 0 * forever the traces are not arming and the change bought nothing; if it reads * healthy while hebb_max stays flat, the bottleneck is somewhere else. That * distinction is the whole reason yesterday's diagnosis took a census instead * of a guess. */ static int _eg_act_hebb_warm = 0; static float* _eg_ctx_c = NULL; static int32_t _eg_ctx_dim = 0; static double _eg_act_ctx_cos = -2.0; static int _eg_embed_consec_fail = 0; static int64_t _eg_embed_breaker_until = 0; /* ── Activation observability counters (2026-07-27 self-review) ────────────── * The executive-filter pathologies this system has repeatedly debugged by * inference (WM flooded with breakthrough-floor nodes, silent cap evictions, * embedder wedged behind the breaker) were all invisible: no ISE, no stats * field. engram_act_stats_json() exposes them so the soul heartbeat can emit * them. * * CUMULATIVE CONTRACT (2026-07-31 self-review): these are monotonic totals * for the process lifetime, like pulse/sync_added_total — NOT per-call. The * original per-call reset meant engram_act_stats_json only reported the LAST * activate call, and the 60s heartbeat (2 curiosity activates per 30s in * between) missed nearly every eviction/breakthrough event. Consumers wanting * rates keep the previous reading and diff. Restart legitimately resets to 0. */ static int64_t _eg_act_breakthroughs = 0; /* forced promotions at the floor, cumulative */ static int64_t _eg_act_wm_evicted = 0; /* ALL WM evictions, cumulative (see below) */ /* Redundancy suppression counters (2026-08-05 self-review) — see * ENGRAM_DEDUP_COS. dup_seeds = semantic seed slots reclaimed from redundant * copies; dup_wm = WM candidates dropped for duplicating a higher-ranked * candidate's content. Both cumulative for the process lifetime. */ static int64_t _eg_act_dup_seeds = 0; static int64_t _eg_act_dup_wm = 0; /* Redundant WM residents evicted by the GLOBAL pass (2026-08-06). Counted * separately from _eg_act_dup_wm on purpose: dup_wm measures duplicates caught * among this call's candidates, dup_wm_global measures duplicates that reached * the persisted WM population through the carry-over path, which is the leak * Pass 3½ structurally could not see. Merging them would hide whether the new * pass is doing anything. */ static int64_t _eg_act_dup_wm_global = 0; /* 2026-08-02 self-review: this counted only the three Pass 4 / Pass 5 floor * and rank paths. The two carry-over eviction paths (base-level below τ, and * decayed weight below WM_FLOOR) were silent, so the reported eviction rate * was an undercount of unknown magnitude — which mattered precisely while * diagnosing the breakthrough storm. All five paths now increment. */ static int64_t engram_now_ms(void); /* defined in the store section below */ static const char* eg_embed_url(void) { const char* s = getenv("EL_EMBED_URL"); return (s && *s) ? s : "http://localhost:11434/api/embeddings"; } static const char* eg_embed_model(void) { const char* s = getenv("EL_EMBED_MODEL"); return (s && *s) ? s : "nomic-embed-text"; } /* Cosine similarity over raw (unnormalized — nomic emits magnitudes >1) * float vectors. Returns -2.0 on dim mismatch / null input so callers can * distinguish "orthogonal" (0.0) from "not comparable". */ static double eg_cosine(const float* a, const float* b, int32_t dim) { if (!a || !b || dim <= 0) return -2.0; double dot = 0.0, na = 0.0, nb = 0.0; for (int32_t i = 0; i < dim; i++) { dot += (double)a[i] * (double)b[i]; na += (double)a[i] * (double)a[i]; nb += (double)b[i] * (double)b[i]; } if (na <= 0.0 || nb <= 0.0) return -2.0; return dot / (sqrt(na) * sqrt(nb)); } /* ── Redundancy suppression (2026-08-05 self-review) ───────────────────────── * MEASUREMENT, not intuition. Content-hash census of the live snapshot * (13,216 nodes / 41,213 edges) on 2026-08-05: * * non-ISE nodes 4,138 * duplicate content groups 1,489 * REDUNDANT copies 1,858 (44.9% of the non-ISE graph) * redundant copies embedded 1,856 * * The copies were all created in a single June 2026 import (1,854 of 1,858; * 2 in July) — an id-scheme migration re-added nodes under fresh UUIDs * instead of matching on content. Generation has stopped. The copies have * not: they are byte-identical, so they carry IDENTICAL embeddings, and * therefore identical cosine to any query. * * That is the damage. Semantic seeding takes the top-K by cosine * (ENGRAM_EMBED_SEED_K = 8, ≥ ENGRAM_EMBED_SEED_MIN). A document with six * copies does not compete for one of those eight slots — it takes six. * Measured over 50 real query probes against the live 3,998-vector set: * * seed slots filled 400 * slots consumed by redundant copies 161 (40.2%) * probes affected 46/50 (92%) * effective DISTINCT seeds per scan 4.78 of 8 * * Two fifths of every retrieval was spent re-reading the same page. The * same collapse hits WM: duplicates score identically, so they promote * together and hold multiple of the 24 slots for one thought. * * The fix is not to delete nodes — data repair is a separate, reversible * operation with its own backup discipline. The fix is that redundancy must * never buy a scarce slot, whatever the graph's state. Enforced at both * scarcity points: semantic seed selection, and WM admission (Pass 3½). * * Identity test, cheapest first: * 1. type|content FNV-1a — exact; catches the import duplicates * 2. cosine ≥ ENGRAM_DEDUP_COS — catches copies differing only in * whitespace/punctuation, which hash differently but embed identically * 0.995 is deliberately severe: at 768 dimensions this admits only * near-verbatim text. Distinct-but-related nodes (the associative structure * this system exists to traverse) sit far below it and are untouched. * This suppresses REDUNDANCY, never similarity. */ #define ENGRAM_DEDUP_COS 0.995 /* eg_content_key — FNV-1a over node_type|content. Identical prose under a * different node_type is not a duplicate (a Knowledge note and the * BacklogItem quoting it are different objects), so the type is folded in. */ static uint64_t eg_content_key(const EngramNode* n) { uint64_t h = 14695981039346656037ULL; const char* s = n->node_type; if (s) while (*s) { h ^= (unsigned char)*s++; h *= 1099511628211ULL; } h ^= (unsigned char)'|'; h *= 1099511628211ULL; s = n->content; if (s) while (*s) { h ^= (unsigned char)*s++; h *= 1099511628211ULL; } return h; } /* eg_same_content — is node `a` redundant with node `b`? * Hash equality first (one pass, no allocation); embedding near-identity as * the fallback for copies that differ only in insignificant characters. * Deliberately does NOT strcmp on hash collision: a 64-bit FNV-1a collision * across ~4k candidates is ~1e-13, and the cost of being wrong is one node * losing one slot on one call — not corruption. */ static int eg_same_content(const EngramNode* a, const EngramNode* b, uint64_t ka, uint64_t kb) { if (ka == kb) return 1; if (a->emb && b->emb && a->emb_dim > 0 && a->emb_dim == b->emb_dim) { if (eg_cosine(a->emb, b->emb, a->emb_dim) >= ENGRAM_DEDUP_COS) return 1; } return 0; } /* Weight-carrying index for the Pass 3½ descending walk. */ typedef struct { double w; int64_t idx; } EgDupCand; static int eg_dupcand_cmp_desc(const void* a, const void* b) { double wa = ((const EgDupCand*)a)->w, wb = ((const EgDupCand*)b)->w; if (wa < wb) return 1; if (wa > wb) return -1; return 0; } /* eg_ctx_blend — fold one touch embedding into the context centroid: * c ← normalize(μ·c + (1−μ)·e). Initializes the centroid (normalized copy) * on first touch or dim change; silently skips degenerate vectors. */ static void eg_ctx_blend(const float* e, int32_t dim) { if (!e || dim <= 0) return; double ne = 0.0; for (int32_t i = 0; i < dim; i++) ne += (double)e[i] * (double)e[i]; if (ne <= 0.0) return; ne = sqrt(ne); if (!_eg_ctx_c || _eg_ctx_dim != dim) { float* c = malloc((size_t)dim * sizeof(float)); if (!c) return; for (int32_t i = 0; i < dim; i++) c[i] = (float)((double)e[i] / ne); free(_eg_ctx_c); _eg_ctx_c = c; _eg_ctx_dim = dim; return; } double nc = 0.0; for (int32_t i = 0; i < dim; i++) { double v = ENGRAM_CTX_MU * (double)_eg_ctx_c[i] + (1.0 - ENGRAM_CTX_MU) * ((double)e[i] / ne); _eg_ctx_c[i] = (float)v; nc += v * v; } if (nc > 0.0) { nc = sqrt(nc); for (int32_t i = 0; i < dim; i++) _eg_ctx_c[i] = (float)((double)_eg_ctx_c[i] / nc); } } /* Fetch an embedding from Ollama. Returns malloc'd float[dim] or NULL. * Truncates input to ENGRAM_EMBED_MAX_CHARS and JSON-escapes it. Honors the * circuit breaker; a NULL return is always safe to ignore (fail-soft). */ static float* eg_embed_fetch(const char* text, int32_t* out_dim) { *out_dim = 0; if (!text || !*text) return NULL; int64_t now = engram_now_ms(); if (now < _eg_embed_breaker_until) return NULL; /* Build request body with escaped, truncated prompt. */ size_t tlen = strlen(text); if (tlen > ENGRAM_EMBED_MAX_CHARS) tlen = ENGRAM_EMBED_MAX_CHARS; char* esc = malloc(tlen * 6 + 1); if (!esc) return NULL; size_t w = 0; for (size_t i = 0; i < tlen; i++) { unsigned char c = (unsigned char)text[i]; if (c == '"' || c == '\\') { esc[w++] = '\\'; esc[w++] = (char)c; } else if (c == '\n') { esc[w++] = '\\'; esc[w++] = 'n'; } else if (c == '\r') { esc[w++] = '\\'; esc[w++] = 'r'; } else if (c == '\t') { esc[w++] = '\\'; esc[w++] = 't'; } else if (c < 0x20) { w += (size_t)snprintf(esc + w, 7, "\\u%04x", c); } else esc[w++] = (char)c; } esc[w] = '\0'; size_t blen = w + strlen(eg_embed_model()) + 64; char* body = malloc(blen); if (!body) { free(esc); return NULL; } snprintf(body, blen, "{\"model\":\"%s\",\"prompt\":\"%s\"}", eg_embed_model(), esc); free(esc); struct curl_slist* h = curl_slist_append(NULL, "Content-Type: application/json"); el_val_t resp = http_do_t("POST", eg_embed_url(), body, h, ENGRAM_EMBED_TIMEOUT_MS); curl_slist_free_all(h); free(body); const char* r = EL_CSTR(resp); const char* arr = r ? strstr(r, "\"embedding\"") : NULL; if (!arr) { if (++_eg_embed_consec_fail >= ENGRAM_EMBED_BREAKER_LIMIT) { _eg_embed_breaker_until = now + ENGRAM_EMBED_BREAKER_COOLDOWN_MS; _eg_embed_consec_fail = 0; } return NULL; } arr = strchr(arr, '['); if (!arr) return NULL; arr++; int32_t cap = 1024, dim = 0; float* v = malloc((size_t)cap * sizeof(float)); if (!v) return NULL; const char* p = arr; while (*p && *p != ']') { char* endp = NULL; double d = strtod(p, &endp); if (endp == p) break; if (dim >= cap) { break; } /* >1024 dims: refuse, model mismatch */ v[dim++] = (float)d; p = endp; while (*p == ',' || *p == ' ' || *p == '\n') p++; } if (dim < 8) { free(v); return NULL; } /* junk response */ _eg_embed_consec_fail = 0; *out_dim = dim; return v; } /* Node types that never receive embeddings: pure telemetry and structural * plumbing. Everything else (Knowledge, Memory, BacklogItem, Entity, ...) * is eligible. */ static int eg_embed_eligible(const EngramNode* n) { if (!n->content || strlen(n->content) < 8) return 0; if (!n->node_type) return 1; if (strcmp(n->node_type, "InternalStateEvent") == 0) return 0; if (strcmp(n->node_type, "Tag") == 0) return 0; return 1; } /* Parse a persisted comma-separated float list into node->emb. */ static void eg_parse_emb(EngramNode* nn, const char* s) { if (!s || !*s) return; int32_t cap = 1024, dim = 0; float* v = malloc((size_t)cap * sizeof(float)); if (!v) return; const char* p = s; while (*p) { char* endp = NULL; double d = strtod(p, &endp); if (endp == p) break; if (dim >= cap) break; v[dim++] = (float)d; p = endp; while (*p == ',' || *p == ' ') p++; } if (dim < 8) { free(v); return; } nn->emb = v; nn->emb_dim = dim; } typedef struct EngramEdge { char* id; char* from_id; char* to_id; char* relation; char* metadata; double weight; /* Hebbian co-activation potentiation, learned at runtime and persisted. * Strictly separate from `weight`, which is authored and never mutated by * activation. Reads as "fraction of recent activation calls in which both * endpoints were in working memory together". See ENGRAM_HEBB_DECAY. */ double hebb; double confidence; int64_t created_at; int64_t updated_at; int64_t last_fired; /* Inhibitory flag: when 1, activating the source node SUPPRESSES the * working_memory_weight of the target node rather than exciting it. * Models attentional inhibition: "I am focused on code work" creates * inhibitory edges to personal/emotional nodes, preventing them from * surfacing even if they have high background_activation. */ int inhibitory; /* Layered consciousness — edges carry a layer assignment for * categorization/visualization. Pass 2 inhibitory gating is decided by * the TARGET node's layer (whether it's suppressible), not by the edge * layer. Defaults to ENGRAM_LAYER_DEFAULT. */ uint32_t layer_id; } EngramEdge; /* Layered consciousness — runtime layer registry entry. */ typedef struct EngramLayer { uint32_t layer_id; /* 0 = deepest (safety/limbic) */ char* name; /* persistent — owned by the store */ uint32_t activation_priority; /* lower = fires earlier; safety = 0 */ int suppressible; /* can higher layers suppress nodes here? */ int transparent; /* invisible to introspection queries? */ int injectable; /* can be added/removed at runtime? */ } EngramLayer; /* ID → index hash map. Open-addressing with linear probing. * Slots hold a strdup'd key and the array index of that node. * Tombstones (deleted entries) use key=ENGRAM_IDMAP_TOMB and idx=-1. * Rebuild required after engram_forget (shift-delete changes all indices * above the deleted position). */ #define ENGRAM_IDMAP_TOMB ((char*)1) /* sentinel pointer, never dereferenced */ #define ENGRAM_IDMAP_LOAD_NUM 3 /* grow when count*3 >= capacity*2 */ #define ENGRAM_IDMAP_LOAD_DEN 2 typedef struct { char* key; /* NULL = empty, ENGRAM_IDMAP_TOMB = deleted, else strdup'd */ int64_t idx; } EngramIdSlot; typedef struct EngramStore { EngramNode* nodes; int64_t node_count; int64_t node_capacity; EngramEdge* edges; int64_t edge_count; int64_t edge_capacity; /* Layer registry — see engram_init_layers. The five canonical layers * are always present; injectable layers (imprint, suit) are extended * via engram_add_layer at runtime. layer_id values are assigned * monotonically; removed injectable layers leave a NULL `name` slot * (tombstone) so existing layer_id references on nodes stay stable. */ EngramLayer* layers; size_t layer_count; size_t layer_capacity; /* O(1) node-id lookup: open-addressing hash map over node IDs. * Maintained in sync with the nodes array. Null until first use. */ EngramIdSlot* id_map; size_t id_map_cap; /* power-of-2 slot count */ size_t id_map_used; /* live entries (excluding tombstones) */ /* Per-node adjacency index: for each node i, adj_from[i] lists edges * where nodes[i] is the 'from' end; adj_to[i] lists edges where it is * the 'to' end. Both store edge indices into g->edges[]. * Rebuilt lazily via engram_adj_rebuild() before any BFS call. Set * adj_dirty=1 whenever an edge is added, deleted, or nodes shift. */ int** adj_from; /* adj_from[node_idx] → int* array of edge indices */ int* adj_from_len; int** adj_to; int* adj_to_len; int adj_dirty; /* 1 = rebuild needed before next BFS */ int64_t adj_node_count; /* node_count at time of last adj_rebuild */ } EngramStore; static EngramStore* engram_global = NULL; /* Initialize the five canonical layers on a fresh store. Called once from * engram_get(). Layer ids 0..4 are reserved; runtime-injected imprint/suit * layers (engram_add_layer) get ids 5+. */ static void engram_init_layers(EngramStore* g) { g->layer_capacity = 16; g->layers = calloc(g->layer_capacity, sizeof(EngramLayer)); if (!g->layers) { fputs("el_runtime: out of memory\n", stderr); exit(1); } g->layer_count = 0; /* Layer 0 — safety. Structural floor. Non-suppressible; transparent * (filtered out of introspection but still shapes output); not * runtime-injectable. */ g->layers[g->layer_count++] = (EngramLayer){ .layer_id = ENGRAM_LAYER_SAFETY, .name = el_strdup_persist("safety"), .activation_priority = 0, .suppressible = 0, .transparent = 1, .injectable = 0 }; /* Layer 1 — core-identity. The default home for legacy nodes. */ g->layers[g->layer_count++] = (EngramLayer){ .layer_id = ENGRAM_LAYER_CORE_IDENTITY, .name = el_strdup_persist("core-identity"), .activation_priority = 10, .suppressible = 1, .transparent = 0, .injectable = 0 }; /* Layer 2 — domain-knowledge. */ g->layers[g->layer_count++] = (EngramLayer){ .layer_id = ENGRAM_LAYER_DOMAIN, .name = el_strdup_persist("domain-knowledge"), .activation_priority = 20, .suppressible = 1, .transparent = 0, .injectable = 0 }; /* Layer 3 — imprint. Injectable: an imprint package adds/removes this * layer (and the nodes assigned to it) as a unit. */ g->layers[g->layer_count++] = (EngramLayer){ .layer_id = ENGRAM_LAYER_IMPRINT, .name = el_strdup_persist("imprint"), .activation_priority = 30, .suppressible = 1, .transparent = 0, .injectable = 1 }; /* Layer 4 — suit. Injectable: a Suit overlays domain skill (e.g. * "enterprise advisor", "divorce lawyer") and can be detached. */ g->layers[g->layer_count++] = (EngramLayer){ .layer_id = ENGRAM_LAYER_SUIT, .name = el_strdup_persist("suit"), .activation_priority = 40, .suppressible = 1, .transparent = 0, .injectable = 1 }; } static EngramStore* engram_get(void) { if (engram_global) return engram_global; engram_global = calloc(1, sizeof(EngramStore)); if (!engram_global) { fputs("el_runtime: out of memory\n", stderr); exit(1); } engram_global->node_capacity = 16; engram_global->nodes = calloc((size_t)engram_global->node_capacity, sizeof(EngramNode)); engram_global->edge_capacity = 16; engram_global->edges = calloc((size_t)engram_global->edge_capacity, sizeof(EngramEdge)); engram_init_layers(engram_global); return engram_global; } /* Resolve a layer record by id. Returns NULL if no layer with that id * exists (e.g. a removed injectable layer or a malformed snapshot). */ static EngramLayer* engram_find_layer(uint32_t layer_id) { EngramStore* g = engram_get(); for (size_t i = 0; i < g->layer_count; i++) { EngramLayer* L = &g->layers[i]; if (!L->name) continue; /* tombstone for removed injectable layer */ if (L->layer_id == layer_id) return L; } return NULL; } /* Resolve a layer record by name. Returns NULL if not found. */ static EngramLayer* engram_find_layer_by_name(const char* name) { if (!name || !*name) return NULL; EngramStore* g = engram_get(); for (size_t i = 0; i < g->layer_count; i++) { EngramLayer* L = &g->layers[i]; if (!L->name) continue; if (strcmp(L->name, name) == 0) return L; } return NULL; } /* Allocate the next layer id. Skips ids that are still in use. */ static uint32_t engram_next_layer_id(void) { EngramStore* g = engram_get(); uint32_t maxid = 0; for (size_t i = 0; i < g->layer_count; i++) { if (g->layers[i].layer_id > maxid) maxid = g->layers[i].layer_id; } return maxid + 1; } /* Whether a node in `layer_id` may be silenced by inhibitory edges in pass 2. */ static int engram_layer_is_suppressible(uint32_t layer_id) { EngramLayer* L = engram_find_layer(layer_id); if (!L) return 1; /* unknown layer → safe default: standard suppression */ return L->suppressible ? 1 : 0; } /* Whether a layer is transparent (its content shapes output but is filtered * from introspection queries). Currently used to mark Layer 0 as invisible * to "what do you know about yourself" lookups while still letting it * dominate the prompt context. */ static int engram_layer_is_transparent(uint32_t layer_id) { EngramLayer* L = engram_find_layer(layer_id); if (!L) return 0; return L->transparent ? 1 : 0; } static int64_t engram_now_ms(void) { struct timeval tv; gettimeofday(&tv, NULL); return (int64_t)tv.tv_sec * 1000LL + (int64_t)tv.tv_usec / 1000LL; } /* Forward declaration: engram_find_node_index is defined after the id_map * helpers but called here. Without this, C99 -Wimplicit-function-declaration * treats the call as an implicit non-static declaration, then conflicts with * the later `static` definition. (2026-07-01 self-review: pre-existing) */ static int64_t engram_find_node_index(const char* id); static EngramNode* engram_find_node(const char* id) { if (!id) return NULL; EngramStore* g = engram_get(); int64_t idx = engram_find_node_index(id); if (idx >= 0) return &g->nodes[idx]; return NULL; } /* ── ID hash map helpers ───────────────────────────────────────────────────── * Open-addressing, linear-probing hash map. Keys are node-id C strings. * Values are int64_t indices into g->nodes[]. * * Rules: * - id_map is NULL until the first insertion (lazy init). * - Capacity is always a power of two. * - Load factor kept below 2/3: when used*3 >= cap*2, rehash to 2*cap. * - Deletion uses ENGRAM_IDMAP_TOMB sentinels (key == (char*)1). * - After engram_forget (shift-delete) the whole map is rebuilt from * scratch because all indices above the deleted position change. */ static uint64_t engram_id_hash(const char* s) { /* FNV-1a 64-bit */ uint64_t h = 14695981039346656037ULL; while (*s) { h ^= (unsigned char)*s++; h *= 1099511628211ULL; } return h; } /* Allocate a zeroed id_map of `cap` slots (cap must be power-of-two). */ static EngramIdSlot* engram_idmap_alloc(size_t cap) { return calloc(cap, sizeof(EngramIdSlot)); } /* Low-level insert (no rehash check, no free of existing). Used during * rehash and initial build where we know the load is controlled. */ static void engram_idmap_put_raw(EngramIdSlot* map, size_t cap, char* key, int64_t idx) { size_t mask = cap - 1; size_t slot = (size_t)engram_id_hash(key) & mask; while (map[slot].key != NULL && map[slot].key != ENGRAM_IDMAP_TOMB) { slot = (slot + 1) & mask; } map[slot].key = key; map[slot].idx = idx; } /* Insert or update id → idx into the store's id_map. Rehashes if needed. */ static void engram_idmap_put(EngramStore* g, const char* id, int64_t idx) { if (!id || !*id) return; /* Lazy init */ if (!g->id_map) { g->id_map_cap = 64; g->id_map_used = 0; g->id_map = engram_idmap_alloc(g->id_map_cap); if (!g->id_map) return; /* OOM: fall back to linear scan */ } /* Rehash if load factor would exceed 2/3 */ if ((g->id_map_used + 1) * ENGRAM_IDMAP_LOAD_NUM >= g->id_map_cap * ENGRAM_IDMAP_LOAD_DEN) { size_t new_cap = g->id_map_cap * 2; EngramIdSlot* new_map = engram_idmap_alloc(new_cap); if (!new_map) return; /* OOM: keep old map, insert below */ for (size_t s = 0; s < g->id_map_cap; s++) { if (g->id_map[s].key && g->id_map[s].key != ENGRAM_IDMAP_TOMB) { engram_idmap_put_raw(new_map, new_cap, g->id_map[s].key, g->id_map[s].idx); } } free(g->id_map); g->id_map = new_map; g->id_map_cap = new_cap; } /* Probe for existing key or empty/tomb slot */ size_t mask = g->id_map_cap - 1; size_t slot = (size_t)engram_id_hash(id) & mask; size_t tomb_slot = SIZE_MAX; while (g->id_map[slot].key != NULL) { if (g->id_map[slot].key == ENGRAM_IDMAP_TOMB) { if (tomb_slot == SIZE_MAX) tomb_slot = slot; } else if (strcmp(g->id_map[slot].key, id) == 0) { g->id_map[slot].idx = idx; /* update */ return; } slot = (slot + 1) & mask; } /* Use tombstone slot if found (avoids growing used count unnecessarily) */ if (tomb_slot != SIZE_MAX) slot = tomb_slot; /* MUST be el_strdup_persist: idmap keys outlive the request/tick arena. * (2026-07-16 self-review) This was el_strdup (arena-tracked): every node * created inside an HTTP request left its idmap key DANGLING as soon as * el_request_end() freed the arena — subsequent lookups strcmp'd freed * memory, and any idmap_free/rebuild in a later request double-freed it * (SIGABRT in http_worker; found via ASAN when engram_prune_telemetry * triggered an in-request rebuild). Same allocation-discipline class as * the 2026-07-15 EngramNode fix — see the store-persistent comment above * engram_new_id(). */ g->id_map[slot].key = el_strdup_persist(id); g->id_map[slot].idx = idx; g->id_map_used++; } /* Look up id in the store's id_map. Returns index or -1 if not found. */ static int64_t engram_idmap_get(const EngramStore* g, const char* id) { if (!g->id_map || !id || !*id) return -1; size_t mask = g->id_map_cap - 1; size_t slot = (size_t)engram_id_hash(id) & mask; while (g->id_map[slot].key != NULL) { if (g->id_map[slot].key != ENGRAM_IDMAP_TOMB && strcmp(g->id_map[slot].key, id) == 0) { return g->id_map[slot].idx; } slot = (slot + 1) & mask; } return -1; } /* Free and null-out the id_map (called on full reset). */ static void engram_idmap_free(EngramStore* g) { if (!g->id_map) return; for (size_t s = 0; s < g->id_map_cap; s++) { if (g->id_map[s].key && g->id_map[s].key != ENGRAM_IDMAP_TOMB) free(g->id_map[s].key); } free(g->id_map); g->id_map = NULL; g->id_map_cap = 0; g->id_map_used = 0; } /* Rebuild id_map from scratch after a structural change (e.g. shift-delete). * Frees old map and constructs a fresh one. */ static void engram_idmap_rebuild(EngramStore* g) { engram_idmap_free(g); for (int64_t i = 0; i < g->node_count; i++) { if (g->nodes[i].id && *g->nodes[i].id) engram_idmap_put(g, g->nodes[i].id, i); } } /* ── Adjacency index helpers ───────────────────────────────────────────────── * Per-node adjacency lists: adj_from[i] holds edge indices where * g->edges[ei].from_id == g->nodes[i].id, adj_to[i] for the 'to' side. * BFS uses these instead of scanning all edges on every hop. * Called once per activation call when adj_dirty != 0. */ static void engram_adj_free(EngramStore* g) { int64_t old_nc = g->adj_node_count; if (g->adj_from) { for (int64_t i = 0; i < old_nc; i++) free(g->adj_from[i]); free(g->adj_from); g->adj_from = NULL; free(g->adj_from_len); g->adj_from_len = NULL; } if (g->adj_to) { for (int64_t i = 0; i < old_nc; i++) free(g->adj_to[i]); free(g->adj_to); g->adj_to = NULL; free(g->adj_to_len); g->adj_to_len = NULL; } g->adj_node_count = 0; g->adj_dirty = 1; } static void engram_adj_rebuild(EngramStore* g) { /* Free old adjacency arrays */ if (g->adj_from) { /* Use adj_node_count (count at build time) not current node_count — * nodes may have been added since the last rebuild, and adj arrays * only have adj_node_count entries. */ int64_t old_nc = g->adj_node_count; for (int64_t i = 0; i < old_nc; i++) { free(g->adj_from[i]); free(g->adj_to[i]); } free(g->adj_from); free(g->adj_from_len); free(g->adj_to); free(g->adj_to_len); } g->adj_from = NULL; g->adj_from_len = NULL; g->adj_to = NULL; g->adj_to_len = NULL; g->adj_node_count = 0; if (g->node_count == 0) { g->adj_dirty = 0; return; } /* Count degree per node */ int* from_cnt = calloc((size_t)g->node_count, sizeof(int)); int* to_cnt = calloc((size_t)g->node_count, sizeof(int)); if (!from_cnt || !to_cnt) { free(from_cnt); free(to_cnt); return; } for (int64_t ei = 0; ei < g->edge_count; ei++) { EngramEdge* e = &g->edges[ei]; if (!e->from_id || !e->to_id) continue; int64_t fi = engram_idmap_get(g, e->from_id); int64_t ti = engram_idmap_get(g, e->to_id); if (fi >= 0) from_cnt[fi]++; if (ti >= 0) to_cnt[ti]++; } /* Allocate per-node arrays */ g->adj_from = calloc((size_t)g->node_count, sizeof(int*)); g->adj_from_len = calloc((size_t)g->node_count, sizeof(int)); g->adj_to = calloc((size_t)g->node_count, sizeof(int*)); g->adj_to_len = calloc((size_t)g->node_count, sizeof(int)); if (!g->adj_from || !g->adj_from_len || !g->adj_to || !g->adj_to_len) { free(from_cnt); free(to_cnt); free(g->adj_from); g->adj_from = NULL; free(g->adj_from_len); g->adj_from_len = NULL; free(g->adj_to); g->adj_to = NULL; free(g->adj_to_len); g->adj_to_len = NULL; return; } for (int64_t i = 0; i < g->node_count; i++) { if (from_cnt[i] > 0) g->adj_from[i] = malloc((size_t)from_cnt[i] * sizeof(int)); if (to_cnt[i] > 0) g->adj_to[i] = malloc((size_t)to_cnt[i] * sizeof(int)); } /* Fill */ int* from_pos = calloc((size_t)g->node_count, sizeof(int)); int* to_pos = calloc((size_t)g->node_count, sizeof(int)); if (!from_pos || !to_pos) { free(from_cnt); free(to_cnt); free(from_pos); free(to_pos); return; } for (int64_t ei = 0; ei < g->edge_count; ei++) { EngramEdge* e = &g->edges[ei]; if (!e->from_id || !e->to_id) continue; int64_t fi = engram_idmap_get(g, e->from_id); int64_t ti = engram_idmap_get(g, e->to_id); if (fi >= 0 && g->adj_from[fi]) g->adj_from[fi][from_pos[fi]++] = (int)ei; if (ti >= 0 && g->adj_to[ti]) g->adj_to[ti][to_pos[ti]++] = (int)ei; } /* Copy counts */ for (int64_t i = 0; i < g->node_count; i++) { g->adj_from_len[i] = from_cnt[i]; g->adj_to_len[i] = to_cnt[i]; } free(from_cnt); free(to_cnt); free(from_pos); free(to_pos); g->adj_node_count = g->node_count; g->adj_dirty = 0; } static int64_t engram_find_node_index(const char* id) { if (!id) return -1; EngramStore* g = engram_get(); /* Fast O(1) path via id_map */ int64_t fast = engram_idmap_get(g, id); if (fast >= 0) return fast; /* Fallback linear scan (id_map not yet built or OOM) */ for (int64_t i = 0; i < g->node_count; i++) { if (g->nodes[i].id && strcmp(g->nodes[i].id, id) == 0) return i; } return -1; } static void engram_grow_nodes(void) { EngramStore* g = engram_get(); if (g->node_count < g->node_capacity) return; int64_t nc = g->node_capacity * 2; g->nodes = realloc(g->nodes, (size_t)nc * sizeof(EngramNode)); if (!g->nodes) { fputs("el_runtime: out of memory\n", stderr); exit(1); } memset(g->nodes + g->node_capacity, 0, (size_t)(nc - g->node_capacity) * sizeof(EngramNode)); g->node_capacity = nc; } static void engram_grow_edges(void) { EngramStore* g = engram_get(); if (g->edge_count < g->edge_capacity) return; int64_t nc = g->edge_capacity * 2; g->edges = realloc(g->edges, (size_t)nc * sizeof(EngramEdge)); if (!g->edges) { fputs("el_runtime: out of memory\n", stderr); exit(1); } memset(g->edges + g->edge_capacity, 0, (size_t)(nc - g->edge_capacity) * sizeof(EngramEdge)); g->edge_capacity = nc; } /* Build a fresh UUID string. Reuses uuid_new but takes the underlying char*. */ /* ── Store-persistent allocation discipline ───────────────────────────────── * (2026-07-15 self-review) EngramNode/EngramEdge string fields OUTLIVE the * request/tick arena they were created in. The 2026-07-13 leak-fix made * el_strdup arena-tracked, which silently turned every node created inside * an HTTP request (route_emit_ise, route_create_node, knowledge capture) or * inside the soul's per-tick arena (engram_load_merge in the refresh cycle) * into a bag of dangling pointers the moment the arena popped: readback by * id returned {}, type-filtered scans skipped them, search returned request * memory reused as node content, and snapshots persisted garbage ("numeric * tier strings"). Everything written into the store must go through * el_strdup_persist / el_strbuf_persist (plain malloc — free() in * engram_forget/evolve remains valid). Arena-tracked el_strdup remains * correct for RETURN values handed back to EL code. */ static char* engram_new_id(void) { el_val_t v = uuid_new(); const char* s = EL_CSTR(v); return el_strdup_persist(s ? s : ""); } /* Convert a node into an ElMap of its fields. */ static el_val_t engram_node_to_map(const EngramNode* n) { el_val_t m = el_map_new(0); m = el_map_set(m, EL_STR(el_strdup("id")), EL_STR(el_strdup(n->id ? n->id : ""))); m = el_map_set(m, EL_STR(el_strdup("content")), EL_STR(el_strdup(n->content ? n->content : ""))); m = el_map_set(m, EL_STR(el_strdup("node_type")), EL_STR(el_strdup(n->node_type ? n->node_type : ""))); m = el_map_set(m, EL_STR(el_strdup("label")), EL_STR(el_strdup(n->label ? n->label : ""))); m = el_map_set(m, EL_STR(el_strdup("tier")), EL_STR(el_strdup(n->tier ? n->tier : "Working"))); m = el_map_set(m, EL_STR(el_strdup("tags")), EL_STR(el_strdup(n->tags ? n->tags : ""))); m = el_map_set(m, EL_STR(el_strdup("metadata")), EL_STR(el_strdup(n->metadata ? n->metadata : "{}"))); m = el_map_set(m, EL_STR(el_strdup("salience")), el_from_float(n->salience)); m = el_map_set(m, EL_STR(el_strdup("importance")), el_from_float(n->importance)); m = el_map_set(m, EL_STR(el_strdup("confidence")), el_from_float(n->confidence)); m = el_map_set(m, EL_STR(el_strdup("temporal_decay_rate")), el_from_float(n->temporal_decay_rate)); m = el_map_set(m, EL_STR(el_strdup("activation_count")), (el_val_t)n->activation_count); m = el_map_set(m, EL_STR(el_strdup("last_activated")), (el_val_t)n->last_activated); m = el_map_set(m, EL_STR(el_strdup("created_at")), (el_val_t)n->created_at); m = el_map_set(m, EL_STR(el_strdup("updated_at")), (el_val_t)n->updated_at); m = el_map_set(m, EL_STR(el_strdup("background_activation")), el_from_float(n->background_activation)); m = el_map_set(m, EL_STR(el_strdup("working_memory_weight")), el_from_float(n->working_memory_weight)); m = el_map_set(m, EL_STR(el_strdup("suppression_count")), (el_val_t)n->suppression_count); m = el_map_set(m, EL_STR(el_strdup("layer_id")), (el_val_t)(int64_t)n->layer_id); /* Observability (2026-07-22): expose the current ACT-R base-level and * promotion anchor so heartbeat ISEs / API consumers can see decay state. */ m = el_map_set(m, EL_STR(el_strdup("wm_anchor")), el_from_float(n->wm_anchor)); m = el_map_set(m, EL_STR(el_strdup("base_level")), el_from_float(engram_bll_base_level(n, engram_now_ms()))); /* emb_dim only — the vector itself is too large for map/API output. * 0 = not yet embedded by the lazy backfill. (2026-07-24) */ m = el_map_set(m, EL_STR(el_strdup("emb_dim")), (el_val_t)(int64_t)n->emb_dim); return m; } /* (Node JSON serialization is provided by `engram_emit_node_json` further * down in the persistence section — reused by the *_json builtins below.) */ static void engram_emit_node_json(JsonBuf* b, const EngramNode* n, int include_emb); static void engram_emit_edge_json(JsonBuf* b, const EngramEdge* e); /* Salience may arrive either as a float bit-pattern or as a small integer * (e.g. 1, meaning 1.0). Heuristic: if interpreted as double it's in * [0.0, 100.0] use it; otherwise treat as int and convert. */ static double engram_decode_score(el_val_t v) { double f = el_to_float(v); if (!isnan(f) && !isinf(f) && f >= 0.0 && f <= 100.0) return f; int64_t n = (int64_t)v; return (double)n; } static char* engram_first_n_chars(const char* s, size_t n) { if (!s) return el_strdup(""); size_t l = strlen(s); if (l > n) l = n; char* out = el_strbuf(l); memcpy(out, s, l); out[l] = '\0'; return out; } el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) { EngramStore* g = engram_get(); engram_grow_nodes(); EngramNode* n = &g->nodes[g->node_count]; memset(n, 0, sizeof(*n)); n->id = engram_new_id(); const char* c = EL_CSTR(content); const char* nt = EL_CSTR(node_type); n->content = el_strdup_persist(c ? c : ""); n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory"); n->label = el_strdup_persist(engram_first_n_chars(c, 60)); n->tier = el_strdup_persist("Working"); n->tags = el_strdup_persist(""); n->metadata = el_strdup_persist("{}"); n->salience = engram_decode_score(salience); if (n->salience <= 0.0 || n->salience > 1.0) n->salience = 0.5; n->importance = 0.5; n->confidence = 1.0; n->temporal_decay_rate = 0.0; /* 0 = use global default ENGRAM_DECAY_LAMBDA */ n->activation_count = 0; int64_t now = engram_now_ms(); n->last_activated = now; n->created_at = now; n->updated_at = now; n->layer_id = ENGRAM_LAYER_DEFAULT; int64_t new_idx = g->node_count; g->node_count++; engram_idmap_put(g, n->id, new_idx); g->adj_dirty = 1; return el_wrap_str(el_strdup(n->id)); } /* ── Text-integrity instrumentation (2026-08-08 self-review) ─────────────── * * WHY THIS EXISTS. The JSON parser silently replaced every \uXXXX escape with * '?' for at least two months (see jp_parse_string_raw). 3,119 of 4,081 * non-telemetry nodes — 76%, including the self traversal root and all 13 * values nodes — were damaged before anything noticed, and nothing noticed * because nothing measured. Working memory, Hebbian potentiation, embedding * coverage, sync age, and the breaker were all instrumented to four decimal * places; the actual TEXT was not instrumented at all. Every gauge answered * "is the machinery running" and none answered "is what it carries intact." * * The damage is unrecoverable in place (a 3-byte codepoint collapses to one * byte), and no snapshot on disk predates it, so this cannot be undone. What * it can be is *impossible to repeat quietly*. Two numbers, split by the * question each answers: * * stock — engram_text_health_json(), a full O(total bytes) census. Too * expensive for the 60s heartbeat, exactly right for the daily * self-review. Answers "how much damage is in the store". * flow — _eg_txt_write_damaged, incremented per damaged node at creation. * O(len) on a path that already copies the string, so it is free. * Rides the heartbeat. Answers "is a write path damaging things * RIGHT NOW" — which is the regression question, and the one that * would have caught this in a day instead of two months. * * SIGNATURE. Conservative on purpose — a false alarm that cries corruption * over ordinary punctuation is worse than useless. Two patterns, both of * which are essentially absent from well-formed English prose: * (a) alnum '?' alnum — "na?ve", "caf?s", "don?t". A real question mark * never sits between two word characters. * (b) ' ? ' followed by a lowercase letter — a lost em/en dash. A real * question mark is not preceded by a space, and * what follows one starts a new sentence. * Deliberately NOT flagged: a trailing '?' after a word, '? ' before a * capital, or '?' at end of string — all legitimate. This under-counts (it * cannot see a mangled 'café ' where the '?' landed before a space), so the * census is a floor on the damage, never an exaggeration of it. */ static int eg_text_loss_signature(const char* s) { if (!s) return 0; for (const char* p = s; *p; p++) { if (*p != '?') continue; unsigned char prev = (p == s) ? 0 : (unsigned char)p[-1]; unsigned char next = (unsigned char)p[1]; /* (a) sandwiched between word characters. */ if (isalnum(prev) && isalnum(next)) return 1; /* (b) spaced, with lowercase continuation — a lost dash. */ if (prev == ' ' && next == ' ' && islower((unsigned char)p[2])) return 1; } return 0; } /* Damaged-node creations since process start. See the block comment above. */ static int64_t _eg_txt_write_damaged = 0; /* engram_text_health_json — full text-integrity census over the store. * O(total content bytes); call it on demand (daily self-review / a route), * never per heartbeat. `damaged` counts nodes carrying the loss signature, * `multibyte` counts nodes holding valid multi-byte UTF-8 — the two together * separate "no damage" from "no non-ASCII text to damage", which a single * number cannot do. Telemetry is excluded: ISE payloads are machine-written * ASCII JSON and would dilute the ratio that matters. */ el_val_t engram_text_health_json(void) { EngramStore* g = engram_get(); int64_t scanned = 0, damaged = 0, multibyte = 0; for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; if (n->node_type && (strcmp(n->node_type, "InternalStateEvent") == 0 || strcmp(n->node_type, "Tag") == 0)) continue; scanned++; if (eg_text_loss_signature(n->content)) damaged++; for (const char* p = n->content; p && *p; p++) { if ((unsigned char)*p >= 0x80) { multibyte++; break; } } } char buf[256]; snprintf(buf, sizeof(buf), "{\"scanned\":%lld,\"damaged\":%lld,\"multibyte\":%lld," "\"damaged_pct\":%.2f,\"write_damaged\":%lld}", (long long)scanned, (long long)damaged, (long long)multibyte, scanned > 0 ? (100.0 * (double)damaged / (double)scanned) : 0.0, (long long)_eg_txt_write_damaged); return el_wrap_str(el_strdup(buf)); } el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label, el_val_t salience, el_val_t importance, el_val_t confidence, el_val_t tier, el_val_t tags) { EngramStore* g = engram_get(); engram_grow_nodes(); EngramNode* n = &g->nodes[g->node_count]; memset(n, 0, sizeof(*n)); n->id = engram_new_id(); const char* c = EL_CSTR(content); const char* nt = EL_CSTR(node_type); const char* lb = EL_CSTR(label); const char* ti = EL_CSTR(tier); const char* tg = EL_CSTR(tags); n->content = el_strdup_persist(c ? c : ""); n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory"); /* Flow half of the text-integrity gauge — see eg_text_loss_signature. * Telemetry is machine-written ASCII and is excluded so the counter stays * a clean signal about content-bearing write paths. */ if (strcmp(n->node_type, "InternalStateEvent") != 0 && eg_text_loss_signature(n->content)) _eg_txt_write_damaged++; n->label = el_strdup_persist(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : "")); n->tier = el_strdup_persist(ti && *ti ? ti : "Working"); n->tags = el_strdup_persist(tg ? tg : ""); n->metadata = el_strdup_persist("{}"); n->salience = engram_decode_score(salience); n->importance = engram_decode_score(importance); n->confidence = engram_decode_score(confidence); if (n->salience <= 0.0 || n->salience > 1.0) n->salience = 0.5; if (n->importance <= 0.0 || n->importance > 1.0) n->importance = 0.5; if (n->confidence <= 0.0 || n->confidence > 1.0) n->confidence = 1.0; n->temporal_decay_rate = 0.0; /* 0 = use global default ENGRAM_DECAY_LAMBDA */ n->activation_count = 0; int64_t now = engram_now_ms(); n->last_activated = now; n->created_at = now; n->updated_at = now; n->layer_id = ENGRAM_LAYER_DEFAULT; int64_t new_idx_full = g->node_count; g->node_count++; engram_idmap_put(g, n->id, new_idx_full); g->adj_dirty = 1; return el_wrap_str(el_strdup(n->id)); } /* engram_node_layered — like engram_node_full but with explicit layer * assignment and an additional `status` slot reserved for callers that * track lifecycle state in metadata. The signature mirrors the public API * defined in the layered consciousness design doc: * * engram_node_layered(content, node_type, label, * salience, certainty, confidence, * status, tags, layer_id) * * `certainty` is folded into `importance` (it occupies the same axis in * the existing schema). `status` is recorded under metadata.status; an * empty status leaves metadata as the default "{}". * * If `layer_id` does not resolve to a known layer the call falls back to * ENGRAM_LAYER_DEFAULT — better to keep the node addressable than to drop * it because of a stale layer reference. Callers wanting strict validation * should engram_list_layers first. */ el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t label, el_val_t salience, el_val_t certainty, el_val_t confidence, el_val_t status, el_val_t tags, el_val_t layer_id) { EngramStore* g = engram_get(); engram_grow_nodes(); EngramNode* n = &g->nodes[g->node_count]; memset(n, 0, sizeof(*n)); n->id = engram_new_id(); const char* c = EL_CSTR(content); const char* nt = EL_CSTR(node_type); const char* lb = EL_CSTR(label); const char* tg = EL_CSTR(tags); const char* st = EL_CSTR(status); n->content = el_strdup_persist(c ? c : ""); n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory"); n->label = el_strdup_persist(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : "")); n->tier = el_strdup_persist("Working"); n->tags = el_strdup_persist(tg ? tg : ""); if (st && *st) { /* Minimal metadata payload: {"status":"..."}. Keep it cheap so * callers using `status` don't pay JSON parse cost on every read. */ size_t sl = strlen(st) + 16; char* meta = el_strbuf_persist(sl); snprintf(meta, sl, "{\"status\":\"%s\"}", st); n->metadata = meta; } else { n->metadata = el_strdup_persist("{}"); } n->salience = engram_decode_score(salience); n->importance = engram_decode_score(certainty); n->confidence = engram_decode_score(confidence); if (n->salience <= 0.0 || n->salience > 1.0) n->salience = 0.5; if (n->importance <= 0.0 || n->importance > 1.0) n->importance = 0.5; if (n->confidence <= 0.0 || n->confidence > 1.0) n->confidence = 1.0; n->temporal_decay_rate = 0.0; n->activation_count = 0; int64_t now = engram_now_ms(); n->last_activated = now; n->created_at = now; n->updated_at = now; /* Resolve layer assignment. Caller passes either a numeric layer_id or * a stringified id; el_to_float / int cast tolerates both. */ int64_t lid = (int64_t)layer_id; if (lid < 0) lid = (int64_t)ENGRAM_LAYER_DEFAULT; if (!engram_find_layer((uint32_t)lid)) lid = (int64_t)ENGRAM_LAYER_DEFAULT; n->layer_id = (uint32_t)lid; int64_t new_idx_layered = g->node_count; g->node_count++; engram_idmap_put(g, n->id, new_idx_layered); g->adj_dirty = 1; return el_wrap_str(el_strdup(n->id)); } /* ── Layer registry public API ────────────────────────────────────────────── * * The five canonical layers are seeded at engram_get() initialization. * Runtime code (typically imprint/suit injection logic at the EL level) * can extend the registry with engram_add_layer() — only layers marked * `injectable=1` may be removed via engram_remove_layer(). Removing a * layer leaves a tombstone slot so existing layer_id references on nodes * stay valid; orphaned references resolve to "unknown layer" and inherit * the default suppression behavior. */ /* engram_add_layer — register a new layer at runtime. * Returns the assigned layer_id as an el_val_t int (cast back via int64_t). * Conflicting names are rejected (returns 0). */ el_val_t engram_add_layer(el_val_t name, el_val_t priority, el_val_t suppressible, el_val_t transparent, el_val_t injectable) { EngramStore* g = engram_get(); const char* nm = EL_CSTR(name); if (!nm || !*nm) return (el_val_t)0; if (engram_find_layer_by_name(nm)) { /* Name collision — return existing id so callers are idempotent. */ return (el_val_t)(int64_t)engram_find_layer_by_name(nm)->layer_id; } if (g->layer_count >= g->layer_capacity) { size_t nc = g->layer_capacity ? g->layer_capacity * 2 : 16; EngramLayer* grown = realloc(g->layers, nc * sizeof(EngramLayer)); if (!grown) { fputs("el_runtime: out of memory\n", stderr); exit(1); } memset(grown + g->layer_capacity, 0, (nc - g->layer_capacity) * sizeof(EngramLayer)); g->layers = grown; g->layer_capacity = nc; } EngramLayer* L = &g->layers[g->layer_count++]; L->layer_id = engram_next_layer_id(); L->name = el_strdup_persist(nm); L->activation_priority = (uint32_t)(int64_t)priority; L->suppressible = (int)(int64_t)suppressible ? 1 : 0; L->transparent = (int)(int64_t)transparent ? 1 : 0; L->injectable = (int)(int64_t)injectable ? 1 : 0; return (el_val_t)(int64_t)L->layer_id; } /* engram_remove_layer — remove an injectable layer by id. * Built-in (non-injectable) layers cannot be removed. Nodes still tagged * with the removed layer's id keep their tag but resolve to "unknown * layer" thereafter and inherit standard (suppressible) behavior. * Returns 1 on success, 0 on failure (unknown id, non-injectable). */ el_val_t engram_remove_layer(el_val_t layer_id) { EngramStore* g = engram_get(); int64_t lid = (int64_t)layer_id; for (size_t i = 0; i < g->layer_count; i++) { EngramLayer* L = &g->layers[i]; if (!L->name) continue; if ((int64_t)L->layer_id != lid) continue; if (!L->injectable) return (el_val_t)0; free(L->name); L->name = NULL; /* tombstone */ /* Leave layer_id, priority, flags intact so debug snapshots can * still distinguish "removed at runtime" from "never existed". */ return (el_val_t)1; } return (el_val_t)0; } /* engram_list_layers — enumerate the active layer registry. * Returns an ElList of maps, one per non-tombstone layer, sorted by * activation_priority ascending (deepest layer first). */ el_val_t engram_list_layers(void) { EngramStore* g = engram_get(); el_val_t lst = el_list_empty(); if (g->layer_count == 0) return lst; /* Build an index sorted by activation_priority ascending. */ size_t* idx = malloc(g->layer_count * sizeof(size_t)); if (!idx) return lst; size_t live = 0; for (size_t i = 0; i < g->layer_count; i++) { if (g->layers[i].name) idx[live++] = i; } /* Insertion sort — N is small (≤ a few dozen layers). */ for (size_t i = 1; i < live; i++) { size_t key = idx[i]; uint32_t kp = g->layers[key].activation_priority; size_t j = i; while (j > 0 && g->layers[idx[j - 1]].activation_priority > kp) { idx[j] = idx[j - 1]; j--; } idx[j] = key; } for (size_t i = 0; i < live; i++) { EngramLayer* L = &g->layers[idx[i]]; el_val_t m = el_map_new(0); m = el_map_set(m, EL_STR(el_strdup("layer_id")), (el_val_t)(int64_t)L->layer_id); m = el_map_set(m, EL_STR(el_strdup("name")), EL_STR(el_strdup(L->name ? L->name : ""))); m = el_map_set(m, EL_STR(el_strdup("activation_priority")), (el_val_t)(int64_t)L->activation_priority); m = el_map_set(m, EL_STR(el_strdup("suppressible")), (el_val_t)(int64_t)(L->suppressible ? 1 : 0)); m = el_map_set(m, EL_STR(el_strdup("transparent")), (el_val_t)(int64_t)(L->transparent ? 1 : 0)); m = el_map_set(m, EL_STR(el_strdup("injectable")), (el_val_t)(int64_t)(L->injectable ? 1 : 0)); lst = el_list_append(lst, m); } free(idx); return lst; } el_val_t engram_get_node(el_val_t id) { const char* sid = EL_CSTR(id); EngramNode* n = engram_find_node(sid); if (!n) return el_map_new(0); return engram_node_to_map(n); } void engram_strengthen(el_val_t node_id) { const char* sid = EL_CSTR(node_id); EngramNode* n = engram_find_node(sid); if (!n) return; n->salience += 0.05; if (n->salience > 1.0) n->salience = 1.0; n->activation_count++; n->last_activated = engram_now_ms(); n->updated_at = n->last_activated; /* 2026-07-26 self-review: REMOVED the BLL access record added on * 2026-07-22 ("explicit strengthen is a presentation too"). The * 2026-07-25 STI multiplier reads the same access ring — so an * explicit strengthen crushed the strengthened node's promotion * score by ×t_n/(t_n+120) for the next ~2 minutes. The awareness * loop strengthens exactly when a node NEWLY reaches WM top * (novelty gating); recording an access here made that * reinforcement self-defeating. Salience/activation_count bumps * above carry the reinforcement; the access ring stays reserved * for genuine retrieval events (promotions in engram_activate). */ } void engram_forget(el_val_t node_id) { const char* sid = EL_CSTR(node_id); if (!sid) return; EngramStore* g = engram_get(); int64_t idx = engram_find_node_index(sid); if (idx < 0) return; /* Free node strings */ EngramNode* n = &g->nodes[idx]; free(n->id); free(n->content); free(n->node_type); free(n->label); free(n->tier); free(n->tags); free(n->metadata); free(n->emb); /* Shift remaining nodes down */ for (int64_t i = idx + 1; i < g->node_count; i++) { g->nodes[i - 1] = g->nodes[i]; } g->node_count--; memset(&g->nodes[g->node_count], 0, sizeof(EngramNode)); /* Remove all incident edges */ int64_t w = 0; for (int64_t r = 0; r < g->edge_count; r++) { EngramEdge* e = &g->edges[r]; int incident = (e->from_id && strcmp(e->from_id, sid) == 0) || (e->to_id && strcmp(e->to_id, sid) == 0); if (incident) { free(e->id); free(e->from_id); free(e->to_id); free(e->relation); free(e->metadata); } else { if (w != r) g->edges[w] = g->edges[r]; w++; } } g->edge_count = w; /* Shift-delete changed all indices above the removed position. * Rebuild id_map and mark adjacency index dirty. */ engram_idmap_rebuild(g); engram_adj_free(g); } el_val_t engram_node_count(void) { return (el_val_t)engram_get()->node_count; } /* ── Telemetry retention ──────────────────────────────────────────────────── * (2026-07-16 self-review) InternalStateEvent nodes are append-only telemetry * (heartbeat, curiosity_scan, engram_sync) written ~3/min by the awareness * loop. Nothing ever removed them: by July 16 they were 10,175 of 13,522 * nodes — 75% of the store was telemetry. They are already force-excluded * from WM promotion (engram_activate), so their only effect was store bloat, * snapshot bloat, and lexical-search noise. * * engram_prune_telemetry(older_than_ms) batch-removes ISE nodes whose * created_at is older than now - older_than_ms, EXCEPT durable markers: * - label "session-start" (boot history) * - content containing "self_review" (daily review trail) * Unlike repeated engram_forget (O(n) shift each), this is a single * compaction pass over nodes plus one pass over edges, with one idmap * rebuild — O(nodes + edges) total, safe to call on every ISE insert. * Returns the number of nodes removed. */ /* FNV-1a hash for the removed-id set used by the edge sweep. */ static uint64_t eg_fnv1a(const char* s) { uint64_t h = 1469598103934665603ULL; while (*s) { h ^= (unsigned char)*s++; h *= 1099511628211ULL; } return h; } el_val_t engram_prune_telemetry(el_val_t older_than_ms) { int64_t horizon = (int64_t)older_than_ms; if (horizon <= 0) horizon = 172800000; /* default 48h */ EngramStore* g = engram_get(); int64_t cutoff = engram_now_ms() - horizon; /* Pass 1: mark. Collect ids of prunable nodes (ownership transferred — * strings freed after the edge sweep). */ int64_t cap = 0; for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; if (n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0 && n->created_at < cutoff) cap++; } if (cap == 0) return 0; char** removed_ids = malloc((size_t)cap * sizeof(char*)); if (!removed_ids) return 0; int64_t removed = 0, w = 0; for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; int prunable = n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0 && n->created_at < cutoff && !(n->label && strcmp(n->label, "session-start") == 0) && !(n->content && strstr(n->content, "self_review")); if (prunable && removed < cap) { removed_ids[removed++] = n->id; /* keep id for edge sweep */ free(n->content); free(n->node_type); free(n->label); free(n->tier); free(n->tags); free(n->metadata); free(n->emb); } else { if (w != i) g->nodes[w] = g->nodes[i]; w++; } } g->node_count = w; if (removed == 0) { free(removed_ids); return 0; } /* Removed-id hash set (open addressing, power-of-two >= 2*removed). */ size_t set_cap = 16; while (set_cap < (size_t)removed * 2) set_cap <<= 1; const char** set = calloc(set_cap, sizeof(char*)); if (set) { for (int64_t i = 0; i < removed; i++) { size_t slot = eg_fnv1a(removed_ids[i]) & (set_cap - 1); while (set[slot]) slot = (slot + 1) & (set_cap - 1); set[slot] = removed_ids[i]; } } /* Pass 2: drop edges incident to any removed node (defensive — ISEs * currently have no edges, but callers may connect them later). */ if (set) { int64_t ew = 0; for (int64_t r = 0; r < g->edge_count; r++) { EngramEdge* e = &g->edges[r]; int incident = 0; const char* ends[2] = { e->from_id, e->to_id }; for (int k = 0; k < 2 && !incident; k++) { if (!ends[k]) continue; size_t slot = eg_fnv1a(ends[k]) & (set_cap - 1); while (set[slot]) { if (strcmp(set[slot], ends[k]) == 0) { incident = 1; break; } slot = (slot + 1) & (set_cap - 1); } } if (incident) { free(e->id); free(e->from_id); free(e->to_id); free(e->relation); free(e->metadata); } else { if (ew != r) g->edges[ew] = g->edges[r]; ew++; } } g->edge_count = ew; free(set); } for (int64_t i = 0; i < removed; i++) free(removed_ids[i]); free(removed_ids); engram_idmap_rebuild(g); engram_adj_free(g); return (el_val_t)removed; } static int istr_contains(const char* hay, const char* needle) { if (!hay || !needle || !*needle) return 0; size_t nl = strlen(needle); for (const char* p = hay; *p; p++) { if (strncasecmp(p, needle, nl) == 0) return 1; } return 0; } /* ── Tokenized query matching ─────────────────────────────────────────── * The engram query surface (search / activate / goal-bias) historically * matched the ENTIRE raw query string as a single case-insensitive * substring via istr_contains(field, q). That is Ctrl-F, not search: * a multi-word query like "windows msi signing" only matched a node whose * text contained that exact contiguous run, so real multi-word queries * returned zero. istr_contains stays as the per-TOKEN primitive; these * helpers split the query on whitespace and match ANY token, then rank by * how many DISTINCT tokens a node covers. Single-token queries are a strict * special case (score is 0 or 1) so single-word callers never regress. * (Ported 2026-07-19 from the el-compiler runtime copy, where the 2026-07-14 * fix landed but never reached this release runtime — the copy the engram * binary actually builds against.) */ #define ENGRAM_MAX_QTOKENS 32 #define ENGRAM_QTOK_LEN 256 /* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct * (case-insensitive) tokens. Returns the token count. Over-long tokens are * truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */ static int engram_tokenize_query(const char* q, char toks[][ENGRAM_QTOK_LEN], int maxtok) { int n = 0; if (!q) return 0; const char* p = q; while (*p && n < maxtok) { while (*p && isspace((unsigned char)*p)) p++; if (!*p) break; char buf[ENGRAM_QTOK_LEN]; size_t tl = 0; while (*p && !isspace((unsigned char)*p)) { if (tl < sizeof(buf) - 1) buf[tl++] = *p; p++; } buf[tl] = '\0'; if (tl == 0) continue; int dup = 0; for (int s = 0; s < n; s++) { if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; } } if (dup) continue; memcpy(toks[n], buf, tl + 1); n++; } return n; } /* Count how many of the ntok distinct query tokens appear (case-insensitive) * in the node's content, label, or tags. 0 == no match. */ static int engram_node_match_score(const EngramNode* n, char toks[][ENGRAM_QTOK_LEN], int ntok) { int score = 0; for (int t = 0; t < ntok; t++) { if (istr_contains(n->content, toks[t]) || istr_contains(n->label, toks[t]) || istr_contains(n->tags, toks[t])) score++; } return score; } /* Rank entry: distinct-token match count (primary, desc) then salience * (tiebreak, desc). */ typedef struct { int64_t idx; int score; double salience; } EngramRankEntry; static int engram_rank_cmp(const void* a, const void* b) { const EngramRankEntry* ea = (const EngramRankEntry*)a; const EngramRankEntry* eb = (const EngramRankEntry*)b; if (ea->score != eb->score) return eb->score - ea->score; /* desc */ if (ea->salience < eb->salience) return 1; if (ea->salience > eb->salience) return -1; return 0; } el_val_t engram_search(el_val_t query, el_val_t limit) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); int64_t lim = (int64_t)limit; if (lim <= 0) lim = 100; el_val_t lst = el_list_empty(); if (!q || !*q) return lst; char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS); if (ntok == 0) return lst; EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry)); if (!hits) return lst; int64_t nhits = 0; for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; /* Filter transparent layers: nodes whose layer is `transparent=1` * shape output but are invisible to introspection ("what do you * know about yourself"). They still surface via engram_activate * + engram_compile_layered_json — that's the legitimate path. */ if (engram_layer_is_transparent(n->layer_id)) continue; int sc = engram_node_match_score(n, toks, ntok); if (sc > 0) { hits[nhits].idx = i; hits[nhits].score = sc; hits[nhits].salience = n->salience; nhits++; } } /* Rank by distinct tokens matched (desc) then salience (desc), then cap. */ qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); int64_t end = nhits < lim ? nhits : lim; for (int64_t k = 0; k < end; k++) { lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx])); } free(hits); return lst; } /* Sort node indices by salience desc, tie-break created_at desc (newest * first). The tie-break matters for telemetry: InternalStateEvent nodes all * share salience 0.3, so before this the listing routes returned them in * store order — OLDEST first — and any limited query (/api/nodes?...&limit=N) * silently returned a stale window. A 41-hour-old heartbeat series read as a * live outage during the 2026-07-22 self-review. Newest-first ties make * limited scans return the recent window consumers actually want. * (Small N, insertion sort is fine.) */ static void engram_sort_indices_by_salience(int64_t* arr, int64_t n, const EngramNode* nodes) { for (int64_t i = 1; i < n; i++) { int64_t key = arr[i]; double ks = nodes[key].salience; int64_t kc = nodes[key].created_at; int64_t j = i - 1; while (j >= 0 && (nodes[arr[j]].salience < ks || (nodes[arr[j]].salience == ks && nodes[arr[j]].created_at < kc))) { arr[j + 1] = arr[j]; j--; } arr[j + 1] = key; } } el_val_t engram_scan_nodes(el_val_t limit, el_val_t offset) { EngramStore* g = engram_get(); int64_t lim = (int64_t)limit; if (lim <= 0) lim = 100; int64_t off = (int64_t)offset; if (off < 0) off = 0; el_val_t lst = el_list_empty(); if (g->node_count == 0) return lst; int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t)); if (!idx) return lst; /* Skip transparent layers — same introspection-filter rationale as * engram_search above. */ int64_t live = 0; for (int64_t i = 0; i < g->node_count; i++) { if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue; idx[live++] = i; } engram_sort_indices_by_salience(idx, live, g->nodes); int64_t end = off + lim; if (end > live) end = live; for (int64_t i = off; i < end; i++) { lst = el_list_append(lst, engram_node_to_map(&g->nodes[idx[i]])); } free(idx); return lst; } void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t relation) { EngramStore* g = engram_get(); const char* f = EL_CSTR(from_id); const char* t = EL_CSTR(to_id); const char* r = EL_CSTR(relation); if (!f || !t) return; engram_grow_edges(); EngramEdge* e = &g->edges[g->edge_count]; memset(e, 0, sizeof(*e)); e->id = engram_new_id(); e->from_id = el_strdup_persist(f); e->to_id = el_strdup_persist(t); e->relation = el_strdup_persist(r && *r ? r : "associate"); e->metadata = el_strdup_persist("{}"); e->weight = engram_decode_score(weight); if (e->weight <= 0.0 || e->weight > 1.0) e->weight = 0.5; e->confidence = 1.0; int64_t now = engram_now_ms(); e->created_at = now; e->updated_at = now; e->last_fired = 0; e->layer_id = ENGRAM_LAYER_DEFAULT; g->edge_count++; g->adj_dirty = 1; } el_val_t engram_edge_between(el_val_t from_id, el_val_t to_id) { EngramStore* g = engram_get(); const char* f = EL_CSTR(from_id); const char* t = EL_CSTR(to_id); if (!f || !t) return 0; for (int64_t i = 0; i < g->edge_count; i++) { EngramEdge* e = &g->edges[i]; if (e->from_id && e->to_id && strcmp(e->from_id, f) == 0 && strcmp(e->to_id, t) == 0) return 1; } return 0; } /* Reserved helper: edge -> ElMap. Kept around for future builtins. */ static el_val_t engram_edge_to_map(const EngramEdge* e) __attribute__((unused)); static el_val_t engram_edge_to_map(const EngramEdge* e) { el_val_t m = el_map_new(0); m = el_map_set(m, EL_STR(el_strdup("id")), EL_STR(el_strdup(e->id ? e->id : ""))); m = el_map_set(m, EL_STR(el_strdup("from_id")), EL_STR(el_strdup(e->from_id ? e->from_id : ""))); m = el_map_set(m, EL_STR(el_strdup("to_id")), EL_STR(el_strdup(e->to_id ? e->to_id : ""))); m = el_map_set(m, EL_STR(el_strdup("relation")), EL_STR(el_strdup(e->relation ? e->relation : ""))); m = el_map_set(m, EL_STR(el_strdup("metadata")), EL_STR(el_strdup(e->metadata ? e->metadata : "{}"))); m = el_map_set(m, EL_STR(el_strdup("weight")), el_from_float(e->weight)); m = el_map_set(m, EL_STR(el_strdup("confidence")), el_from_float(e->confidence)); m = el_map_set(m, EL_STR(el_strdup("created_at")), (el_val_t)e->created_at); m = el_map_set(m, EL_STR(el_strdup("updated_at")), (el_val_t)e->updated_at); m = el_map_set(m, EL_STR(el_strdup("hebb")), el_from_float(e->hebb)); m = el_map_set(m, EL_STR(el_strdup("last_fired")), (el_val_t)e->last_fired); m = el_map_set(m, EL_STR(el_strdup("inhibitory")), (el_val_t)(e->inhibitory ? 1 : 0)); m = el_map_set(m, EL_STR(el_strdup("layer_id")), (el_val_t)(int64_t)e->layer_id); return m; } el_val_t engram_neighbors(el_val_t node_id) { EngramStore* g = engram_get(); const char* sid = EL_CSTR(node_id); el_val_t lst = el_list_empty(); if (!sid) return lst; for (int64_t i = 0; i < g->edge_count; i++) { EngramEdge* e = &g->edges[i]; const char* other = NULL; if (e->from_id && strcmp(e->from_id, sid) == 0) other = e->to_id; else if (e->to_id && strcmp(e->to_id, sid) == 0) other = e->from_id; if (!other) continue; EngramNode* n = engram_find_node(other); if (n) lst = el_list_append(lst, engram_node_to_map(n)); } return lst; } el_val_t engram_neighbors_filtered(el_val_t node_id, el_val_t max_depth, el_val_t direction) { EngramStore* g = engram_get(); const char* sid = EL_CSTR(node_id); int64_t md = (int64_t)max_depth; if (md <= 0) md = 1; const char* dir = EL_CSTR(direction); /* "out" | "in" | "both" (default) */ el_val_t lst = el_list_empty(); if (!sid || g->node_count == 0) return lst; int64_t start = engram_find_node_index(sid); if (start < 0) return lst; /* BFS with depth tracking */ int64_t* visited = calloc((size_t)g->node_count, sizeof(int64_t)); int64_t* queue = calloc((size_t)g->node_count, sizeof(int64_t)); int64_t* depths = calloc((size_t)g->node_count, sizeof(int64_t)); if (!visited || !queue || !depths) { free(visited); free(queue); free(depths); return lst; } int64_t qh = 0, qt = 0; queue[qt++] = start; visited[start] = 1; depths[start] = 0; while (qh < qt) { int64_t cur = queue[qh++]; const char* cur_id = g->nodes[cur].id; int64_t cur_depth = depths[cur]; if (cur_depth >= md) continue; for (int64_t i = 0; i < g->edge_count; i++) { EngramEdge* e = &g->edges[i]; const char* other = NULL; int outgoing = e->from_id && strcmp(e->from_id, cur_id) == 0; int incoming = e->to_id && strcmp(e->to_id, cur_id) == 0; if (dir && strcmp(dir, "out") == 0 && !outgoing) continue; if (dir && strcmp(dir, "in") == 0 && !incoming) continue; if (outgoing) other = e->to_id; else if (incoming) other = e->from_id; else continue; int64_t oi = engram_find_node_index(other); if (oi < 0 || visited[oi]) continue; visited[oi] = 1; depths[oi] = cur_depth + 1; queue[qt++] = oi; } } /* Emit all visited except the seed */ for (int64_t i = 0; i < g->node_count; i++) { if (visited[i] && i != start) { lst = el_list_append(lst, engram_node_to_map(&g->nodes[i])); } } free(visited); free(queue); free(depths); return lst; } el_val_t engram_edge_count(void) { return (el_val_t)engram_get()->edge_count; } /* Compute temporal decay factor for a node given current time. * effective contribution = salience * exp(-lambda * age_hours / T_half) * Clamped to [0.05, 1.0] so very old nodes retain a meaningful floor. */ /* eg_edge_eff_weight — the weight spreading activation actually propagates * through: the authored weight, potentiated by learned co-activation. * hebb == 0 (fresh edge, cold graph, or feature effectively disabled) returns * exactly e->weight, so this is a strict no-op until the graph has learned * something. Clamped to 1.0 so a potentiated edge can never amplify a signal * above its source. See the ENGRAM_HEBB_* block for the full rationale. */ static double eg_edge_eff_weight(const EngramEdge* e) { double w = e->weight; if (e->hebb > 0.0) { w *= (1.0 + ENGRAM_HEBB_GAIN * e->hebb); if (w > 1.0) w = 1.0; } return w; } /* eg_wm_carry_over — the ACT-R/Petrov retention rule for a node that already * holds a working-memory slot and was not re-promoted on this call. Hard-evict * below the base-level threshold τ (Soar-style forgetting); otherwise hold a * weight shaped by the retrieval-probability logistic and decayed by how long * the slot has been held (occupancy inhibition). Pure function of wall-clock * time, so it is idempotent no matter how often activate is called. * * Extracted 2026-08-04: this logic was inline and applied to exactly ONE of * the two paths that need it. See the call sites. */ static void eg_wm_carry_over(EngramNode* cn, int64_t now_ms, int64_t* evict_ctr) { double anchor = (cn->wm_anchor > 0.0) ? cn->wm_anchor : cn->working_memory_weight; double B = engram_bll_base_level(cn, now_ms); double w = 0.0; if (B >= ENGRAM_BLL_TAU) { double keep = 1.0 / (1.0 + exp(-(B - ENGRAM_BLL_TAU) / ENGRAM_BLL_S)); double hold_s = (double)(now_ms - cn->last_activated) / 1000.0; if (hold_s < 0.0) hold_s = 0.0; double occ = ENGRAM_CARRY_TC / (ENGRAM_CARRY_TC + hold_s); w = anchor * keep * occ; } if (w < ENGRAM_WM_FLOOR) { cn->working_memory_weight = 0.0; cn->wm_anchor = 0.0; if (evict_ctr) (*evict_ctr)++; } else { cn->working_memory_weight = w; } } /* ── Hebbian candidate-pair table helpers ─────────────────────────────────── * Pairs are order-normalized by strcmp so (a,b) and (b,a) always resolve to * the same slot. Collisions are resolved by strength: an incumbent that has * decayed to nothing yields its slot, a live one keeps it and the challenger * simply loses this round. That is a lossy table by design — consolidation * should favor associations that recur, and a pair that keeps losing a * collision is by definition not recurring often enough to matter. */ /* eg_hebb_trace — the node's eligibility trace right now, in [0,1]. * Stored as (amplitude, timestamp) and decayed on read, so the value is a pure * function of wall-clock time: idempotent no matter how often activate runs. * Snapped to 0 below ENGRAM_HEBB_TRACE_MIN. See ENGRAM_HEBB_TRACE_TC. */ static double eg_hebb_trace(const EngramNode* n, int64_t now_ms) { if (n->hebb_elig <= 0.0 || n->hebb_elig_ts <= 0) return 0.0; double dt = (double)(now_ms - n->hebb_elig_ts) / 1000.0; if (dt < 0.0) dt = 0.0; /* clock skew ⇒ treat as fresh */ double t = n->hebb_elig * exp(-dt / ENGRAM_HEBB_TRACE_TC); return (t < ENGRAM_HEBB_TRACE_MIN) ? 0.0 : t; } static int eg_hebb_slot(const char* a, const char* b) { if (!a || !b) return -1; const char* lo = (strcmp(a, b) <= 0) ? a : b; const char* hi = (lo == a) ? b : a; uint64_t h = engram_id_hash(lo) * 1000003u ^ engram_id_hash(hi); return (int)(h % (uint64_t)ENGRAM_HEBB_CAND_SLOTS); } static int eg_hebb_slot_holds(const EgHebbCand* c, const char* a, const char* b) { if (!c->a || !c->b) return 0; return (strcmp(c->a, a) == 0 && strcmp(c->b, b) == 0) || (strcmp(c->a, b) == 0 && strcmp(c->b, a) == 0); } static void eg_hebb_slot_clear(EgHebbCand* c) { free(c->a); free(c->b); c->a = NULL; c->b = NULL; c->score = 0.0; } /* eg_hebb_cand_bump — reinforce the (a,b) candidate association by `inc`. * Extracted 2026-08-06 so the same collision policy serves both the co-resident * pairs and the eligibility-trace pairs; two copies of this logic would have * drifted. Collision policy is unchanged: claim a free slot, reinforce our own, * evict an incumbent only once it has decayed to nothing, otherwise lose the * round. `inc` is graded by the partner's trace, so an incumbent is never * displaced by a challenger carrying less weight than one full co-activation. */ static void eg_hebb_cand_bump(const char* a, const char* b, double inc) { if (!a || !b || inc <= 0.0) return; int s = eg_hebb_slot(a, b); if (s < 0) return; EgHebbCand* c = &_eg_hebb_cand[s]; if (!c->a) { /* free slot: claim */ c->a = el_strdup_persist(a); c->b = el_strdup_persist(b); c->score = inc; } else if (eg_hebb_slot_holds(c, a, b)) { c->score += inc; /* ours: reinforce */ } else if (c->score <= ENGRAM_HEBB_ETA) { eg_hebb_slot_clear(c); /* dead incumbent: take the slot */ c->a = el_strdup_persist(a); c->b = el_strdup_persist(b); c->score = inc; } /* else: live incumbent keeps the slot this round. */ } /* Does any edge already connect these two nodes, in either direction? * Linear over the edge array, but called at most ENGRAM_HEBB_LINK_PER_CALL * times per activation and only for pairs that already cleared the * consolidation threshold — a handful of scans per day, not per hop. */ static int eg_edge_exists_between(EngramStore* g, const char* a, const char* b) { for (int64_t i = 0; i < g->edge_count; i++) { const EngramEdge* e = &g->edges[i]; if (!e->from_id || !e->to_id) continue; if ((strcmp(e->from_id, a) == 0 && strcmp(e->to_id, b) == 0) || (strcmp(e->from_id, b) == 0 && strcmp(e->to_id, a) == 0)) return 1; } return 0; } /* engram_temporal_decay — recency shaping on the activation path. * * MEASURED FAILURE (2026-08-05 self-review). Census of the live graph under * the previous form (uniform 168 h half-life, floor 0.05): * * node type n median tdecay % pinned at the 0.05 floor * Memory 1233 0.0500 81% * Knowledge 1183 0.0500 58% * BacklogItem 1057 0.0500 91% * Project 321 0.0500 98% * Tag 135 0.0500 100% * * The median value for EVERY node type was the clamp. A function whose median * output is its floor is not a signal — it is a constant with exceptions, and * the exceptions were exactly the nodes touched in the last few days. * * What that cost, concretely: 10 of the 13 grounded value nodes — "Precision * Over Brute Force", "Honesty Before Comfort", "The System Must Accumulate" — * sat at 0.05, a 20x activation penalty, while Knowledge ingested overnight * sat near 1.0 and held the working-memory top slots. The decay function was * quietly erasing the accumulated library in favour of whatever arrived last * night. That is a direct inversion of the system's purpose. * * Worse, tdecay multiplies at EVERY hop (seed activation and each propagation * step), so a 2-hop path through settled knowledge compounded to 0.05^2 = * 0.0025. Old regions of the graph were not disfavoured; they were unreachable. * * EXTERNAL EVIDENCE. "Not All Memories Age the Same" (arXiv:2604.26970) * measures retrieval under different decay regimes: * * no temporal weighting NDCG@5 0.274 * uniform exponential decay NDCG@5 0.015 <- 18x WORSE than none * domain-adaptive decay NDCG@5 0.241 * full adaptive hierarchy NDCG@5 0.260 * * Uniform exponential decay is not merely suboptimal — it is worse than having * no decay at all, because it penalises stable knowledge (rarely accessed, * heavily load-bearing) while failing to suppress stale volatile facts. Notably * not even the full adaptive hierarchy beat switching decay off. * * THE FIX: make the half-life a function of how established a node is, and * make the floor a preference rather than a cliff. * * T_eff = T_HALF * (1 + ln(1 + activation_count)) * * Frequently-retrieved nodes age slowly; nodes nothing has ever asked for age * at the original rate. This is the spacing effect and the Lindy property in * one line, it is monotone and log-bounded (a 10,000-activation node gets only * a ~10x longer half-life, not a permanent exemption), and it is built from * activation_count — which is measured, unlike `tier`, whose assignments are * inconsistent enough to be untrustworthy here (the values node is tagged * Episodic). * * The floor moves 0.05 -> 0.25. Given the evidence that no decay outperforms * uniform decay, the honest maximum penalty for age alone is 4x, not 20x. Age * should express a preference for the recent; it should never make a region of * the graph structurally unreachable. * * Explicit per-node temporal_decay_rate still overrides lambda (77 nodes carry * one) — that path is untouched and remains the escape hatch for content that * genuinely should expire fast. */ #define ENGRAM_DECAY_FLOOR 0.25 static double engram_temporal_decay(const EngramNode* n, int64_t now_ms) { int64_t age_ms = now_ms - n->last_activated; if (age_ms <= 0) return 1.0; double lambda = (n->temporal_decay_rate > 0.0) ? n->temporal_decay_rate : ENGRAM_DECAY_LAMBDA; double age_hours = (double)age_ms / 3600000.0; double t_half = ENGRAM_T_HALF_HOURS * (1.0 + log(1.0 + (double)n->activation_count)); double factor = exp(-lambda * age_hours / t_half); if (factor < ENGRAM_DECAY_FLOOR) factor = ENGRAM_DECAY_FLOOR; return factor; } /* Activation dampening: high activation_count nodes are "well-known" context * and get less marginal boost per firing. * count=0 → 1.0, count=2 → ~0.74, count=9 → ~0.59, count=99 → ~0.43 */ static double engram_activation_dampen(const EngramNode* n) { return 1.0 / (1.0 + log(1.0 + (double)n->activation_count)); } /* Temporal proximity bonus: boost propagation along edges connecting * co-temporal nodes. Returns a multiplier bonus in [0, 0.2]. */ static double engram_temporal_proximity_bonus(int64_t node_created, int64_t seed_epoch) { int64_t diff = node_created - seed_epoch; if (diff < 0) diff = -diff; if (diff < 86400000LL) return 0.20; /* within 1 day */ if (diff < 604800000LL) return 0.10; /* within 7 days */ return 0.0; } /* ── Two-layer activation (biologically-motivated) ─────────────────────────── * * Layer 1 — Broad fan-out (background activation): * BFS + spreading activation fires on ALL nodes reachable from seeds, * regardless of relevance to the current goal. Every reachable node gets * a background_activation score. Nothing is filtered here. Models the * brain's massive parallel sub-threshold activation of all associated * content in response to a stimulus. Temporal decay and activation * dampening are applied at this layer (as before), but no threshold gate. * * Layer 2 — Executive filter (working memory promotion): * A second pass asks: given the query (goal intent), attentional bias, * and inhibitory edge topology — which background-activated nodes should * break through into working memory? * * wm_weight = bg_activation * goal_bias(node, query) * confidence * * inhibitory_suppression_factor * * Only nodes where wm_weight >= ENGRAM_WM_THRESHOLD are promoted to * working memory (working_memory_weight > 0). Background-activated nodes * that don't cross the threshold accumulate suppression_count. After * ENGRAM_SUPPRESSION_BREAKTHROUGH consecutive suppressed turns, the node * force-breaks through at ENGRAM_BREAKTHROUGH_WEIGHT (latent tension * surfacing — models intrusive memory / unresolved cognitive load). * * Inhibitory edges: * An edge with inhibitory=1 suppresses the TARGET node's working memory * promotion when the SOURCE is background-activated. Background activation * of the target is NOT affected — the node fires in layer 1. Only the * executive filter (layer 2) is gated. Models attentional inhibition: * "focused on code work" suppresses personal memories from surfacing * even if they have high background_activation. * * Goal bias: * A lightweight heuristic rates how well each background-activated node * aligns with the apparent intent of the current query. Technical queries * boost Belief/Canonical/Lesson nodes; relational queries boost Memory/ * Entity nodes. Direct lexical overlap gives a 50% bonus. * * Working memory persistence (turn continuity): * Nodes promoted in the previous turn retain a decayed working_memory_weight * (ACT-R base-level carry-over, 2026-07-22) without needing re-activation. This models * conversational thread continuity — once a topic is in working memory, * it persists slightly into the next turn. * * Returns ElList of {node, activation_strength, working_memory_weight, * epistemic_confidence, hops, promoted}. * "promoted" = 1 if working_memory_weight > 0, 0 if background-only. * Context compilation uses ONLY nodes with promoted=1. * * Temporal decay (preserved from prior implementation): * effective_salience = salience * exp(-lambda * age_hours / T_half) * where T_half = 168 h (one week), lambda = ln(2) * * Activation dampening (preserved): * dampen = 1 / (1 + log(1 + activation_count)) * * Temporal proximity bonus (preserved): * edge_strength *= (1 + tbonus) where tbonus ∈ {0, 0.10, 0.20} * * Per-type threshold gates apply only to working memory promotion (layer 2): * Safety/DharmaSelf: 0.05 Canonical: 0.15 Lesson: 0.25 * Belief/Entity: 0.30 Note/Memory/Working: 0.40 */ /* Compute goal-state bias multiplier for a node given the query. * Returns a value in [0.3, 2.0]. This is a lightweight heuristic — * a production implementation may use LLM-derived intent classification. */ static double engram_goal_bias(const EngramNode* n, const char* query) { if (!query || !*query) return 1.0; double bias = 1.0; /* Direct lexical overlap, graded by token coverage: a node covering all * query tokens gets the full +0.5; partial coverage gets a proportional * share. Single-token queries → full +0.5 on match, identical to before. * (2026-07-19 port of the 2026-07-14 tokenized-search fix) */ { char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; int ntok = engram_tokenize_query(query, toks, ENGRAM_MAX_QTOKENS); int sc = engram_node_match_score(n, toks, ntok); if (sc > 0 && ntok > 0) bias += 0.5 * ((double)sc / (double)ntok); } /* Node-type resonance with query intent. */ int technical_query = istr_contains(query, "code") || istr_contains(query, "function") || istr_contains(query, "implement") || istr_contains(query, "error") || istr_contains(query, "bug") || istr_contains(query, "build") || istr_contains(query, "system") || istr_contains(query, "design") || istr_contains(query, "architecture") || /* Curiosity-scan seeds: without these, idle-loop * activation queries ("decision pattern lesson", * "memory knowledge context") produce no goal-bias * differentiation at all. Ported from dev-line fix * d53516b (2026-06-14). (2026-07-19 self-review) */ istr_contains(query, "knowledge") || istr_contains(query, "pattern") || istr_contains(query, "decision") || istr_contains(query, "memory") || istr_contains(query, "lesson"); int personal_query = istr_contains(query, "feel") || istr_contains(query, "emotion") || istr_contains(query, "remember") || istr_contains(query, "personal") || istr_contains(query, "story") || istr_contains(query, "relationship"); if (n->node_type) { int is_knowledge = (strcmp(n->node_type, "Belief") == 0) || (strcmp(n->node_type, "DharmaSelf") == 0) || (strcmp(n->node_type, "Safety") == 0) || /* The primary knowledge-capture type was absent * from its own bias class: captureKnowledge() and * the world ingestor write node_type "Knowledge", * which competed at neutral bias on technical * queries. Ported from dev-line fix d53516b. * (2026-07-19 self-review) */ (strcmp(n->node_type, "Knowledge") == 0); int is_personal = (strcmp(n->node_type, "Memory") == 0) || (strcmp(n->node_type, "Entity") == 0); if (technical_query && is_knowledge) bias += 0.3; if (technical_query && is_personal) bias -= 0.3; if (personal_query && is_personal) bias += 0.3; if (personal_query && is_knowledge) bias -= 0.1; } /* Tier-based bonus: promote higher-confidence knowledge nodes. */ if (n->tier) { if (strcmp(n->tier, "Canonical") == 0) bias += 0.2; if (strcmp(n->tier, "Lesson") == 0) bias += 0.1; } if (bias < 0.3) bias = 0.3; if (bias > 2.0) bias = 2.0; return bias; } el_val_t engram_activate(el_val_t query, el_val_t depth) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); int64_t max_depth = (int64_t)depth; if (max_depth <= 0) max_depth = 2; el_val_t out = el_list_empty(); if (!q || g->node_count == 0) return out; /* Rebuild adjacency index if the edge/node topology changed since the * last activation call. This is O(E) one-time cost vs O(E) per BFS step * without the index. On a 40K-edge graph this drops BFS from O(frontier * * E) to O(frontier * avg_degree). (2026-07-01 self-review) */ if (g->adj_dirty || !g->adj_from) engram_adj_rebuild(g); int64_t now_ms = engram_now_ms(); /* Observability counters: _eg_act_breakthroughs/_eg_act_wm_evicted are * CUMULATIVE for the process lifetime and intentionally NOT reset here * (2026-07-31 self-review — the old per-call reset made the 60s heartbeat * miss nearly all events between beats; see the definition site). * ctx_cos stays per-call: it is a gauge of THIS query vs the centroid. */ _eg_act_ctx_cos = -2.0; /* ── Embedding backfill + query embedding (2026-07-24, bl-b2d1c944) ── * Backfill: embed up to N un-embedded eligible nodes per call, newest * first (append order ≈ creation order), so fresh content is semantic- * searchable within one scan cycle and the historical store fills in * gradually — ~16 nodes/min under the 30s curiosity cadence, no bulk * hammering of Ollama, no latency on any create path. */ { int backfilled = 0; for (int64_t i = g->node_count - 1; i >= 0 && backfilled < ENGRAM_EMBED_BACKFILL_PER_CALL; i--) { EngramNode* n = &g->nodes[i]; if (n->emb || !eg_embed_eligible(n)) continue; int32_t d = 0; float* v = eg_embed_fetch(n->content, &d); if (!v) break; /* embedder down / breaker open — stop this call */ n->emb = v; n->emb_dim = d; backfilled++; } } /* Query embedding, cached single-slot: the curiosity loop re-issues the * same 4 rotating phrases, so consecutive identical queries skip the * HTTP round-trip entirely. */ static char* _eg_qcache_text = NULL; static float* _eg_qcache_emb = NULL; static int32_t _eg_qcache_dim = 0; float* q_emb = NULL; int32_t q_dim = 0; if (_eg_qcache_text && strcmp(_eg_qcache_text, q) == 0) { q_emb = _eg_qcache_emb; q_dim = _eg_qcache_dim; } else { int32_t d = 0; float* v = eg_embed_fetch(q, &d); if (v) { free(_eg_qcache_text); free(_eg_qcache_emb); _eg_qcache_text = strdup(q); _eg_qcache_emb = v; _eg_qcache_dim = d; q_emb = v; q_dim = d; } } /* ── Context centroid fold-in (2026-07-29) ────────────────────────── * Record drift BEFORE blending (cos of the query against yesterday's * context), then fold the query in as a touch, then build the * query-dominant effective scoring vector. See the ENGRAM_CTX_* block * for the design and the feedback-loop guard rationale. */ float* e_eff = NULL; if (q_emb) { if (_eg_ctx_c && _eg_ctx_dim == q_dim) _eg_act_ctx_cos = eg_cosine(q_emb, _eg_ctx_c, q_dim); eg_ctx_blend(q_emb, q_dim); if (_eg_ctx_c && _eg_ctx_dim == q_dim) { e_eff = malloc((size_t)q_dim * sizeof(float)); if (e_eff) { double nq = 0.0; for (int32_t i = 0; i < q_dim; i++) nq += (double)q_emb[i] * (double)q_emb[i]; nq = (nq > 0.0) ? sqrt(nq) : 1.0; double nn = 0.0; for (int32_t i = 0; i < q_dim; i++) { double v = ENGRAM_CTX_QALPHA * ((double)q_emb[i] / nq) + (1.0 - ENGRAM_CTX_QALPHA) * (double)_eg_ctx_c[i]; e_eff[i] = (float)v; nn += v * v; } if (nn <= 0.0) { free(e_eff); e_eff = NULL; } } } } /* Per-node cosine vs the effective query (query ⊕ context centroid; * plain query on cold start), computed once, consumed twice: semantic * seeding below and the additive WM term in Pass 2 (use similarity * twice, coherently — HippoRAG). cosq stays NULL when the embedder is * unavailable; every consumer degrades to pure lexical behavior. */ double* cosq = NULL; if (q_emb) { const float* qv = e_eff ? e_eff : q_emb; cosq = calloc((size_t)g->node_count, sizeof(double)); if (cosq) { for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; cosq[i] = (n->emb && n->emb_dim == q_dim) ? eg_cosine(n->emb, qv, q_dim) : -2.0; } } } free(e_eff); e_eff = NULL; /* only needed to fill cosq */ /* Per-node layer-1 tracking. */ double* best_bg = calloc((size_t)g->node_count, sizeof(double)); int64_t* best_hops = calloc((size_t)g->node_count, sizeof(int64_t)); int* reached = calloc((size_t)g->node_count, sizeof(int)); if (!best_bg || !best_hops || !reached) { free(best_bg); free(best_hops); free(reached); free(cosq); return out; } /* ── LAYER 1: broad fan-out (background activation) ───────────────── * Find seeds, apply temporal decay + dampening, BFS with edge weights. * Inhibitory edges propagate activation normally at this layer — they * only gate working memory promotion in layer 2. */ typedef struct { int64_t idx; double act; int64_t created_at; } SeedEntry; SeedEntry* seeds = malloc((size_t)g->node_count * sizeof(SeedEntry)); int64_t seed_count = 0; if (!seeds) { free(best_bg); free(best_hops); free(reached); free(cosq); return out; } /* Tokenize once: a node seeds if it matches ANY query token, and its seed * activation is scaled by token coverage (fraction of distinct query * tokens it contains) so a node matching all words seeds more strongly * than one matching a single word. Single-word queries → coverage 1.0, * identical to the prior whole-query behavior. Before this, the soul's * rotating 3-word curiosity seeds ("working project active") activated * ZERO nodes almost every scan — idle cognition firing blanks. * (2026-07-19 port of the 2026-07-14 tokenized-search fix; NOTE the * el-compiler copy of this fix dropped the ISE seed exclusion below — * kept here deliberately, do not "sync" it away.) */ char qtoks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; int qntok = engram_tokenize_query(q, qtoks, ENGRAM_MAX_QTOKENS); for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; /* InternalStateEvent nodes are observability-only telemetry — never * seed activation from them. Their JSON payloads contain common words * ("memory", "context", ...) that lexically match almost any query, * turning telemetry into a spreading-activation ignition source. They * are already excluded from WM promotion in pass 2; exclude them from * seeding here too. */ if (n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0) continue; int msc = engram_node_match_score(n, qtoks, qntok); if (msc > 0) { double tdecay = engram_temporal_decay(n, now_ms); double dampen = engram_activation_dampen(n); double cover = qntok > 0 ? (double)msc / (double)qntok : 1.0; double act = n->salience * tdecay * dampen * cover; seeds[seed_count].idx = i; seeds[seed_count].act = act; seeds[seed_count].created_at = n->created_at; seed_count++; best_bg[i] = act; best_hops[i] = 0; reached[i] = 1; } } /* ── Semantic seed supplement (2026-07-24, bl-b2d1c944) ───────────── * Top-K nodes by cosine ≥ SEED_MIN join the seed set with initial * activation = similarity × the same decay/dampen shaping the lexical * seeds get. This is the fix for "idle cognition firing blanks": a * curiosity phrase like "decision pattern lesson" now ignites nodes * that MEAN decisions and lessons, not just nodes that contain those * literal substrings. Lexically-seeded nodes are skipped — the lexical * path already gave them coverage-scaled activation. */ if (cosq) { /* Redundancy-suppressed top-K (2026-08-05 self-review; see * ENGRAM_DEDUP_COS for the measurement that motivated it). A rejected * candidate does NOT consume one of the K slots — the loop retries for * the next-best distinct node, so K distinct meanings are seeded rather * than K copies of one. Rejects are recorded in seed_dup[] rather than * reached[] or cosq[]: marking reached[] would suppress the node's * propagation, and clobbering cosq[] would change the downstream * query-aware propagation gate. Neither belongs in a seeding decision. * `guard` bounds the retries so a pathological duplicate cluster can * never turn seed selection into an O(K·N²) scan. */ unsigned char* seed_dup = calloc((size_t)g->node_count, 1); int64_t sel[ENGRAM_EMBED_SEED_K]; uint64_t selkey[ENGRAM_EMBED_SEED_K]; int nsel = 0; int guard = ENGRAM_EMBED_SEED_K * 8; while (nsel < ENGRAM_EMBED_SEED_K && guard-- > 0) { int64_t bi = -1; double bc = ENGRAM_EMBED_SEED_MIN; for (int64_t i = 0; i < g->node_count; i++) { if (reached[i]) continue; if (seed_dup && seed_dup[i]) continue; if (cosq[i] > bc) { bc = cosq[i]; bi = i; } } if (bi < 0) break; EngramNode* n = &g->nodes[bi]; uint64_t key = eg_content_key(n); int dup = 0; for (int s = 0; s < nsel; s++) { if (eg_same_content(n, &g->nodes[sel[s]], key, selkey[s])) { dup = 1; break; } } if (dup) { _eg_act_dup_seeds++; if (seed_dup) { seed_dup[bi] = 1; continue; } break; /* OOM on the skip map: stop rather than spin */ } double tdecay = engram_temporal_decay(n, now_ms); double dampen = engram_activation_dampen(n); double act = bc * tdecay * dampen; seeds[seed_count].idx = bi; seeds[seed_count].act = act; seeds[seed_count].created_at = n->created_at; seed_count++; best_bg[bi] = act; best_hops[bi] = 0; reached[bi] = 1; sel[nsel] = bi; selkey[nsel] = key; nsel++; } free(seed_dup); } /* Compute mean seed created_at for temporal proximity bonus. * Was a running pairwise average — seed_epoch = (seed_epoch + t_s)/2 — * which is NOT the arithmetic mean: it exponentially over-weights the * later seeds (last seed gets weight 1/2, second-to-last 1/4, ...), so * the temporal-proximity bonus skewed toward whichever seeds happened * to sit later in the scan order. True mean via int64 sum: ms epochs * (~1.8e12) times any plausible seed_count stays far below INT64_MAX. * (2026-07-19 self-review) */ int64_t seed_epoch = 0; if (seed_count > 0) { int64_t epoch_sum = 0; for (int64_t s = 0; s < seed_count; s++) epoch_sum += seeds[s].created_at; seed_epoch = epoch_sum / seed_count; } typedef struct { int64_t idx; int64_t hops; double act; } Frontier; Frontier* fr = malloc((size_t)(g->node_count * (max_depth + 1)) * sizeof(Frontier) + 16 * sizeof(Frontier)); if (!fr) { free(best_bg); free(best_hops); free(reached); free(seeds); free(cosq); return out; } int64_t fhead = 0, ftail = 0; int64_t fcap = (int64_t)((size_t)(g->node_count * (max_depth + 1)) + 16); for (int64_t s = 0; s < seed_count; s++) { if (ftail >= fcap) break; fr[ftail].idx = seeds[s].idx; fr[ftail].hops = 0; fr[ftail].act = seeds[s].act; ftail++; } const double SPREAD_DECAY = 0.7; while (fhead < ftail) { Frontier f = fr[fhead++]; if (f.hops >= max_depth) continue; int64_t cur = f.idx; int64_t new_hops = f.hops + 1; /* Use adjacency index: iterate only edges incident to `cur`. * adj_from[cur] holds edge indices where cur is the 'from' node; * adj_to[cur] holds edge indices where cur is the 'to' node. * If adj index is unavailable (OOM during rebuild), fall back to * full edge scan so activation is never silently wrong. */ int use_adj = (g->adj_from != NULL && g->adj_to != NULL); int from_len = use_adj ? g->adj_from_len[cur] : 0; int to_len = use_adj ? g->adj_to_len[cur] : 0; int edge_scan_count = use_adj ? (from_len + to_len) : (int)g->edge_count; for (int scan_i = 0; scan_i < edge_scan_count; scan_i++) { int64_t ei; int64_t oi; if (use_adj) { ei = (scan_i < from_len) ? g->adj_from[cur][scan_i] : g->adj_to[cur][scan_i - from_len]; EngramEdge* e = &g->edges[ei]; oi = (scan_i < from_len) ? engram_idmap_get(g, e->to_id) : engram_idmap_get(g, e->from_id); } else { /* Fallback: linear scan */ ei = scan_i; EngramEdge* e = &g->edges[ei]; const char* other = NULL; const char* cur_id = g->nodes[cur].id; if (e->from_id && strcmp(e->from_id, cur_id) == 0) other = e->to_id; else if (e->to_id && strcmp(e->to_id, cur_id) == 0) other = e->from_id; else continue; oi = engram_find_node_index(other); } if (oi < 0 || oi >= g->node_count) continue; EngramEdge* e = &g->edges[ei]; EngramNode* on = &g->nodes[oi]; /* Never propagate INTO InternalStateEvent nodes. They are already * barred from WM promotion (pass 2) and from seeding (above), but * as high-degree hubs they still relayed activation across the * graph. Skipping here keeps them out of the frontier entirely. */ if (on->node_type && strcmp(on->node_type, "InternalStateEvent") == 0) continue; double tbonus = engram_temporal_proximity_bonus(on->created_at, seed_epoch); double tdecay = engram_temporal_decay(on, now_ms); double dampen = engram_activation_dampen(on); /* ── Query-aware propagation gate (2026-07-27 self-review) ── * Prior behavior was "query-blind" spreading: the query chose * the seeds, but propagation depended only on graph structure, * so high-degree hubs relayed activation into branches with no * semantic relation to the query. Per arXiv:2606.30133, gating * each increment by the TARGET node's query similarity * (sigma(v) = max(cos(e_v, e_q), 0)) prunes low-information * branches at every hop (+3.6..+7.4 F1 over uniform spreading, * 1.5-4.9x faster via a shrinking working set). * * Adaptation for partial embedding coverage: the paper skips * unembedded targets outright, but only eligible non-ISE/Tag * nodes carry embeddings here — a hard gate would sever purely * lexical/structural pathways. So: embedded targets get a soft * gate FLOOR + (1-FLOOR)*clip(cos) (dissimilar nodes damped * ~4x, never killed); unembedded targets pass ungated (no * information, no penalty); cosq == NULL (embedder down) means * no gating at all — same graceful degradation as seeding. */ double qgate = 1.0; if (cosq && cosq[oi] > -1.5) { double c = cosq[oi] > 0.0 ? cosq[oi] : 0.0; qgate = ENGRAM_QGATE_FLOOR + (1.0 - ENGRAM_QGATE_FLOOR) * c; } /* eg_edge_eff_weight, not e->weight: edges that have repeatedly * carried co-activated pairs propagate more strongly. Identity on * an unlearned edge. (2026-08-04 self-review.) */ double new_act = f.act * eg_edge_eff_weight(e) * SPREAD_DECAY * (1.0 + tbonus) * tdecay * dampen * qgate; /* Firing threshold per classic spreading-activation: sub-threshold * activation neither updates the target nor enqueues it, so weak * signals die out instead of flooding the whole graph with tiny * nonzero background activation. */ if (new_act < 0.02) continue; if (!reached[oi] || new_act > best_bg[oi]) { best_bg[oi] = new_act; best_hops[oi] = new_hops; reached[oi] = 1; if (ftail < fcap) { fr[ftail].idx = oi; fr[ftail].hops = new_hops; fr[ftail].act = new_act; ftail++; } } } } /* Persist layer-1 background_activation to node store. */ for (int64_t i = 0; i < g->node_count; i++) { g->nodes[i].background_activation = reached[i] ? best_bg[i] : 0.0; } /* ── PASS 2: executive filter → working memory promotion ──────────── */ /* Step A: collect inhibitory suppressions from fired inhibitory edges. * Layered consciousness: inhibition is ONLY recorded against targets * whose layer is `suppressible == 1`. Nodes in non-suppressible layers * (Layer 0 / safety) ignore inhibitory edges entirely — their working * memory weight cannot be silenced by attentional suppression. */ double* inhibition = calloc((size_t)g->node_count, sizeof(double)); if (!inhibition) { free(best_bg); free(best_hops); free(reached); free(seeds); free(fr); free(cosq); return out; } for (int64_t ei = 0; ei < g->edge_count; ei++) { EngramEdge* e = &g->edges[ei]; if (!e->inhibitory) continue; int64_t src = engram_find_node_index(e->from_id); int64_t tgt = engram_find_node_index(e->to_id); if (src < 0 || tgt < 0) continue; if (!reached[src] || best_bg[src] <= 0.0) continue; /* Skip if target layer is non-suppressible: Layer 0 / safety nodes * are immune to inhibitory edges from any source. The pass-3 * override below also force-promotes them, but recording inhibition * against them at all would be wasted work and could confuse * downstream debugging output. */ if (!engram_layer_is_suppressible(g->nodes[tgt].layer_id)) continue; /* Inhibition strength proportional to source background activation * and edge weight. Takes the maximum if multiple inhibitory edges * target the same node. */ double inh = best_bg[src] * e->weight; if (inh > inhibition[tgt]) inhibition[tgt] = inh; } /* Step B: compute working_memory_weight per candidate node. */ double* wm_weights = calloc((size_t)g->node_count, sizeof(double)); if (!wm_weights) { free(best_bg); free(best_hops); free(reached); free(seeds); free(fr); free(inhibition); free(cosq); return out; } /* Per-call breakthrough budget (2026-08-02) — see ENGRAM_BREAKTHROUGH_BUDGET. */ int64_t bt_budget = ENGRAM_BREAKTHROUGH_BUDGET; for (int64_t i = 0; i < g->node_count; i++) { if (!reached[i] || best_bg[i] <= 0.0) continue; EngramNode* n = &g->nodes[i]; /* InternalStateEvent nodes are observability-only — never admit to WM. * Their JSON content (curiosity seeds, heartbeat payloads) contains common * words that trigger lexical seeding (e.g. "knowledge" in curiosity ISEs), * leading to repeated suppression and eventual breakthrough at the floor. * ISEs surfacing in context compilation are noise, not signal. Clear their * suppression_count so they don't build toward breakthrough, then skip. * (2026-06-30 self-review: porting fix from 2026-06-26 branch; SYNAPSE * paper confirms WM should hold only semantically relevant content.) */ if (n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0) { n->suppression_count = 0; wm_weights[i] = 0.0; continue; } /* Per-type threshold: safety nodes break through more easily. */ double type_threshold = engram_type_threshold(n->node_type, n->tier); /* Goal bias weights the node's relevance to current intent. */ double bias = engram_goal_bias(n, q); /* Raw working memory score. * Importance factor (2026-07-28 self-review): n->importance was * stored, serialized, and clamped at creation (default 0.5) but * never read by any scoring path — a curated importance=1.0 node * competed identically with a throwaway note at equal activation. * Map it to a gentle multiplier centered on the 0.5 default: * impf = 0.5 + importance → default nodes unchanged (×1.0), * critical (1.0) ×1.5, low (0.1) ×0.6. Nodes loaded from legacy * snapshots with importance<=0 stay neutral rather than being * silently suppressed. */ double impf = (n->importance > 0.0) ? (0.5 + n->importance) : 1.0; double raw_wm = best_bg[i] * bias * n->confidence * impf; /* Apply inhibitory suppression. Full inhibition → scale by factor. */ double inh = inhibition[i]; if (inh > 1.0) inh = 1.0; double suppress = 1.0 - (1.0 - ENGRAM_INHIBITION_FACTOR) * inh; raw_wm *= suppress; /* Short-term inhibition-of-return (2026-07-25, Lebiere-Best): * damp by t_n/(t_n + t_s) where t_n = seconds since the most * recent recorded access (WM promotion / strengthen). A node that * just held a WM slot yields it even to structurally stronger * competitors, and recovers as t_n grows. access_ts is recorded * at promotion, so persistent WM residents self-inhibit. Nodes * with no access history (never promoted) are uninhibited. * Layer-0 override in Pass 3 still floors safety nodes. */ if (n->access_filled > 0) { int32_t sti_last = (n->access_head + ENGRAM_BLL_K - 1) % ENGRAM_BLL_K; double sti_tn = (double)(now_ms - n->access_ts[sti_last]) / 1000.0; if (sti_tn < 0.1) sti_tn = 0.1; raw_wm *= sti_tn / (sti_tn + ENGRAM_STI_TS); } /* Additive semantic-relevance term (2026-07-24, bl-b2d1c944): * shift-and-floor at S0 — nomic-embed scores unrelated pairs * 0.4–0.5, so raw cosine in a weighted sum would be a constant * bias swamping the decayed base-level signal. Above S0 the term * ramps 0 → WM_WEIGHT, breaking ties among structurally equivalent * candidates in favor of nodes that mean what the query means. * * MOVED AFTER the STI damper (2026-08-02 self-review). It used to * be added BEFORE, so the recency multiplier scaled the semantic * term too: an incumbent re-reached 30s later took t_n/(t_n+120) * = 0.2×, cutting the cosine term's ceiling from 0.20 to 0.04 — * below every per-type threshold (0.15–0.40). Meaning-match was * being punished for having been recently useful. Inhibition-of- * return should rotate the STRUCTURAL score (what the graph * dragged in), not the semantic one (what the query actually * means); relevance to the current query is not stale merely * because the node was in WM a moment ago. */ if (cosq && cosq[i] > ENGRAM_EMBED_S0) { raw_wm += ENGRAM_EMBED_WM_WEIGHT * (cosq[i] - ENGRAM_EMBED_S0) / (1.0 - ENGRAM_EMBED_S0); } /* Threshold gate: must exceed per-type threshold to enter working * memory. Type threshold replaces the old flat 0.2 filter. */ if (raw_wm >= type_threshold) { wm_weights[i] = raw_wm > 1.0 ? 1.0 : raw_wm; if (n->suppression_count > 0) n->suppression_count = 0; } else { /* Node didn't make it through — increment suppression counter. * After N consecutive suppressions it MAY force a breakthrough, * subject to the per-call budget and the negative-count cooldown * (2026-08-02 — see ENGRAM_BREAKTHROUGH_BUDGET/_COOLDOWN). */ n->suppression_count++; if (n->suppression_count >= ENGRAM_SUPPRESSION_BREAKTHROUGH && bt_budget > 0) { /* Graded breakthrough weight (2026-08-02): previously every * breakthrough landed on exactly ENGRAM_BREAKTHROUGH_WEIGHT, * so hundreds tied at 0.10 and the rank-cap tie-break at the * cutoff degenerated to node-array index order — i.e. whoever * was inserted earliest won, which is not a cognitive * criterion. Scale within ±10% by how close the node came to * its own threshold, so a near-miss outranks a node that was * nowhere near. Stays strictly below min(type_threshold) * (0.15) and strictly above ENGRAM_WM_FLOOR (0.05), which is * the invariant ENGRAM_BREAKTHROUGH_WEIGHT documents. */ double near = (type_threshold > 0.0) ? (raw_wm / type_threshold) : 0.0; if (near < 0.0) near = 0.0; if (near > 1.0) near = 1.0; wm_weights[i] = ENGRAM_BREAKTHROUGH_WEIGHT * (0.9 + 0.2 * near); /* Negative = cooldown. Must climb back through the cooldown * before it can breach again. */ n->suppression_count = -ENGRAM_BREAKTHROUGH_COOLDOWN; bt_budget--; _eg_act_breakthroughs++; } else { wm_weights[i] = 0.0; /* Budget-starved or cooling down: do NOT reset the counter — * let it saturate so the node surfaces on a later call rather * than restarting its climb from zero. Cap the ceiling so the * int32 cannot drift unbounded over a long uptime. */ if (n->suppression_count > ENGRAM_SUPPRESSION_BREAKTHROUGH * 4) n->suppression_count = ENGRAM_SUPPRESSION_BREAKTHROUGH * 4; } } } /* ── PASS 3: Layer 0 override (the sacred fire) ───────────────────── * Every node in a non-suppressible layer that received any background * activation is force-promoted to AT LEAST ENGRAM_LAYER0_OVERRIDE_WEIGHT. * This runs LAST and overrides whatever Pass 2 decided — Layer 0 cannot * be silenced by inhibitory edges, by goal-bias misalignment, by * confidence weighting, or by per-type threshold gates. If the seed * fan-out reached a structural-floor node, that node surfaces. * * Note: this also clears the suppression_count when an override fires, * since the node DID surface this turn — it just took the override path * rather than the standard threshold path. Without this, a Layer 0 * node with persistent inhibitory pressure would accumulate * suppression_count forever and never reach the breakthrough state. */ for (int64_t i = 0; i < g->node_count; i++) { if (!reached[i] || best_bg[i] <= 0.0) continue; EngramNode* n = &g->nodes[i]; if (engram_layer_is_suppressible(n->layer_id)) continue; if (wm_weights[i] < ENGRAM_LAYER0_OVERRIDE_WEIGHT) { wm_weights[i] = ENGRAM_LAYER0_OVERRIDE_WEIGHT; } n->suppression_count = 0; } /* ── PASS 3½: redundancy suppression ──────────────────────────────────── * (2026-08-05 self-review; see ENGRAM_DEDUP_COS for the census.) Runs * BEFORE the capacity cap, so the 24 slots are contested by 24 distinct * meanings rather than by however many copies of one document the graph * happens to hold. Byte-identical nodes score byte-identically, so without * this they promote as a block — a six-copy document could hold a quarter * of working memory while saying one thing. * * Walk candidates in descending weight; the first occurrence of a content * survives, later ones are evicted. Highest-weight copy wins, which keeps * the survivor choice deterministic and preserves the strongest activation. * * Bounded work: once WM_CAP distinct survivors are held, every remaining * candidate is weaker than the weakest survivor and Pass 4 would evict it * anyway — so the walk stops at the first candidate STRICTLY below the * cap-th survivor's weight. Ties keep being processed, because a tie can * still take a slot through Pass 4's at_cutoff_slots path. Typical cost is * a few dozen content hashes per call, not a full-graph sweep. * * This runs after Pass 3 deliberately: Layer 0 (safety) force-promotions * are already in wm_weights and are ranked like anything else. If safety * content is genuinely duplicated, one copy still holds a slot — the * guarantee is that the content is present, not that every copy of it is. */ { int64_t nc = 0; for (int64_t i = 0; i < g->node_count; i++) if (wm_weights[i] > 0.0) nc++; if (nc > 1) { EgDupCand* dc = malloc((size_t)nc * sizeof(EgDupCand)); if (dc) { int64_t ci = 0; for (int64_t i = 0; i < g->node_count; i++) { if (wm_weights[i] > 0.0) { dc[ci].w = wm_weights[i]; dc[ci].idx = i; ci++; } } qsort(dc, (size_t)nc, sizeof(EgDupCand), eg_dupcand_cmp_desc); int64_t keep[ENGRAM_WM_CAP]; uint64_t kkey[ENGRAM_WM_CAP]; int nk = 0; double cap_w = 0.0; /* weight of the WM_CAP-th survivor */ for (int64_t z = 0; z < nc; z++) { if (nk >= ENGRAM_WM_CAP && dc[z].w < cap_w) break; int64_t i = dc[z].idx; EngramNode* n = &g->nodes[i]; uint64_t key = eg_content_key(n); int dup = 0; for (int s = 0; s < nk; s++) { if (eg_same_content(n, &g->nodes[keep[s]], key, kkey[s])) { dup = 1; break; } } if (dup) { wm_weights[i] = 0.0; _eg_act_wm_evicted++; _eg_act_dup_wm++; continue; } if (nk < ENGRAM_WM_CAP) { keep[nk] = i; kkey[nk] = key; nk++; if (nk == ENGRAM_WM_CAP) cap_w = dc[z].w; } } free(dc); } /* malloc failure: skip suppression — duplicates may share slots * this call, which is the pre-2026-08-05 behavior. No corruption. */ } } /* ── PASS 4: WM capacity cap (per-call) ───────────────────────────────── * Enforce ENGRAM_WM_CAP as a hard upper bound on nodes promoted in this * activation call. Without this, broad curiosity seeds like "knowledge" * promote 500+ nodes simultaneously — wm_avg_weight collapses to the * breakthrough floor, goal-bias differentiation is lost, and working memory * becomes useless. (Ported from 2026-06-26 self-review branch; observed * 525 promoted for "knowledge", 524 at breakthrough floor 0.25, 1 natural.) */ { /* Absolute admission floor (2026-07-30): drop sub-floor candidates * BEFORE rank-trimming, so the cap is filled only by nodes that clear * an absolute bar — fill below ENGRAM_WM_CAP becomes reachable and * wm_saturated becomes an informative signal. See ENGRAM_WM_FLOOR. */ for (int64_t i = 0; i < g->node_count; i++) { if (wm_weights[i] > 0.0 && wm_weights[i] < ENGRAM_WM_FLOOR) { wm_weights[i] = 0.0; _eg_act_wm_evicted++; } } int64_t cap_count = 0; for (int64_t i = 0; i < g->node_count; i++) { if (wm_weights[i] > 0.0) cap_count++; } if (cap_count > ENGRAM_WM_CAP) { double* cap_vals = malloc((size_t)cap_count * sizeof(double)); if (cap_vals) { int64_t ci = 0; for (int64_t i = 0; i < g->node_count; i++) { if (wm_weights[i] > 0.0) cap_vals[ci++] = wm_weights[i]; } qsort(cap_vals, (size_t)cap_count, sizeof(double), engram_cmp_double_desc); /* cap_vals[ENGRAM_WM_CAP-1] is the lowest weight that still * fits inside the cap when sorted descending. */ double cutoff = cap_vals[ENGRAM_WM_CAP - 1]; free(cap_vals); /* Count strictly above cutoff to handle ties correctly. */ int64_t above = 0; for (int64_t i = 0; i < g->node_count; i++) { if (wm_weights[i] > cutoff) above++; } int64_t at_cutoff_slots = ENGRAM_WM_CAP - above; /* Evict nodes that don't make the cut. */ for (int64_t i = 0; i < g->node_count; i++) { if (wm_weights[i] <= 0.0) continue; /* not promoted */ if (wm_weights[i] > cutoff) continue; /* above cutoff */ if (at_cutoff_slots > 0) { at_cutoff_slots--; continue; /* fills a slot */ } wm_weights[i] = 0.0; /* over cap: evict */ _eg_act_wm_evicted++; } } /* If malloc failed, skip cap — WM unbounded this call, no corruption. */ } } /* Pre-persist residency snapshot (2026-07-30): record which nodes held a * WM slot BEFORE this call's results are written back. Used below to fold * only NEW WM entrants into the context centroid — an incumbent that gets * re-promoted every scan no longer re-entrenches the centroid each time, * which was the remaining positive-feedback path in the WM→centroid→ * e_eff→re-selection loop (fixation driver; cf. wm_top0_streak=1407 * incident). NULL on OOM → fold falls back to previous behavior. */ unsigned char* was_wm = malloc((size_t)g->node_count); if (was_wm) { for (int64_t i = 0; i < g->node_count; i++) was_wm[i] = (g->nodes[i].working_memory_weight > 0.0) ? 1 : 0; } /* Persist working_memory_weight (post Pass 4) to node store. * * Conversational thread continuity (ENGRAM_WM_DECAY): * Nodes promoted in a previous turn but NOT reached by the current BFS * fan-out retain a decayed weight rather than being zeroed. This models * the brain's ability to maintain recent context across successive turns * without requiring explicit re-activation. A node that was relevant one * query ago stays weakly present in working memory; a node from two * queries ago retains 0.7² ≈ 0.49 of its original weight; after ~5 quiet * turns it falls below 0.01 and is effectively evicted (set to 0.0). * * NOTE: this was documented in the ENGRAM_WM_DECAY constant comment since * the two-layer architecture was introduced, but was never implemented — * unreached nodes were always zeroed unconditionally. Fixed 2026-06-30 * self-review. */ for (int64_t i = 0; i < g->node_count; i++) { if (!reached[i] && g->nodes[i].working_memory_weight > 0.0) { /* Carry-over decay: node held WM weight from prior activation but * the current query's BFS fan-out did not reach it. * * 2026-07-22 self-review (bl-b17facdd): replaced the per-call * multiplicative decay (weight *= ENGRAM_WM_DECAY) with the * ACT-R/Petrov base-level scheme. The old form was call-rate- * dependent — carried context died in seconds under rapid * curiosity scans and lingered for hours under quiet loops. * Now: hard-evict when base-level < τ (Soar-style forgetting); * above τ, shape the weight held at promotion (wm_anchor) by the * retrieval-probability logistic. Pure function of wall-clock * time — idempotent no matter how often activate is called. */ eg_wm_carry_over(&g->nodes[i], now_ms, &_eg_act_wm_evicted); } else if (wm_weights[i] > 0.0) { g->nodes[i].working_memory_weight = wm_weights[i]; /* Anchor the promotion weight: carry-over decay above computes * from this fixed point rather than compounding per call. */ g->nodes[i].wm_anchor = wm_weights[i]; } else if (was_wm && was_wm[i]) { /* ── Reached but sub-threshold (2026-08-04 self-review) ────────── * This case used to fall into the unconditional `= wm_weights[i]` * below, zeroing the slot outright — no carry-over, no base-level * check, not even counted as an eviction. The asymmetry was exactly * backwards: a node the current query did NOT reach got the full * ACT-R retention treatment, while a node the query DID reach, but * which landed a hair under its type threshold, was dropped * instantly. Being found was punished relative to not being found. * * MEASURED CONSEQUENCE: working memory turned over 100% on every * call. Three consecutive activations with a byte-identical query * gave |A∩B| = |B∩C| = 0 — no node survived a single call — while * wm_evicted stayed at 0 the whole time, because this path never * incremented it. WM was not a working set at all; it was six fresh * suppression-breakthrough nodes per call, re-drawn each time. That * silently defeated every mechanism built on WM continuity: the * conversational-thread carry-over documented since the two-layer * architecture landed, the wm_anchor fixed point, and (this * session) any possibility of learning from co-activation, since no * pair can co-activate twice if nothing survives one call. * * Same helper as the unreached path: one retention rule, both ways * out of a WM slot. The existing guards (τ hard-evict, WM_FLOOR, * occupancy decay, Pass 5 global cap) all still apply — routing * into them is why this is safe rather than merely sticky. */ eg_wm_carry_over(&g->nodes[i], now_ms, &_eg_act_wm_evicted); } else { g->nodes[i].working_memory_weight = 0.0; /* Zero the anchor when the slot empties (2026-07-30): a stale * anchor on an evicted node was a latent resurrection bug if the * carry-over entry guard ever changes. */ g->nodes[i].wm_anchor = 0.0; } } /* ── PASS 5: Global WM cap enforcement ─────────────────────────────────── * Pass 4 capped this call's new candidates. But nodes already in WM from * prior calls retain their persisted working_memory_weight (via the decay * carry-over above). Over multiple activation calls total WM can grow well * above ENGRAM_WM_CAP. This pass enforces the cap globally across ALL * nodes in the store, keeping only the top ENGRAM_WM_CAP by current weight. * Correct cognitive model: WM capacity is global (Cowan 2001); more recent * activations outcompete older decayed ones. (Ported from 2026-06-26 * self-review branch.) */ { /* Absolute admission floor, global pass (2026-07-30): see * ENGRAM_WM_FLOOR. Sub-floor residents are dropped even when the * global population is under cap — this is what lets wm_active * drain below 24 during quiet periods. */ for (int64_t i = 0; i < g->node_count; i++) { EngramNode* fn = &g->nodes[i]; if (fn->working_memory_weight > 0.0 && fn->working_memory_weight < ENGRAM_WM_FLOOR) { fn->working_memory_weight = 0.0; fn->wm_anchor = 0.0; _eg_act_wm_evicted++; } } /* ── Global redundancy suppression (2026-08-06 self-review) ────────── * Pass 3½ deduplicates THIS CALL'S candidates. But the WM population * that actually exists after persist is a UNION of two sets: nodes * promoted this call, and nodes carried over from earlier calls by * eg_wm_carry_over. Pass 3½ never sees the second set, so the dedup * guarantee it advertises does not hold for the thing it is a * guarantee about. * * OBSERVED, not inferred. A live WM census caught two byte-identical * Knowledge nodes — f93d5f90 and b578d6a7, the same 3,193-character * "# Memory Integration" document under the same node_type — both * holding slots at 0.289 and 0.271, having arrived by the two * different routes. Pass 3½ had run and had correctly passed, because * only one of the two was a candidate that call. * * The cost is small and constant: 1–2 of 24 slots, ~4–8% of working * memory, indefinitely. It is worth fixing anyway, because the failure * is silent and self-reinforcing — a duplicate that holds a slot gets * reinforced for holding it, and (as of this session) now also earns * Hebbian eligibility, so redundancy would start teaching the graph * that a document is associated with itself. * * Runs BEFORE the cap count below, so slots freed here are reclaimed * by distinct content in the same pass rather than left empty. Same * identity test and same highest-weight-survives rule as Pass 3½. */ { int64_t gn = 0; for (int64_t i = 0; i < g->node_count; i++) if (g->nodes[i].working_memory_weight > 0.0) gn++; if (gn > 1) { EgDupCand* gd = malloc((size_t)gn * sizeof(EgDupCand)); if (gd) { int64_t gi = 0; for (int64_t i = 0; i < g->node_count; i++) { if (g->nodes[i].working_memory_weight > 0.0) { gd[gi].w = g->nodes[i].working_memory_weight; gd[gi].idx = i; gi++; } } qsort(gd, (size_t)gn, sizeof(EgDupCand), eg_dupcand_cmp_desc); int64_t gkeep[ENGRAM_WM_CAP]; uint64_t gkey[ENGRAM_WM_CAP]; int gnk = 0; double gcap_w = 0.0; for (int64_t z = 0; z < gn; z++) { if (gnk >= ENGRAM_WM_CAP && gd[z].w < gcap_w) break; int64_t i = gd[z].idx; EngramNode* n = &g->nodes[i]; uint64_t key = eg_content_key(n); int dup = 0; for (int s = 0; s < gnk; s++) { if (eg_same_content(n, &g->nodes[gkeep[s]], key, gkey[s])) { dup = 1; break; } } if (dup) { n->working_memory_weight = 0.0; n->wm_anchor = 0.0; _eg_act_wm_evicted++; _eg_act_dup_wm_global++; continue; } if (gnk < ENGRAM_WM_CAP) { gkeep[gnk] = i; gkey[gnk] = key; gnk++; if (gnk == ENGRAM_WM_CAP) gcap_w = gd[z].w; } } free(gd); } /* malloc failure: skip — duplicates may share slots this call, * which is the pre-2026-08-06 behavior. No corruption. */ } } int64_t global_wm_count = 0; for (int64_t i = 0; i < g->node_count; i++) { if (g->nodes[i].working_memory_weight > 0.0) global_wm_count++; } if (global_wm_count > ENGRAM_WM_CAP) { double* gvals = malloc((size_t)global_wm_count * sizeof(double)); if (gvals) { int64_t gi = 0; for (int64_t i = 0; i < g->node_count; i++) { if (g->nodes[i].working_memory_weight > 0.0) gvals[gi++] = g->nodes[i].working_memory_weight; } qsort(gvals, (size_t)global_wm_count, sizeof(double), engram_cmp_double_desc); double gcutoff = gvals[ENGRAM_WM_CAP - 1]; free(gvals); int64_t gabove = 0; for (int64_t i = 0; i < g->node_count; i++) { if (g->nodes[i].working_memory_weight > gcutoff) gabove++; } int64_t gslots_at_cutoff = ENGRAM_WM_CAP - gabove; for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; if (n->working_memory_weight <= 0.0) continue; if (n->working_memory_weight > gcutoff) continue; if (gslots_at_cutoff > 0) { gslots_at_cutoff--; continue; /* fills a slot */ } n->working_memory_weight = 0.0; /* evict: over global cap */ n->wm_anchor = 0.0; /* keep anchor coherent */ _eg_act_wm_evicted++; /* was uncounted before 2026-08-02 */ } } /* If malloc failed, skip — WM over cap this call, no data corruption. */ } } /* ── Retrieval reinforcement (2026-07-18 self-review) ─────────────────── * ACT-R base-level learning: retrieval strengthens memory. Before this, * NOTHING in engram_activate updated last_activated/activation_count — * only the rarely-called engram_strengthen did. Consequence: temporal * decay aged every node from its last explicit strengthen (usually * creation), so frequently-retrieved memories decayed identically to * abandoned ones, and engram_activation_dampen() saw a frozen count. * * Scope deliberately narrow: reinforce ONLY nodes promoted to WM in THIS * call that survived both capacity caps (wm_weights[i] > 0 = promoted this * call; working_memory_weight > 0 = survived Pass 4/5 eviction). BFS * fan-out touches thousands of nodes per curiosity scan — reinforcing all * of them would flatten dampening and freeze decay globally. Promotion to * working memory is the analog of actual retrieval (executive access), * matching ACT-R where only completed retrievals add a base-level * presentation. Carry-over nodes (reached[i]==0) are NOT reinforced: they * persist by decay, they were not re-retrieved. */ for (int64_t i = 0; i < g->node_count; i++) { if (!reached[i] || wm_weights[i] <= 0.0) continue; EngramNode* n = &g->nodes[i]; if (n->working_memory_weight <= 0.0) continue; /* evicted by cap */ n->last_activated = now_ms; n->activation_count++; /* Base-level presentation: WM promotion is the retrieval event. * (2026-07-22 self-review — feeds engram_bll_base_level.) */ engram_bll_record_access(n, now_ms); } /* ── Hebbian edge potentiation (2026-08-04 self-review) ───────────────── * The edge-level counterpart of the node-level reinforcement immediately * above. That loop says "this memory was retrieved"; this one says "these * two memories were retrieved TOGETHER, so the path between them is worth * more than it was". Runs on final post-Pass-5 working memory, so only * pairs that survived both capacity caps count as co-active — the same * "promotion to WM is the analog of actual retrieval" standard the ACT-R * reinforcement above uses. Consistency matters: two mechanisms disagreeing * about what counts as a retrieval would drift apart invisibly. * * Three steps: (1) decay every edge, so disuse fades; (2) increment * co-active pairs; (3) homeostatic scaling so no node accumulates * unbounded associative mass. All three are required — see ENGRAM_HEBB_*. */ { unsigned char* in_wm = calloc((size_t)g->node_count, 1); if (in_wm) { int64_t wm_n = 0; for (int64_t i = 0; i < g->node_count; i++) { if (g->nodes[i].working_memory_weight > 0.0) { in_wm[i] = 1; wm_n++; } } /* Step 0 (2026-08-06): refresh the eligibility trace of everything * currently in WM. Done BEFORE the edge pass so a pair that is * co-resident right now reads trace 1.0 on both ends and receives * exactly ENGRAM_HEBB_ETA — identical to the pre-trace rule. */ for (int64_t i = 0; i < g->node_count; i++) { if (!in_wm[i]) continue; g->nodes[i].hebb_elig = 1.0; g->nodes[i].hebb_elig_ts = now_ms; } /* Steps 1+2: decay all, potentiate co-active. Fused into one O(E) * pass. Edges whose endpoints resolve to nothing still decay — * a dangling edge should not hold learned strength forever. * * Co-activation is now graded by the product of the endpoints' * eligibility traces rather than gated on same-call WM residency; * see the ENGRAM_HEBB_TRACE_* block for the census that forced * this. The old `wm_n > 1` guard is gone because a single node * entering WM can now legitimately potentiate against a partner * that was in WM moments ago — that asymmetric case is precisely * the signal the simultaneity rule was throwing away. When no * trace is warm, co is 0 and the pass degenerates to pure decay. */ for (int64_t ei = 0; ei < g->edge_count; ei++) { EngramEdge* e = &g->edges[ei]; double h = e->hebb * ENGRAM_HEBB_DECAY; if (!e->inhibitory) { int64_t a = engram_idmap_get(g, e->from_id); int64_t b = engram_idmap_get(g, e->to_id); if (a >= 0 && a < g->node_count && b >= 0 && b < g->node_count) { double co = eg_hebb_trace(&g->nodes[a], now_ms) * eg_hebb_trace(&g->nodes[b], now_ms); if (co > 0.0) { h += ENGRAM_HEBB_ETA * co; e->last_fired = now_ms; /* first real writer outside dharma_strengthen */ } } } e->hebb = (h < ENGRAM_HEBB_MIN) ? 0.0 : h; } /* Step 3: homeostatic scaling. Sum incident hebb per node; any node * over budget scales ALL its incident edges down proportionally. * An edge is scaled by the stronger (smaller) of its two endpoints' * factors, so one pass satisfies both endpoints' constraints — * conservative, non-iterative, and stable. Skipped on OOM: the * potentiation above is still correct, just uncompensated for one * call, and the next call re-normalizes. */ double* mass = calloc((size_t)g->node_count, sizeof(double)); if (mass) { for (int64_t ei = 0; ei < g->edge_count; ei++) { EngramEdge* e = &g->edges[ei]; if (e->hebb <= 0.0) continue; int64_t a = engram_idmap_get(g, e->from_id); int64_t b = engram_idmap_get(g, e->to_id); if (a >= 0 && a < g->node_count) mass[a] += e->hebb; if (b >= 0 && b < g->node_count) mass[b] += e->hebb; } for (int64_t ei = 0; ei < g->edge_count; ei++) { EngramEdge* e = &g->edges[ei]; if (e->hebb <= 0.0) continue; int64_t a = engram_idmap_get(g, e->from_id); int64_t b = engram_idmap_get(g, e->to_id); double s = 1.0; if (a >= 0 && a < g->node_count && mass[a] > ENGRAM_HEBB_NODE_BUDGET) s = ENGRAM_HEBB_NODE_BUDGET / mass[a]; if (b >= 0 && b < g->node_count && mass[b] > ENGRAM_HEBB_NODE_BUDGET) { double sb = ENGRAM_HEBB_NODE_BUDGET / mass[b]; if (sb < s) s = sb; } if (s < 1.0) { double h = e->hebb * s; e->hebb = (h < ENGRAM_HEBB_MIN) ? 0.0 : h; } } free(mass); } /* ── Associative link formation ────────────────────────────── * Everything above reweights edges that already exist. This part * grows the ones that don't. See the ENGRAM_HEBB_LINK_* block for * why this is gated as hard as it is. */ { /* Decay every candidate slot, exactly as edges decay, so an * association that stops recurring loses ground at the same * rate whether or not it has been consolidated yet. */ for (int s = 0; s < ENGRAM_HEBB_CAND_SLOTS; s++) { EgHebbCand* c = &_eg_hebb_cand[s]; if (!c->a) continue; c->score *= ENGRAM_HEBB_DECAY; if (c->score < ENGRAM_HEBB_MIN) eg_hebb_slot_clear(c); } /* Gather this call's WM members (bounded by ENGRAM_WM_CAP, so * at most 276 pairs — the O(n²) here is over ≤24 items). */ int64_t wm_idx[ENGRAM_WM_CAP]; int wm_k = 0; for (int64_t i = 0; i < g->node_count && wm_k < ENGRAM_WM_CAP; i++) { if (in_wm[i]) wm_idx[wm_k++] = i; } /* Warm set (2026-08-06): nodes NOT in WM right now but whose * eligibility trace is still up. These are the partners the * simultaneity rule could never see. Bounded by * ENGRAM_HEBB_WARM_MAX; the scan is O(node_count), which is * an order of magnitude cheaper than the O(edge_count) pass * already running above it. */ int64_t warm_idx[ENGRAM_HEBB_WARM_MAX]; double warm_t[ENGRAM_HEBB_WARM_MAX]; int warm_k = 0; for (int64_t i = 0; i < g->node_count && warm_k < ENGRAM_HEBB_WARM_MAX; i++) { if (in_wm[i]) continue; double t = eg_hebb_trace(&g->nodes[i], now_ms); if (t <= 0.0) continue; warm_idx[warm_k] = i; warm_t[warm_k] = t; warm_k++; } _eg_act_hebb_warm = warm_k; (void)wm_n; /* superseded as a gate by the trace product */ /* Reinforce candidate scores. Two families, one rule: * WM × WM — both traces are 1.0 ⇒ increment ETA, exactly * the pre-2026-08-06 behavior. * WM × warm — increment ETA·trace, so a partner that left * working memory one cycle ago still earns most * of the credit and one that left ten cycles ago * earns a third of it. * warm × warm is deliberately NOT paired: with nothing currently * active there is no event to be eligible FOR, and pairing decayed * residue against decayed residue would manufacture associations * out of two absences. Eligibility gates on something happening * now — that is the whole content of the three-factor rule. */ for (int x = 0; x < wm_k; x++) { for (int y = x + 1; y < wm_k; y++) { eg_hebb_cand_bump(g->nodes[wm_idx[x]].id, g->nodes[wm_idx[y]].id, ENGRAM_HEBB_ETA); } for (int w = 0; w < warm_k; w++) { eg_hebb_cand_bump(g->nodes[wm_idx[x]].id, g->nodes[warm_idx[w]].id, ENGRAM_HEBB_ETA * warm_t[w]); } } /* Consolidate the strongest qualifying candidates into real * edges. Done last and separately because engram_grow_edges() * may realloc g->edges — no EngramEdge* may be held across * this point. */ int formed = 0; /* Count existing self-formed edges once, up front: the cap is * on total learned structure, not on this call's rate. */ int64_t hebb_edge_total = 0; for (int64_t i = 0; i < g->edge_count; i++) { if (g->edges[i].relation && strcmp(g->edges[i].relation, "hebbian-associate") == 0) hebb_edge_total++; } int64_t hebb_edge_cap = (int64_t)((double)g->edge_count * ENGRAM_HEBB_LINK_MAX_FRAC); for (int s = 0; s < ENGRAM_HEBB_CAND_SLOTS && formed < ENGRAM_HEBB_LINK_PER_CALL && hebb_edge_total < hebb_edge_cap; s++) { EgHebbCand* c = &_eg_hebb_cand[s]; if (!c->a || c->score < ENGRAM_HEBB_LINK_MIN) continue; if (engram_idmap_get(g, c->a) < 0 || engram_idmap_get(g, c->b) < 0) { /* node gone */ eg_hebb_slot_clear(c); continue; } if (eg_edge_exists_between(g, c->a, c->b)) { eg_hebb_slot_clear(c); continue; /* already wired */ } engram_grow_edges(); EngramEdge* ne = &g->edges[g->edge_count]; memset(ne, 0, sizeof(*ne)); ne->id = engram_new_id(); ne->from_id = el_strdup_persist(c->a); ne->to_id = el_strdup_persist(c->b); ne->relation = el_strdup_persist("hebbian-associate"); ne->metadata = el_strdup_persist("{\"origin\":\"co-activation\"}"); ne->weight = ENGRAM_HEBB_LINK_W0; /* Carry the earned score across so a freshly consolidated * edge starts where the association already is, rather * than restarting a climb it has already made. */ ne->hebb = c->score; ne->confidence = 1.0; ne->created_at = now_ms; ne->updated_at = now_ms; ne->last_fired = now_ms; ne->layer_id = ENGRAM_LAYER_DEFAULT; g->edge_count++; g->adj_dirty = 1; _eg_hebb_links_formed++; hebb_edge_total++; formed++; /* Hand the association to the durable store. In the soul * daemon this local edge is the ONLY copy and dies with the * process — see the ENGRAM_HEBB_WB_SLOTS block. Enqueue * from ne->from_id/to_id rather than c->a/c->b: the slot is * cleared on the next line and the edge now owns the ids. */ eg_hebb_wb_push(ne->from_id, ne->to_id, ne->weight, ne->hebb); eg_hebb_slot_clear(c); /* the edge is the record now */ } } free(in_wm); } } /* ── Collect all background-activated nodes for the return value ──── * Callers see both layers. Context compilation uses only promoted nodes * (working_memory_weight > 0). Sort: promoted first by wm_weight desc, * then background-only by background_activation desc. */ typedef struct { int64_t idx; double bg; double wm; double epist; int64_t hops; } Result; Result* results = malloc((size_t)g->node_count * sizeof(Result)); int64_t rcount = 0; if (!results) { free(best_bg); free(best_hops); free(reached); free(seeds); free(fr); free(inhibition); free(wm_weights); free(cosq); free(was_wm); return out; } for (int64_t i = 0; i < g->node_count; i++) { if (!reached[i]) continue; double epist = best_bg[i] * g->nodes[i].confidence; /* Include if promoted to working memory OR if background activation * is meaningful enough to report (epist >= 0.1). */ if (epist < 0.1 && wm_weights[i] <= 0.0) continue; results[rcount].idx = i; results[rcount].bg = best_bg[i]; results[rcount].wm = wm_weights[i]; results[rcount].epist = epist; results[rcount].hops = best_hops[i]; rcount++; } /* Sort: promoted nodes first (by wm_weight desc), then background-only * by background_activation desc. */ for (int64_t i = 1; i < rcount; i++) { Result key = results[i]; int64_t j = i - 1; while (j >= 0 && (results[j].wm < key.wm || (results[j].wm == key.wm && results[j].bg < key.bg))) { results[j + 1] = results[j]; j--; } results[j + 1] = key; } /* ── Context centroid: fold in the touched nodes (2026-07-29) ─────── * Results are sorted promoted-first by wm_weight desc, so the first * ENGRAM_CTX_TOUCH_MAX embedded entries with wm > 0 are exactly the * strongest WM survivors of THIS call — the same "promotion is the * retrieval event" rule the BLL reinforcement pass uses. μ=0.9 EMA * keeps any single scan's touches a minority contribution. */ { int touched = 0; for (int64_t i = 0; i < rcount && touched < ENGRAM_CTX_TOUCH_MAX; i++) { if (results[i].wm <= 0.0) break; /* promoted block exhausted */ EngramNode* n = &g->nodes[results[i].idx]; if (!n->emb || n->emb_dim <= 0) continue; /* New-entrant gate (2026-07-30): skip nodes that already held a * WM slot before this call — incumbents must not keep pulling * the centroid toward themselves. Fresh topical shifts (new * entrants + the query fold at call start) steer it instead. */ if (was_wm && was_wm[results[i].idx]) continue; eg_ctx_blend(n->emb, n->emb_dim); touched++; } } free(was_wm); for (int64_t i = 0; i < rcount; i++) { el_val_t entry = el_map_new(0); entry = el_map_set(entry, EL_STR(el_strdup("node")), engram_node_to_map(&g->nodes[results[i].idx])); entry = el_map_set(entry, EL_STR(el_strdup("activation_strength")), el_from_float(results[i].bg)); entry = el_map_set(entry, EL_STR(el_strdup("working_memory_weight")), el_from_float(results[i].wm)); entry = el_map_set(entry, EL_STR(el_strdup("epistemic_confidence")), el_from_float(results[i].epist)); entry = el_map_set(entry, EL_STR(el_strdup("hops")), (el_val_t)results[i].hops); entry = el_map_set(entry, EL_STR(el_strdup("promoted")), (el_val_t)(results[i].wm > 0.0 ? 1 : 0)); out = el_list_append(out, entry); } free(best_bg); free(best_hops); free(reached); free(seeds); free(fr); free(inhibition); free(wm_weights); free(results); free(cosq); return out; } /* ── Engram persistence (JSON snapshot) ─────────────────────────────────── */ /* include_emb (2026-07-31 self-review): the ~5.7KB "emb" vector belongs ONLY * in persistence/replication output (engram_save → snapshot.json, which also * backs /api/sync and /api/edges via scratch exports). Consumer read routes * (/api/nodes, /api/search, activation results, neighbors, compiled context) * were shipping it on every node — responses 10-50x oversized, blowing MCP * token limits. Pass include_emb=1 only from engram_save. */ static void engram_emit_node_json(JsonBuf* b, const EngramNode* n, int include_emb) { jb_putc(b, '{'); jb_puts(b, "\"id\":"); jb_emit_escaped(b, n->id ? n->id : ""); jb_puts(b, ",\"content\":"); jb_emit_escaped(b, n->content ? n->content : ""); jb_puts(b, ",\"node_type\":"); jb_emit_escaped(b, n->node_type ? n->node_type : ""); jb_puts(b, ",\"label\":"); jb_emit_escaped(b, n->label ? n->label : ""); jb_puts(b, ",\"tier\":"); jb_emit_escaped(b, n->tier ? n->tier : "Working"); jb_puts(b, ",\"tags\":"); jb_emit_escaped(b, n->tags ? n->tags : ""); jb_puts(b, ",\"metadata\":"); jb_emit_escaped(b, n->metadata ? n->metadata : "{}"); char tmp[80]; snprintf(tmp, sizeof(tmp), ",\"salience\":%g", n->salience); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"importance\":%g", n->importance); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"confidence\":%g", n->confidence); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"temporal_decay_rate\":%g", n->temporal_decay_rate); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"activation_count\":%lld", (long long)n->activation_count); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"last_activated\":%lld", (long long)n->last_activated); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"created_at\":%lld", (long long)n->created_at); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"updated_at\":%lld", (long long)n->updated_at); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"background_activation\":%g", n->background_activation); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"working_memory_weight\":%g", n->working_memory_weight); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"suppression_count\":%d", n->suppression_count); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"layer_id\":%u", n->layer_id); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"wm_anchor\":%g", n->wm_anchor); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"base_level\":%g", engram_bll_base_level(n, engram_now_ms())); jb_puts(b, tmp); /* Base-level access history: chronological (oldest→newest) compact * string. Loaders replay it through engram_bll_record_access; absent * field = empty ring (optimized-form fallback). (2026-07-22) */ if (n->access_filled > 0) { jb_puts(b, ",\"access_ts\":\""); for (int32_t j = 0; j < n->access_filled; j++) { int32_t idx = (n->access_head - n->access_filled + j + 2 * ENGRAM_BLL_K) % ENGRAM_BLL_K; snprintf(tmp, sizeof(tmp), "%s%lld", j ? "," : "", (long long)n->access_ts[idx]); jb_puts(b, tmp); } jb_putc(b, '"'); } /* Semantic embedding (2026-07-24): compact %.4g comma list — cosine is * insensitive to 4-sig-fig rounding, and this keeps snapshot bloat to * ~5KB per embedded node without a base64 codec. Absent field = not * embedded; the lazy backfill re-embeds eventually if dropped. */ if (include_emb && n->emb && n->emb_dim > 0) { jb_puts(b, ",\"emb\":\""); for (int32_t j = 0; j < n->emb_dim; j++) { snprintf(tmp, sizeof(tmp), "%s%.4g", j ? "," : "", (double)n->emb[j]); jb_puts(b, tmp); } jb_putc(b, '"'); } jb_putc(b, '}'); } static void engram_emit_edge_json(JsonBuf* b, const EngramEdge* e) { jb_putc(b, '{'); jb_puts(b, "\"id\":"); jb_emit_escaped(b, e->id ? e->id : ""); jb_puts(b, ",\"from_id\":"); jb_emit_escaped(b, e->from_id ? e->from_id : ""); jb_puts(b, ",\"to_id\":"); jb_emit_escaped(b, e->to_id ? e->to_id : ""); jb_puts(b, ",\"relation\":"); jb_emit_escaped(b, e->relation ? e->relation : ""); jb_puts(b, ",\"metadata\":"); jb_emit_escaped(b, e->metadata ? e->metadata : "{}"); char tmp[64]; snprintf(tmp, sizeof(tmp), ",\"weight\":%g", e->weight); jb_puts(b, tmp); /* Learned potentiation is persisted: it is the graph's accumulated * associative experience and must survive restarts, or the system relearns * from zero every boot and never accumulates. Emitted only when nonzero so * a cold snapshot stays byte-comparable to the pre-change format. * (2026-08-04 self-review.) */ if (e->hebb > 0.0) { snprintf(tmp, sizeof(tmp), ",\"hebb\":%.6g", e->hebb); jb_puts(b, tmp); } snprintf(tmp, sizeof(tmp), ",\"confidence\":%g", e->confidence); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"created_at\":%lld", (long long)e->created_at); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"updated_at\":%lld", (long long)e->updated_at); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"last_fired\":%lld", (long long)e->last_fired); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"inhibitory\":%d", e->inhibitory ? 1 : 0); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"layer_id\":%u", e->layer_id); jb_puts(b, tmp); jb_putc(b, '}'); } el_val_t engram_save(el_val_t path) { const char* p = EL_CSTR(path); if (!p || !*p) return 0; EngramStore* g = engram_get(); JsonBuf b; jb_init(&b); jb_puts(&b, "{\"nodes\":["); for (int64_t i = 0; i < g->node_count; i++) { if (i > 0) jb_putc(&b, ','); engram_emit_node_json(&b, &g->nodes[i], 1); } jb_puts(&b, "],\"edges\":["); for (int64_t i = 0; i < g->edge_count; i++) { if (i > 0) jb_putc(&b, ','); engram_emit_edge_json(&b, &g->edges[i]); } /* Layered consciousness — emit the layer registry under "layers". * Older readers that don't know about this top-level key will simply * ignore it (forward compatible). Tombstoned (removed-injectable) * layers are skipped — they have no name and can't be re-created * meaningfully on load anyway. */ jb_puts(&b, "],\"layers\":["); int first_layer = 1; for (size_t i = 0; i < g->layer_count; i++) { EngramLayer* L = &g->layers[i]; if (!L->name) continue; if (!first_layer) jb_putc(&b, ','); first_layer = 0; jb_putc(&b, '{'); char tmp[80]; snprintf(tmp, sizeof(tmp), "\"layer_id\":%u", L->layer_id); jb_puts(&b, tmp); jb_puts(&b, ",\"name\":"); jb_emit_escaped(&b, L->name); snprintf(tmp, sizeof(tmp), ",\"activation_priority\":%u", L->activation_priority); jb_puts(&b, tmp); snprintf(tmp, sizeof(tmp), ",\"suppressible\":%d", L->suppressible ? 1 : 0); jb_puts(&b, tmp); snprintf(tmp, sizeof(tmp), ",\"transparent\":%d", L->transparent ? 1 : 0); jb_puts(&b, tmp); snprintf(tmp, sizeof(tmp), ",\"injectable\":%d", L->injectable ? 1 : 0); jb_puts(&b, tmp); jb_putc(&b, '}'); } jb_puts(&b, "]}"); FILE* f = fopen(p, "wb"); if (!f) { free(b.buf); return 0; } size_t w = fwrite(b.buf, 1, b.len, f); fclose(f); int ok = (w == b.len); free(b.buf); return ok ? 1 : 0; } /* Helper: extract a string field from a JSON object substring. */ static char* eg_get_str_field(const char* obj, const char* key) { /* Returns a STORE-PERSISTENT string: callers assign the result directly * into EngramNode/EngramEdge fields, and engram_load_merge runs inside * the soul's per-tick arena. jp_parse_string_raw's success buffer is * plain malloc (persist-safe, returned as-is); the empty/error returns * were arena-tracked el_strdup("") — those dangled after the tick arena * popped and free()ing them in callers was a latent double-free. */ const char* p = json_find_key(obj, key); if (!p) return el_strdup_persist(""); if (*p != '"') return el_strdup_persist(""); JsonParser jp = { .p = p, .end = p + strlen(p), .err = 0 }; char* out = jp_parse_string_raw(&jp); /* *p == '"' is guaranteed above, so jp_parse_string_raw took its malloc * path (its arena-tracked early-return only fires on a non-'"' start) — * free(out) on error is safe here. */ if (jp.err) { free(out); return el_strdup_persist(""); } return out; } static double eg_get_num_field(const char* obj, const char* key) { const char* p = json_find_key(obj, key); if (!p || *p == '"' || *p == '{' || *p == '[') return 0.0; return strtod(p, NULL); } static int64_t eg_get_int_field(const char* obj, const char* key) { const char* p = json_find_key(obj, key); if (!p || *p == '"' || *p == '{' || *p == '[') return 0; return strtoll(p, NULL, 10); } /* Iterate the top-level nodes/edges arrays in a saved snapshot. */ static const char* eg_skip_ws(const char* p) { while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; return p; } /* eg_enforce_wm_cap_on_load — clamp a snapshot-restored working-memory * population to ENGRAM_WM_CAP. * * WHY (2026-07-15 self-review): engram_load restored working_memory_weight * verbatim from the snapshot with no cap. Pass 4/5 cap enforcement only runs * inside engram_activate — so a snapshot frozen in the pre-2026-06-30 era * (87-111 promoted nodes, all at the old 0.25 breakthrough floor) reloaded * as-is on every boot, and heartbeats faithfully reported the stale * population (wm_active 87-111, wm_avg pinned at exactly 0.25) forever. * The cap must hold at every entry point that materializes WM, not just * the activation path. Same top-K-by-weight logic as engram_activate * Pass 5 (Cowan 2001: WM capacity is global). */ static void eg_enforce_wm_cap_on_load(EngramStore* g) { /* Absolute admission floor (2026-07-30): see ENGRAM_WM_FLOOR. */ for (int64_t i = 0; i < g->node_count; i++) { EngramNode* fn = &g->nodes[i]; if (fn->working_memory_weight > 0.0 && fn->working_memory_weight < ENGRAM_WM_FLOOR) { fn->working_memory_weight = 0.0; fn->wm_anchor = 0.0; } } int64_t wm_count = 0; for (int64_t i = 0; i < g->node_count; i++) { if (g->nodes[i].working_memory_weight > 0.0) wm_count++; } if (wm_count <= ENGRAM_WM_CAP) return; double* vals = malloc((size_t)wm_count * sizeof(double)); if (!vals) return; /* skip on OOM — over cap this boot, no corruption */ int64_t vi = 0; for (int64_t i = 0; i < g->node_count; i++) { if (g->nodes[i].working_memory_weight > 0.0) vals[vi++] = g->nodes[i].working_memory_weight; } qsort(vals, (size_t)wm_count, sizeof(double), engram_cmp_double_desc); double cutoff = vals[ENGRAM_WM_CAP - 1]; free(vals); int64_t above = 0; for (int64_t i = 0; i < g->node_count; i++) { if (g->nodes[i].working_memory_weight > cutoff) above++; } int64_t slots_at_cutoff = ENGRAM_WM_CAP - above; for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; if (n->working_memory_weight <= 0.0) continue; if (n->working_memory_weight > cutoff) continue; if (slots_at_cutoff > 0) { slots_at_cutoff--; continue; } n->working_memory_weight = 0.0; /* evict: over cap at load */ n->wm_anchor = 0.0; /* keep anchor coherent */ } } el_val_t engram_load(el_val_t path) { const char* p = EL_CSTR(path); if (!p || !*p) return 0; FILE* f = fopen(p, "rb"); if (!f) return 0; fseek(f, 0, SEEK_END); long sz = ftell(f); rewind(f); if (sz <= 0) { fclose(f); return 0; } char* data = malloc((size_t)sz + 1); if (!data) { fclose(f); return 0; } size_t got = fread(data, 1, (size_t)sz, f); fclose(f); data[got] = '\0'; /* Reset store */ EngramStore* g = engram_get(); for (int64_t i = 0; i < g->node_count; i++) { free(g->nodes[i].id); free(g->nodes[i].content); free(g->nodes[i].node_type); free(g->nodes[i].label); free(g->nodes[i].tier); free(g->nodes[i].tags); free(g->nodes[i].metadata); /* 2026-07-26 self-review: emb was the one heap field not freed * here — ~3 KB leaked per embedded node per reload (~11 MB per * reload at 3.7k embedded). forget/prune already free it. */ free(g->nodes[i].emb); g->nodes[i].emb = NULL; g->nodes[i].emb_dim = 0; } g->node_count = 0; for (int64_t i = 0; i < g->edge_count; i++) { free(g->edges[i].id); free(g->edges[i].from_id); free(g->edges[i].to_id); free(g->edges[i].relation); free(g->edges[i].metadata); } g->edge_count = 0; engram_idmap_free(g); engram_adj_free(g); /* Walk nodes array */ const char* nodes_p = json_find_key(data, "nodes"); if (nodes_p) { nodes_p = eg_skip_ws(nodes_p); if (*nodes_p == '[') { nodes_p++; nodes_p = eg_skip_ws(nodes_p); while (*nodes_p && *nodes_p != ']') { if (*nodes_p != '{') { nodes_p++; continue; } const char* end = json_skip_value(nodes_p); size_t n = (size_t)(end - nodes_p); char* obj = malloc(n + 1); memcpy(obj, nodes_p, n); obj[n] = '\0'; engram_grow_nodes(); EngramNode* nn = &g->nodes[g->node_count]; memset(nn, 0, sizeof(*nn)); nn->id = eg_get_str_field(obj, "id"); nn->content = eg_get_str_field(obj, "content"); nn->node_type = eg_get_str_field(obj, "node_type"); nn->label = eg_get_str_field(obj, "label"); nn->tier = eg_get_str_field(obj, "tier"); nn->tags = eg_get_str_field(obj, "tags"); nn->metadata = eg_get_str_field(obj, "metadata"); if (!nn->metadata || !*nn->metadata) { free(nn->metadata); nn->metadata = el_strdup_persist("{}"); } nn->salience = eg_get_num_field(obj, "salience"); nn->importance = eg_get_num_field(obj, "importance"); nn->confidence = eg_get_num_field(obj, "confidence"); nn->temporal_decay_rate = eg_get_num_field(obj, "temporal_decay_rate"); /* temporal_decay_rate defaults to 0 (use global) if absent in snapshot */ nn->activation_count = eg_get_int_field(obj, "activation_count"); nn->last_activated = eg_get_int_field(obj, "last_activated"); nn->created_at = eg_get_int_field(obj, "created_at"); nn->updated_at = eg_get_int_field(obj, "updated_at"); nn->background_activation = eg_get_num_field(obj, "background_activation"); nn->working_memory_weight = eg_get_num_field(obj, "working_memory_weight"); /* Launder persisted WM weights across restarts: snapshots carry * legacy breakthrough-floor 0.25 weights (pinned by the old * suppression-breakthrough path) and nothing else ever launders * them. Halve on every boot so genuine working-memory state has * continuity across a restart while stale pinned weights decay * out over successive boots; sub-0.05 residue drops to zero. */ nn->working_memory_weight *= 0.5; if (nn->working_memory_weight < ENGRAM_WM_FLOOR) nn->working_memory_weight = 0.0; nn->suppression_count = (int32_t)eg_get_int_field(obj, "suppression_count"); /* layer_id defaults to ENGRAM_LAYER_DEFAULT (core-identity) * for snapshots that predate the layered schema. We can't * tell "explicit 0" from "missing field" using the helper * directly, so probe for the key — if absent, fall back. */ if (json_find_key(obj, "layer_id")) { nn->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); } else { nn->layer_id = ENGRAM_LAYER_DEFAULT; } /* Base-level state (2026-07-22): absent fields leave the ring * empty (memset) → optimized-form fallback. */ nn->wm_anchor = eg_get_num_field(obj, "wm_anchor"); { char* ats = eg_get_str_field(obj, "access_ts"); if (ats) { engram_bll_parse_access(nn, ats); free(ats); } } { char* es = eg_get_str_field(obj, "emb"); if (es) { eg_parse_emb(nn, es); free(es); } } int64_t load_idx = g->node_count; g->node_count++; if (nn->id && *nn->id) engram_idmap_put(g, nn->id, load_idx); free(obj); nodes_p = end; nodes_p = eg_skip_ws(nodes_p); if (*nodes_p == ',') { nodes_p++; nodes_p = eg_skip_ws(nodes_p); } } } } g->adj_dirty = 1; /* Walk edges array */ const char* edges_p = json_find_key(data, "edges"); if (edges_p) { edges_p = eg_skip_ws(edges_p); if (*edges_p == '[') { edges_p++; edges_p = eg_skip_ws(edges_p); while (*edges_p && *edges_p != ']') { if (*edges_p != '{') { edges_p++; continue; } const char* end = json_skip_value(edges_p); size_t n = (size_t)(end - edges_p); char* obj = malloc(n + 1); memcpy(obj, edges_p, n); obj[n] = '\0'; engram_grow_edges(); EngramEdge* ee = &g->edges[g->edge_count]; memset(ee, 0, sizeof(*ee)); ee->id = eg_get_str_field(obj, "id"); ee->from_id = eg_get_str_field(obj, "from_id"); ee->to_id = eg_get_str_field(obj, "to_id"); ee->relation = eg_get_str_field(obj, "relation"); ee->metadata = eg_get_str_field(obj, "metadata"); if (!ee->metadata || !*ee->metadata) { free(ee->metadata); ee->metadata = el_strdup_persist("{}"); } ee->weight = eg_get_num_field(obj, "weight"); ee->hebb = eg_get_num_field(obj, "hebb"); /* absent ⇒ 0 */ ee->confidence = eg_get_num_field(obj, "confidence"); ee->created_at = eg_get_int_field(obj, "created_at"); ee->updated_at = eg_get_int_field(obj, "updated_at"); ee->last_fired = eg_get_int_field(obj, "last_fired"); ee->inhibitory = (int)eg_get_int_field(obj, "inhibitory"); if (json_find_key(obj, "layer_id")) { ee->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); } else { ee->layer_id = ENGRAM_LAYER_DEFAULT; } g->edge_count++; free(obj); edges_p = end; edges_p = eg_skip_ws(edges_p); if (*edges_p == ',') { edges_p++; edges_p = eg_skip_ws(edges_p); } } } } /* Walk layers array (optional — older snapshots omit this). * If present we replace the canonical registry entirely; if absent we * keep whatever the engram_get() init established. */ const char* layers_p = json_find_key(data, "layers"); if (layers_p) { layers_p = eg_skip_ws(layers_p); if (*layers_p == '[') { /* Reset existing layer registry. Free strdup'd names; the * struct array itself can be reused. */ for (size_t i = 0; i < g->layer_count; i++) { if (g->layers[i].name) free(g->layers[i].name); g->layers[i].name = NULL; } g->layer_count = 0; layers_p++; layers_p = eg_skip_ws(layers_p); while (*layers_p && *layers_p != ']') { if (*layers_p != '{') { layers_p++; continue; } const char* end = json_skip_value(layers_p); size_t n = (size_t)(end - layers_p); char* obj = malloc(n + 1); memcpy(obj, layers_p, n); obj[n] = '\0'; if (g->layer_count >= g->layer_capacity) { size_t nc = g->layer_capacity ? g->layer_capacity * 2 : 16; EngramLayer* grown = realloc(g->layers, nc * sizeof(EngramLayer)); if (!grown) { fputs("el_runtime: out of memory\n", stderr); exit(1); } memset(grown + g->layer_capacity, 0, (nc - g->layer_capacity) * sizeof(EngramLayer)); g->layers = grown; g->layer_capacity = nc; } EngramLayer* L = &g->layers[g->layer_count]; memset(L, 0, sizeof(*L)); L->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); L->activation_priority = (uint32_t)eg_get_int_field(obj, "activation_priority"); L->suppressible = (int)eg_get_int_field(obj, "suppressible") ? 1 : 0; L->transparent = (int)eg_get_int_field(obj, "transparent") ? 1 : 0; L->injectable = (int)eg_get_int_field(obj, "injectable") ? 1 : 0; char* nm = eg_get_str_field(obj, "name"); if (nm && *nm) { L->name = el_strdup_persist(nm); free(nm); } else { free(nm); L->name = el_strdup_persist(""); } g->layer_count++; free(obj); layers_p = end; layers_p = eg_skip_ws(layers_p); if (*layers_p == ',') { layers_p++; layers_p = eg_skip_ws(layers_p); } } } } /* WM cap discipline applies to every entry point that materializes WM, * including snapshot restore (see eg_enforce_wm_cap_on_load). */ eg_enforce_wm_cap_on_load(g); free(data); return 1; } /* engram_load_merge — like engram_load but WITHOUT resetting the store. * Reads a JSON snapshot from `path` and adds any nodes/edges not already * present in the in-memory graph. Dedup is by node id (for nodes) and by * (from_id, to_id, relation) tuple (for edges). * * Returns (as an EL int) the count of new nodes added. Used by the soul * daemon's periodic refresh cycle to keep its in-process Engram in sync * with the HTTP Engram store without losing current working memory state. * Ported from el-compiler/runtime on 2026-06-30 self-review. */ el_val_t engram_load_merge(el_val_t path) { const char* p = EL_CSTR(path); if (!p || !*p) return 0; FILE* f = fopen(p, "rb"); if (!f) return 0; fseek(f, 0, SEEK_END); long sz = ftell(f); rewind(f); if (sz <= 0) { fclose(f); return 0; } char* data = malloc((size_t)sz + 1); if (!data) { fclose(f); return 0; } size_t got = fread(data, 1, (size_t)sz, f); fclose(f); data[got] = '\0'; EngramStore* g = engram_get(); int64_t added_nodes = 0; /* Walk nodes array — skip any node whose id already exists */ const char* nodes_p = json_find_key(data, "nodes"); if (nodes_p) { nodes_p = eg_skip_ws(nodes_p); if (*nodes_p == '[') { nodes_p++; nodes_p = eg_skip_ws(nodes_p); while (*nodes_p && *nodes_p != ']') { if (*nodes_p != '{') { nodes_p++; continue; } const char* end = json_skip_value(nodes_p); size_t n = (size_t)(end - nodes_p); char* obj = malloc(n + 1); memcpy(obj, nodes_p, n); obj[n] = '\0'; char* nid = eg_get_str_field(obj, "id"); /* Nodes with an empty/unparseable id can never dedup against * the idmap, so without this guard they were re-added on EVERY * merge cycle — unbounded store growth. Skip them entirely. */ int has_id = (nid && *nid); int already = (has_id && engram_find_node(nid) != NULL); free(nid); if (has_id && !already) { engram_grow_nodes(); EngramNode* nn = &g->nodes[g->node_count]; memset(nn, 0, sizeof(*nn)); nn->id = eg_get_str_field(obj, "id"); nn->content = eg_get_str_field(obj, "content"); nn->node_type = eg_get_str_field(obj, "node_type"); nn->label = eg_get_str_field(obj, "label"); nn->tier = eg_get_str_field(obj, "tier"); nn->tags = eg_get_str_field(obj, "tags"); nn->metadata = eg_get_str_field(obj, "metadata"); if (!nn->metadata || !*nn->metadata) { free(nn->metadata); nn->metadata = strdup("{}"); } nn->salience = eg_get_num_field(obj, "salience"); nn->importance = eg_get_num_field(obj, "importance"); nn->confidence = eg_get_num_field(obj, "confidence"); nn->temporal_decay_rate = eg_get_num_field(obj, "temporal_decay_rate"); nn->activation_count = eg_get_int_field(obj, "activation_count"); nn->last_activated = eg_get_int_field(obj, "last_activated"); nn->created_at = eg_get_int_field(obj, "created_at"); nn->updated_at = eg_get_int_field(obj, "updated_at"); nn->background_activation = eg_get_num_field(obj, "background_activation"); /* Nodes arriving via merge were never part of THIS store's * working memory — importing the source store's WM weight * would inject foreign (often legacy 0.25 breakthrough- * floor) weights into local WM every sync cycle. */ nn->working_memory_weight = 0.0; nn->suppression_count = (int32_t)eg_get_int_field(obj, "suppression_count"); if (json_find_key(obj, "layer_id")) { nn->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); } else { nn->layer_id = ENGRAM_LAYER_DEFAULT; } /* Base-level history merges with the node (2026-07-22); * wm_anchor stays 0 — WM state is local-only (see the * working_memory_weight comment above). */ { char* ats = eg_get_str_field(obj, "access_ts"); if (ats) { engram_bll_parse_access(nn, ats); free(ats); } } { char* es = eg_get_str_field(obj, "emb"); if (es) { eg_parse_emb(nn, es); free(es); } } int64_t merge_idx = g->node_count; g->node_count++; added_nodes++; if (nn->id && *nn->id) engram_idmap_put(g, nn->id, merge_idx); g->adj_dirty = 1; } free(obj); nodes_p = end; nodes_p = eg_skip_ws(nodes_p); if (*nodes_p == ',') { nodes_p++; nodes_p = eg_skip_ws(nodes_p); } } } } /* Walk edges array — skip if (from_id, to_id, relation) already present */ const char* edges_p = json_find_key(data, "edges"); if (edges_p) { edges_p = eg_skip_ws(edges_p); if (*edges_p == '[') { edges_p++; edges_p = eg_skip_ws(edges_p); while (*edges_p && *edges_p != ']') { if (*edges_p != '{') { edges_p++; continue; } const char* end = json_skip_value(edges_p); size_t n = (size_t)(end - edges_p); char* obj = malloc(n + 1); memcpy(obj, edges_p, n); obj[n] = '\0'; char* efrom = eg_get_str_field(obj, "from_id"); char* eto = eg_get_str_field(obj, "to_id"); char* erel = eg_get_str_field(obj, "relation"); int dup = 0; if (efrom && eto && erel) { for (int64_t ei = 0; ei < g->edge_count; ei++) { EngramEdge* ex = &g->edges[ei]; if (ex->from_id && ex->to_id && ex->relation && strcmp(ex->from_id, efrom) == 0 && strcmp(ex->to_id, eto) == 0 && strcmp(ex->relation, erel) == 0) { dup = 1; break; } } } if (!dup) { engram_grow_edges(); EngramEdge* ee = &g->edges[g->edge_count]; memset(ee, 0, sizeof(*ee)); ee->id = eg_get_str_field(obj, "id"); ee->from_id = efrom ? efrom : strdup(""); ee->to_id = eto ? eto : strdup(""); ee->relation = erel ? erel : strdup(""); ee->metadata = eg_get_str_field(obj, "metadata"); if (!ee->metadata || !*ee->metadata) { free(ee->metadata); ee->metadata = strdup("{}"); } ee->weight = eg_get_num_field(obj, "weight"); ee->hebb = eg_get_num_field(obj, "hebb"); /* absent ⇒ 0 */ ee->confidence = eg_get_num_field(obj, "confidence"); ee->created_at = eg_get_int_field(obj, "created_at"); ee->updated_at = eg_get_int_field(obj, "updated_at"); ee->last_fired = eg_get_int_field(obj, "last_fired"); ee->inhibitory = (int)eg_get_int_field(obj, "inhibitory"); if (json_find_key(obj, "layer_id")) { ee->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); } else { ee->layer_id = ENGRAM_LAYER_DEFAULT; } g->edge_count++; efrom = NULL; eto = NULL; erel = NULL; } else { free(efrom); free(eto); free(erel); } free(obj); edges_p = end; edges_p = eg_skip_ws(edges_p); if (*edges_p == ',') { edges_p++; edges_p = eg_skip_ws(edges_p); } } } } /* Merged nodes can carry snapshot WM weights too — hold the cap here * as well (see eg_enforce_wm_cap_on_load). */ eg_enforce_wm_cap_on_load(g); free(data); return (el_val_t)added_nodes; } /* ══════════════════════════════════════════════════════════════════════════ * Engram WAL (write-ahead log) + compaction + integrity hardening. * * Design: docs/architecture/design/engram-storage-engine-wal.md §§3-14, §18. * Whole feature is gated behind ENGRAM_WAL=on (default off → byte-identical * to the snapshot-as-store behavior). When on, structural mutations append a * framed, CRC'd record to /engram.wal instead of rewriting the full * snapshot; boot replays the WAL over the base snapshot; a size threshold * triggers compaction (fresh snapshot + WAL truncate). * * Record framing (little-endian native; single-machine prototype): * [u32 magic 'EWL1'][u32 payload_len][u8 op][u8 flags][u64 lsn] * [u32 crc32][payload…] * crc32 covers op|flags|lsn|payload. A record whose magic/length/crc fails * validation marks end-of-valid-log (torn tail from a crash → replay stops * cleanly, never crashes). * ══════════════════════════════════════════════════════════════════════════ */ #define EG_WAL_MAGIC 0x314C5745u /* 'E''W''L''1' */ #define EG_WAL_HDR_LEN 22 /* 4+4+1+1+8+4 */ /* Op codes (§5). NODE_PUT/EDGE_PUT are id-keyed upserts (replay-idempotent). */ enum { EG_OP_NODE_PUT = 1, EG_OP_EDGE_PUT = 2, EG_OP_TOMBSTONE = 3, EG_OP_SUPERSEDE = 4, EG_OP_LAYER_PUT = 5, EG_OP_LAYER_DEL = 6, EG_OP_FORGET = 7, EG_OP_HEBB_BATCH = 8, EG_OP_COMPACT_MARK = 9 }; /* ── crc32 (IEEE 802.3, reflected, poly 0xEDB88320) ────────────────────────── * init 0xFFFFFFFF, final XOR 0xFFFFFFFF. Known answers: * crc32("") == 0x00000000 * crc32("123456789") == 0xCBF43926 (the canonical check value) * eg_crc32_update takes/returns the *internal* (pre-final-xor) running value * so a checksum can be computed across several buffers. */ static uint32_t eg_crc32_table[256]; static int eg_crc32_ready = 0; static void eg_crc32_init(void) { for (uint32_t i = 0; i < 256; i++) { uint32_t c = i; for (int k = 0; k < 8; k++) c = (c & 1) ? (0xEDB88320u ^ (c >> 1)) : (c >> 1); eg_crc32_table[i] = c; } eg_crc32_ready = 1; } static uint32_t eg_crc32_update(uint32_t crc, const void* data, size_t n) { if (!eg_crc32_ready) eg_crc32_init(); const uint8_t* d = (const uint8_t*)data; for (size_t i = 0; i < n; i++) crc = eg_crc32_table[(crc ^ d[i]) & 0xFF] ^ (crc >> 8); return crc; } static uint32_t eg_crc32(const void* data, size_t n) { return eg_crc32_update(0xFFFFFFFFu, data, n) ^ 0xFFFFFFFFu; } /* crc over a record's covered bytes: op|flags|lsn|payload. */ static uint32_t eg_wal_record_crc(uint8_t op, uint8_t flags, uint64_t lsn, const char* payload, size_t plen) { uint32_t c = 0xFFFFFFFFu; c = eg_crc32_update(c, &op, 1); c = eg_crc32_update(c, &flags, 1); c = eg_crc32_update(c, &lsn, sizeof(lsn)); if (plen) c = eg_crc32_update(c, payload, plen); return c ^ 0xFFFFFFFFu; } /* EL builtin: crc32 of a string (known-answer unit tests / debugging). */ el_val_t engram_crc32(el_val_t s) { const char* p = EL_CSTR(s); if (!p) return (el_val_t)0; return (el_val_t)(int64_t)(uint32_t)eg_crc32(p, strlen(p)); } /* ── WAL runtime state (single log per process — engram is single-threaded) ── */ typedef struct { FILE* fp; /* append handle, or NULL when closed */ char path[1024]; /* /engram.wal */ uint64_t lsn; /* last assigned lsn; next record = lsn+1 */ uint64_t uncommitted; /* records appended since last fsync */ int64_t last_sync_ms; /* wall clock of last fsync */ int64_t bytes; /* current WAL size (for compaction trigger) */ } EngramWal; static EngramWal eg_wal = { NULL, {0}, 0, 0, 0, 0 }; static int eg_wal_sync_mode(void) { /* ENGRAM_WAL_SYNC=always|group|off (default group). */ const char* m = getenv("ENGRAM_WAL_SYNC"); if (m && strcmp(m, "always") == 0) return 2; if (m && strcmp(m, "off") == 0) return 0; return 1; /* group */ } static int64_t eg_wal_group_ms(void) { const char* v = getenv("ENGRAM_WAL_SYNC_MS"); if (v && *v) { long n = strtol(v, NULL, 10); if (n > 0) return n; } return 50; /* §5: N≈50ms */ } static int64_t eg_wal_compact_bytes(void) { const char* v = getenv("ENGRAM_WAL_COMPACT_BYTES"); if (v && *v) { long long n = strtoll(v, NULL, 10); if (n > 0) return n; } return 32LL * 1024 * 1024; /* §7 default 32 MB */ } int engram_wal_enabled(void) { const char* f = getenv("ENGRAM_WAL"); return (f && (strcmp(f, "on") == 0 || strcmp(f, "1") == 0)) ? 1 : 0; } /* Force the WAL to durable storage per the commit policy. force=1 (used by * compaction / explicit commit) always fsyncs; otherwise group policy. */ static void eg_wal_commit(int force) { if (!eg_wal.fp) return; int mode = eg_wal_sync_mode(); if (mode == 0 && !force) { fflush(eg_wal.fp); return; } if (!force && mode == 1) { int64_t now = engram_now_ms(); if (eg_wal.uncommitted == 0) return; if (now - eg_wal.last_sync_ms < eg_wal_group_ms()) { fflush(eg_wal.fp); return; } } fflush(eg_wal.fp); fsync(fileno(eg_wal.fp)); eg_wal.uncommitted = 0; eg_wal.last_sync_ms = engram_now_ms(); } /* Append one framed+CRC'd record. Returns 1 on success, 0 on I/O failure * (caller keeps the mutation in RAM; no corruption — §11 disk-full row). */ static int eg_wal_write(uint8_t op, uint8_t flags, const char* payload, size_t plen) { if (!eg_wal.fp) return 0; /* Remember the pre-record offset so a partial write (e.g. ENOSPC mid- * record) can be rolled back — otherwise a torn record would sit in the * MIDDLE of the log and prematurely end replay of everything after it. * We roll back to a clean record boundary and report failure; the caller * keeps the mutation in RAM (§11 disk-full row). */ long start = ftell(eg_wal.fp); uint64_t lsn = ++eg_wal.lsn; uint32_t magic = EG_WAL_MAGIC; uint32_t len32 = (uint32_t)plen; uint32_t crc = eg_wal_record_crc(op, flags, lsn, payload, plen); uint8_t hdr[EG_WAL_HDR_LEN]; memcpy(hdr + 0, &magic, 4); memcpy(hdr + 4, &len32, 4); hdr[8] = op; hdr[9] = flags; memcpy(hdr + 10, &lsn, 8); memcpy(hdr + 18, &crc, 4); int ok = (fwrite(hdr, 1, EG_WAL_HDR_LEN, eg_wal.fp) == EG_WAL_HDR_LEN); if (ok && plen) ok = (fwrite(payload, 1, plen, eg_wal.fp) == plen); if (!ok) { eg_wal.lsn--; /* reclaim the lsn */ fflush(eg_wal.fp); if (start >= 0) { if (ftruncate(fileno(eg_wal.fp), start) == 0) {} /* drop torn bytes */ fseek(eg_wal.fp, 0, SEEK_END); } return 0; } eg_wal.bytes += EG_WAL_HDR_LEN + (int64_t)plen; eg_wal.uncommitted++; eg_wal_commit(0); return 1; } /* ── Single-record apply (replay + live are the same code path) ───────────── */ /* Populate an EngramNode (freshly memset OR an existing node being overwritten * in place) from a node JSON object. Mirrors engram_load's field set exactly, * WITHOUT the boot-time working_memory_weight laundering, so replay reproduces * the exact logged state (parity with the direct-apply oracle). Caller owns * freeing prior heap fields when overwriting. */ static void eg_fill_node_from_json(EngramNode* nn, const char* obj) { nn->id = eg_get_str_field(obj, "id"); nn->content = eg_get_str_field(obj, "content"); nn->node_type = eg_get_str_field(obj, "node_type"); nn->label = eg_get_str_field(obj, "label"); nn->tier = eg_get_str_field(obj, "tier"); nn->tags = eg_get_str_field(obj, "tags"); nn->metadata = eg_get_str_field(obj, "metadata"); if (!nn->metadata || !*nn->metadata) { free(nn->metadata); nn->metadata = el_strdup_persist("{}"); } nn->salience = eg_get_num_field(obj, "salience"); nn->importance = eg_get_num_field(obj, "importance"); nn->confidence = eg_get_num_field(obj, "confidence"); nn->temporal_decay_rate = eg_get_num_field(obj, "temporal_decay_rate"); nn->activation_count = eg_get_int_field(obj, "activation_count"); nn->last_activated = eg_get_int_field(obj, "last_activated"); nn->created_at = eg_get_int_field(obj, "created_at"); nn->updated_at = eg_get_int_field(obj, "updated_at"); nn->background_activation = eg_get_num_field(obj, "background_activation"); nn->working_memory_weight = eg_get_num_field(obj, "working_memory_weight"); nn->suppression_count = (int32_t)eg_get_int_field(obj, "suppression_count"); if (json_find_key(obj, "layer_id")) nn->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); else nn->layer_id = ENGRAM_LAYER_DEFAULT; nn->wm_anchor = eg_get_num_field(obj, "wm_anchor"); { char* ats = eg_get_str_field(obj, "access_ts"); if (ats) { engram_bll_parse_access(nn, ats); free(ats); } } { char* es = eg_get_str_field(obj, "emb"); if (es) { eg_parse_emb(nn, es); free(es); } } } static void eg_free_node_heap(EngramNode* n) { free(n->id); free(n->content); free(n->node_type); free(n->label); free(n->tier); free(n->tags); free(n->metadata); free(n->emb); } static void eg_free_edge_heap(EngramEdge* e) { free(e->id); free(e->from_id); free(e->to_id); free(e->relation); free(e->metadata); } /* Upsert a node by id. Idempotent: replaying the same NODE_PUT twice yields * the same single node. */ static void eg_apply_node_put(const char* obj) { EngramStore* g = engram_get(); char* id = eg_get_str_field(obj, "id"); if (!id || !*id) { free(id); return; } int64_t idx = engram_find_node_index(id); if (idx >= 0) { EngramNode* n = &g->nodes[idx]; int32_t emb_dim = n->emb_dim; (void)emb_dim; eg_free_node_heap(n); memset(n, 0, sizeof(*n)); eg_fill_node_from_json(n, obj); /* id unchanged → id_map entry (which owns its own key copy) stays * valid; no re-put needed. */ } else { engram_grow_nodes(); EngramNode* n = &g->nodes[g->node_count]; memset(n, 0, sizeof(*n)); eg_fill_node_from_json(n, obj); int64_t ni = g->node_count; g->node_count++; if (n->id && *n->id) engram_idmap_put(g, n->id, ni); } g->adj_dirty = 1; free(id); } static int64_t eg_find_edge_index(EngramStore* g, const char* id) { if (!id || !*id) return -1; for (int64_t i = 0; i < g->edge_count; i++) if (g->edges[i].id && strcmp(g->edges[i].id, id) == 0) return i; return -1; } static void eg_fill_edge_from_json(EngramEdge* ee, const char* obj) { ee->id = eg_get_str_field(obj, "id"); ee->from_id = eg_get_str_field(obj, "from_id"); ee->to_id = eg_get_str_field(obj, "to_id"); ee->relation = eg_get_str_field(obj, "relation"); ee->metadata = eg_get_str_field(obj, "metadata"); if (!ee->metadata || !*ee->metadata) { free(ee->metadata); ee->metadata = el_strdup_persist("{}"); } ee->weight = eg_get_num_field(obj, "weight"); ee->hebb = eg_get_num_field(obj, "hebb"); ee->confidence = eg_get_num_field(obj, "confidence"); ee->created_at = eg_get_int_field(obj, "created_at"); ee->updated_at = eg_get_int_field(obj, "updated_at"); ee->last_fired = eg_get_int_field(obj, "last_fired"); ee->inhibitory = (int)eg_get_int_field(obj, "inhibitory"); if (json_find_key(obj, "layer_id")) ee->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); else ee->layer_id = ENGRAM_LAYER_DEFAULT; } static void eg_apply_edge_put(const char* obj) { EngramStore* g = engram_get(); char* id = eg_get_str_field(obj, "id"); int64_t idx = eg_find_edge_index(g, id); if (idx >= 0) { EngramEdge* e = &g->edges[idx]; eg_free_edge_heap(e); memset(e, 0, sizeof(*e)); eg_fill_edge_from_json(e, obj); } else { engram_grow_edges(); EngramEdge* e = &g->edges[g->edge_count]; memset(e, 0, sizeof(*e)); eg_fill_edge_from_json(e, obj); g->edge_count++; } g->adj_dirty = 1; free(id); } /* TOMBSTONE / SUPERSEDE — store-level soft markers on the node's metadata. * (The server's DELETE route uses the higher-level marker-node model; these * ops exist for completeness and are replay-idempotent.) */ static void eg_apply_meta_marker(const char* obj, const char* markerfield, const char* from) { EngramStore* g = engram_get(); char* id = eg_get_str_field(obj, "id"); int64_t idx = engram_find_node_index(id); if (idx >= 0) { EngramNode* n = &g->nodes[idx]; char* by = from ? eg_get_str_field(obj, from) : NULL; size_t need = strlen(markerfield) + (by ? strlen(by) : 4) + 32; char* meta = (char*)malloc(need); if (by && *by) snprintf(meta, need, "{\"%s\":\"%s\"}", markerfield, by); else snprintf(meta, need, "{\"%s\":1}", markerfield); free(n->metadata); n->metadata = el_strdup_persist(meta); free(meta); free(by); } free(id); } static void eg_apply_layer_put(const char* obj) { EngramStore* g = engram_get(); uint32_t lid = (uint32_t)eg_get_int_field(obj, "layer_id"); EngramLayer* L = NULL; for (size_t i = 0; i < g->layer_count; i++) if (g->layers[i].layer_id == lid) { L = &g->layers[i]; break; } if (!L) { if (g->layer_count >= g->layer_capacity) { size_t nc = g->layer_capacity ? g->layer_capacity * 2 : 16; EngramLayer* grown = realloc(g->layers, nc * sizeof(EngramLayer)); if (!grown) return; memset(grown + g->layer_capacity, 0, (nc - g->layer_capacity) * sizeof(EngramLayer)); g->layers = grown; g->layer_capacity = nc; } L = &g->layers[g->layer_count++]; memset(L, 0, sizeof(*L)); } else if (L->name) { free(L->name); L->name = NULL; } L->layer_id = lid; L->activation_priority = (uint32_t)eg_get_int_field(obj, "activation_priority"); L->suppressible = (int)eg_get_int_field(obj, "suppressible") ? 1 : 0; L->transparent = (int)eg_get_int_field(obj, "transparent") ? 1 : 0; L->injectable = (int)eg_get_int_field(obj, "injectable") ? 1 : 0; char* nm = eg_get_str_field(obj, "name"); L->name = el_strdup_persist(nm && *nm ? nm : ""); free(nm); } static void eg_apply_layer_del(const char* obj) { EngramStore* g = engram_get(); uint32_t lid = (uint32_t)eg_get_int_field(obj, "layer_id"); for (size_t i = 0; i < g->layer_count; i++) if (g->layers[i].layer_id == lid && g->layers[i].name) { free(g->layers[i].name); g->layers[i].name = NULL; break; } } static void eg_apply_forget(const char* obj) { char* id = eg_get_str_field(obj, "id"); if (id && *id) engram_forget((el_val_t)(uintptr_t)id); free(id); } /* Apply one decoded record to the in-RAM store. HEBB_BATCH payload is a JSON * object {"edges":[edge,…]} — one record, one fsync, N edge upserts (§5-B). */ static void eg_wal_apply(uint8_t op, const char* payload, size_t plen) { char* obj = (char*)malloc(plen + 1); if (!obj) return; memcpy(obj, payload, plen); obj[plen] = '\0'; switch (op) { case EG_OP_NODE_PUT: eg_apply_node_put(obj); break; case EG_OP_EDGE_PUT: eg_apply_edge_put(obj); break; case EG_OP_TOMBSTONE: eg_apply_meta_marker(obj, "tombstoned", NULL); break; case EG_OP_SUPERSEDE: eg_apply_meta_marker(obj, "superseded_by", "by"); break; case EG_OP_LAYER_PUT: eg_apply_layer_put(obj); break; case EG_OP_LAYER_DEL: eg_apply_layer_del(obj); break; case EG_OP_FORGET: eg_apply_forget(obj); break; case EG_OP_HEBB_BATCH: { const char* arr = json_find_key(obj, "edges"); if (arr) { arr = eg_skip_ws(arr); if (*arr == '[') { arr++; arr = eg_skip_ws(arr); while (*arr && *arr != ']') { if (*arr != '{') { arr++; continue; } const char* end = json_skip_value(arr); size_t en = (size_t)(end - arr); char* eobj = (char*)malloc(en + 1); memcpy(eobj, arr, en); eobj[en] = '\0'; eg_apply_edge_put(eobj); free(eobj); arr = eg_skip_ws(end); if (*arr == ',') { arr++; arr = eg_skip_ws(arr); } } } } break; } case EG_OP_COMPACT_MARK: break; /* boundary marker; no state change */ default: break; } free(obj); } /* ── Replay (§6) ───────────────────────────────────────────────────────────── * Reads records in lsn order, applies each intact one, and stops at the first * record that fails magic/length/crc validation (torn tail) — never crashes. * Returns the number of records applied; sets *out_last_lsn to the highest * good lsn seen (0 if none). */ static int64_t eg_wal_replay_file(const char* path, uint64_t* out_last_lsn) { if (out_last_lsn) *out_last_lsn = 0; FILE* f = fopen(path, "rb"); if (!f) return 0; fseek(f, 0, SEEK_END); long fsz = ftell(f); rewind(f); if (fsz <= 0) { fclose(f); return 0; } uint8_t* buf = (uint8_t*)malloc((size_t)fsz); if (!buf) { fclose(f); return 0; } size_t got = fread(buf, 1, (size_t)fsz, f); fclose(f); int64_t applied = 0; size_t off = 0; while (off + EG_WAL_HDR_LEN <= got) { uint32_t magic, len32, crc; uint64_t lsn; uint8_t op, flags; memcpy(&magic, buf + off + 0, 4); if (magic != EG_WAL_MAGIC) break; /* garbage / torn */ memcpy(&len32, buf + off + 4, 4); op = buf[off + 8]; flags = buf[off + 9]; memcpy(&lsn, buf + off + 10, 8); memcpy(&crc, buf + off + 18, 4); size_t plen = (size_t)len32; if (off + EG_WAL_HDR_LEN + plen > got) break; /* short final record */ const char* payload = (const char*)(buf + off + EG_WAL_HDR_LEN); if (eg_wal_record_crc(op, flags, lsn, payload, plen) != crc) break; /* torn */ eg_wal_apply(op, payload, plen); if (out_last_lsn) *out_last_lsn = lsn; applied++; off += EG_WAL_HDR_LEN + plen; } free(buf); return applied; } static void eg_wal_build_path(const char* dir, char* out, size_t cap) { snprintf(out, cap, "%s/engram.wal", dir ? dir : "."); } /* Open (create if absent) the WAL for appending. Idempotent for a given dir. */ static int eg_wal_open(const char* dir) { char path[1024]; eg_wal_build_path(dir, path, sizeof(path)); if (eg_wal.fp && strcmp(eg_wal.path, path) == 0) return 1; /* already open */ if (eg_wal.fp) { fclose(eg_wal.fp); eg_wal.fp = NULL; } FILE* f = fopen(path, "ab"); if (!f) return 0; eg_wal.fp = f; snprintf(eg_wal.path, sizeof(eg_wal.path), "%s", path); fseek(f, 0, SEEK_END); eg_wal.bytes = ftell(f); eg_wal.last_sync_ms = engram_now_ms(); eg_wal.uncommitted = 0; return 1; } /* ── EL-facing builtins ──────────────────────────────────────────────────── */ /* Boot: replay /engram.wal over the already-loaded base snapshot, then * open the WAL for appending, continuing the lsn sequence. Idempotent replay * makes overlap with the base harmless (§6). Returns records replayed. */ el_val_t engram_wal_boot(el_val_t dir) { const char* d = EL_CSTR(dir); char path[1024]; eg_wal_build_path(d, path, sizeof(path)); uint64_t last = 0; int64_t applied = eg_wal_replay_file(path, &last); eg_wal.lsn = last; /* continue monotonically */ eg_wal_open(d); return (el_val_t)applied; } el_val_t engram_wal_open_dir(el_val_t dir) { return (el_val_t)(int64_t)eg_wal_open(EL_CSTR(dir)); } /* Append a NODE_PUT for node `id` (serialized via the shared emitter, incl. * emb). Returns 1 on success. */ el_val_t engram_wal_node_put(el_val_t dir, el_val_t id) { if (!eg_wal_open(EL_CSTR(dir))) return (el_val_t)0; EngramNode* n = engram_find_node(EL_CSTR(id)); if (!n) return (el_val_t)0; JsonBuf b; jb_init(&b); engram_emit_node_json(&b, n, 1); int ok = eg_wal_write(EG_OP_NODE_PUT, 0, b.buf, b.len); free(b.buf); return (el_val_t)(int64_t)ok; } /* Append an EDGE_PUT for every edge at index >= start_count. Covers both the * single-edge route and any append-only batch. Returns edges logged. */ el_val_t engram_wal_edges_since(el_val_t dir, el_val_t start_count) { if (!eg_wal_open(EL_CSTR(dir))) return (el_val_t)0; EngramStore* g = engram_get(); int64_t start = (int64_t)start_count; if (start < 0) start = 0; int64_t logged = 0; for (int64_t i = start; i < g->edge_count; i++) { JsonBuf b; jb_init(&b); engram_emit_edge_json(&b, &g->edges[i]); if (eg_wal_write(EG_OP_EDGE_PUT, 0, b.buf, b.len)) logged++; free(b.buf); } return (el_val_t)logged; } /* Append ONE HEBB_BATCH record covering every edge at index >= start_count * (single fsync for the whole consolidation batch — §5 tier B). */ el_val_t engram_wal_hebb_batch(el_val_t dir, el_val_t start_count) { if (!eg_wal_open(EL_CSTR(dir))) return (el_val_t)0; EngramStore* g = engram_get(); int64_t start = (int64_t)start_count; if (start < 0) start = 0; if (start >= g->edge_count) return (el_val_t)0; JsonBuf b; jb_init(&b); jb_puts(&b, "{\"edges\":["); int first = 1; for (int64_t i = start; i < g->edge_count; i++) { if (!first) jb_putc(&b, ','); first = 0; engram_emit_edge_json(&b, &g->edges[i]); } jb_puts(&b, "]}"); int ok = eg_wal_write(EG_OP_HEBB_BATCH, 0, b.buf, b.len); free(b.buf); return (el_val_t)(int64_t)ok; } /* Append a FORGET (hard remove). Internal-GC only — NOT wired to HTTP DELETE. */ el_val_t engram_wal_forget(el_val_t dir, el_val_t id) { if (!eg_wal_open(EL_CSTR(dir))) return (el_val_t)0; const char* sid = EL_CSTR(id); JsonBuf b; jb_init(&b); jb_puts(&b, "{\"id\":"); jb_emit_escaped(&b, sid ? sid : ""); jb_putc(&b, '}'); int ok = eg_wal_write(EG_OP_FORGET, 0, b.buf, b.len); free(b.buf); return (el_val_t)(int64_t)ok; } /* Compaction (§7). Crash-safe ordering: fresh base snapshot renamed into place * (engram_save is atomic temp+fsync+rename) BEFORE the WAL is truncated. A * crash in the window replays the still-present WAL over the (old or new) * base; idempotent apply converges. */ static int eg_wal_compact(const char* dir) { char snap[1024], waltmp[1024], walpath[1024]; snprintf(snap, sizeof(snap), "%s/snapshot.json", dir); eg_wal_build_path(dir, walpath, sizeof(walpath)); snprintf(waltmp, sizeof(waltmp), "%s/engram.wal.tmp", dir); /* 1. new base (atomic) */ if (!engram_save((el_val_t)(uintptr_t)snap)) return 0; /* fsync the directory so the rename is durable before we touch the WAL */ { int dfd = open(dir, O_RDONLY); if (dfd >= 0) { fsync(dfd); close(dfd); } } /* 2. fresh WAL containing only a COMPACT_MARK, atomically swapped in */ uint64_t base_lsn = eg_wal.lsn; FILE* tf = fopen(waltmp, "wb"); if (!tf) return 0; { char pl[64]; int pn = snprintf(pl, sizeof(pl), "{\"base_lsn\":%llu}", (unsigned long long)base_lsn); uint64_t lsn = ++eg_wal.lsn; uint32_t magic = EG_WAL_MAGIC, len32 = (uint32_t)pn; uint32_t crc = eg_wal_record_crc(EG_OP_COMPACT_MARK, 0, lsn, pl, pn); uint8_t hdr[EG_WAL_HDR_LEN]; memcpy(hdr, &magic, 4); memcpy(hdr + 4, &len32, 4); hdr[8] = EG_OP_COMPACT_MARK; hdr[9] = 0; memcpy(hdr + 10, &lsn, 8); memcpy(hdr + 18, &crc, 4); fwrite(hdr, 1, EG_WAL_HDR_LEN, tf); fwrite(pl, 1, pn, tf); fflush(tf); fsync(fileno(tf)); } fclose(tf); if (eg_wal.fp) { fclose(eg_wal.fp); eg_wal.fp = NULL; } if (rename(waltmp, walpath) != 0) { return 0; } { int dfd = open(dir, O_RDONLY); if (dfd >= 0) { fsync(dfd); close(dfd); } } /* 3. reopen the truncated WAL for appending */ eg_wal_open(dir); return 1; } el_val_t engram_wal_compact(el_val_t dir) { return (el_val_t)(int64_t)eg_wal_compact(EL_CSTR(dir)); } /* Compact iff the WAL has crossed the size threshold. Returns 1 if compacted. */ el_val_t engram_wal_maybe_compact(el_val_t dir) { if (!eg_wal.fp) eg_wal_open(EL_CSTR(dir)); if (eg_wal.bytes > eg_wal_compact_bytes()) return (el_val_t)(int64_t)eg_wal_compact(EL_CSTR(dir)); return (el_val_t)0; } /* ── Integrity: safe data-dir resolution (§18.2) ───────────────────────────── * Unset ENGRAM_DATA_DIR → $HOME/.neuron/engram (created if absent). If HOME is * also unresolvable, FAIL LOUD (exit) rather than silently persisting to an * ephemeral /tmp. Prod (ENGRAM_DATA_DIR=/data) is unaffected. */ el_val_t engram_resolve_data_dir(void) { const char* d = getenv("ENGRAM_DATA_DIR"); if (d && *d) return el_wrap_str(el_strdup(d)); const char* home = getenv("HOME"); if (!home || !*home) { fprintf(stderr, "[engram] FATAL: ENGRAM_DATA_DIR unset and HOME unresolved; " "refusing to persist to an ephemeral dir. Set ENGRAM_DATA_DIR.\n"); exit(1); } char neuron[1024], engramdir[1024]; snprintf(neuron, sizeof(neuron), "%s/.neuron", home); snprintf(engramdir, sizeof(engramdir), "%s/.neuron/engram", home); mkdir(neuron, 0700); mkdir(engramdir, 0700); return el_wrap_str(el_strdup(engramdir)); } /* ── Integrity: store-level write-protection (§18.1, §18.3) ────────────────── * The protected set is DERIVED from the self-graph at call time, not hardcoded: * the self root and the values hub, plus every node adjacent to either (in * either direction). Adjacency of the values hub yields all value nodes; of * the self root yields the identity children — so new values/identity stay * protected automatically. */ #define EG_SELF_ROOT "kn-efeb4a5b" #define EG_VALUES_HUB "kn-5b606390" static int eg_is_neighbor_of(EngramStore* g, const char* hub, const char* id) { for (int64_t i = 0; i < g->edge_count; i++) { EngramEdge* e = &g->edges[i]; if (e->from_id && e->to_id) { if (strcmp(e->from_id, hub) == 0 && strcmp(e->to_id, id) == 0) return 1; if (strcmp(e->to_id, hub) == 0 && strcmp(e->from_id, id) == 0) return 1; } } return 0; } static int eg_is_protected(const char* id) { if (!id || !*id) return 0; if (strcmp(id, EG_SELF_ROOT) == 0) return 1; if (strcmp(id, EG_VALUES_HUB) == 0) return 1; EngramStore* g = engram_get(); if (eg_is_neighbor_of(g, EG_VALUES_HUB, id)) return 1; if (eg_is_neighbor_of(g, EG_SELF_ROOT, id)) return 1; return 0; } el_val_t engram_is_protected(el_val_t id) { return (el_val_t)(int64_t)eg_is_protected(EL_CSTR(id)); } /* Derived protected set as a JSON array (self root + values hub + all adjacent * ids). For diagnostics and the integrity tests (§18.5). */ el_val_t engram_protected_json(void) { EngramStore* g = engram_get(); JsonBuf b; jb_init(&b); jb_putc(&b, '['); int first = 1; const char* seeds[2] = { EG_SELF_ROOT, EG_VALUES_HUB }; /* emit the two hubs themselves */ for (int s = 0; s < 2; s++) { if (!first) jb_putc(&b, ','); first = 0; jb_emit_escaped(&b, seeds[s]); } for (int64_t i = 0; i < g->node_count; i++) { const char* nid = g->nodes[i].id; if (!nid || !*nid) continue; if (strcmp(nid, EG_SELF_ROOT) == 0 || strcmp(nid, EG_VALUES_HUB) == 0) continue; if (eg_is_neighbor_of(g, EG_VALUES_HUB, nid) || eg_is_neighbor_of(g, EG_SELF_ROOT, nid)) { if (!first) jb_putc(&b, ','); first = 0; jb_emit_escaped(&b, nid); } } jb_putc(&b, ']'); return el_wrap_str(b.buf); } /* ── Engram JSON-string accessors ───────────────────────────────────────── * These return pre-serialized JSON strings so callers (especially HTTP * handlers) don't have to round-trip ElList/ElMap through json_stringify * — which can't reliably distinguish those structures from raw pointers * due to el_val_t's type erasure. The runtime knows the real C types and * can serialize directly. */ el_val_t engram_get_node_json(el_val_t id) { const char* sid = EL_CSTR(id); EngramNode* n = engram_find_node(sid); if (!n) return el_wrap_str(el_strdup("{}")); JsonBuf b; jb_init(&b); engram_emit_node_json(&b, n, 0); return el_wrap_str(b.buf); } /* engram_get_node_by_label — find the first node whose label field exactly * matches the given string. Returns the node as a JSON object string, or "{}" * if no match is found. * * Used by chat.el to retrieve well-known nodes (e.g. "conv:history", * "session:summary") by their stable label rather than by ID, which is immune * to vector index drift across restarts. * * Exact match (strcmp, not istr_contains) because labels like "conv:history" * must not collide with nodes whose content happens to contain that substring. * * Added 2026-07-01 self-review: was called in chat.el but never defined, * causing build failure since June 30. */ el_val_t engram_get_node_by_label(el_val_t label) { const char* lbl = EL_CSTR(label); if (!lbl || !*lbl) return el_wrap_str(el_strdup("{}")); EngramStore* g = engram_get(); for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; if (n->label && strcmp(n->label, lbl) == 0) { JsonBuf b; jb_init(&b); engram_emit_node_json(&b, n, 0); return el_wrap_str(b.buf); } } return el_wrap_str(el_strdup("{}")); } el_val_t engram_search_json(el_val_t query, el_val_t limit) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); int64_t lim = (int64_t)limit; if (lim <= 0) lim = 100; JsonBuf b; jb_init(&b); jb_putc(&b, '['); int first = 1; if (q && *q) { /* Tokenized + ranked, same scheme as engram_search: match ANY query * token, rank by distinct-token coverage then salience, cap at lim. * (2026-07-19 port of the 2026-07-14 tokenized-search fix) */ char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS); if (ntok > 0) { EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry)); if (hits) { int64_t nhits = 0; for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; /* Filter transparent layers — same as engram_search. */ if (engram_layer_is_transparent(n->layer_id)) continue; int sc = engram_node_match_score(n, toks, ntok); if (sc > 0) { hits[nhits].idx = i; hits[nhits].score = sc; hits[nhits].salience = n->salience; nhits++; } } qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); int64_t end = nhits < lim ? nhits : lim; for (int64_t k = 0; k < end; k++) { if (!first) jb_putc(&b, ','); engram_emit_node_json(&b, &g->nodes[hits[k].idx], 0); first = 0; } free(hits); } } } jb_putc(&b, ']'); return el_wrap_str(b.buf); } el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset) { EngramStore* g = engram_get(); int64_t lim = (int64_t)limit; if (lim <= 0) lim = 100; int64_t off = (int64_t)offset; if (off < 0) off = 0; JsonBuf b; jb_init(&b); jb_putc(&b, '['); if (g->node_count == 0) { jb_putc(&b, ']'); return el_wrap_str(b.buf); } int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t)); if (!idx) { jb_putc(&b, ']'); return el_wrap_str(b.buf); } /* Skip transparent layers — introspection filter, same as engram_scan_nodes. */ int64_t live = 0; for (int64_t i = 0; i < g->node_count; i++) { if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue; idx[live++] = i; } engram_sort_indices_by_salience(idx, live, g->nodes); int64_t end = off + lim; if (end > live) end = live; int first = 1; for (int64_t i = off; i < end; i++) { if (!first) jb_putc(&b, ','); engram_emit_node_json(&b, &g->nodes[idx[i]], 0); first = 0; } free(idx); jb_putc(&b, ']'); return el_wrap_str(b.buf); } /* engram_scan_nodes_by_type_json — filter by node_type before paginating. * Empty / NULL type_v falls back to the unfiltered scan (existing behaviour). * Result is JSON array, salience-sorted, transparent layers skipped. */ el_val_t engram_scan_nodes_by_type_json(el_val_t type_v, el_val_t limit, el_val_t offset) { const char* type_filter = EL_CSTR(type_v); if (!type_filter || !*type_filter) { return engram_scan_nodes_json(limit, offset); } EngramStore* g = engram_get(); int64_t lim = (int64_t)limit; if (lim <= 0) lim = 100; int64_t off = (int64_t)offset; if (off < 0) off = 0; JsonBuf b; jb_init(&b); jb_putc(&b, '['); if (g->node_count == 0) { jb_putc(&b, ']'); return el_wrap_str(b.buf); } int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t)); if (!idx) { jb_putc(&b, ']'); return el_wrap_str(b.buf); } int64_t live = 0; for (int64_t i = 0; i < g->node_count; i++) { if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue; const char* nt = g->nodes[i].node_type; if (!nt || strcmp(nt, type_filter) != 0) continue; idx[live++] = i; } engram_sort_indices_by_salience(idx, live, g->nodes); int64_t end = off + lim; if (end > live) end = live; int first = 1; for (int64_t i = off; i < end; i++) { if (!first) jb_putc(&b, ','); engram_emit_node_json(&b, &g->nodes[idx[i]], 0); first = 0; } free(idx); jb_putc(&b, ']'); return el_wrap_str(b.buf); } el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction) { /* Re-implement here directly so we serialize without going through * the ElList path. Walks BFS to max_depth, emits {node, edge, hops} * triples. */ EngramStore* g = engram_get(); const char* sid = EL_CSTR(node_id); int64_t depth = (int64_t)max_depth; if (depth <= 0) depth = 1; const char* dir = EL_CSTR(direction); if (!dir) dir = "both"; int allow_out = (strcmp(dir, "out") == 0) || (strcmp(dir, "both") == 0); int allow_in = (strcmp(dir, "in") == 0) || (strcmp(dir, "both") == 0); JsonBuf b; jb_init(&b); jb_putc(&b, '['); if (!sid || !*sid) { jb_putc(&b, ']'); return el_wrap_str(b.buf); } /* Frontier of (node_id, hops). Cap to a sane size. */ char** frontier = calloc(1024, sizeof(char*)); int64_t* frontier_h = calloc(1024, sizeof(int64_t)); int64_t fc = 0; char** visited = calloc(1024, sizeof(char*)); int64_t vc = 0; if (!frontier || !frontier_h || !visited) { free(frontier); free(frontier_h); free(visited); jb_putc(&b, ']'); return el_wrap_str(b.buf); } /* MUST be el_strdup_persist: this function frees frontier/visited strings * manually (lines below). el_strdup would ALSO register them in the * per-request arena, so el_request_end() double-freed every one of them * at the end of the HTTP request — SIGABRT in http_worker under load. * (2026-07-18 self-review; reproduced via ASAN on /api/neuron/session/begin * and /api/neuron/graph. Same allocation-discipline class as the * 2026-07-15 EngramNode and 2026-07-16 idmap-key fixes: never mix arena * tracking with manual free.) */ frontier[fc] = el_strdup_persist(sid); frontier_h[fc] = 0; fc++; visited[vc++] = el_strdup_persist(sid); int first = 1; while (fc > 0) { char* cur = frontier[0]; int64_t h = frontier_h[0]; for (int64_t k = 1; k < fc; k++) { frontier[k-1] = frontier[k]; frontier_h[k-1] = frontier_h[k]; } fc--; if (h >= depth) { free(cur); continue; } for (int64_t i = 0; i < g->edge_count; i++) { EngramEdge* e = &g->edges[i]; const char* peer = NULL; if (allow_out && e->from_id && strcmp(e->from_id, cur) == 0) peer = e->to_id; else if (allow_in && e->to_id && strcmp(e->to_id, cur) == 0) peer = e->from_id; if (!peer) continue; int seen = 0; for (int64_t v = 0; v < vc; v++) { if (strcmp(visited[v], peer) == 0) { seen = 1; break; } } if (seen) continue; EngramNode* n = engram_find_node(peer); if (!n) continue; if (!first) jb_putc(&b, ','); jb_puts(&b, "{\"node\":"); engram_emit_node_json(&b, n, 0); jb_puts(&b, ",\"edge\":"); engram_emit_edge_json(&b, e); char tmp[64]; snprintf(tmp, sizeof(tmp), ",\"hops\":%lld}", (long long)(h + 1)); jb_puts(&b, tmp); first = 0; if (vc < 1024) visited[vc++] = el_strdup_persist(peer); if (fc < 1024 && h + 1 < depth) { frontier[fc] = el_strdup_persist(peer); frontier_h[fc] = h + 1; fc++; } } free(cur); } for (int64_t i = 0; i < fc; i++) free(frontier[i]); for (int64_t i = 0; i < vc; i++) free(visited[i]); free(frontier); free(frontier_h); free(visited); jb_putc(&b, ']'); return el_wrap_str(b.buf); } el_val_t engram_activate_json(el_val_t query, el_val_t depth) { /* Run two-layer engram_activate and serialize the result list to JSON. * Each entry includes both activation_strength (layer 1 background) and * working_memory_weight (layer 2 executive filter), plus promoted flag. * Callers performing context compilation should filter to promoted=1. */ el_val_t lst = engram_activate(query, depth); ElList* arr = (ElList*)(uintptr_t)lst; JsonBuf b; jb_init(&b); jb_putc(&b, '['); if (arr) { for (int64_t i = 0; i < arr->length; i++) { if (!arr->elems[i]) continue; el_val_t node_map = el_map_get(arr->elems[i], EL_STR("node")); el_val_t strength_v = el_map_get(arr->elems[i], EL_STR("activation_strength")); el_val_t wm_v = el_map_get(arr->elems[i], EL_STR("working_memory_weight")); el_val_t epist_v = el_map_get(arr->elems[i], EL_STR("epistemic_confidence")); el_val_t hops_v = el_map_get(arr->elems[i], EL_STR("hops")); el_val_t promoted_v = el_map_get(arr->elems[i], EL_STR("promoted")); /* Look up underlying EngramNode by id to emit canonical JSON. */ el_val_t id_v = el_map_get(node_map, EL_STR("id")); const char* id_s = EL_CSTR(id_v); EngramNode* n = id_s ? engram_find_node(id_s) : NULL; if (i > 0) jb_putc(&b, ','); jb_puts(&b, "{\"node\":"); if (n) { engram_emit_node_json(&b, n, 0); } else { jb_puts(&b, "{}"); } char tmp[80]; snprintf(tmp, sizeof(tmp), ",\"activation_strength\":%g", el_to_float(strength_v)); jb_puts(&b, tmp); snprintf(tmp, sizeof(tmp), ",\"working_memory_weight\":%g", el_to_float(wm_v)); jb_puts(&b, tmp); snprintf(tmp, sizeof(tmp), ",\"epistemic_confidence\":%g", el_to_float(epist_v)); jb_puts(&b, tmp); snprintf(tmp, sizeof(tmp), ",\"hops\":%lld", (long long)(int64_t)hops_v); jb_puts(&b, tmp); snprintf(tmp, sizeof(tmp), ",\"promoted\":%d}", (int)(int64_t)promoted_v); jb_puts(&b, tmp); } } jb_putc(&b, ']'); return el_wrap_str(b.buf); } /* ── Working memory introspection helpers ──────────────────────────────────── * * These three functions give the soul daemon visibility into WM composition * without re-running activation. Used in heartbeat ISEs and curiosity scans. * Ported from el-compiler/runtime to releases/v1.0.0-20260501 on 2026-06-30 * self-review (they were missing from the release build, breaking soul daemon * compilation). */ el_val_t engram_wm_count(void) { EngramStore* g = engram_get(); int64_t count = 0; for (int64_t i = 0; i < g->node_count; i++) { if (g->nodes[i].working_memory_weight > 0.0) count++; } return (el_val_t)count; } /* Average working_memory_weight across all promoted nodes (wm > 0). * Returns the float bit-pattern via el_from_float so EL can use it with * float_to_str / float_gt. Returns 0.0 when no nodes are promoted. * Useful in heartbeat ISEs to distinguish "many weak activations" from * "few strong activations". Added 2026-06-04 self-review. */ el_val_t engram_wm_avg_weight(void) { EngramStore* g = engram_get(); double sum = 0.0; int64_t count = 0; for (int64_t i = 0; i < g->node_count; i++) { double w = g->nodes[i].working_memory_weight; /* Skip corrupt/out-of-range values so a single bad snapshot node * doesn't produce a garbage average. */ if (w > 0.0 && w <= 1.0 && isfinite(w)) { sum += w; count++; } } double avg = (count > 0) ? (sum / (double)count) : 0.0; return el_from_float(avg); } /* engram_wm_top_json — return top N working-memory nodes (by wm weight) as a * compact JSON array for ISE heartbeat reporting. * Each element: {"label":"...","node_type":"...","tier":"...","wm":0.42} * InternalStateEvent nodes are excluded — they're observation artifacts that * would bury substantive WM content. Added 2026-06-05 self-review. */ el_val_t engram_wm_top_json(el_val_t n_v) { int64_t top_n = (int64_t)n_v; if (top_n <= 0) top_n = 10; if (top_n > 50) top_n = 50; EngramStore* g = engram_get(); int64_t* idx = malloc((size_t)(g->node_count + 1) * sizeof(int64_t)); if (!idx) return el_wrap_str(el_strdup("[]")); int64_t mc = 0; for (int64_t i = 0; i < g->node_count; i++) { if (g->nodes[i].working_memory_weight > 0.0) { const char* nt = g->nodes[i].node_type; if (nt && strcmp(nt, "InternalStateEvent") == 0) continue; idx[mc++] = i; } } /* Insertion-sort descending by wm weight (mc is typically small). */ for (int64_t i = 1; i < mc; i++) { int64_t key = idx[i]; double kw = g->nodes[key].working_memory_weight; int64_t j = i; while (j > 0 && g->nodes[idx[j-1]].working_memory_weight < kw) { idx[j] = idx[j-1]; j--; } idx[j] = key; } int64_t emit = mc < top_n ? mc : top_n; JsonBuf b; jb_init(&b); jb_putc(&b, '['); for (int64_t k = 0; k < emit; k++) { EngramNode* n = &g->nodes[idx[k]]; if (k > 0) jb_putc(&b, ','); jb_putc(&b, '{'); /* 2026-07-26 self-review: id was never emitted here, so the * awareness heartbeat's wm_top0_streak compared ""=="" and * incremented unconditionally — the streak metric measured * uptime, not fixation. */ jb_puts(&b, "\"id\":"); jb_emit_escaped(&b, n->id ? n->id : ""); jb_puts(&b, ",\"label\":"); jb_emit_escaped(&b, n->label ? n->label : ""); jb_puts(&b, ",\"node_type\":"); jb_emit_escaped(&b, n->node_type ? n->node_type : ""); jb_puts(&b, ",\"tier\":"); jb_emit_escaped(&b, n->tier ? n->tier : ""); char tmp[48]; snprintf(tmp, sizeof(tmp), ",\"wm\":%.3f", n->working_memory_weight); jb_puts(&b, tmp); jb_putc(&b, '}'); } free(idx); jb_putc(&b, ']'); return el_wrap_str(b.buf); } el_val_t engram_stats_json(void) { EngramStore* g = engram_get(); /* embedded_count: how far the lazy backfill has progressed. The single * observable that tells the daily self-review whether semantic * activation is actually accumulating coverage. (2026-07-24) * * embed_eligible_count (2026-07-27): embedded_count alone misleads — * ~70%+ of the store is ISE/Tag/short-content nodes that are permanently * ineligible for embedding, so raw embedded/node_count reads as "~30% * coverage, something is broken" when eligible coverage may be complete. * This exact misdiagnosis happened in today's self-review. Report the * true denominator so coverage = embedded_count / embed_eligible_count. */ int64_t embedded = 0, eligible = 0; for (int64_t i = 0; i < g->node_count; i++) { if (g->nodes[i].emb) embedded++; if (eg_embed_eligible(&g->nodes[i])) eligible++; } char buf[256]; snprintf(buf, sizeof(buf), "{\"node_count\":%lld,\"edge_count\":%lld,\"layer_count\":%zu," "\"embedded_count\":%lld,\"embed_eligible_count\":%lld}", (long long)g->node_count, (long long)g->edge_count, g->layer_count, (long long)embedded, (long long)eligible); return el_wrap_str(el_strdup(buf)); } /* engram_act_stats_json — activation observability + embedder breaker state. * (2026-07-27 self-review; counters made cumulative 2026-07-31.) * wm_evicted/breakthroughs are monotonic process-lifetime totals across ALL * engram_activate calls on this store (diff successive readings for rates; * they reset to 0 only on restart); embed_breaker_open=1 means * eg_embed_fetch is currently refusing calls (semantic activation silently * degraded to lexical until the cooldown expires). The soul heartbeat folds * this into its ISE so the pathologies are diagnosable from telemetry * instead of inferred from wm_avg_weight hovering at the floor. */ el_val_t engram_act_stats_json(void) { int64_t now = engram_now_ms(); int breaker_open = (now < _eg_embed_breaker_until) ? 1 : 0; /* Hebbian potentiation gauges (2026-08-04 self-review). Three numbers, * each answering a question the mechanism can fail on: * hebb_edges — is it learning at all? (0 forever ⇒ co-activation never * happens, or the pass is dead) * hebb_max — is any single association saturating? (persistent 1.0 ⇒ * homeostasis is not biting) * hebb_mass — total associative mass; the aggregate that runaway * potentiation would show up in first. Should plateau, not * climb without bound. * O(E) per call, and this is called once per 60s heartbeat. */ EngramStore* g = engram_get(); int64_t hebb_edges = 0; double hebb_max = 0.0, hebb_mass = 0.0; for (int64_t i = 0; i < g->edge_count; i++) { double h = g->edges[i].hebb; if (h <= 0.0) continue; hebb_edges++; hebb_mass += h; if (h > hebb_max) hebb_max = h; } /* Candidate-table gauges: hebb_cands is how many associations are being * tracked toward consolidation, hebb_cand_max how close the leader is to * ENGRAM_HEBB_LINK_MIN. Together they answer "is anything about to be * learned, and if nothing ever consolidates, is it because nothing * co-activates or because the threshold is set too high?" — the question * the zero-potentiated-edges measurement had to be instrumented to answer. */ int hebb_cands = 0; double hebb_cand_max = 0.0; for (int i = 0; i < ENGRAM_HEBB_CAND_SLOTS; i++) { if (!_eg_hebb_cand[i].a) continue; hebb_cands++; if (_eg_hebb_cand[i].score > hebb_cand_max) hebb_cand_max = _eg_hebb_cand[i].score; } /* 768, not 512: the write-back gauges added 2026-08-07 push the worst-case * rendering past the old bound, and snprintf would truncate the JSON into * an unparseable tail rather than fail loudly. */ char buf[896]; /* ctx_cos (2026-07-29): cos(query, context centroid) at the LAST * activate call, measured before the query was folded in. ~1.0 = * context aligned with current query; low = divergence (expected at * curiosity domain-rotation boundaries); -2.0 = no centroid yet or * embedder down. The drift gauge for the context-centroid mechanism. */ snprintf(buf, sizeof(buf), "{\"wm_evicted\":%lld,\"breakthroughs\":%lld," "\"embed_breaker_open\":%d,\"embed_consec_fail\":%d," "\"ctx_cos\":%.3f," "\"hebb_edges\":%lld,\"hebb_max\":%.4f,\"hebb_mass\":%.3f," "\"hebb_cands\":%d,\"hebb_cand_max\":%.4f,\"hebb_links\":%lld," "\"hebb_warm\":%d," "\"hebb_wb_pending\":%d,\"hebb_wb_drained\":%lld," "\"hebb_wb_dropped\":%lld," "\"dup_seeds\":%lld,\"dup_wm\":%lld,\"dup_wm_global\":%lld," /* txt_damaged: nodes created THIS process whose content carries * the character-loss signature. Steady 0 is the healthy state; * any climb means a write path is mangling text again. Cheap * (counted at creation) — the full census lives in * engram_text_health_json. (2026-08-08 self-review) */ "\"txt_damaged\":%lld}", (long long)_eg_act_wm_evicted, (long long)_eg_act_breakthroughs, breaker_open, _eg_embed_consec_fail, _eg_act_ctx_cos, (long long)hebb_edges, hebb_max, hebb_mass, hebb_cands, hebb_cand_max, (long long)_eg_hebb_links_formed, _eg_act_hebb_warm, _eg_hebb_wb_len, (long long)_eg_hebb_wb_drained, (long long)_eg_hebb_wb_dropped, (long long)_eg_act_dup_seeds, (long long)_eg_act_dup_wm, (long long)_eg_act_dup_wm_global, (long long)_eg_txt_write_damaged); return el_wrap_str(el_strdup(buf)); } /* engram_hebb_drain_json — pop up to `max` newly-formed Hebbian associations * off the write-back queue and return them as a JSON array: * * [{"from_id":"...","to_id":"...","weight":0.15,"hebb":0.31}, ...] * * Draining is DESTRUCTIVE: entries returned here are gone from the queue. The * caller owns delivery from that point on. That is deliberate — the alternative * (peek, deliver, ack) needs a second round trip and a retry ledger to be * correct, and the payload is an association that will re-form from live * co-activation if it genuinely matters. Losing one is cheap; a queue that * silently refills forever because acks never land is not. * * Empty queue returns "[]". See the ENGRAM_HEBB_WB_SLOTS block for why this * exists at all: the process that does the learning is not the process that * owns persistence. (2026-08-07 self-review.) */ el_val_t engram_hebb_drain_json(el_val_t max_v) { int64_t max_n = (int64_t)max_v; if (max_n <= 0) max_n = 64; if (max_n > ENGRAM_HEBB_WB_SLOTS) max_n = ENGRAM_HEBB_WB_SLOTS; JsonBuf b; jb_init(&b); jb_puts(&b, "["); int emitted = 0; while (_eg_hebb_wb_len > 0 && emitted < (int)max_n) { EgHebbWB* e = &_eg_hebb_wb[_eg_hebb_wb_head]; if (e->a && e->b) { if (emitted > 0) jb_puts(&b, ","); /* relation is emitted here, not stamped on by the caller: the * payload should be postable to /api/edges/batch verbatim. A * consumer that has to rewrite the JSON to make it valid is a * consumer that will eventually rewrite it wrong. */ jb_puts(&b, "{\"relation\":\"hebbian-associate\",\"from_id\":"); jb_emit_escaped(&b, e->a); jb_puts(&b, ",\"to_id\":"); jb_emit_escaped(&b, e->b); char tmp[96]; snprintf(tmp, sizeof(tmp), ",\"weight\":%.6g,\"hebb\":%.6g}", e->w, e->hebb); jb_puts(&b, tmp); emitted++; _eg_hebb_wb_drained++; } /* Free and advance whether or not the entry rendered — a NULL id is a * strdup failure at push time, not a retryable condition. */ free(e->a); free(e->b); e->a = NULL; e->b = NULL; _eg_hebb_wb_head = (_eg_hebb_wb_head + 1) % ENGRAM_HEBB_WB_SLOTS; _eg_hebb_wb_len--; } jb_puts(&b, "]"); return el_wrap_str(b.buf ? b.buf : el_strdup("[]")); } /* engram_cosine_sim — cosine similarity between two nodes' embeddings. * Returns a float in [-1, 1], or -2.0 when either node is missing or not * yet embedded. Exposed so EL code (and the introspection API) can probe * semantic distance directly. (2026-07-24, bl-b2d1c944) */ el_val_t engram_cosine_sim(el_val_t id_a, el_val_t id_b) { EngramStore* g = engram_get(); int64_t ia = engram_find_node_index(EL_CSTR(id_a)); int64_t ib = engram_find_node_index(EL_CSTR(id_b)); if (ia < 0 || ib < 0) return el_from_float(-2.0); EngramNode* a = &g->nodes[ia]; EngramNode* b = &g->nodes[ib]; if (!a->emb || !b->emb || a->emb_dim != b->emb_dim) return el_from_float(-2.0); return el_from_float(eg_cosine(a->emb, b->emb, a->emb_dim)); } /* engram_label_df — document frequency of `term` across node LABELS. * Returns the count of nodes whose label contains term (case-insensitive), * or the total node count for an empty/NULL term so callers treat "no term" * as maximally unspecific (i.e. reject it). * * WHY THIS EXISTS (2026-08-03 self-review). The soul's auto-term extractor * (awareness.el:auto_term_try_slot) picks a curiosity seed by taking the * FIRST WORD of a top-WM node label. A first-word extractor has no notion of * term quality, so three consecutive self-reviews each bolted another * hand-curated blocklist onto it — genre words (07-23), quoted titles * (07-25), English stopwords (07-30). Every one of those was written * REACTIVELY, after observing a flood in the live ISE stream. The mechanism * is whack-a-mole: the list can only ever contain floods that already * happened. * * Measured live on this store (13,370 nodes) while two unanticipated floods * were in flight and unfixed: * "