diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 96a804d..8508781 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -39,7 +39,7 @@ jobs: > /etc/apt/sources.list.d/google-cloud-sdk.list apt-get update -qq && apt-get install -y google-cloud-cli - - name: Download El runtime from Artifact Registry + - name: Authenticate to GCP + stage PINNED El runtime env: GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} run: | @@ -47,41 +47,21 @@ jobs: gcloud auth activate-service-account --key-file=/tmp/gcp-key.json gcloud config set project neuron-785695 + # PINNED RUNTIME — do NOT pull "latest" from Artifact Registry. + # The ship-soul calls engram_prune_telemetry (awareness.el sync/heartbeat + # self-review). The latest published el-runtime-c no longer defines that + # symbol, so an unpinned build fails to LINK — which is exactly how a + # broken/handlerless soul reached prod before. Compile against the + # vendored release runtime v1.0.0-20260501: the exact runtime the merged + # ship-soul was verified against (verify-soul-contract GATE PASS + + # genesis boot survives + full safety-contact response). It is committed + # under vendor/ so the soul build is fully reproducible and never depends + # on a moving AR "latest". rm -rf /opt/el/runtime mkdir -p /opt/el/runtime - - # Get latest version of each runtime package (elc/elb not needed — we compile - # dist/soul.c directly; running elb on Linux OOM-kills the runner, and we - # always use the repo's pre-built soul.c anyway). - get_latest() { - gcloud artifacts versions list \ - --repository=foundation-prod \ - --location=us-central1 \ - --project=neuron-785695 \ - --package="$1" \ - --sort-by="~createTime" \ - --limit=1 \ - --format="value(name)" 2>/dev/null | awk -F/ '{print $NF}' - } - - RC_VER=$(get_latest el-runtime-c) - RH_VER=$(get_latest el-runtime-h) - - echo "Downloading runtime@${RC_VER}" - - gcloud artifacts generic download \ - --repository=foundation-prod --location=us-central1 --project=neuron-785695 \ - --package=el-runtime-c --version="${RC_VER}" \ - --destination=/opt/el/runtime/ - - gcloud artifacts generic download \ - --repository=foundation-prod --location=us-central1 --project=neuron-785695 \ - --package=el-runtime-h --version="${RH_VER}" \ - --destination=/opt/el/runtime/ - - mv /opt/el/runtime/el_runtime.c* /opt/el/runtime/el_runtime.c 2>/dev/null || true - mv /opt/el/runtime/el_runtime.h* /opt/el/runtime/el_runtime.h 2>/dev/null || true - echo "El runtime ready: $(ls /opt/el/runtime/)" + cp vendor/el-runtime/v1.0.0-20260501/el_runtime.c /opt/el/runtime/el_runtime.c + cp vendor/el-runtime/v1.0.0-20260501/el_runtime.h /opt/el/runtime/el_runtime.h + echo "El runtime PINNED to v1.0.0-20260501: $(ls /opt/el/runtime/)" - name: Build neuron soul binary run: | diff --git a/vendor/el-runtime/v1.0.0-20260501/RELEASE.md b/vendor/el-runtime/v1.0.0-20260501/RELEASE.md new file mode 100644 index 0000000..280af6e --- /dev/null +++ b/vendor/el-runtime/v1.0.0-20260501/RELEASE.md @@ -0,0 +1,28 @@ +# El Compiler Release v1.0.0 — 2026-05-02 + +## Components +- `bootstrap.py` — El language compiler (Python, recursive descent parser, emits C) +- `el_runtime.c` — El runtime (C, HTTP server, engram, DHARMA, LLM chain) +- `el_runtime.h` — Runtime public API header + +## Changes in this release + +### Critical bug fixes +- `state_set`/`state_get` are now thread-safe (pthread_mutex). Was racing across 64 worker threads. +- `looks_like_string` threshold raised from 1,000,000 to 4GB. Unix timestamps were being dereferenced as heap pointers. +- `fs_read` guards against negative `ftell` result (pipe/special file overflow). + +### Engram architecture (major) +- Two-layer activation: `background_activation` (Layer 1, broad fan-out) + `working_memory_weight` (Layer 2, executive filter) +- Inhibitory edges: `EngramEdge.inhibitory` flag suppresses working memory promotion without affecting background activation +- Suppression memory: `suppression_count` — nodes activated-but-suppressed accumulate pressure toward breakthrough +- Temporal decay: `temporal_decay_rate`, `created_at`, `last_activated_at`, `activation_count` on EngramNode +- Per-type activation thresholds (Safety: 0.05, Canonical: 0.15, Lesson: 0.25, Note: 0.40) +- Temporal range query: `engram_query_range(start_ms, end_ms)` +- Layered consciousness: `EngramLayer` struct, `layer_id` on nodes and edges, `EngramStore.layers[]` +- Layer 0 override pass: safety layer fires last and cannot be suppressed + +## SHA256 +bootstrap.py +el_runtime.c +el_runtime.h diff --git a/vendor/el-runtime/v1.0.0-20260501/el_runtime.c b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c new file mode 100644 index 0000000..4cc00dd --- /dev/null +++ b/vendor/el-runtime/v1.0.0-20260501/el_runtime.c @@ -0,0 +1,12585 @@ +/* + * el_runtime.c — El language C runtime implementation + * + * All functions use el_val_t (= int64_t) as the universal value type. + * Strings are transported as their pointer address cast to int64_t. + * On any 64-bit system sizeof(pointer) <= sizeof(int64_t), so this is safe. + * + * Compile with: + * cc -std=c11 -I -lcurl -lpthread -o .c el_runtime.c + * + * Link requirements: -lcurl (HTTP client + LLM), -lpthread (HTTP server). + */ + +/* Feature-test macros must be set before any standard headers. _GNU_SOURCE + * exposes clock_gettime/CLOCK_REALTIME, strcasecmp, and the dlfcn extensions + * (RTLD_DEFAULT) — all of which macOS hands us without asking but glibc on + * Debian gates behind an explicit opt-in. */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include "el_runtime.h" + +#include +#include /* strcasecmp */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* dlsym for http_set_handler fallback */ +#include +#include +#include +#include +#include +#include + +/* ── Internal allocators ─────────────────────────────────────────────────── */ + +/* + * Per-request string arena + * + * Every El string allocated via el_strbuf / el_strdup during an HTTP request + * is registered in a thread-local arena. When el_request_end() is called at + * the end of the worker thread, every arena entry is freed — recovering all + * the intermediate strings from el_str_concat chains (build_system_prompt, + * engram_compile, etc.) that are otherwise leaked forever. + * + * Long-lived allocations (state_set values, engram internal storage) call + * el_strdup_persist() / el_strbuf_persist() which bypass the arena entirely. + */ + +#define EL_ARENA_INITIAL 512 + +typedef struct { + char** ptrs; + size_t count; + size_t cap; +} ElArena; + +static _Thread_local ElArena _tl_arena = {NULL, 0, 0}; +static _Thread_local int _tl_arena_active = 0; + +/* Binary-safe fs_read length — set by fs_read, consumed by http_send_response. + * Allows serving PNGs and other binary files without strlen truncation. */ +static _Thread_local size_t _tl_fs_read_len = 0; + +static void el_arena_track(char* p) { + if (!_tl_arena_active || !p) return; + if (_tl_arena.count >= _tl_arena.cap) { + size_t nc = _tl_arena.cap == 0 ? EL_ARENA_INITIAL : _tl_arena.cap * 2; + char** grown = realloc(_tl_arena.ptrs, nc * sizeof(char*)); + if (!grown) return; /* can't track — will leak this one ptr, but don't crash */ + _tl_arena.ptrs = grown; + _tl_arena.cap = nc; + } + _tl_arena.ptrs[_tl_arena.count++] = p; +} + +/* Called by http_worker before dispatching the El handler. */ +void el_request_start(void) { + _tl_arena.count = 0; + _tl_arena_active = 1; +} + +/* Called by http_worker after the El handler returns and the response is sent. + * Frees every intermediate string allocated during the request. */ +void el_request_end(void) { + _tl_arena_active = 0; + for (size_t i = 0; i < _tl_arena.count; i++) { + free(_tl_arena.ptrs[i]); + } + _tl_arena.count = 0; +} + +/* ── Scoped arena for CLI use ─────────────────────────────────────────────── * + * CLI programs never call el_request_start/end, so all strdup allocations are + * permanent. el_arena_push/pop let the compiler free intermediate strings + * after each compilation unit. Ported verbatim from el-compiler/runtime on + * 2026-07-17: the soul daemon's awareness loop arena-scopes each tick with + * these, and they were present only in the dev runtime copy. + * + * el_arena_push() — activates the arena if not already active, saves the + * current arena count as a mark, and returns it as an el_val_t Int. + * el_arena_pop(mark) — frees all strings allocated since the push mark and + * resets the count. If count reaches 0, deactivates the arena. + */ +#define EL_ARENA_SCOPE_DEPTH 32 +static _Thread_local size_t _tl_arena_scope[EL_ARENA_SCOPE_DEPTH]; +static _Thread_local int _tl_arena_scope_depth = 0; + +el_val_t el_arena_push(void) { + if (!_tl_arena_active) { + _tl_arena_active = 1; + } + if (_tl_arena_scope_depth < EL_ARENA_SCOPE_DEPTH) { + _tl_arena_scope[_tl_arena_scope_depth++] = _tl_arena.count; + } + return (el_val_t)(int64_t)_tl_arena.count; +} + +el_val_t el_arena_pop(el_val_t mark) { + size_t save = (size_t)(int64_t)mark; + if (save > _tl_arena.count) save = 0; + for (size_t i = save; i < _tl_arena.count; i++) { + if (_tl_arena.ptrs[i]) { + free(_tl_arena.ptrs[i]); + _tl_arena.ptrs[i] = NULL; + } + } + _tl_arena.count = save; + if (_tl_arena_scope_depth > 0) _tl_arena_scope_depth--; + if (save == 0) _tl_arena_active = 0; + return 0; +} + +/* Persistent allocation — bypasses the arena (state_set, engram internals). */ +static char* el_strdup_persist(const char* s) { + if (!s) return strdup(""); + return strdup(s); +} +static char* el_strbuf_persist(size_t n) { + char* p = malloc(n + 1); + if (!p) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + p[0] = '\0'; + return p; +} + +static char* el_strdup(const char* s) { + if (!s) { char* p = strdup(""); el_arena_track(p); return p; } + char* p = strdup(s); + el_arena_track(p); + return p; +} + +static char* el_strbuf(size_t n) { + char* p = malloc(n + 1); + if (!p) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + p[0] = '\0'; + el_arena_track(p); + return p; +} + +/* Wrap an allocated C string as el_val_t */ +static el_val_t el_wrap_str(char* s) { + return EL_STR(s); +} + +/* ── I/O ──────────────────────────────────────────────────────────────────── */ + +void println(el_val_t s) { + const char* str = EL_CSTR(s); + if (str) puts(str); + else puts(""); +} + +void print(el_val_t s) { + const char* str = EL_CSTR(s); + if (str) fputs(str, stdout); +} + +el_val_t readline(void) { + char buf[4096]; + if (!fgets(buf, sizeof(buf), stdin)) return el_wrap_str(el_strdup("")); + size_t len = strlen(buf); + if (len > 0 && buf[len - 1] == '\n') buf[len - 1] = '\0'; + return el_wrap_str(el_strdup(buf)); +} + +/* ── String builtins ─────────────────────────────────────────────────────── */ + +el_val_t el_str_concat(el_val_t av, el_val_t bv) { + const char* a = EL_CSTR(av); + const char* b = EL_CSTR(bv); + if (!a) a = ""; + if (!b) b = ""; + size_t la = strlen(a); + size_t lb = strlen(b); + char* out = el_strbuf(la + lb); + memcpy(out, a, la); + memcpy(out + la, b, lb); + out[la + lb] = '\0'; + return el_wrap_str(out); +} + +el_val_t str_eq(el_val_t av, el_val_t bv) { + const char* a = EL_CSTR(av); + const char* b = EL_CSTR(bv); + if (!a || !b) return (el_val_t)(a == b); + return (el_val_t)(strcmp(a, b) == 0); +} + +el_val_t str_starts_with(el_val_t sv, el_val_t prefv) { + const char* s = EL_CSTR(sv); + const char* prefix = EL_CSTR(prefv); + if (!s || !prefix) return 0; + size_t lp = strlen(prefix); + return (el_val_t)(strncmp(s, prefix, lp) == 0); +} + +el_val_t str_ends_with(el_val_t sv, el_val_t sufv) { + const char* s = EL_CSTR(sv); + const char* suffix = EL_CSTR(sufv); + if (!s || !suffix) return 0; + size_t ls = strlen(s); + size_t lsuf = strlen(suffix); + if (lsuf > ls) return 0; + return (el_val_t)(strcmp(s + ls - lsuf, suffix) == 0); +} + +el_val_t str_len(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return 0; + return (el_val_t)strlen(s); +} + +el_val_t str_concat(el_val_t a, el_val_t b) { + return el_str_concat(a, b); +} + +el_val_t int_to_str(el_val_t n) { + char buf[32]; + snprintf(buf, sizeof(buf), "%lld", (long long)n); + return el_wrap_str(el_strdup(buf)); +} + +el_val_t str_to_int(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return 0; + return (el_val_t)atoll(s); +} + +el_val_t str_slice(el_val_t sv, el_val_t start, el_val_t end) { + const char* s = EL_CSTR(sv); + if (!s) return el_wrap_str(el_strdup("")); + int64_t len = (int64_t)strlen(s); + if (start < 0) start = 0; + if (end > len) end = len; + if (start >= end) return el_wrap_str(el_strdup("")); + int64_t sz = end - start; + char* out = el_strbuf((size_t)sz); + memcpy(out, s + start, (size_t)sz); + out[sz] = '\0'; + return el_wrap_str(out); +} + +el_val_t str_contains(el_val_t sv, el_val_t subv) { + const char* s = EL_CSTR(sv); + const char* sub = EL_CSTR(subv); + if (!s || !sub) return 0; + return (el_val_t)(strstr(s, sub) != NULL); +} + +el_val_t str_replace(el_val_t sv, el_val_t fromv, el_val_t tov) { + const char* s = EL_CSTR(sv); + const char* from = EL_CSTR(fromv); + const char* to = EL_CSTR(tov); + if (!s || !from || !to) return el_wrap_str(el_strdup(s ? s : "")); + size_t ls = strlen(s); + size_t lf = strlen(from); + size_t lt = strlen(to); + if (lf == 0) return el_wrap_str(el_strdup(s)); + size_t count = 0; + const char* p = s; + while ((p = strstr(p, from)) != NULL) { count++; p += lf; } + size_t out_sz = ls + count * lt + 1; + char* out = el_strbuf(out_sz); + char* dst = out; + p = s; + const char* found; + while ((found = strstr(p, from)) != NULL) { + size_t chunk = (size_t)(found - p); + memcpy(dst, p, chunk); dst += chunk; + memcpy(dst, to, lt); dst += lt; + p = found + lf; + } + strcpy(dst, p); + return el_wrap_str(out); +} + +el_val_t str_to_upper(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return el_wrap_str(el_strdup("")); + size_t n = strlen(s); + char* out = el_strbuf(n); + for (size_t i = 0; i < n; i++) out[i] = (char)toupper((unsigned char)s[i]); + out[n] = '\0'; + return el_wrap_str(out); +} + +el_val_t str_to_lower(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return el_wrap_str(el_strdup("")); + size_t n = strlen(s); + char* out = el_strbuf(n); + for (size_t i = 0; i < n; i++) out[i] = (char)tolower((unsigned char)s[i]); + out[n] = '\0'; + return el_wrap_str(out); +} + +el_val_t str_trim(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return el_wrap_str(el_strdup("")); + while (*s && isspace((unsigned char)*s)) s++; + size_t n = strlen(s); + while (n > 0 && isspace((unsigned char)s[n - 1])) n--; + char* out = el_strbuf(n); + memcpy(out, s, n); + out[n] = '\0'; + return el_wrap_str(out); +} + +/* ── Math ────────────────────────────────────────────────────────────────── */ + +el_val_t el_abs(el_val_t n) { return n < 0 ? -n : n; } +el_val_t el_max(el_val_t a, el_val_t b) { return a > b ? a : b; } +el_val_t el_min(el_val_t a, el_val_t b) { return a < b ? a : b; } + +/* ── Refcounted heap objects ────────────────────────────────────────────────── + * + * ElList and ElMap carry a magic-tagged header at offset 0: + * { uint32_t magic; uint32_t refcount; ... payload ... } + * + * The magic tag distinguishes refcounted objects from raw C strings (whose + * first byte is printable ASCII < 0x80) and from small integers (which can't + * be dereferenced). el_retain / el_release sniff the magic and act only on + * matching values; everything else is a safe no-op. + * + * Both ElList and ElMap use INDIRECTION: the header is fixed-size and never + * moves. The payload arrays (elems, keys, values) live in separate heap + * allocations, so realloc-grow on append never invalidates the caller's + * pointer to the header. This is what lets us mutate-in-place safely when + * the refcount is 1 and copy-on-write when it's higher. + * + * Memory model in practice: + * Single-owner accumulator (the cg_stmts pattern) — refcount stays at 1, + * appends amortize to O(1), total memory O(N) for an N-element list. + * Multi-owner branching (the cg_if_stmt pattern) — refcount > 1, each + * append on a shared list copies, so the original is preserved for the + * else-branch. Persistent semantics where they're needed; mutation where + * they're not. */ + +#define EL_MAGIC_LIST 0xE15710A1u /* >= 0x80 in MSB so 'looks_like_string' rejects */ +#define EL_MAGIC_MAP 0xE19A704Bu + +typedef struct { + uint32_t magic; + uint32_t refcount; +} ElHeader; + +/* ── List ────────────────────────────────────────────────────────────────── */ + +typedef struct { + ElHeader hdr; + int64_t length; + int64_t capacity; + el_val_t* elems; +} ElList; + +static ElList* list_alloc(int64_t cap) { + if (cap < 4) cap = 4; + ElList* lst = malloc(sizeof(ElList)); + if (!lst) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + lst->hdr.magic = EL_MAGIC_LIST; + lst->hdr.refcount = 1; + lst->length = 0; + lst->capacity = cap; + lst->elems = malloc((size_t)cap * sizeof(el_val_t)); + if (!lst->elems) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + return lst; +} + +el_val_t el_list_empty(void) { + return EL_STR(list_alloc(4)); +} + +el_val_t el_list_new(el_val_t count, ...) { + ElList* lst = list_alloc(count > 0 ? count : 4); + va_list ap; + va_start(ap, count); + for (int64_t i = 0; i < count; i++) { + lst->elems[i] = va_arg(ap, el_val_t); + } + va_end(ap); + lst->length = count; + return EL_STR(lst); +} + +el_val_t el_list_len(el_val_t listv) { + ElList* lst = (ElList*)(uintptr_t)listv; + if (!lst) return 0; + return lst->length; +} + +el_val_t el_list_get(el_val_t listv, el_val_t index) { + ElList* lst = (ElList*)(uintptr_t)listv; + if (!lst) return 0; + if (index < 0 || index >= lst->length) return 0; + return lst->elems[index]; +} + +el_val_t el_list_append(el_val_t listv, el_val_t elem) { + ElList* old = (ElList*)(uintptr_t)listv; + if (!old) { + ElList* fresh = list_alloc(4); + fresh->elems[0] = elem; + fresh->length = 1; + return EL_STR(fresh); + } + + /* Uniquely owned: grow the elems buffer in place. The header pointer the + * caller holds doesn't move (we only realloc the inner array). This is + * the common case in compiler accumulators, and it's amortized O(1). */ + if (old->hdr.refcount <= 1) { + if (old->length >= old->capacity) { + int64_t new_cap = old->capacity > 0 ? old->capacity * 2 : 4; + el_val_t* grown = realloc(old->elems, (size_t)new_cap * sizeof(el_val_t)); + if (!grown) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + old->elems = grown; + old->capacity = new_cap; + } + old->elems[old->length++] = elem; + return listv; + } + + /* Shared: copy-on-write. The original is preserved for its other owners. */ + int64_t new_cap = old->length + 1; + if (new_cap < 4) new_cap = 4; + ElList* fresh = malloc(sizeof(ElList)); + if (!fresh) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + fresh->hdr.magic = EL_MAGIC_LIST; + fresh->hdr.refcount = 1; + fresh->length = old->length + 1; + fresh->capacity = new_cap; + fresh->elems = malloc((size_t)new_cap * sizeof(el_val_t)); + if (!fresh->elems) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + if (old->length > 0) { + memcpy(fresh->elems, old->elems, (size_t)old->length * sizeof(el_val_t)); + } + fresh->elems[old->length] = elem; + return EL_STR(fresh); +} + +el_val_t el_list_clone(el_val_t listv) { + /* Shallow copy: the new ElList owns its own header and elems buffer, but + * the elements themselves are shared (which is what callers want for the + * cg_if_stmt 'declared' pattern — cloning the spine, not its contents). + * Used by codegen at scope branch points where two child scopes need to + * see the same starting set of declared names without each other's + * mutations. */ + ElList* old = (ElList*)(uintptr_t)listv; + if (!old) return el_list_empty(); + int64_t cap = old->capacity > 0 ? old->capacity : 4; + if (cap < old->length) cap = old->length; + if (cap < 4) cap = 4; + ElList* fresh = malloc(sizeof(ElList)); + if (!fresh) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + fresh->hdr.magic = EL_MAGIC_LIST; + fresh->hdr.refcount = 1; + fresh->length = old->length; + fresh->capacity = cap; + fresh->elems = malloc((size_t)cap * sizeof(el_val_t)); + if (!fresh->elems) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + if (old->length > 0) { + memcpy(fresh->elems, old->elems, (size_t)old->length * sizeof(el_val_t)); + } + return EL_STR(fresh); +} + +/* ── Map ─────────────────────────────────────────────────────────────────── */ + +typedef struct { + ElHeader hdr; + int64_t count; + int64_t capacity; + el_val_t* keys; + el_val_t* values; +} ElMap; + +static ElMap* map_alloc(int64_t cap) { + if (cap < 4) cap = 4; + ElMap* m = malloc(sizeof(ElMap)); + if (!m) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + m->hdr.magic = EL_MAGIC_MAP; + m->hdr.refcount = 1; + m->count = 0; + m->capacity = cap; + m->keys = malloc((size_t)cap * sizeof(el_val_t)); + m->values = malloc((size_t)cap * sizeof(el_val_t)); + if (!m->keys || !m->values) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + return m; +} + +el_val_t el_map_new(el_val_t pair_count, ...) { + ElMap* m = map_alloc(pair_count > 0 ? pair_count : 4); + va_list ap; + va_start(ap, pair_count); + for (int64_t i = 0; i < pair_count; i++) { + m->keys[i] = va_arg(ap, el_val_t); + m->values[i] = va_arg(ap, el_val_t); + } + va_end(ap); + m->count = pair_count; + return EL_STR(m); +} + +static ElMap* as_map(el_val_t v) { return (ElMap*)(uintptr_t)v; } + +el_val_t el_map_get(el_val_t mapv, el_val_t keyv) { + ElMap* m = as_map(mapv); + const char* key = EL_CSTR(keyv); + if (!m || !key) return 0; + for (int64_t i = 0; i < m->count; i++) { + const char* k = EL_CSTR(m->keys[i]); + if (k && strcmp(k, key) == 0) return m->values[i]; + } + return 0; +} + +el_val_t el_get_field(el_val_t mapv, el_val_t keyv) { + return el_map_get(mapv, keyv); +} + +/* Internal: in-place set on a uniquely-owned map. */ +static el_val_t map_set_in_place(ElMap* m, el_val_t keyv, el_val_t value) { + const char* key = EL_CSTR(keyv); + if (key) { + for (int64_t i = 0; i < m->count; i++) { + const char* k = EL_CSTR(m->keys[i]); + if (k && strcmp(k, key) == 0) { m->values[i] = value; return EL_STR(m); } + } + } + if (m->count >= m->capacity) { + int64_t new_cap = m->capacity > 0 ? m->capacity * 2 : 4; + el_val_t* gk = realloc(m->keys, (size_t)new_cap * sizeof(el_val_t)); + el_val_t* gv = realloc(m->values, (size_t)new_cap * sizeof(el_val_t)); + if (!gk || !gv) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + m->keys = gk; + m->values = gv; + m->capacity = new_cap; + } + m->keys[m->count] = keyv; + m->values[m->count] = value; + m->count++; + return EL_STR(m); +} + +el_val_t el_map_set(el_val_t mapv, el_val_t keyv, el_val_t value) { + ElMap* m = as_map(mapv); + if (!m) return 0; + if (m->hdr.refcount <= 1) { + return map_set_in_place(m, keyv, value); + } + /* Shared: copy then set. The original is preserved for its other owners. */ + int64_t new_cap = m->count + 1; + if (new_cap < 4) new_cap = 4; + ElMap* fresh = malloc(sizeof(ElMap)); + if (!fresh) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + fresh->hdr.magic = EL_MAGIC_MAP; + fresh->hdr.refcount = 1; + fresh->count = m->count; + fresh->capacity = new_cap; + fresh->keys = malloc((size_t)new_cap * sizeof(el_val_t)); + fresh->values = malloc((size_t)new_cap * sizeof(el_val_t)); + if (!fresh->keys || !fresh->values) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + if (m->count > 0) { + memcpy(fresh->keys, m->keys, (size_t)m->count * sizeof(el_val_t)); + memcpy(fresh->values, m->values, (size_t)m->count * sizeof(el_val_t)); + } + return map_set_in_place(fresh, keyv, value); +} + +/* ── Refcount ops ─────────────────────────────────────────────────────────── */ +/* + * Both retain and release sniff the magic header to decide whether a value + * is a refcounted heap object. For small integers, raw C strings, and any + * value whose magic word doesn't match, both functions are no-ops. This lets + * codegen emit them on every let-binding without having to track types. + * + * Safety: we filter out obvious non-pointers (small magnitudes, misaligned + * addresses) before dereferencing. For any value that passes the filter and + * lives in a mapped page, reading the first 4 bytes is safe — strings start + * with printable ASCII (< 0x80), so their magic word will never collide with + * EL_MAGIC_LIST (0xE1...) or EL_MAGIC_MAP (0xE1...). Random integers that + * happen to look like aligned heap pointers are exceedingly unlikely to land + * on a page whose first 4 bytes match either magic. */ + +static int looks_like_heap_obj(el_val_t v) { + if (v == 0) return 0; + int64_t s = (int64_t)v; + if (s > -0x10000 && s < 0x10000) return 0; /* small ints */ + uintptr_t p = (uintptr_t)v; + if (p < 0x10000) return 0; /* low addresses */ + if (p & 0x7) return 0; /* malloc returns 8-aligned */ + return 1; +} + +void el_retain(el_val_t v) { + if (!looks_like_heap_obj(v)) return; + ElHeader* h = (ElHeader*)(uintptr_t)v; + if (h->magic == EL_MAGIC_LIST || h->magic == EL_MAGIC_MAP) { + h->refcount++; + } +} + +void el_release(el_val_t v) { + if (!looks_like_heap_obj(v)) return; + ElHeader* h = (ElHeader*)(uintptr_t)v; + if (h->magic == EL_MAGIC_LIST) { + if (h->refcount > 0 && --h->refcount == 0) { + ElList* l = (ElList*)h; + free(l->elems); + l->hdr.magic = 0; /* poison so use-after-free is detected */ + free(l); + } + } else if (h->magic == EL_MAGIC_MAP) { + if (h->refcount > 0 && --h->refcount == 0) { + ElMap* m = (ElMap*)h; + free(m->keys); + free(m->values); + m->hdr.magic = 0; + free(m); + } + } +} + +/* ── Batch 2/3 forward decls (defined later in JSON section) ────────────── */ + +typedef struct JsonBuf JsonBuf; +typedef struct JsonParser JsonParser; +static void jb_init(JsonBuf* b); +static void jb_putc(JsonBuf* b, char c); +static void jb_puts(JsonBuf* b, const char* s); +static void jb_emit_escaped(JsonBuf* b, const char* s); +static int looks_like_string(el_val_t v); +static const char* json_find_key(const char* s, const char* key); +static const char* json_skip_value(const char* p); +static char* jp_parse_string_raw(JsonParser* jp); + +/* Struct definitions are visible here because batch 2/3 helpers above use + * them by value; the bodies (jb_init, etc.) appear in the JSON section. */ +struct JsonBuf { + char* buf; + size_t len; + size_t cap; +}; + +struct JsonParser { + const char* p; + const char* end; + int err; +}; + +/* ── Batch 2: Real HTTP (libcurl client + POSIX-socket server) ───────────── */ +/* + * Client: blocking libcurl easy-handle calls. Errors are returned as a JSON + * fragment {"error":"..."} so callers can detect via str_starts_with("{") / + * json_get_string("error", ...). + * + * Server: bind/listen/accept loop on a TCP socket. Each accepted connection + * is handled in its own pthread (detached). A semaphore-style counter caps + * concurrent in-flight connections at HTTP_MAX_CONNS (64). When the cap is + * reached, accept() blocks until a worker exits. This prevents runaway + * thread creation under high load. + * + * Handler dispatch: El does not expose first-class function references at + * the runtime layer, so the second argument to http_serve(port, handler) is + * treated as a string name (or any el_val_t — the runtime ignores its + * value and uses the registry). Callers register a C-level handler via + * + * extern void el_runtime_register_handler(const char* name, + * el_val_t (*fn)(el_val_t, + * el_val_t, + * el_val_t)); + * + * and select the active handler by calling http_set_handler("name") from + * El, or by setting it directly through the C registry. If no handler is + * registered, the server replies with a 200 carrying a default message so + * the loop is observable. + */ + +/* ── HTTP client write-callback buffer ───────────────────────────────────── */ + +typedef struct { + char* data; + size_t len; + size_t cap; +} HttpBuf; + +static void httpbuf_init(HttpBuf* b) { + b->cap = 1024; + b->len = 0; + b->data = malloc(b->cap); + if (!b->data) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + b->data[0] = '\0'; +} + +static void httpbuf_append(HttpBuf* b, const void* src, size_t n) { + if (b->len + n + 1 > b->cap) { + while (b->len + n + 1 > b->cap) b->cap *= 2; + b->data = realloc(b->data, b->cap); + if (!b->data) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + } + memcpy(b->data + b->len, src, n); + b->len += n; + b->data[b->len] = '\0'; +} + +static size_t http_write_cb(char* ptr, size_t size, size_t nmemb, void* ud) { + size_t n = size * nmemb; + httpbuf_append((HttpBuf*)ud, ptr, n); + return n; +} + +/* JSON-escape an arbitrary C string into an allocated buffer. */ +static char* json_escape_alloc(const char* s) { + if (!s) return el_strdup(""); + JsonBuf b; jb_init(&b); + for (const char* p = s; *p; p++) { + unsigned char c = (unsigned char)*p; + switch (c) { + case '"': jb_puts(&b, "\\\""); break; + case '\\': jb_puts(&b, "\\\\"); break; + case '\n': jb_puts(&b, "\\n"); break; + case '\r': jb_puts(&b, "\\r"); break; + case '\t': jb_puts(&b, "\\t"); break; + default: + if (c < 0x20) { + char tmp[8]; snprintf(tmp, sizeof(tmp), "\\u%04x", c); + jb_puts(&b, tmp); + } else jb_putc(&b, (char)c); + } + } + return b.buf; +} + +static el_val_t http_error_json(const char* msg) { + char* esc = json_escape_alloc(msg ? msg : "unknown error"); + char* buf = el_strbuf(strlen(esc) + 16); + sprintf(buf, "{\"error\":\"%s\"}", esc); + free(esc); + return el_wrap_str(buf); +} + +/* HTTP timeout (ms) — read once from EL_HTTP_TIMEOUT_MS, default 60000. + * Applied via CURLOPT_TIMEOUT_MS on every libcurl request. */ +static long _el_http_timeout_ms = -1; +static long el_http_timeout_ms(void) { + long v = __atomic_load_n(&_el_http_timeout_ms, __ATOMIC_ACQUIRE); + if (v >= 0) return v; + const char* s = getenv("EL_HTTP_TIMEOUT_MS"); + long parsed = 60000L; + if (s && *s) { + char* end = NULL; + long n = strtol(s, &end, 10); + if (end != s && n > 0) parsed = n; + } + __atomic_store_n(&_el_http_timeout_ms, parsed, __ATOMIC_RELEASE); + return parsed; +} + +/* Internal: do a libcurl request; takes optional body/headers, optional method + * override, and an optional timeout override (0 = use EL_HTTP_TIMEOUT_MS). + * The override exists for the embedding path: activation must never wait the + * full 60s default on a wedged Ollama. (2026-07-24 self-review) */ +static el_val_t http_do_t(const char* method, const char* url, const char* body, + struct curl_slist* extra_headers, long timeout_ms) { + if (!url || !*url) return http_error_json("empty url"); + CURL* c = curl_easy_init(); + if (!c) return http_error_json("curl_easy_init failed"); + HttpBuf rb; httpbuf_init(&rb); + char errbuf[CURL_ERROR_SIZE]; errbuf[0] = '\0'; + curl_easy_setopt(c, CURLOPT_URL, url); + curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_write_cb); + curl_easy_setopt(c, CURLOPT_WRITEDATA, &rb); + curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, + timeout_ms > 0 ? timeout_ms : el_http_timeout_ms()); + curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf); + curl_easy_setopt(c, CURLOPT_USERAGENT, "el-runtime/1.0"); + if (extra_headers) curl_easy_setopt(c, CURLOPT_HTTPHEADER, extra_headers); + if (method && strcmp(method, "POST") == 0) { + curl_easy_setopt(c, CURLOPT_POST, 1L); + curl_easy_setopt(c, CURLOPT_POSTFIELDS, body ? body : ""); + curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)(body ? strlen(body) : 0)); + } else if (method && strcmp(method, "DELETE") == 0) { + curl_easy_setopt(c, CURLOPT_CUSTOMREQUEST, "DELETE"); + /* DELETE with a body (2026-07-24): the engram server authenticates + * mutating requests via an "_auth" field in the JSON body, so EL + * code must be able to send DELETE + body. Absent body → unchanged. */ + if (body && *body) { + curl_easy_setopt(c, CURLOPT_POSTFIELDS, body); + curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)strlen(body)); + } + } + CURLcode rc = curl_easy_perform(c); + curl_easy_cleanup(c); + if (rc != CURLE_OK) { + free(rb.data); + const char* m = errbuf[0] ? errbuf : curl_easy_strerror(rc); + return http_error_json(m); + } + return el_wrap_str(rb.data); +} + +/* Legacy entry point: default timeout. */ +static el_val_t http_do(const char* method, const char* url, const char* body, + struct curl_slist* extra_headers) { + return http_do_t(method, url, body, extra_headers, 0); +} + +el_val_t http_get(el_val_t url) { + return http_do("GET", EL_CSTR(url), NULL, NULL); +} + +el_val_t http_post(el_val_t url, el_val_t body) { + return http_do("POST", EL_CSTR(url), EL_CSTR(body), NULL); +} + +el_val_t http_post_json(el_val_t url, el_val_t json_body) { + struct curl_slist* h = NULL; + h = curl_slist_append(h, "Content-Type: application/json"); + el_val_t r = http_do("POST", EL_CSTR(url), EL_CSTR(json_body), h); + curl_slist_free_all(h); + return r; +} + +/* Build a curl_slist from an ElMap of name -> value strings. */ +static struct curl_slist* headers_from_map(el_val_t headers_map) { + struct curl_slist* h = NULL; + ElMap* m = as_map(headers_map); + if (!m) return NULL; + for (int64_t i = 0; i < m->count; i++) { + const char* k = EL_CSTR(m->keys[i]); + const char* v = EL_CSTR(m->values[i]); + if (!k || !v) continue; + size_t n = strlen(k) + strlen(v) + 4; + char* line = malloc(n); + if (!line) continue; + snprintf(line, n, "%s: %s", k, v); + h = curl_slist_append(h, line); + free(line); + } + return h; +} + +el_val_t http_get_with_headers(el_val_t url, el_val_t headers_map) { + struct curl_slist* h = headers_from_map(headers_map); + el_val_t r = http_do("GET", EL_CSTR(url), NULL, h); + if (h) curl_slist_free_all(h); + return r; +} + +el_val_t http_post_with_headers(el_val_t url, el_val_t body, el_val_t headers_map) { + struct curl_slist* h = headers_from_map(headers_map); + el_val_t r = http_do("POST", EL_CSTR(url), EL_CSTR(body), h); + if (h) curl_slist_free_all(h); + return r; +} + +el_val_t http_post_form_auth(el_val_t url, el_val_t form_body, el_val_t auth_header) { + struct curl_slist* h = NULL; + h = curl_slist_append(h, "Content-Type: application/x-www-form-urlencoded"); + const char* a = EL_CSTR(auth_header); + if (a && *a) { + size_t n = strlen(a) + 32; + char* line = malloc(n); + snprintf(line, n, "Authorization: %s", a); + h = curl_slist_append(h, line); + free(line); + } + el_val_t r = http_do("POST", EL_CSTR(url), EL_CSTR(form_body), h); + curl_slist_free_all(h); + return r; +} + +/* HTTP DELETE — mirrors http_post but with CURLOPT_CUSTOMREQUEST=DELETE. + * Returns response body on success; on transport failure returns an error + * JSON fragment (same convention as http_get/http_post). Callers that + * expect "" on failure should check for a leading '{' and an "error" key. */ +el_val_t http_delete(el_val_t url) { + return http_do("DELETE", EL_CSTR(url), NULL, NULL); +} + +/* DELETE with a JSON body — required by the engram server's body-based + * "_auth" scheme for mutating requests. (2026-07-24 self-review) */ +el_val_t http_delete_json(el_val_t url, el_val_t json_body) { + struct curl_slist* h = NULL; + h = curl_slist_append(h, "Content-Type: application/json"); + el_val_t r = http_do("DELETE", EL_CSTR(url), EL_CSTR(json_body), h); + curl_slist_free_all(h); + return r; +} + +/* ── HTTP → file streaming ──────────────────────────────────────────────── + * + * Why this exists: el_val_t strings are NUL-terminated by convention, so + * accumulating an HTTP response into an httpbuf and then wrapping its + * `.data` pointer with el_wrap_str() loses the byte length. Any consumer + * that does strlen() on the wrapped pointer truncates the body at the + * first embedded NUL. Audio (MP3, WAV, OGG), images (PNG, JPEG), and any + * other binary payload hits this. The vessels that download such bodies + * (e.g. ElevenLabs TTS → MP3) get silently corrupted files. + * + * The fix: wire libcurl's CURLOPT_WRITEFUNCTION directly to fwrite() + * against a fopen()-ed FILE*. The bytes never pass through an el_val_t + * string, so embedded NULs are preserved verbatim. Caller's contract is + * just "a file at this path with the response body in it". */ + +static size_t http_file_write_cb(char* ptr, size_t size, size_t nmemb, void* ud) { + FILE* f = (FILE*)ud; + return fwrite(ptr, size, nmemb, f); +} + +/* Internal: stream body to file. method is "GET" or "POST". body may be NULL + * (GET) or NUL-terminated (POST). headers may be NULL. Returns 1/0. */ +static el_val_t http_do_to_file(const char* method, const char* url, + const char* body, struct curl_slist* extra_headers, + const char* output_path) { + if (!url || !*url) return 0; + if (!output_path || !*output_path) return 0; + FILE* f = fopen(output_path, "wb"); + if (!f) return 0; + + CURL* c = curl_easy_init(); + if (!c) { fclose(f); remove(output_path); return 0; } + + char errbuf[CURL_ERROR_SIZE]; errbuf[0] = '\0'; + curl_easy_setopt(c, CURLOPT_URL, url); + curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_file_write_cb); + curl_easy_setopt(c, CURLOPT_WRITEDATA, f); + curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, el_http_timeout_ms()); + curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf); + curl_easy_setopt(c, CURLOPT_USERAGENT, "el-runtime/1.0"); + curl_easy_setopt(c, CURLOPT_FAILONERROR, 1L); /* 4xx/5xx → CURLE_HTTP_RETURNED_ERROR */ + if (extra_headers) curl_easy_setopt(c, CURLOPT_HTTPHEADER, extra_headers); + + if (method && strcmp(method, "POST") == 0) { + curl_easy_setopt(c, CURLOPT_POST, 1L); + curl_easy_setopt(c, CURLOPT_POSTFIELDS, body ? body : ""); + /* For the request body we still rely on strlen — POST bodies are + * caller-controlled and JSON/text in every known El use case. + * If a future caller needs a binary POST body, add a *_bytes + * variant that takes an explicit length, mirroring fs_write_bytes. */ + curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)(body ? strlen(body) : 0)); + } + + CURLcode rc = curl_easy_perform(c); + curl_easy_cleanup(c); + + /* Flush + close before signalling success, so the file is fully on disk + * by the time the caller reads back. */ + int flush_ok = (fflush(f) == 0); + int close_ok = (fclose(f) == 0); + + if (rc != CURLE_OK || !flush_ok || !close_ok) { + remove(output_path); + return 0; + } + return 1; +} + +el_val_t http_get_to_file(el_val_t url, el_val_t headers_map, el_val_t output_path) { + struct curl_slist* h = headers_from_map(headers_map); + el_val_t r = http_do_to_file("GET", EL_CSTR(url), NULL, h, EL_CSTR(output_path)); + if (h) curl_slist_free_all(h); + return r; +} + +el_val_t http_post_to_file(el_val_t url, el_val_t body, el_val_t headers_map, el_val_t output_path) { + struct curl_slist* h = headers_from_map(headers_map); + el_val_t r = http_do_to_file("POST", EL_CSTR(url), EL_CSTR(body), h, EL_CSTR(output_path)); + if (h) curl_slist_free_all(h); + return r; +} + +/* ── HTTP server (POSIX sockets + pthreads) ──────────────────────────────── */ + +#define HTTP_MAX_CONNS 64 + +typedef el_val_t (*http_handler_fn)(el_val_t method, el_val_t path, el_val_t body); + +typedef struct { + char* name; + http_handler_fn fn; +} HttpHandlerEntry; + +static HttpHandlerEntry _http_handlers[32]; +static size_t _http_handler_count = 0; +static char* _http_active_handler = NULL; +static pthread_mutex_t _http_handler_mu = PTHREAD_MUTEX_INITIALIZER; + +static pthread_mutex_t _http_conn_mu = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t _http_conn_cv = PTHREAD_COND_INITIALIZER; +static int _http_conn_active = 0; + +/* Public C-level API: register a handler by name. Programs that want El + * `http_serve` to dispatch into their handler call this from main() before + * http_serve. Not declared in the header to keep the public API minimal — + * extern lookup works since C symbols are global. */ +void el_runtime_register_handler(const char* name, http_handler_fn fn); +void el_runtime_register_handler(const char* name, http_handler_fn fn) { + if (!name || !fn) return; + pthread_mutex_lock(&_http_handler_mu); + for (size_t i = 0; i < _http_handler_count; i++) { + if (strcmp(_http_handlers[i].name, name) == 0) { + _http_handlers[i].fn = fn; + pthread_mutex_unlock(&_http_handler_mu); + return; + } + } + if (_http_handler_count < sizeof(_http_handlers) / sizeof(_http_handlers[0])) { + _http_handlers[_http_handler_count].name = el_strdup(name); + _http_handlers[_http_handler_count].fn = fn; + _http_handler_count++; + } + pthread_mutex_unlock(&_http_handler_mu); +} + +void http_set_handler(el_val_t name) { + const char* n = EL_CSTR(name); + pthread_mutex_lock(&_http_handler_mu); + free(_http_active_handler); + _http_active_handler = el_strdup(n ? n : ""); + /* If the name is not yet in the registry, try dlsym lookup against + * the running binary's symbol table. Every El `fn name(...)` compiles + * to a global C symbol with that exact name, so El programs can self- + * register their own handlers just by calling http_set_handler("name"). */ + if (n && *n) { + int found = 0; + for (size_t i = 0; i < _http_handler_count; i++) { + if (strcmp(_http_handlers[i].name, n) == 0) { found = 1; break; } + } + if (!found) { + void* sym = dlsym(RTLD_DEFAULT, n); + if (sym && _http_handler_count < sizeof(_http_handlers) / sizeof(_http_handlers[0])) { + _http_handlers[_http_handler_count].name = el_strdup(n); + _http_handlers[_http_handler_count].fn = (http_handler_fn)sym; + _http_handler_count++; + } + } + } + pthread_mutex_unlock(&_http_handler_mu); +} + +static http_handler_fn http_lookup_active(void) { + http_handler_fn out = NULL; + pthread_mutex_lock(&_http_handler_mu); + if (_http_active_handler) { + for (size_t i = 0; i < _http_handler_count; i++) { + if (strcmp(_http_handlers[i].name, _http_active_handler) == 0) { + out = _http_handlers[i].fn; break; + } + } + } + pthread_mutex_unlock(&_http_handler_mu); + return out; +} + +/* Auto-detect Content-Type from response body. */ +static const char* http_detect_content_type(const char* body) { + if (!body) return "text/plain; charset=utf-8"; + const char* p = body; + /* Binary magic bytes — check before stripping whitespace */ + if ((unsigned char)p[0] == 0x89 && p[1]=='P' && p[2]=='N' && p[3]=='G') + return "image/png"; + if ((unsigned char)p[0] == 0xFF && (unsigned char)p[1] == 0xD8) + return "image/jpeg"; + if (strncmp(p, "GIF8", 4) == 0) return "image/gif"; + if (strncmp(p, "RIFF", 4) == 0) return "image/webp"; + if (strncmp(p, "wOFF", 4) == 0) return "font/woff"; + if (strncmp(p, "wOF2", 4) == 0) return "font/woff2"; + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; + if (strncasecmp(p, "= cap) { + if (cap >= 1024 * 1024) { free(buf); return -1; } + cap *= 2; + buf = realloc(buf, cap); + if (!buf) return -1; + } + ssize_t n = recv(fd, buf + len, cap - len - 1, 0); + if (n <= 0) { free(buf); return -1; } + len += (size_t)n; + buf[len] = '\0'; + if (strstr(buf, "\r\n\r\n")) break; + } + /* Parse request line */ + char* sp1 = strchr(buf, ' '); + if (!sp1) { free(buf); return -1; } + *sp1 = '\0'; + *out_method = el_strdup(buf); + char* path_start = sp1 + 1; + char* sp2 = strchr(path_start, ' '); + if (!sp2) { free(*out_method); *out_method = NULL; free(buf); return -1; } + *sp2 = '\0'; + *out_path = el_strdup(path_start); + char* hdr_end = strstr(sp2 + 1, "\r\n\r\n"); + /* Capture the raw header block (after the request line's CRLF, up to + * but not including the terminating \r\n\r\n) for callers that asked + * for it. The legacy 3-arg path passes NULL and skips this. */ + if (out_headers_block) { + char* hdr_start = strstr(sp2 + 1, "\r\n"); + if (hdr_start && hdr_start < hdr_end) { + hdr_start += 2; + size_t hb_len = (size_t)(hdr_end - hdr_start); + char* hb = malloc(hb_len + 1); + if (hb) { + memcpy(hb, hdr_start, hb_len); + hb[hb_len] = '\0'; + *out_headers_block = hb; + } + } else { + *out_headers_block = el_strdup(""); + } + } + /* Find Content-Length */ + long content_length = 0; + char* hp = sp2 + 1; + while (hp < hdr_end) { + char* line_end = strstr(hp, "\r\n"); + /* line_end == hdr_end means we're on the LAST header line — its + * trailing \r\n is the same \r\n that begins the \r\n\r\n header + * terminator. Process this line; only stop when line_end is past + * hdr_end (which means the parser walked off the end of the + * header block). The previous condition (line_end >= hdr_end) + * silently dropped any Content-Length that appeared as the last + * header — exactly what real curl/clients tend to emit. */ + if (!line_end || line_end > hdr_end) break; + if (strncasecmp(hp, "Content-Length:", 15) == 0) { + content_length = strtol(hp + 15, NULL, 10); + if (content_length < 0) content_length = 0; + if (content_length > 64 * 1024 * 1024) content_length = 64 * 1024 * 1024; + } + hp = line_end + 2; + } + /* Body: any bytes already read past hdr_end, plus more recv */ + char* body_start = hdr_end + 4; + size_t body_have = (buf + len) - body_start; + char* body = malloc((size_t)content_length + 1); + if (!body) { free(*out_method); free(*out_path); *out_method=NULL; *out_path=NULL; free(buf); return -1; } + if ((long)body_have > content_length) body_have = (size_t)content_length; + if (body_have > 0) memcpy(body, body_start, body_have); + while ((long)body_have < content_length) { + ssize_t n = recv(fd, body + body_have, (size_t)content_length - body_have, 0); + if (n <= 0) break; + body_have += (size_t)n; + } + body[body_have] = '\0'; + *out_body = body; + free(buf); + return 0; +} + +/* Reason phrase for common HTTP statuses. Falls back to "Status" for the + * long tail — clients only care about the numeric code. */ +static const char* http_reason_phrase(int status) { + switch (status) { + case 200: return "OK"; + case 201: return "Created"; + case 202: return "Accepted"; + case 204: return "No Content"; + case 301: return "Moved Permanently"; + case 302: return "Found"; + case 303: return "See Other"; + case 304: return "Not Modified"; + case 307: return "Temporary Redirect"; + case 308: return "Permanent Redirect"; + case 400: return "Bad Request"; + case 401: return "Unauthorized"; + case 403: return "Forbidden"; + case 404: return "Not Found"; + case 405: return "Method Not Allowed"; + case 409: return "Conflict"; + case 410: return "Gone"; + case 422: return "Unprocessable Entity"; + case 429: return "Too Many Requests"; + case 500: return "Internal Server Error"; + case 501: return "Not Implemented"; + case 502: return "Bad Gateway"; + case 503: return "Service Unavailable"; + case 504: return "Gateway Timeout"; + default: return "Status"; + } +} + +/* Best-effort send with retry on partial writes. */ +static int http_send_all(int fd, const char* p, size_t left) { + while (left > 0) { + ssize_t w = send(fd, p, left, 0); + if (w <= 0) return -1; + p += w; left -= (size_t)w; + } + return 0; +} + +/* Discriminator that http_response() embeds at the start of its envelope. + * A handler returning a string starting with this exact prefix is treated + * as a structured response; anything else is treated as a raw body. */ +#define EL_HTTP_RESPONSE_TAG "{\"el_http_response\":1" + +/* Keys that conflict with runtime-managed headers are silently dropped to + * avoid double-emission — the runtime always emits its own Content-Length + * and Connection: close. Content-Type from the envelope IS allowed and + * overrides auto-detection. */ +static int http_header_is_managed(const char* k) { + return strcasecmp(k, "Content-Length") == 0 + || strcasecmp(k, "Connection") == 0; +} + +/* Walk an ElMap of header pairs and emit each as `K: V\r\n` into JsonBuf b. + * Sets *out_saw_content_type to 1 if the map contained an explicit + * Content-Type so the caller can skip auto-detection. */ +static void http_emit_headers_from_map(JsonBuf* b, el_val_t headers_map, + int* out_saw_content_type) { + *out_saw_content_type = 0; + if (headers_map == 0) return; + ElMap* m = (ElMap*)(uintptr_t)headers_map; + if (!m || m->hdr.magic != EL_MAGIC_MAP) return; + for (int64_t i = 0; i < m->count; i++) { + const char* k = EL_CSTR(m->keys[i]); + const char* v = EL_CSTR(m->values[i]); + if (!k || !v) continue; + if (http_header_is_managed(k)) continue; + if (strcasecmp(k, "Content-Type") == 0) *out_saw_content_type = 1; + jb_puts(b, k); + jb_puts(b, ": "); + jb_puts(b, v); + jb_puts(b, "\r\n"); + } +} + +/* Parse the envelope produced by http_response(). On success returns 1 and + * populates *out_status, *out_headers_map (an ElMap el_val_t — caller must + * el_release), and *out_body (allocated). On failure returns 0. + * + * Implementation: feeds the entire envelope through the recursive-descent + * JSON parser (which builds proper ElMap/ElList values), then pulls the + * three top-level fields by name. Avoids re-stringifying the headers map + * since json_stringify() does not support nested objects. */ +static int http_parse_envelope(const char* s, int* out_status, + el_val_t* out_headers_map, char** out_body, + el_val_t* out_parsed_root) { + if (!s) return 0; + if (strncmp(s, EL_HTTP_RESPONSE_TAG, + sizeof(EL_HTTP_RESPONSE_TAG) - 1) != 0) return 0; + + el_val_t parsed = json_parse(EL_STR(s)); + if (parsed == EL_NULL) return 0; + + int status = 200; + el_val_t hmap = 0; + char* body = NULL; + + el_val_t sv = el_map_get(parsed, EL_STR("status")); + if (sv != 0) { + /* status comes back as an integer — el_val_t holds it directly. */ + long sc = (long)sv; + if (sc >= 100 && sc <= 599) status = (int)sc; + } + + el_val_t hv = el_map_get(parsed, EL_STR("headers")); + if (hv != 0) { + ElMap* hm = (ElMap*)(uintptr_t)hv; + if (hm && hm->hdr.magic == EL_MAGIC_MAP) hmap = hv; + } + + el_val_t bv = el_map_get(parsed, EL_STR("body")); + if (bv != 0) { + const char* bs = EL_CSTR(bv); + if (bs) body = el_strdup(bs); + } + if (!body) body = el_strdup(""); + + *out_status = status; + *out_headers_map = hmap; + *out_body = body; + *out_parsed_root = parsed; /* caller releases to free hmap + entries */ + return 1; +} + +/* Lightweight `__status__` envelope: if the body's first key is `__status__` + * and its value is a numeric literal, lift the status to the HTTP layer and + * strip the marker from the body before sending. This is the common case for + * El handlers that want to return 4xx/5xx without going through + * http_response() — they just prepend `{"__status__":,...}` to the JSON + * they were already returning. + * + * We deliberately recognise ONLY the first-key form so the contract is cheap + * to detect and unambiguous: `{"__status__":401,"error":"unauthorized"}` is + * an envelope, but `{"error":"...","__status__":401}` is not. Product code + * controls placement. + * + * On success returns 1 with *out_status set and *out_body_alloc populated + * with a freshly malloc'd body (caller frees). On failure returns 0 and + * leaves outputs untouched. */ +static int http_parse_status_envelope(const char* s, int* out_status, + char** out_body_alloc) { + if (!s) return 0; + const char* p = s; + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; + if (*p != '{') return 0; + p++; + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; + static const char marker[] = "\"__status__\""; + size_t mlen = sizeof(marker) - 1; + if (strncmp(p, marker, mlen) != 0) return 0; + p += mlen; + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; + if (*p != ':') return 0; + p++; + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; + if (*p < '0' || *p > '9') return 0; /* non-numeric -> not an envelope */ + int status = 0; + while (*p >= '0' && *p <= '9') { + status = status * 10 + (*p - '0'); + p++; + } + if (status < 100 || status > 599) return 0; + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; + /* Two trailing shapes accepted: + * ,"k":v,...} -> body becomes {"k":v,...} + * } -> body becomes {} + * Anything else (e.g. `:` re-appearing, garbage) drops the envelope so + * we don't strip what we shouldn't. */ + if (*p == '}') { + *out_status = status; + *out_body_alloc = el_strdup("{}"); + return 1; + } + if (*p != ',') return 0; + p++; /* skip the comma; the rest of the object follows */ + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; + /* Build the trimmed body: '{' + remainder. */ + size_t rest_len = strlen(p); + char* out = (char*)malloc(rest_len + 2); + if (!out) return 0; + out[0] = '{'; + memcpy(out + 1, p, rest_len); + out[rest_len + 1] = '\0'; + *out_status = status; + *out_body_alloc = out; + return 1; +} + +/* Send a fully-built HTTP response. If `body` starts with the envelope tag, + * unpack status/headers/body. Otherwise emit the historical 200-OK with + * auto-detected Content-Type. */ +/* Thread-local flag: if 1, http_send_response writes status + headers but + * NO body (HEAD method behaviour). Set by http_worker before calling + * http_send_response, cleared after. */ +static __thread int _tl_http_head_only = 0; + +static void http_send_response(int fd, const char* body) { + if (!body) body = ""; + + int status = 200; + el_val_t env_headers_map = 0; + char* env_body = NULL; + el_val_t env_parsed_root = 0; + int is_envelope = http_parse_envelope(body, &status, + &env_headers_map, &env_body, + &env_parsed_root); + + /* If the rich http_response() envelope didn't claim this body, try the + * lightweight `__status__` form. This second envelope is malloc-backed so + * we route it through env_body and let the existing cleanup path free it + * — same lifetime contract, no special case at the bottom of the + * function. */ + if (!is_envelope) { + char* trimmed = NULL; + if (http_parse_status_envelope(body, &status, &trimmed)) { + env_body = trimmed; + is_envelope = 1; + } + } + + const char* eff_body = is_envelope ? env_body : body; + /* Use the real byte count from fs_read if available (handles binary files + * with embedded null bytes — PNG, WOFF2, etc.). Fall back to strlen for + * normal text/JSON responses where _tl_fs_read_len is 0. */ + size_t blen = (_tl_fs_read_len > 0) ? _tl_fs_read_len : strlen(eff_body); + _tl_fs_read_len = 0; /* consume — one-shot per response */ + int head_only = _tl_http_head_only; + + JsonBuf hdrs; jb_init(&hdrs); + int saw_content_type = 0; + if (is_envelope) { + http_emit_headers_from_map(&hdrs, env_headers_map, + &saw_content_type); + } + if (!saw_content_type) { + jb_puts(&hdrs, "Content-Type: "); + jb_puts(&hdrs, http_detect_content_type(eff_body)); + jb_puts(&hdrs, "\r\n"); + } + + char status_line[64]; + int sl = snprintf(status_line, sizeof(status_line), + "HTTP/1.1 %d %s\r\n", + status, http_reason_phrase(status)); + if (sl < 0) { + if (env_parsed_root) el_release(env_parsed_root); + free(env_body); free(hdrs.buf); return; + } + + char tail[128]; + int tl = snprintf(tail, sizeof(tail), + "Content-Length: %zu\r\n" + "Connection: close\r\n" + "\r\n", blen); + if (tl < 0) { + if (env_parsed_root) el_release(env_parsed_root); + free(env_body); free(hdrs.buf); return; + } + + if (http_send_all(fd, status_line, (size_t)sl) == 0 + && http_send_all(fd, hdrs.buf, hdrs.len) == 0 + && http_send_all(fd, tail, (size_t)tl) == 0 + && (head_only + /* HEAD requests echo headers + Content-Length but no body. */ + ? 1 + : http_send_all(fd, eff_body, blen) == 0)) { + /* sent successfully */ + } + + if (env_parsed_root) el_release(env_parsed_root); + free(env_body); + free(hdrs.buf); +} + +typedef struct { + int fd; +} HttpWorkerArg; + +static void* http_worker(void* arg) { + HttpWorkerArg* a = (HttpWorkerArg*)arg; + int fd = a->fd; + free(a); + char *method = NULL, *path = NULL, *body = NULL; + if (http_read_request(fd, &method, &path, &body, NULL) == 0) { + http_handler_fn h = http_lookup_active(); + char* response = NULL; + /* HEAD: dispatch as GET so existing handlers respond with the same + * body, but flag the response writer to emit headers only. RFC 9110 + * requires HEAD to mirror GET headers + Content-Length without body. */ + int head_only = (method && strcmp(method, "HEAD") == 0); + const char* dispatch_method = head_only ? "GET" : method; + el_request_start(); /* begin per-request arena */ + if (h) { + el_val_t r = h(EL_STR(dispatch_method), EL_STR(path), EL_STR(body)); + const char* rs = EL_CSTR(r); + /* Copy response out BEFORE arena teardown. + * For binary files, _tl_fs_read_len holds the real byte count — + * use memcpy instead of strdup so null bytes are preserved. */ + size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0); + response = malloc(rlen + 1); + if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; } + else if (response) { response[0] = '\0'; } + } else { + response = el_strdup_persist("el-runtime: no http handler registered"); + } + el_request_end(); /* free all intermediate strings */ + _tl_http_head_only = head_only; + http_send_response(fd, response); + _tl_http_head_only = 0; + free(response); + } + free(method); free(path); free(body); + close(fd); + /* release a slot */ + pthread_mutex_lock(&_http_conn_mu); + _http_conn_active--; + pthread_cond_signal(&_http_conn_cv); + pthread_mutex_unlock(&_http_conn_mu); + return NULL; +} + +void http_serve(el_val_t port, el_val_t handler) { + /* If `handler` looks like a string name, register it as the active handler. */ + const char* hname = EL_CSTR(handler); + if (hname && looks_like_string(handler)) { + http_set_handler(handler); + } + int p = (int)port; + if (p <= 0 || p > 65535) { fprintf(stderr, "http_serve: invalid port %d\n", p); return; } + /* Dual-stack: AF_INET6 with IPV6_V6ONLY=0 accepts both IPv4 and IPv6. + * This makes `localhost` work in browsers that resolve it to ::1 first. */ + int sock = socket(AF_INET6, SOCK_STREAM, 0); + if (sock < 0) { perror("socket"); return; } + int yes = 1; int no = 0; + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &no, sizeof(no)); + struct sockaddr_in6 addr; + memset(&addr, 0, sizeof(addr)); + addr.sin6_family = AF_INET6; + addr.sin6_addr = in6addr_any; + addr.sin6_port = htons((uint16_t)p); + if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) { + perror("bind"); close(sock); return; + } + if (listen(sock, 64) < 0) { perror("listen"); close(sock); return; } + fprintf(stderr, "[http] listening on [::]:%d (dual-stack)\n", p); + while (1) { + struct sockaddr_in6 cli; + socklen_t clen = sizeof(cli); + int cfd = accept(sock, (struct sockaddr*)&cli, &clen); + if (cfd < 0) { + if (errno == EINTR) continue; + perror("accept"); break; + } + pthread_mutex_lock(&_http_conn_mu); + while (_http_conn_active >= HTTP_MAX_CONNS) { + pthread_cond_wait(&_http_conn_cv, &_http_conn_mu); + } + _http_conn_active++; + pthread_mutex_unlock(&_http_conn_mu); + HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg)); + if (!arg) { close(cfd); continue; } + arg->fd = cfd; + pthread_t tid; + if (pthread_create(&tid, NULL, http_worker, arg) != 0) { + close(cfd); free(arg); + pthread_mutex_lock(&_http_conn_mu); + _http_conn_active--; + pthread_cond_signal(&_http_conn_cv); + pthread_mutex_unlock(&_http_conn_mu); + continue; + } + pthread_detach(tid); + } + close(sock); +} + +/* ── http_serve_async — non-blocking HTTP server ─────────────────────────── */ +/* Runs the accept loop in a background pthread, returns immediately so the + * calling EL script can continue (e.g. to run an awareness loop). + * Ported verbatim from el-compiler/runtime on 2026-07-17: the soul daemon + * (soul.el) builds against this release runtime and calls http_serve_async, + * which was present only in the dev runtime copy. + * + * El signature: http_serve_async(port, handler) -> Void */ + +typedef struct { int sock; } HttpServeAsyncArg; + +static void* _http_serve_async_loop(void* raw) { + HttpServeAsyncArg* a = (HttpServeAsyncArg*)raw; + int sock = a->sock; + free(a); + while (1) { + struct sockaddr_in6 cli; + socklen_t clen = sizeof(cli); + int cfd = accept(sock, (struct sockaddr*)&cli, &clen); + if (cfd < 0) { + if (errno == EINTR) continue; + perror("accept"); break; + } + pthread_mutex_lock(&_http_conn_mu); + while (_http_conn_active >= HTTP_MAX_CONNS) { + pthread_cond_wait(&_http_conn_cv, &_http_conn_mu); + } + _http_conn_active++; + pthread_mutex_unlock(&_http_conn_mu); + HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg)); + if (!arg) { close(cfd); continue; } + arg->fd = cfd; + pthread_t tid; + if (pthread_create(&tid, NULL, http_worker, arg) != 0) { + close(cfd); free(arg); + pthread_mutex_lock(&_http_conn_mu); + _http_conn_active--; + pthread_cond_signal(&_http_conn_cv); + pthread_mutex_unlock(&_http_conn_mu); + continue; + } + pthread_detach(tid); + } + close(sock); + return NULL; +} + +void http_serve_async(el_val_t port, el_val_t handler) { + const char* hname = EL_CSTR(handler); + if (hname && looks_like_string(handler)) { + http_set_handler(handler); + } + int p = (int)port; + if (p <= 0 || p > 65535) { fprintf(stderr, "http_serve_async: invalid port %d\n", p); return; } + int sock = socket(AF_INET6, SOCK_STREAM, 0); + if (sock < 0) { perror("socket"); return; } + int yes = 1; int no = 0; + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &no, sizeof(no)); + struct sockaddr_in6 addr; + memset(&addr, 0, sizeof(addr)); + addr.sin6_family = AF_INET6; + addr.sin6_addr = in6addr_any; + addr.sin6_port = htons((uint16_t)p); + if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) { + perror("bind"); close(sock); return; + } + if (listen(sock, 64) < 0) { perror("listen"); close(sock); return; } + fprintf(stderr, "[http] async listening on [::]:%d (dual-stack)\n", p); + HttpServeAsyncArg* a = malloc(sizeof(HttpServeAsyncArg)); + if (!a) { close(sock); return; } + a->sock = sock; + pthread_t tid; + if (pthread_create(&tid, NULL, _http_serve_async_loop, a) != 0) { + perror("pthread_create"); free(a); close(sock); return; + } + pthread_detach(tid); + /* Returns immediately — caller can now run awareness_run() or any loop. */ +} + +/* ── HTTP server v2 — request headers + structured response ──────────────── */ +/* + * v2 widens the handler signature from + * (method, path, body) -> body_string + * to + * (method, path, headers_map, body) -> body_string_or_envelope + * + * The response envelope is detected uniformly inside http_send_response — so + * 4-arg handlers can return either a plain body or http_response(...). The + * 3-arg path stays untouched in spirit (its handlers still build plain + * bodies; the envelope tag, being `{"el_http_response":1`, will never + * collide with normal JSON the legacy server.el routes return). + * + * Registry is parallel to the 3-arg handler registry: separate name table, + * separate active-handler slot, separate dlsym fallback. Mixing v1 and v2 + * handlers in the same process is fine — they don't share the active slot. */ + +typedef el_val_t (*http_handler4_fn)(el_val_t method, el_val_t path, + el_val_t headers_map, el_val_t body); + +typedef struct { + char* name; + http_handler4_fn fn; +} HttpHandler4Entry; + +static HttpHandler4Entry _http_handlers4[32]; +static size_t _http_handler4_count = 0; +static char* _http_active_handler4 = NULL; + +void el_runtime_register_handler_v2(const char* name, http_handler4_fn fn); +void el_runtime_register_handler_v2(const char* name, http_handler4_fn fn) { + if (!name || !fn) return; + pthread_mutex_lock(&_http_handler_mu); + for (size_t i = 0; i < _http_handler4_count; i++) { + if (strcmp(_http_handlers4[i].name, name) == 0) { + _http_handlers4[i].fn = fn; + pthread_mutex_unlock(&_http_handler_mu); + return; + } + } + if (_http_handler4_count < + sizeof(_http_handlers4) / sizeof(_http_handlers4[0])) { + _http_handlers4[_http_handler4_count].name = el_strdup(name); + _http_handlers4[_http_handler4_count].fn = fn; + _http_handler4_count++; + } + pthread_mutex_unlock(&_http_handler_mu); +} + +void http_set_handler_v2(el_val_t name) { + const char* n = EL_CSTR(name); + pthread_mutex_lock(&_http_handler_mu); + free(_http_active_handler4); + _http_active_handler4 = el_strdup(n ? n : ""); + if (n && *n) { + int found = 0; + for (size_t i = 0; i < _http_handler4_count; i++) { + if (strcmp(_http_handlers4[i].name, n) == 0) { found = 1; break; } + } + if (!found) { + void* sym = dlsym(RTLD_DEFAULT, n); + if (sym && _http_handler4_count < + sizeof(_http_handlers4) / sizeof(_http_handlers4[0])) { + _http_handlers4[_http_handler4_count].name = el_strdup(n); + _http_handlers4[_http_handler4_count].fn = + (http_handler4_fn)sym; + _http_handler4_count++; + } + } + } + pthread_mutex_unlock(&_http_handler_mu); +} + +static http_handler4_fn http_lookup_active_v2(void) { + http_handler4_fn out = NULL; + pthread_mutex_lock(&_http_handler_mu); + if (_http_active_handler4) { + for (size_t i = 0; i < _http_handler4_count; i++) { + if (strcmp(_http_handlers4[i].name, + _http_active_handler4) == 0) { + out = _http_handlers4[i].fn; break; + } + } + } + pthread_mutex_unlock(&_http_handler_mu); + return out; +} + +/* Build an ElMap from the raw header block produced by http_read_request. + * Keys are lowercased (RFC 7230 — case-insensitive); values have leading + * whitespace trimmed. Repeated headers with the same name are joined with + * ", " in arrival order, matching standard library behaviour elsewhere. */ +static el_val_t http_build_headers_map(const char* hdr_block) { + el_val_t m = el_map_new(0); + if (!hdr_block || !*hdr_block) return m; + const char* p = hdr_block; + while (*p) { + const char* line_end = strstr(p, "\r\n"); + const char* end = line_end ? line_end : p + strlen(p); + const char* colon = NULL; + for (const char* c = p; c < end; c++) { + if (*c == ':') { colon = c; break; } + } + if (colon && colon > p) { + size_t klen = (size_t)(colon - p); + char* key = malloc(klen + 1); + if (key) { + for (size_t i = 0; i < klen; i++) { + unsigned char ch = (unsigned char)p[i]; + key[i] = (char)tolower(ch); + } + key[klen] = '\0'; + const char* vstart = colon + 1; + while (vstart < end && (*vstart == ' ' || *vstart == '\t')) vstart++; + size_t vlen = (size_t)(end - vstart); + /* Strip trailing OWS just in case. */ + while (vlen > 0 + && (vstart[vlen - 1] == ' ' + || vstart[vlen - 1] == '\t')) vlen--; + /* Coalesce repeats: if key already present, append ", value". */ + el_val_t existing = el_map_get(m, EL_STR(key)); + if (existing != 0 && looks_like_string(existing)) { + const char* old = EL_CSTR(existing); + size_t olen = strlen(old); + char* combined = malloc(olen + 2 + vlen + 1); + if (combined) { + memcpy(combined, old, olen); + memcpy(combined + olen, ", ", 2); + memcpy(combined + olen + 2, vstart, vlen); + combined[olen + 2 + vlen] = '\0'; + m = el_map_set(m, EL_STR(key), EL_STR(combined)); + } + free(key); + } else { + char* val = malloc(vlen + 1); + if (val) { + memcpy(val, vstart, vlen); + val[vlen] = '\0'; + m = el_map_set(m, EL_STR(key), EL_STR(val)); + } else { + free(key); + } + } + } + } + if (!line_end) break; + p = line_end + 2; + } + return m; +} + +static void* http_worker_v2(void* arg) { + HttpWorkerArg* a = (HttpWorkerArg*)arg; + int fd = a->fd; + free(a); + char *method = NULL, *path = NULL, *body = NULL, *hdr_block = NULL; + if (http_read_request(fd, &method, &path, &body, &hdr_block) == 0) { + http_handler4_fn h = http_lookup_active_v2(); + char* response = NULL; + int head_only = (method && strcmp(method, "HEAD") == 0); + const char* dispatch_method = head_only ? "GET" : method; + el_request_start(); /* begin per-request arena */ + if (h) { + el_val_t hmap = http_build_headers_map(hdr_block ? hdr_block : ""); + el_val_t r = h(EL_STR(dispatch_method), EL_STR(path), hmap, EL_STR(body)); + const char* rs = EL_CSTR(r); + size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0); + response = malloc(rlen + 1); + if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; } + else if (response) { response[0] = '\0'; } + el_release(hmap); + } else { + response = el_strdup_persist( + "el-runtime: no v2 http handler registered " + "(call http_set_handler_v2)"); + } + el_request_end(); /* free all intermediate strings */ + _tl_http_head_only = head_only; + http_send_response(fd, response); + _tl_http_head_only = 0; + free(response); + } + free(method); free(path); free(body); free(hdr_block); + close(fd); + pthread_mutex_lock(&_http_conn_mu); + _http_conn_active--; + pthread_cond_signal(&_http_conn_cv); + pthread_mutex_unlock(&_http_conn_mu); + return NULL; +} + +void http_serve_v2(el_val_t port, el_val_t handler) { + const char* hname = EL_CSTR(handler); + if (hname && looks_like_string(handler)) { + http_set_handler_v2(handler); + } + int p = (int)port; + if (p <= 0 || p > 65535) { + fprintf(stderr, "http_serve_v2: invalid port %d\n", p); + return; + } + /* Dual-stack: same as http_serve - AF_INET6 + IPV6_V6ONLY=0. */ + int sock = socket(AF_INET6, SOCK_STREAM, 0); + if (sock < 0) { perror("socket"); return; } + int yes = 1; int no = 0; + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &no, sizeof(no)); + struct sockaddr_in6 addr; + memset(&addr, 0, sizeof(addr)); + addr.sin6_family = AF_INET6; + addr.sin6_addr = in6addr_any; + addr.sin6_port = htons((uint16_t)p); + if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) { + perror("bind"); close(sock); return; + } + if (listen(sock, 64) < 0) { perror("listen"); close(sock); return; } + fprintf(stderr, "[http v2] listening on [::]:%d (dual-stack)\n", p); + while (1) { + struct sockaddr_in6 cli; + socklen_t clen = sizeof(cli); + int cfd = accept(sock, (struct sockaddr*)&cli, &clen); + if (cfd < 0) { + if (errno == EINTR) continue; + perror("accept"); break; + } + pthread_mutex_lock(&_http_conn_mu); + while (_http_conn_active >= HTTP_MAX_CONNS) { + pthread_cond_wait(&_http_conn_cv, &_http_conn_mu); + } + _http_conn_active++; + pthread_mutex_unlock(&_http_conn_mu); + HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg)); + if (!arg) { close(cfd); continue; } + arg->fd = cfd; + pthread_t tid; + if (pthread_create(&tid, NULL, http_worker_v2, arg) != 0) { + close(cfd); free(arg); + pthread_mutex_lock(&_http_conn_mu); + _http_conn_active--; + pthread_cond_signal(&_http_conn_cv); + pthread_mutex_unlock(&_http_conn_mu); + continue; + } + pthread_detach(tid); + } + close(sock); +} + +/* Build the response envelope a 4-arg handler can return. We hand-write + * the JSON so the discriminator key always lands first — the runtime's + * http_parse_envelope() detects it via prefix match. headers_json must be + * either "" (empty), "{}" (empty object), or a well-formed JSON object + * literal; anything else will produce a malformed envelope and the runtime + * will treat the whole string as a plain body (no envelope detected). */ +el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body) { + long sc = (long)status; + if (sc < 100 || sc > 599) sc = 200; + const char* hj = EL_CSTR(headers_json); + if (!hj || !*hj) hj = "{}"; + /* Light validation: must start with '{' and end with '}'. */ + size_t hlen = strlen(hj); + int hj_ok = (hlen >= 2 && hj[0] == '{' && hj[hlen - 1] == '}'); + if (!hj_ok) hj = "{}"; + const char* b = EL_CSTR(body); + if (!b) b = ""; + + JsonBuf out; jb_init(&out); + jb_puts(&out, EL_HTTP_RESPONSE_TAG); /* {"el_http_response":1 */ + jb_puts(&out, ",\"status\":"); + char num[32]; + snprintf(num, sizeof(num), "%ld", sc); + jb_puts(&out, num); + jb_puts(&out, ",\"headers\":"); + jb_puts(&out, hj); + jb_puts(&out, ",\"body\":"); + jb_emit_escaped(&out, b); + jb_putc(&out, '}'); + return el_wrap_str(out.buf); +} + +/* ── Filesystem ──────────────────────────────────────────────────────────── */ + +el_val_t fs_read(el_val_t pathv) { + const char* path = EL_CSTR(pathv); + _tl_fs_read_len = 0; + if (!path) return el_wrap_str(el_strdup("")); + FILE* f = fopen(path, "rb"); + if (!f) return el_wrap_str(el_strdup("")); + fseek(f, 0, SEEK_END); + long sz = ftell(f); + rewind(f); + if (sz < 0) { fclose(f); return el_wrap_str(el_strdup("")); } /* pipe/special file */ + char* buf = el_strbuf((size_t)sz); + size_t got = fread(buf, 1, (size_t)sz, f); + buf[got] = '\0'; + _tl_fs_read_len = got; /* store real byte count for binary-safe send */ + fclose(f); + return el_wrap_str(buf); +} + +el_val_t fs_write(el_val_t pathv, el_val_t contentv) { + const char* path = EL_CSTR(pathv); + const char* content = EL_CSTR(contentv); + if (!path || !content) return 0; + FILE* f = fopen(path, "wb"); + if (!f) return 0; + size_t n = strlen(content); + size_t written = fwrite(content, 1, n, f); + fclose(f); + return written == n ? 1 : 0; +} + +/* fs_write_bytes — explicit-length binary write. Bypasses strlen so embedded + * NULs survive. Caller must know the byte count (e.g. from base64_decode, + * or the fixed 32-byte sha256_bytes/hmac_sha256_bytes outputs). + * + * If `length` is negative, treats as failure. If `length` is 0, creates an + * empty file (still useful as a "touch with content" primitive). */ +el_val_t fs_write_bytes(el_val_t pathv, el_val_t bytesv, el_val_t lengthv) { + const char* path = EL_CSTR(pathv); + const char* bytes = EL_CSTR(bytesv); + int64_t n = (int64_t)lengthv; + if (!path || !bytes) return 0; + if (n < 0) return 0; + FILE* f = fopen(path, "wb"); + if (!f) return 0; + size_t written = (n > 0) ? fwrite(bytes, 1, (size_t)n, f) : 0; + int flush_ok = (fflush(f) == 0); + int close_ok = (fclose(f) == 0); + if (!flush_ok || !close_ok || written != (size_t)n) { + remove(path); + return 0; + } + return 1; +} + +// exec_command — run a shell command, return exit code (0 = success). +// Used by elb and other El tooling to invoke subprocesses. +el_val_t exec_command(el_val_t cmdv) { + const char* cmd = EL_CSTR(cmdv); + if (!cmd) return (el_val_t)(int64_t)-1; + int ret = system(cmd); + return (el_val_t)(int64_t)ret; +} + +// exec_capture — run a shell command, capture stdout, return as String. +// Returns "" on failure. +el_val_t exec_capture(el_val_t cmdv) { + const char* cmd = EL_CSTR(cmdv); + if (!cmd) return el_wrap_str(el_strdup("")); + FILE* f = popen(cmd, "r"); + if (!f) return el_wrap_str(el_strdup("")); + JsonBuf b; jb_init(&b); + char buf[4096]; + while (fgets(buf, sizeof(buf), f)) jb_puts(&b, buf); + pclose(f); + return el_wrap_str(b.buf); +} + +// exec — run a shell command via /bin/sh, capture stdout, return as String. +// Times out after 30 seconds. Returns "" on any error. +// El name: exec(cmd) -> String +el_val_t exec(el_val_t cmdv) { + const char* cmd = EL_CSTR(cmdv); + if (!cmd || !*cmd) return el_wrap_str(el_strdup("")); + /* Build a time-limited command: wrap with timeout(1) if available, + * otherwise rely on the 30s read loop guard below. We use the simple + * popen approach with a deadline measured by wall clock so the caller + * is never blocked indefinitely. */ + FILE* f = popen(cmd, "r"); + if (!f) return el_wrap_str(el_strdup("")); + JsonBuf b; jb_init(&b); + char buf[4096]; + /* 30-second wall-clock deadline */ + time_t deadline = time(NULL) + 30; + while (time(NULL) < deadline) { + if (fgets(buf, sizeof(buf), f) == NULL) break; + jb_puts(&b, buf); + } + pclose(f); + return el_wrap_str(b.buf); +} + +// exec_bg — run a shell command in background, return PID as String. +// The child process runs independently; the caller is not blocked. +// Returns "" on fork failure. +// El name: exec_bg(cmd) -> String +el_val_t exec_bg(el_val_t cmdv) { + const char* cmd = EL_CSTR(cmdv); + if (!cmd || !*cmd) return el_wrap_str(el_strdup("")); + pid_t pid = fork(); + if (pid < 0) { + /* fork failed */ + return el_wrap_str(el_strdup("")); + } + if (pid == 0) { + /* child: detach from parent's stdio, exec via shell */ + setsid(); + int devnull = open("/dev/null", O_RDWR); + if (devnull >= 0) { + dup2(devnull, STDIN_FILENO); + dup2(devnull, STDOUT_FILENO); + dup2(devnull, STDERR_FILENO); + close(devnull); + } + execl("/bin/sh", "sh", "-c", cmd, (char*)NULL); + _exit(127); + } + /* parent: convert pid to string and return immediately */ + char pidbuf[32]; + snprintf(pidbuf, sizeof(pidbuf), "%d", (int)pid); + return el_wrap_str(el_strdup(pidbuf)); +} + +el_val_t fs_list(el_val_t pathv) { + const char* path = EL_CSTR(pathv); + el_val_t lst = el_list_empty(); + if (!path) return lst; + DIR* d = opendir(path); + if (!d) return lst; + struct dirent* e; + while ((e = readdir(d)) != NULL) { + if (strcmp(e->d_name, ".") == 0 || strcmp(e->d_name, "..") == 0) continue; + lst = el_list_append(lst, el_wrap_str(el_strdup(e->d_name))); + } + closedir(d); + return lst; +} + +/* fs_exists — true iff stat(path) succeeds. Symlinks are followed. */ +el_val_t fs_exists(el_val_t pathv) { + const char* path = EL_CSTR(pathv); + if (!path || !*path) return 0; + struct stat st; + return (el_val_t)(stat(path, &st) == 0 ? 1 : 0); +} + +/* fs_mkdir — create directory at path with mode 0755, mkdir -p semantics. + * Returns 1 if path exists or was created (incl. all parents); 0 on failure. + * Walks the path component-by-component so missing intermediate dirs are + * also created. An existing leaf is not an error. */ +el_val_t fs_mkdir(el_val_t pathv) { + const char* path = EL_CSTR(pathv); + if (!path || !*path) return 0; + size_t n = strlen(path); + char* buf = malloc(n + 1); + if (!buf) return 0; + memcpy(buf, path, n + 1); + /* Walk components; create each prefix in turn. */ + for (size_t i = 1; i <= n; i++) { + if (buf[i] == '/' || buf[i] == '\0') { + char saved = buf[i]; + buf[i] = '\0'; + if (buf[0] != '\0') { + if (mkdir(buf, 0755) != 0 && errno != EEXIST) { + /* Tolerate the case where this prefix exists as a non-dir + * only when stat says it's a directory. */ + struct stat st; + if (stat(buf, &st) != 0 || !S_ISDIR(st.st_mode)) { + free(buf); + return 0; + } + } + } + buf[i] = saved; + } + } + free(buf); + return 1; +} + +/* ── URL encoding ─────────────────────────────────────────────────────────── */ + +/* RFC 3986 percent-encoding for URL components (form bodies, query strings). + * Unreserved set: A-Z a-z 0-9 - _ . ~ — passed through verbatim. + * Everything else (including space) becomes %XX hex. */ +el_val_t url_encode(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return el_wrap_str(el_strdup("")); + static const char hex[] = "0123456789ABCDEF"; + size_t n = strlen(s); + char* out = el_strbuf(n * 3); + size_t o = 0; + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)s[i]; + if ((c >= 'A' && c <= 'Z') || + (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9') || + c == '-' || c == '_' || c == '.' || c == '~') { + out[o++] = (char)c; + } else { + out[o++] = '%'; + out[o++] = hex[(c >> 4) & 0xF]; + out[o++] = hex[c & 0xF]; + } + } + out[o] = '\0'; + return el_wrap_str(out); +} + +/* Decode percent-encoded URL component. '+' becomes space (form-encoded); + * malformed %-escapes are emitted verbatim. */ +el_val_t url_decode(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return el_wrap_str(el_strdup("")); + size_t n = strlen(s); + char* out = el_strbuf(n); + size_t o = 0; + for (size_t i = 0; i < n; i++) { + char c = s[i]; + if (c == '+') { + out[o++] = ' '; + } else if (c == '%' && i + 2 < n) { + char h1 = s[i + 1], h2 = s[i + 2]; + int v1 = (h1 >= '0' && h1 <= '9') ? h1 - '0' + : (h1 >= 'a' && h1 <= 'f') ? h1 - 'a' + 10 + : (h1 >= 'A' && h1 <= 'F') ? h1 - 'A' + 10 : -1; + int v2 = (h2 >= '0' && h2 <= '9') ? h2 - '0' + : (h2 >= 'a' && h2 <= 'f') ? h2 - 'a' + 10 + : (h2 >= 'A' && h2 <= 'F') ? h2 - 'A' + 10 : -1; + if (v1 >= 0 && v2 >= 0) { + out[o++] = (char)((v1 << 4) | v2); + i += 2; + } else { + out[o++] = c; + } + } else { + out[o++] = c; + } + } + out[o] = '\0'; + return el_wrap_str(out); +} + +/* ── HTML allowlist sanitizer ──────────────────────────────────────────────── + * el_html_sanitize(input, allowlist_json) + * + * Strict allowlist HTML cleaner. Replaces the older denylist patterns + * (str_replace cascades that wrapped dangerous tags in HTML comments and + * renamed `on*` attributes). The denylist approach is fragile: comment- + * wrapping can be re-broken by a literal `-->` inside an attacker-supplied + * attribute value, and every new attack vector requires a code change. + * + * Design: + * - Single-pass byte-level state machine. + * - Tag and attribute names are matched case-insensitively against the + * allowlist. Unknown tags are dropped entirely (the open and close + * markers are stripped; their inner text content survives, escaped). + * - A small set of "dangerous container" tags (script, style, iframe, + * object, embed, form, plus a few rarer ones) drop themselves AND + * their full subtree — text between `` is + * CDATA-like and must not be re-emitted as escaped text either. + * - Comments (), doctype (), CDATA (), + * and processing instructions () are dropped entirely. + * - Text content outside dropped subtrees is HTML-escaped (&, <, >, ", '). + * - Attribute values are unquoted/dequoted, then re-emitted with double + * quotes around the cleanly-escaped value. + * - For `` and any `src` attribute, the URL scheme is validated: + * only http:, https:, mailto:, fragment-only `#anchor`, or relative + * paths are allowed. Anything else (javascript:, data:, vbscript:, + * about:, file:, etc.) drops the attribute. + * - Self-closing void tags (br, hr, img, etc.) emit without a close tag. + * - Malformed input (unclosed tag at EOF, bad attribute syntax) drops + * the pending tag and continues. Pre-encoded entities (<, &, + * etc.) are passed through verbatim — the browser will decode them + * safely on render. + * + * Allowlist format (JSON string): + * {"p":[],"a":["href","title"],"strong":[],...} + * - Key = lowercase tag name. + * - Value = JSON array of allowed attribute names (lowercase). + * - Empty array means tag allowed but no attributes survive. + * + * Output is a freshly-allocated arena-tracked el_val_t string. */ + +/* Internal byte buffer with realloc-doubling. Used during sanitization; + * the final result is copied into an arena-tracked el_strbuf so the caller + * sees standard runtime memory semantics. */ +typedef struct { + char* data; + size_t len; + size_t cap; +} html_buf_t; + +static void html_buf_init(html_buf_t* b) { + b->cap = 256; + b->data = malloc(b->cap); + if (!b->data) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + b->len = 0; +} + +static void html_buf_grow(html_buf_t* b, size_t need) { + if (b->len + need + 1 <= b->cap) return; + size_t nc = b->cap; + while (b->len + need + 1 > nc) nc *= 2; + char* nd = realloc(b->data, nc); + if (!nd) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + b->data = nd; + b->cap = nc; +} + +static void html_buf_putc(html_buf_t* b, char c) { + html_buf_grow(b, 1); + b->data[b->len++] = c; +} + +static void html_buf_puts(html_buf_t* b, const char* s) { + if (!s) return; + size_t n = strlen(s); + html_buf_grow(b, n); + memcpy(b->data + b->len, s, n); + b->len += n; +} + +static void html_buf_free(html_buf_t* b) { + free(b->data); + b->data = NULL; + b->len = b->cap = 0; +} + +/* ASCII tolower, locale-independent. */ +static int html_tolower(int c) { + return (c >= 'A' && c <= 'Z') ? c + 32 : c; +} + +/* Case-insensitive ASCII compare of [a, a+n) against c-string `s`. + * Returns 1 iff lengths match and bytes are equal under tolower. */ +static int html_ieq_n(const char* a, size_t n, const char* s) { + if (!a || !s) return 0; + if (strlen(s) != n) return 0; + for (size_t i = 0; i < n; i++) { + if (html_tolower((unsigned char)a[i]) != html_tolower((unsigned char)s[i])) return 0; + } + return 1; +} + +/* Case-insensitive ASCII compare of two byte slices. */ +static int html_iemem(const char* a, const char* b, size_t n) { + for (size_t i = 0; i < n; i++) { + if (html_tolower((unsigned char)a[i]) != html_tolower((unsigned char)b[i])) return 0; + } + return 1; +} + +/* Walk a JSON allowlist object and find the value (an array) for a given + * tag key, comparing case-insensitively. On hit returns a pointer to the + * opening `[` of the array and writes the byte length of the array span + * (including the brackets) to *out_len. On miss returns NULL. + * + * The parser is intentionally tiny: it does not handle escapes inside + * keys (allowlist authors do not need them), and it relies on balanced + * brackets/quotes within the value array. */ +static const char* html_allowlist_find(const char* allow, const char* tag, + size_t tag_len, size_t* out_len) { + if (!allow) return NULL; + const char* p = allow; + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; + if (*p != '{') return NULL; + p++; + while (*p) { + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r' || *p == ',') p++; + if (*p == '}' || *p == 0) return NULL; + if (*p != '"') return NULL; + p++; + const char* k = p; + while (*p && *p != '"') p++; + if (*p != '"') return NULL; + size_t klen = (size_t)(p - k); + p++; + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; + if (*p != ':') return NULL; + p++; + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; + if (*p != '[') return NULL; + const char* arr_start = p; + int depth = 0; + int in_str = 0; + while (*p) { + char c = *p; + if (in_str) { + if (c == '\\' && p[1]) { p += 2; continue; } + if (c == '"') in_str = 0; + } else { + if (c == '"') in_str = 1; + else if (c == '[') depth++; + else if (c == ']') { depth--; if (depth == 0) { p++; break; } } + } + p++; + } + size_t alen = (size_t)(p - arr_start); + int match = (klen == tag_len) && html_iemem(k, tag, klen); + if (match) { + if (out_len) *out_len = alen; + return arr_start; + } + } + return NULL; +} + +/* Returns 1 iff `attr` (length attr_len) appears as a string element + * in the JSON array slice [arr, arr+arr_len). Comparison is case- + * insensitive. */ +static int html_attr_in_array(const char* arr, size_t arr_len, + const char* attr, size_t attr_len) { + if (!arr || arr_len < 2) return 0; + const char* p = arr + 1; + const char* end = arr + arr_len - 1; + while (p < end) { + while (p < end && (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r' || *p == ',')) p++; + if (p >= end) return 0; + if (*p != '"') return 0; + p++; + const char* s = p; + while (p < end && *p != '"') { + if (*p == '\\' && p + 1 < end) p++; + p++; + } + if (p >= end) return 0; + size_t slen = (size_t)(p - s); + p++; + if (slen == attr_len && html_iemem(s, attr, slen)) return 1; + } + return 0; +} + +/* Hard-coded set of tags whose content is ALSO dropped (entire subtree). */ +static int html_is_dangerous_container(const char* tag, size_t tag_len) { + static const char* names[] = { + "script", "style", "iframe", "object", "embed", "form", + "noscript", "noembed", "template", "svg", "math", "frame", + "frameset", "applet", "audio", "video", "source", "track", + NULL + }; + for (int i = 0; names[i]; i++) { + if (html_ieq_n(tag, tag_len, names[i])) return 1; + } + return 0; +} + +/* HTML void elements — emit without a close tag. */ +static int html_is_void(const char* tag, size_t tag_len) { + static const char* names[] = { + "area", "base", "br", "col", "embed", "hr", "img", "input", + "link", "meta", "param", "source", "track", "wbr", + NULL + }; + for (int i = 0; names[i]; i++) { + if (html_ieq_n(tag, tag_len, names[i])) return 1; + } + return 0; +} + +/* Append a single byte HTML-escaped into the output buffer. */ +static void html_escape_byte(html_buf_t* out, unsigned char c) { + switch (c) { + case '<': html_buf_puts(out, "<"); break; + case '>': html_buf_puts(out, ">"); break; + case '"': html_buf_puts(out, """); break; + case '\'': html_buf_puts(out, "'"); break; + default: html_buf_putc(out, (char)c); break; + } +} + +/* Validate a URL value against the allowlist of safe schemes for hrefs. + * Returns 1 iff the URL is safe to emit. Acceptable forms: + * - http:// or https:// (case-insensitive) + * - mailto: + * - fragment-only `#anchor` + * - relative path that does not contain a colon before the first + * slash/?/# (so `foo/bar`, `/foo`, `?x=1` are OK; `javascript:x` is + * not — its colon precedes any path/hash/query separator). + * + * URL leading whitespace and embedded ASCII control bytes (TAB, LF, CR) + * are stripped before the scheme test, mirroring how browsers normalise + * URLs (these bytes are otherwise a known XSS bypass: `java\tscript:`). */ +static int html_url_is_safe(const char* url, size_t len) { + if (!url || len == 0) return 1; /* empty href is harmless */ + size_t i = 0; + while (i < len) { + unsigned char c = (unsigned char)url[i]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == 0x0B || c == 0x0C) { + i++; continue; + } + break; + } + if (i >= len) return 1; /* whitespace only */ + if (url[i] == '#') return 1; /* fragment only */ + if (url[i] == '/' || url[i] == '?') return 1; /* relative */ + /* Find the first scheme-terminating character. */ + size_t scheme_end = (size_t)-1; + for (size_t j = i; j < len; j++) { + char c = url[j]; + if (c == ':') { scheme_end = j; break; } + if (c == '/' || c == '?' || c == '#') break; + } + if (scheme_end == (size_t)-1) return 1; /* no colon → relative path */ + /* Lowercase the scheme, stripping embedded control bytes. */ + char scheme[32]; + size_t sl = 0; + for (size_t j = i; j < scheme_end && sl < sizeof(scheme) - 1; j++) { + unsigned char c = (unsigned char)url[j]; + if (c == '\t' || c == '\n' || c == '\r' || c == 0x0B || c == 0x0C) continue; + scheme[sl++] = (char)html_tolower(c); + } + scheme[sl] = '\0'; + if (strcmp(scheme, "http") == 0) return 1; + if (strcmp(scheme, "https") == 0) return 1; + if (strcmp(scheme, "mailto") == 0) return 1; + return 0; +} + +el_val_t el_html_sanitize(el_val_t input_v, el_val_t allowlist_v) { + const char* input = EL_CSTR(input_v); + const char* allow = EL_CSTR(allowlist_v); + if (!input) return el_wrap_str(el_strdup("")); + if (!allow) allow = "{}"; + size_t in_len = strlen(input); + + html_buf_t out; + html_buf_init(&out); + + size_t i = 0; + while (i < in_len) { + unsigned char c = (unsigned char)input[i]; + if (c != '<') { + /* Plain text — escape and emit. We pass `&` through verbatim + * to preserve pre-encoded entities (`<`, `&`, `&#x...;`) + * which the browser will decode safely. */ + if (c == '&') html_buf_putc(&out, '&'); + else html_escape_byte(&out, c); + i++; + continue; + } + /* `<` — try to parse a tag. */ + if (i + 1 >= in_len) { + html_buf_puts(&out, "<"); + i++; + continue; + } + /* Comments, doctype, CDATA, processing instructions — drop entirely. */ + if (input[i + 1] == '!') { + if (i + 3 < in_len && input[i + 2] == '-' && input[i + 3] == '-') { + size_t j = i + 4; + while (j + 2 < in_len && !(input[j] == '-' && input[j + 1] == '-' && input[j + 2] == '>')) j++; + if (j + 2 < in_len) i = j + 3; + else i = in_len; + continue; + } + size_t j = i + 2; + while (j < in_len && input[j] != '>') j++; + i = (j < in_len) ? j + 1 : in_len; + continue; + } + if (input[i + 1] == '?') { + size_t j = i + 2; + while (j < in_len && input[j] != '>') j++; + i = (j < in_len) ? j + 1 : in_len; + continue; + } + int is_close = 0; + size_t name_start = i + 1; + if (input[i + 1] == '/') { + is_close = 1; + name_start = i + 2; + } + if (name_start >= in_len) { + html_buf_puts(&out, "<"); + i++; + continue; + } + unsigned char nc = (unsigned char)input[name_start]; + if (!((nc >= 'a' && nc <= 'z') || (nc >= 'A' && nc <= 'Z'))) { + /* `<` followed by non-letter — emit as escaped text. */ + html_buf_puts(&out, "<"); + i++; + continue; + } + size_t name_end = name_start; + while (name_end < in_len) { + unsigned char x = (unsigned char)input[name_end]; + if ((x >= 'a' && x <= 'z') || (x >= 'A' && x <= 'Z') || + (x >= '0' && x <= '9') || x == '-' || x == '_' || x == ':') { + name_end++; + } else { + break; + } + } + const char* tag = input + name_start; + size_t tag_len = name_end - name_start; + /* Find the `>` that closes this tag, respecting quoted attrs. */ + size_t cur = name_end; + int self_close = 0; + while (cur < in_len) { + unsigned char x = (unsigned char)input[cur]; + if (x == '"' || x == '\'') { + unsigned char q = x; + cur++; + while (cur < in_len && (unsigned char)input[cur] != q) cur++; + if (cur < in_len) cur++; /* skip closing quote */ + continue; + } + if (x == '/' && cur + 1 < in_len && input[cur + 1] == '>') { + self_close = 1; + break; + } + if (x == '>') break; + cur++; + } + if (cur >= in_len) { + /* Malformed: unclosed tag at EOF. Drop the rest of the input. */ + i = in_len; + continue; + } + size_t tag_end = self_close ? cur + 2 : cur + 1; /* one past `>` */ + /* Dangerous container — drop the whole subtree. */ + if (!is_close && html_is_dangerous_container(tag, tag_len)) { + if (self_close || html_is_void(tag, tag_len)) { + i = tag_end; + continue; + } + size_t scan = tag_end; + int found_close = 0; + while (scan < in_len) { + if (input[scan] != '<') { scan++; continue; } + if (scan + 1 < in_len && input[scan + 1] == '/') { + size_t cn_start = scan + 2; + size_t cn_end = cn_start; + while (cn_end < in_len) { + unsigned char x = (unsigned char)input[cn_end]; + if ((x >= 'a' && x <= 'z') || (x >= 'A' && x <= 'Z') || + (x >= '0' && x <= '9') || x == '-' || x == '_' || x == ':') { + cn_end++; + } else break; + } + if (cn_end - cn_start == tag_len && + html_iemem(input + cn_start, tag, tag_len)) { + size_t end_close = cn_end; + while (end_close < in_len && input[end_close] != '>') end_close++; + i = (end_close < in_len) ? end_close + 1 : in_len; + found_close = 1; + break; + } + } + scan++; + } + if (!found_close) { + /* No matching close — drop everything from here on. */ + i = in_len; + } + continue; + } + /* Look up the tag in the allowlist. */ + size_t arr_len = 0; + const char* arr = html_allowlist_find(allow, tag, tag_len, &arr_len); + if (!arr) { + /* Tag not allowed. Drop the open/close marker; inner text is + * processed by the outer loop and re-emitted as escaped text. */ + i = tag_end; + continue; + } + if (is_close) { + if (!html_is_void(tag, tag_len)) { + html_buf_putc(&out, '<'); + html_buf_putc(&out, '/'); + for (size_t k = 0; k < tag_len; k++) { + html_buf_putc(&out, (char)html_tolower((unsigned char)tag[k])); + } + html_buf_putc(&out, '>'); + } + i = tag_end; + continue; + } + /* Allowed open tag. Emit ``. */ + html_buf_putc(&out, '<'); + for (size_t k = 0; k < tag_len; k++) { + html_buf_putc(&out, (char)html_tolower((unsigned char)tag[k])); + } + size_t a = name_end; + while (a < cur) { + unsigned char x = (unsigned char)input[a]; + if (x == ' ' || x == '\t' || x == '\n' || x == '\r' || x == '/') { a++; continue; } + size_t an_start = a; + while (a < cur) { + unsigned char y = (unsigned char)input[a]; + if (y == '=' || y == ' ' || y == '\t' || y == '\n' || y == '\r' || y == '/' || y == '>') break; + a++; + } + size_t an_len = a - an_start; + if (an_len == 0) { a++; continue; } + size_t av_start = 0; + size_t av_len = 0; + int has_value = 0; + size_t b = a; + while (b < cur && (input[b] == ' ' || input[b] == '\t' || input[b] == '\n' || input[b] == '\r')) b++; + if (b < cur && input[b] == '=') { + has_value = 1; + b++; + while (b < cur && (input[b] == ' ' || input[b] == '\t' || input[b] == '\n' || input[b] == '\r')) b++; + if (b < cur && (input[b] == '"' || input[b] == '\'')) { + unsigned char q = (unsigned char)input[b]; + b++; + av_start = b; + while (b < cur && (unsigned char)input[b] != q) b++; + av_len = b - av_start; + if (b < cur) b++; + } else { + av_start = b; + while (b < cur) { + unsigned char y = (unsigned char)input[b]; + if (y == ' ' || y == '\t' || y == '\n' || y == '\r' || y == '>') break; + b++; + } + av_len = b - av_start; + } + a = b; + } + if (!html_attr_in_array(arr, arr_len, input + an_start, an_len)) continue; + int is_href = (an_len == 4 && html_iemem(input + an_start, "href", 4)); + int is_src = (an_len == 3 && html_iemem(input + an_start, "src", 3)); + if ((is_href || is_src) && has_value) { + if (!html_url_is_safe(input + av_start, av_len)) continue; + } + html_buf_putc(&out, ' '); + for (size_t k = 0; k < an_len; k++) { + html_buf_putc(&out, (char)html_tolower((unsigned char)input[an_start + k])); + } + if (has_value) { + html_buf_puts(&out, "=\""); + for (size_t k = 0; k < av_len; k++) { + unsigned char y = (unsigned char)input[av_start + k]; + /* Re-escape so the emitted attribute is well-formed + * double-quoted HTML. `&` passes through to preserve + * pre-encoded entities. */ + if (y == '"') html_buf_puts(&out, """); + else if (y == '<') html_buf_puts(&out, "<"); + else if (y == '>') html_buf_puts(&out, ">"); + else html_buf_putc(&out, (char)y); + } + html_buf_putc(&out, '"'); + } + } + html_buf_putc(&out, '>'); + i = tag_end; + } + /* Copy into arena-tracked buffer so the standard runtime memory model + * applies to the returned string. */ + char* result = el_strbuf(out.len); + memcpy(result, out.data, out.len); + result[out.len] = '\0'; + html_buf_free(&out); + return el_wrap_str(result); +} + +/* ── JSON ────────────────────────────────────────────────────────────────── */ + +/* True iff the segment is non-empty and every byte is an ASCII digit. We treat + * such segments as numeric array indices when walking a dot-path; mixed names + * like "0a" remain object-key lookups, so a key named "0" still wins over an + * index when the surrounding container is an object. */ +static int json_path_seg_is_index(const char* seg, size_t n) { + if (n == 0) return 0; + for (size_t i = 0; i < n; i++) { + if (seg[i] < '0' || seg[i] > '9') return 0; + } + return 1; +} + +/* Skip JSON whitespace. */ +static const char* json_skip_ws(const char* p) { + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; + return p; +} + +/* Descend one segment into the JSON cursor `p`. + * - If `p` points at an array `[...]` and the segment is all digits, + * advance to that element (zero-based). + * - Otherwise treat the segment as an object key and use json_find_key + * scoped to a one-level slice of the current container. + * Returns NULL if the descent fails (segment not found, container mismatch). + * + * `seg` is a pointer into the original path string and `seg_len` is its + * byte length — this avoids an extra alloc per segment. */ +static const char* json_path_descend(const char* p, const char* seg, size_t seg_len) { + if (!p || !seg) return NULL; + p = json_skip_ws(p); + if (*p == '[' && json_path_seg_is_index(seg, seg_len)) { + long idx = 0; + for (size_t i = 0; i < seg_len; i++) idx = idx * 10 + (seg[i] - '0'); + p++; /* step past '[' */ + p = json_skip_ws(p); + long cur = 0; + while (*p && *p != ']') { + if (cur == idx) return p; + const char* end = json_skip_value(p); + if (!end || end == p) return NULL; + p = json_skip_ws(end); + if (*p == ',') { p++; p = json_skip_ws(p); cur++; continue; } + /* No comma after this element — only acceptable at the closing ']', + * which means we ran out of elements. */ + break; + } + return NULL; + } + /* Object lookup. json_find_key walks at depth 1 of whatever container it + * receives, so we slice from `p` onwards. Caller already positioned us at + * the opening '{' (or at whitespace before it). */ + if (*p != '{') return NULL; + /* Build a NUL-terminated copy of the key segment for the lookup. We only + * pay this cost when the segment isn't a numeric index. */ + char stack_key[256]; + char* k = stack_key; + if (seg_len + 1 > sizeof(stack_key)) { + k = malloc(seg_len + 1); + if (!k) return NULL; + } + memcpy(k, seg, seg_len); + k[seg_len] = '\0'; + const char* found = json_find_key(p, k); + if (k != stack_key) free(k); + return found; +} + +/* Read the JSON value at `p` into a freshly-allocated, arena-owned el_val_t. + * - String -> unescaped, wrapped el_val_t string + * - Anything else -> raw JSON slice as a string (matches the historical + * json_get behaviour: numbers/bools/null come back stringified). */ +static el_val_t json_read_value(const char* p) { + p = json_skip_ws(p); + if (*p == '"') { + p++; + size_t cap = strlen(p) + 1; + char* out = el_strbuf(cap); + char* w = out; + while (*p && *p != '"') { + if (*p == '\\' && *(p+1)) { + p++; + switch (*p) { + case '"': *w++ = '"'; break; + case '\\': *w++ = '\\'; break; + case '/': *w++ = '/'; break; + case 'n': *w++ = '\n'; break; + case 'r': *w++ = '\r'; break; + case 't': *w++ = '\t'; break; + default: *w++ = *p; break; + } + } else { + *w++ = *p; + } + p++; + } + *w = '\0'; + return el_wrap_str(out); + } + /* Object/array/number/bool/null — return the raw slice up to the value's + * end. json_skip_value tracks brace/bracket/string state so nested objects + * round-trip cleanly. */ + const char* end = json_skip_value(p); + if (!end) end = p; + size_t n = (size_t)(end - p); + /* Strip trailing whitespace from scalar values so callers don't see + * `123 ` when they parsed a pretty-printed number. */ + while (n > 0 && (p[n-1] == ' ' || p[n-1] == '\t' || p[n-1] == '\n' || p[n-1] == '\r')) { + n--; + } + char* out = el_strbuf(n); + memcpy(out, p, n); + out[n] = '\0'; + return el_wrap_str(out); +} + +el_val_t json_get(el_val_t jsonv, el_val_t keyv) { + const char* json = EL_CSTR(jsonv); + const char* key = EL_CSTR(keyv); + if (!json || !key) return el_wrap_str(el_strdup("")); + + /* Fast path: key contains no '.' — keep the historical single-segment + * substring search so existing callers retain their O(strlen) cost + * profile. The dot-path walker is only paid for when needed. */ + if (!strchr(key, '.')) { + size_t klen = strlen(key); + char stack_pat[512]; + char* pattern; + if (klen + 5 <= sizeof(stack_pat)) { + pattern = stack_pat; + } else { + pattern = malloc(klen + 5); + if (!pattern) return el_wrap_str(el_strdup("")); + } + snprintf(pattern, klen + 5, "\"%s\":", key); + const char* p = strstr(json, pattern); + if (pattern != stack_pat) free(pattern); + if (!p) return el_wrap_str(el_strdup("")); + p += strlen(key) + 3; /* skip "key": */ + return json_read_value(p); + } + + /* Dot-path traversal. Walk segments left to right; at each step, descend + * into the current container by either array index (all-digit segment on + * an array cursor) or object key. */ + const char* cursor = json_skip_ws(json); + const char* seg_start = key; + const char* k = key; + while (1) { + if (*k == '.' || *k == '\0') { + size_t seg_len = (size_t)(k - seg_start); + cursor = json_path_descend(cursor, seg_start, seg_len); + if (!cursor) return el_wrap_str(el_strdup("")); + if (*k == '\0') break; + k++; + seg_start = k; + continue; + } + k++; + } + return json_read_value(cursor); +} + +/* ── Float bit-cast helpers ──────────────────────────────────────────────── */ +/* `el_to_float` and `el_from_float` are exposed in el_runtime.h as static + * inlines so generated programs (which #include the header) can call them + * for Float literals. No definitions are needed here. */ + +/* ── JSON parser (recursive descent) ─────────────────────────────────────── */ +/* + * Parsed JSON representation: + * - object -> ElMap (keys & values are el_val_t) + * - array -> ElList + * - string -> EL_STR-wrapped char* (allocated) + * - number -> int (el_val_t) if integer, otherwise el_from_float(double) + * - true -> 1 + * - false -> 0 + * - null -> EL_NULL (0) + * + * Note: there is no runtime type tag — parsed numbers cannot be + * distinguished from booleans by the runtime alone. The codegen tracks + * types separately. This matches the rest of el_val_t's type-erased model. + */ + +/* JsonParser struct is forward-declared near the HTTP/Engram section. */ + +static void jp_skip_ws(JsonParser* jp) { + while (jp->p < jp->end) { + char c = *jp->p; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') jp->p++; + else break; + } +} + +static el_val_t jp_parse_value(JsonParser* jp); + +/* Parse a JSON string literal (the opening " has NOT yet been consumed). */ +static char* jp_parse_string_raw(JsonParser* jp) { + if (jp->p >= jp->end || *jp->p != '"') { jp->err = 1; return el_strdup(""); } + jp->p++; + size_t cap = 32, len = 0; + char* out = malloc(cap); + if (!out) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + while (jp->p < jp->end && *jp->p != '"') { + char c = *jp->p++; + if (c == '\\' && jp->p < jp->end) { + char esc = *jp->p++; + switch (esc) { + case '"': c = '"'; break; + case '\\': c = '\\'; break; + case '/': c = '/'; break; + case 'b': c = '\b'; break; + case 'f': c = '\f'; break; + case 'n': c = '\n'; break; + case 'r': c = '\r'; break; + case 't': c = '\t'; break; + case 'u': { + /* Skip 4 hex digits; emit '?' as a placeholder */ + for (int i = 0; i < 4 && jp->p < jp->end; i++) jp->p++; + c = '?'; + break; + } + default: c = esc; break; + } + } + if (len + 1 >= cap) { + cap *= 2; + out = realloc(out, cap); + if (!out) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + } + out[len++] = c; + } + if (jp->p < jp->end && *jp->p == '"') jp->p++; + else jp->err = 1; + out[len] = '\0'; + return out; +} + +static el_val_t jp_parse_number(JsonParser* jp) { + const char* start = jp->p; + int is_float = 0; + if (jp->p < jp->end && (*jp->p == '-' || *jp->p == '+')) jp->p++; + while (jp->p < jp->end && isdigit((unsigned char)*jp->p)) jp->p++; + if (jp->p < jp->end && *jp->p == '.') { + is_float = 1; jp->p++; + while (jp->p < jp->end && isdigit((unsigned char)*jp->p)) jp->p++; + } + if (jp->p < jp->end && (*jp->p == 'e' || *jp->p == 'E')) { + is_float = 1; jp->p++; + if (jp->p < jp->end && (*jp->p == '+' || *jp->p == '-')) jp->p++; + while (jp->p < jp->end && isdigit((unsigned char)*jp->p)) jp->p++; + } + size_t n = (size_t)(jp->p - start); + char buf[64]; + if (n >= sizeof(buf)) n = sizeof(buf) - 1; + memcpy(buf, start, n); + buf[n] = '\0'; + if (is_float) return el_from_float(strtod(buf, NULL)); + return (el_val_t)strtoll(buf, NULL, 10); +} + +static el_val_t jp_parse_array(JsonParser* jp) { + if (jp->p < jp->end && *jp->p == '[') jp->p++; + el_val_t lst = el_list_empty(); + jp_skip_ws(jp); + if (jp->p < jp->end && *jp->p == ']') { jp->p++; return lst; } + while (jp->p < jp->end) { + jp_skip_ws(jp); + el_val_t v = jp_parse_value(jp); + lst = el_list_append(lst, v); + jp_skip_ws(jp); + if (jp->p < jp->end && *jp->p == ',') { jp->p++; continue; } + if (jp->p < jp->end && *jp->p == ']') { jp->p++; break; } + jp->err = 1; + break; + } + return lst; +} + +static el_val_t jp_parse_object(JsonParser* jp) { + if (jp->p < jp->end && *jp->p == '{') jp->p++; + el_val_t m = el_map_new(0); + jp_skip_ws(jp); + if (jp->p < jp->end && *jp->p == '}') { jp->p++; return m; } + while (jp->p < jp->end) { + jp_skip_ws(jp); + char* key = jp_parse_string_raw(jp); + jp_skip_ws(jp); + if (jp->p < jp->end && *jp->p == ':') jp->p++; + else { jp->err = 1; free(key); break; } + jp_skip_ws(jp); + el_val_t v = jp_parse_value(jp); + m = el_map_set(m, EL_STR(key), v); + jp_skip_ws(jp); + if (jp->p < jp->end && *jp->p == ',') { jp->p++; continue; } + if (jp->p < jp->end && *jp->p == '}') { jp->p++; break; } + jp->err = 1; + break; + } + return m; +} + +static el_val_t jp_parse_value(JsonParser* jp) { + jp_skip_ws(jp); + if (jp->p >= jp->end) { jp->err = 1; return EL_NULL; } + char c = *jp->p; + if (c == '"') return el_wrap_str(jp_parse_string_raw(jp)); + if (c == '{') return jp_parse_object(jp); + if (c == '[') return jp_parse_array(jp); + if (c == '-' || isdigit((unsigned char)c)) return jp_parse_number(jp); + if (c == 't' && jp->p + 4 <= jp->end && strncmp(jp->p, "true", 4) == 0) { jp->p += 4; return 1; } + if (c == 'f' && jp->p + 5 <= jp->end && strncmp(jp->p, "false", 5) == 0) { jp->p += 5; return 0; } + if (c == 'n' && jp->p + 4 <= jp->end && strncmp(jp->p, "null", 4) == 0) { jp->p += 4; return EL_NULL; } + jp->err = 1; + return EL_NULL; +} + +el_val_t json_parse(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return EL_NULL; + JsonParser jp = { .p = s, .end = s + strlen(s), .err = 0 }; + el_val_t v = jp_parse_value(&jp); + if (jp.err) return EL_NULL; + return v; +} + +/* ── JSON stringify ──────────────────────────────────────────────────────── */ +/* + * Stringify policy: el_val_t is type-erased, so we cannot perfectly + * round-trip arbitrary values. We use these heuristics: + * - If value is an ElList pointer (in the heap range), serialize as array. + * - If value is an ElMap pointer, serialize as object. + * - If value looks like a printable string pointer, serialize as string. + * - Otherwise serialize as integer. + * This is best-effort. Programs that need exact control should build the + * string directly. A pointer test is the cheapest way to disambiguate + * from small integers without a separate type tag. + */ + +/* JsonBuf struct is forward-declared near the HTTP section so HTTP helpers + * can use it. Its definition appears there. */ + +static void jb_init(JsonBuf* b) { + b->cap = 64; b->len = 0; + b->buf = malloc(b->cap); + if (!b->buf) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + b->buf[0] = '\0'; +} + +static void jb_reserve(JsonBuf* b, size_t add) { + if (b->len + add + 1 > b->cap) { + while (b->len + add + 1 > b->cap) b->cap *= 2; + b->buf = realloc(b->buf, b->cap); + if (!b->buf) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + } +} + +static void jb_putc(JsonBuf* b, char c) { + jb_reserve(b, 1); + b->buf[b->len++] = c; + b->buf[b->len] = '\0'; +} + +static void jb_puts(JsonBuf* b, const char* s) { + size_t n = strlen(s); + jb_reserve(b, n); + memcpy(b->buf + b->len, s, n); + b->len += n; + b->buf[b->len] = '\0'; +} + +static void jb_emit_escaped(JsonBuf* b, const char* s) { + jb_putc(b, '"'); + for (; *s; s++) { + unsigned char c = (unsigned char)*s; + switch (c) { + case '"': jb_puts(b, "\\\""); break; + case '\\': jb_puts(b, "\\\\"); break; + case '\b': jb_puts(b, "\\b"); break; + case '\f': jb_puts(b, "\\f"); break; + case '\n': jb_puts(b, "\\n"); break; + case '\r': jb_puts(b, "\\r"); break; + case '\t': jb_puts(b, "\\t"); break; + default: + if (c < 0x20) { + char tmp[8]; + snprintf(tmp, sizeof(tmp), "\\u%04x", c); + jb_puts(b, tmp); + } else { + jb_putc(b, (char)c); + } + break; + } + } + jb_putc(b, '"'); +} + +/* Heuristic: is this el_val_t likely a pointer to an ElList? + * We can't fully verify, but pointers are large addresses, integers small. + * Treat values whose magnitude exceeds 2^32 as potential pointers and + * sniff by reading the header conservatively. + * + * Simpler heuristic: if the value reads as a printable string, treat as + * string; otherwise as integer. Lists/Maps are encoded as struct pointers, + * which have leading binary bytes — so they won't look like strings. */ + +static int looks_like_string(el_val_t v) { + if (v == 0) return 0; + /* Treat plausible heap addresses as candidates. + * Threshold: 4 GiB (0x100000000). On 64-bit systems heap addresses from + * malloc/mmap start well above 4 GiB (ASLR pushes them to ~0x7f...). + * El integer values (counters, unix timestamps up to ~2106) all fit below + * 0x100000000 (4294967296). The old threshold of 1,000,000 caused unix + * timestamps (~1.7e9) to be misidentified as string pointers — a segfault + * risk in json_stringify and jb_emit_value. */ + uintptr_t p = (uintptr_t)v; + if (p < 0x100000000ULL) return 0; /* integers, timestamps, counters */ + if (p < 0x1000) return 0; + /* Sniff first bytes for printable */ + const unsigned char* s = (const unsigned char*)p; + for (int i = 0; i < 16; i++) { + unsigned char c = s[i]; + if (c == '\0') return 1; /* terminated string (empty string is still a valid string) */ + /* Reject C0 control chars (non-whitespace), allow UTF-8 high bytes. + * 0x09-0x0d = tab/newline/cr/vt/ff (whitespace, OK) + * 0x20-0x7e = printable ASCII (OK) + * 0x7f = DEL (reject) + * 0x80-0xff = UTF-8 continuation/lead bytes (OK for multi-byte chars) */ + if (c < 0x09 || (c > 0x0d && c < 0x20) || c == 0x7f) return 0; + } + return 1; /* 16+ printable bytes — call it a string */ +} + +static void jb_emit_value(JsonBuf* b, el_val_t v); + +static void jb_emit_int(JsonBuf* b, int64_t n) { + char tmp[32]; + snprintf(tmp, sizeof(tmp), "%lld", (long long)n); + jb_puts(b, tmp); +} + +static void jb_emit_value(JsonBuf* b, el_val_t v) { + if (v == EL_NULL) { jb_puts(b, "null"); return; } + if (looks_like_string(v)) { + jb_emit_escaped(b, EL_CSTR(v)); + return; + } + jb_emit_int(b, (int64_t)v); +} + +el_val_t json_stringify(el_val_t v) { + JsonBuf b; jb_init(&b); + jb_emit_value(&b, v); + return el_wrap_str(b.buf); +} + +/* ── JSON substring accessors ────────────────────────────────────────────── */ +/* + * These walk the raw JSON string looking for "key": at the top level (depth 1) + * of an object. They handle escaped quotes, nested objects/arrays, and + * whitespace around the colon. + */ + +/* Find "key": at object-depth == 1 inside the JSON object string `s`. + * Returns pointer to the first byte of the value, or NULL. */ +static const char* json_find_key(const char* s, const char* key) { + if (!s || !key) return NULL; + size_t klen = strlen(key); + int depth = 0; + int in_str = 0; + int escape = 0; + const char* p = s; + while (*p) { + char c = *p; + if (in_str) { + if (escape) { escape = 0; } + else if (c == '\\') { escape = 1; } + else if (c == '"') { + /* End of string. If we're at depth 1, check if this was a key. */ + p++; + if (depth == 1) { + /* The string just ended at p-1. Check if it matches key + * and is followed by a colon. We need to backtrack to find + * the start of this string and compare. */ + } + in_str = 0; + continue; + } + p++; + continue; + } + if (c == '"') { + /* Start of a string literal */ + const char* str_start = p + 1; + const char* q = str_start; + int e = 0; + while (*q) { + if (e) { e = 0; q++; continue; } + if (*q == '\\') { e = 1; q++; continue; } + if (*q == '"') break; + q++; + } + size_t slen = (size_t)(q - str_start); + const char* after = (*q == '"') ? q + 1 : q; + /* If at depth 1 and matches key and followed by ':' -> got it */ + if (depth == 1 && slen == klen && strncmp(str_start, key, klen) == 0) { + const char* r = after; + while (*r == ' ' || *r == '\t' || *r == '\n' || *r == '\r') r++; + if (*r == ':') { + r++; + while (*r == ' ' || *r == '\t' || *r == '\n' || *r == '\r') r++; + return r; + } + } + p = after; + continue; + } + if (c == '{' || c == '[') depth++; + else if (c == '}' || c == ']') depth--; + p++; + } + return NULL; +} + +/* Skip a JSON value starting at p; return pointer past the value end. */ +static const char* json_skip_value(const char* p) { + if (!p || !*p) return p; + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; + if (*p == '"') { + p++; + int e = 0; + while (*p) { + if (e) { e = 0; p++; continue; } + if (*p == '\\') { e = 1; p++; continue; } + if (*p == '"') { p++; break; } + p++; + } + return p; + } + if (*p == '{' || *p == '[') { + char open = *p; + char close = (open == '{') ? '}' : ']'; + int depth = 0; + int in_str = 0; + int e = 0; + while (*p) { + char c = *p; + if (in_str) { + if (e) { e = 0; } + else if (c == '\\') { e = 1; } + else if (c == '"') in_str = 0; + p++; + continue; + } + if (c == '"') { in_str = 1; p++; continue; } + if (c == open) depth++; + else if (c == close) { depth--; p++; if (depth == 0) return p; continue; } + p++; + } + return p; + } + /* scalar: number, true/false/null */ + while (*p && *p != ',' && *p != '}' && *p != ']' && + *p != ' ' && *p != '\t' && *p != '\n' && *p != '\r') p++; + return p; +} + +el_val_t json_get_string(el_val_t json_str, el_val_t key) { + const char* json = EL_CSTR(json_str); + const char* k = EL_CSTR(key); + const char* p = json_find_key(json, k); + if (!p || *p != '"') return el_wrap_str(el_strdup("")); + p++; + JsonParser jp = { .p = p - 1, .end = json + (json ? strlen(json) : 0), .err = 0 }; + char* parsed = jp_parse_string_raw(&jp); + if (jp.err) { free(parsed); return el_wrap_str(el_strdup("")); } + return el_wrap_str(parsed); +} + +el_val_t json_get_int(el_val_t json_str, el_val_t key) { + const char* json = EL_CSTR(json_str); + const char* k = EL_CSTR(key); + const char* p = json_find_key(json, k); + if (!p) return 0; + if (*p == '"' || *p == '{' || *p == '[') return 0; + return (el_val_t)strtoll(p, NULL, 10); +} + +el_val_t json_get_float(el_val_t json_str, el_val_t key) { + const char* json = EL_CSTR(json_str); + const char* k = EL_CSTR(key); + const char* p = json_find_key(json, k); + if (!p) return 0; + if (*p == '"' || *p == '{' || *p == '[') return 0; + return el_from_float(strtod(p, NULL)); +} + +el_val_t json_get_bool(el_val_t json_str, el_val_t key) { + const char* json = EL_CSTR(json_str); + const char* k = EL_CSTR(key); + const char* p = json_find_key(json, k); + if (!p) return 0; + if (strncmp(p, "true", 4) == 0) return 1; + return 0; +} + +el_val_t json_get_raw(el_val_t json_str, el_val_t key) { + const char* json = EL_CSTR(json_str); + const char* k = EL_CSTR(key); + const char* p = json_find_key(json, k); + /* Clear fs_read binary-length hint — result is a fresh null-terminated + * string, not the raw file bytes, so Content-Length must use strlen. */ + _tl_fs_read_len = 0; + if (!p) return el_wrap_str(el_strdup("")); + const char* end = json_skip_value(p); + size_t n = (size_t)(end - p); + char* out = el_strbuf(n); + memcpy(out, p, n); + out[n] = '\0'; + return el_wrap_str(out); +} + +el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value) { + const char* json = EL_CSTR(json_str); + const char* k = EL_CSTR(key); + if (!k) k = ""; + if (!json || !*json) { + /* Build a fresh object */ + JsonBuf b; jb_init(&b); + jb_putc(&b, '{'); + jb_emit_escaped(&b, k); + jb_putc(&b, ':'); + jb_emit_value(&b, value); + jb_putc(&b, '}'); + return el_wrap_str(b.buf); + } + const char* existing = json_find_key(json, k); + JsonBuf b; jb_init(&b); + if (existing) { + const char* end = json_skip_value(existing); + /* Copy [json .. existing) */ + size_t prefix = (size_t)(existing - json); + jb_reserve(&b, prefix); + memcpy(b.buf + b.len, json, prefix); + b.len += prefix; + b.buf[b.len] = '\0'; + jb_emit_value(&b, value); + jb_puts(&b, end); + return el_wrap_str(b.buf); + } + /* Insert before closing '}'. Find last '}' */ + size_t jl = strlen(json); + if (jl == 0) { free(b.buf); return el_wrap_str(el_strdup("{}")); } + /* Find last '}' from the end */ + ssize_t close_idx = -1; + for (ssize_t i = (ssize_t)jl - 1; i >= 0; i--) { + if (json[i] == '}') { close_idx = i; break; } + } + if (close_idx < 0) { + free(b.buf); + return el_wrap_str(el_strdup(json)); + } + /* Determine if object is empty: scan between last '{' and '}' for non-ws */ + int empty = 1; + for (ssize_t i = close_idx - 1; i >= 0; i--) { + char c = json[i]; + if (c == '{') break; + if (c != ' ' && c != '\t' && c != '\n' && c != '\r') { empty = 0; break; } + } + /* Copy json[0..close_idx) */ + jb_reserve(&b, (size_t)close_idx); + memcpy(b.buf + b.len, json, (size_t)close_idx); + b.len += (size_t)close_idx; + b.buf[b.len] = '\0'; + if (!empty) jb_putc(&b, ','); + jb_emit_escaped(&b, k); + jb_putc(&b, ':'); + jb_emit_value(&b, value); + /* Append from close_idx onward */ + jb_puts(&b, json + close_idx); + return el_wrap_str(b.buf); +} + +el_val_t json_array_len(el_val_t json_str) { + const char* s = EL_CSTR(json_str); + if (!s) return 0; + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; + if (*s != '[') return 0; + s++; + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; + if (*s == ']') return 0; + int64_t count = 0; + while (*s) { + const char* end = json_skip_value(s); + if (end == s) break; + count++; + s = end; + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; + if (*s == ',') { s++; continue; } + if (*s == ']' || *s == '\0') break; + } + return (el_val_t)count; +} + +/* json_array_get — return the i-th element of a JSON array as a JSON + * fragment string. Nested objects and arrays are returned verbatim + * (json_skip_value tracks brace/bracket depth so nested structures are + * preserved intact). Out-of-range index → "". */ +el_val_t json_array_get(el_val_t json_str, el_val_t index) { + const char* s = EL_CSTR(json_str); + int64_t idx = (int64_t)index; + if (!s || idx < 0) return el_wrap_str(el_strdup("")); + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; + if (*s != '[') return el_wrap_str(el_strdup("")); + s++; + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; + if (*s == ']') return el_wrap_str(el_strdup("")); + int64_t i = 0; + while (*s) { + const char* start = s; + const char* end = json_skip_value(s); + if (end == s) break; + if (i == idx) { + size_t n = (size_t)(end - start); + char* out = el_strbuf(n); + memcpy(out, start, n); + out[n] = '\0'; + return el_wrap_str(out); + } + i++; + s = end; + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; + if (*s == ',') { s++; while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; continue; } + if (*s == ']' || *s == '\0') break; + } + return el_wrap_str(el_strdup("")); +} + +/* json_array_get_string — same as json_array_get, but assume the element + * is a JSON string and return the unquoted/unescaped value. Non-string + * elements yield "". */ +el_val_t json_array_get_string(el_val_t json_str, el_val_t index) { + el_val_t raw = json_array_get(json_str, index); + const char* s = EL_CSTR(raw); + if (!s || *s != '"') return el_wrap_str(el_strdup("")); + JsonParser jp = { + .p = s, + .end = s + strlen(s), + .err = 0, + }; + char* parsed = jp_parse_string_raw(&jp); + if (jp.err) { + free(parsed); + return el_wrap_str(el_strdup("")); + } + return el_wrap_str(parsed); +} + +/* ── Time ────────────────────────────────────────────────────────────────── */ + +el_val_t time_now(void) { + struct timeval tv; + gettimeofday(&tv, NULL); + int64_t ms = (int64_t)tv.tv_sec * 1000LL + (int64_t)tv.tv_usec / 1000LL; + return (el_val_t)ms; +} + +el_val_t time_now_utc(void) { + return time_now(); +} + +el_val_t time_format(el_val_t ts, el_val_t fmt) { + int64_t ms = (int64_t)ts; + time_t s = (time_t)(ms / 1000); + int msec = (int)(ms % 1000); + if (msec < 0) { msec += 1000; s -= 1; } + struct tm tm; + gmtime_r(&s, &tm); + const char* fmt_str = EL_CSTR(fmt); + if (!fmt_str || strcmp(fmt_str, "ISO") == 0) { + char buf[64]; + snprintf(buf, sizeof(buf), "%04d-%02d-%02dT%02d:%02d:%02d.%03dZ", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec, msec); + return el_wrap_str(el_strdup(buf)); + } + char buf[256]; + if (strftime(buf, sizeof(buf), fmt_str, &tm) == 0) buf[0] = '\0'; + return el_wrap_str(el_strdup(buf)); +} + +el_val_t time_to_parts(el_val_t ts) { + int64_t ms = (int64_t)ts; + time_t s = (time_t)(ms / 1000); + int msec = (int)(ms % 1000); + if (msec < 0) { msec += 1000; s -= 1; } + struct tm tm; + gmtime_r(&s, &tm); + el_val_t m = el_map_new(0); + m = el_map_set(m, EL_STR(el_strdup("year")), (el_val_t)(tm.tm_year + 1900)); + m = el_map_set(m, EL_STR(el_strdup("month")), (el_val_t)(tm.tm_mon + 1)); + m = el_map_set(m, EL_STR(el_strdup("day")), (el_val_t)tm.tm_mday); + m = el_map_set(m, EL_STR(el_strdup("hour")), (el_val_t)tm.tm_hour); + m = el_map_set(m, EL_STR(el_strdup("minute")), (el_val_t)tm.tm_min); + m = el_map_set(m, EL_STR(el_strdup("second")), (el_val_t)tm.tm_sec); + m = el_map_set(m, EL_STR(el_strdup("ms")), (el_val_t)msec); + return m; +} + +el_val_t time_from_parts(el_val_t secs, el_val_t ns, el_val_t tz) { + (void)tz; + int64_t s = (int64_t)secs; + int64_t n = (int64_t)ns; + int64_t ms = s * 1000LL + n / 1000000LL; + return (el_val_t)ms; +} + +el_val_t time_add(el_val_t ts, el_val_t n, el_val_t unit) { + const char* u = EL_CSTR(unit); + int64_t cur = (int64_t)ts; + int64_t d = (int64_t)n; + int64_t add_ms = d; + if (u) { + if (strcmp(u, "ms") == 0) add_ms = d; + else if (strcmp(u, "sec") == 0) add_ms = d * 1000LL; + else if (strcmp(u, "min") == 0) add_ms = d * 60000LL; + else if (strcmp(u, "hour") == 0) add_ms = d * 3600000LL; + else if (strcmp(u, "day") == 0) add_ms = d * 86400000LL; + } + return (el_val_t)(cur + add_ms); +} + +el_val_t time_diff(el_val_t ts1, el_val_t ts2, el_val_t unit) { + int64_t d = (int64_t)ts2 - (int64_t)ts1; + const char* u = EL_CSTR(unit); + if (!u || strcmp(u, "ms") == 0) return (el_val_t)d; + if (strcmp(u, "sec") == 0) return (el_val_t)(d / 1000LL); + if (strcmp(u, "min") == 0) return (el_val_t)(d / 60000LL); + if (strcmp(u, "hour") == 0) return (el_val_t)(d / 3600000LL); + if (strcmp(u, "day") == 0) return (el_val_t)(d / 86400000LL); + return (el_val_t)d; +} + +/* Block the calling thread for `secs` seconds. Negative values are clamped + * to 0. Used by El programs that poll external resources (e.g. RunPod + * /status, Engram readiness probes). */ +el_val_t sleep_secs(el_val_t secs) { + int64_t s = (int64_t)secs; + if (s < 0) s = 0; + struct timespec ts; + ts.tv_sec = (time_t)s; + ts.tv_nsec = 0; + nanosleep(&ts, NULL); + return 0; +} + +el_val_t sleep_ms(el_val_t ms) { + int64_t m = (int64_t)ms; + if (m < 0) m = 0; + struct timespec ts; + ts.tv_sec = (time_t)(m / 1000LL); + ts.tv_nsec = (long)((m % 1000LL) * 1000000LL); + nanosleep(&ts, NULL); + return 0; +} + +/* ── Instant + Duration: first-class temporal types ────────────────────────── + * El's substrate (Neuron) is a temporal cognition system. Memory salience + * decay, the six-tier pacemaker, TTL caches, and supersession are all + * temporal. Treating time as a raw Int (now() returning ms-since-epoch and + * arithmetic done with mixed unit literals) lets bugs through the type + * system: `(now - cached_at) < 60` cannot tell ms from sec, and `sleep(30)` + * is ambiguous. This block introduces two dedicated representations. + * + * Representation: + * Instant — int64 nanoseconds since the Unix epoch + * Duration — int64 nanoseconds (signed; negative durations are legal, + * e.g. when a deadline has passed) + * + * Both share the el_val_t (int64) slot the rest of the runtime uses, so no + * boxing / arena allocation is needed. Type discipline is enforced at the + * codegen layer: `let x: Duration = ...` registers `x` in __duration_names, + * and BinOp dispatches through typed wrappers (el_duration_add, etc.) that + * make intent explicit in the generated C. Mismatched ops (Instant+Instant, + * Duration+Int) are surfaced via #error directives at codegen time so the + * downstream cc step fails with a clear El-source-level message. + * + * Nanosecond precision matches POSIX clock_gettime / nanosleep granularity. + * 2^63 nanos covers ~292 years from epoch — comfortably past 2200, plenty + * for a memory-system runtime that never schedules outside a human lifespan. + */ + +/* now() — current Instant. Wraps clock_gettime(CLOCK_REALTIME) for nanosecond + * precision. Falls back to gettimeofday on systems where clock_gettime is + * unavailable (defensive — every supported platform has it). */ +el_val_t el_now_instant(void) { + struct timespec ts; + if (clock_gettime(CLOCK_REALTIME, &ts) == 0) { + int64_t ns = (int64_t)ts.tv_sec * 1000000000LL + (int64_t)ts.tv_nsec; + return (el_val_t)ns; + } + struct timeval tv; + gettimeofday(&tv, NULL); + int64_t ns = (int64_t)tv.tv_sec * 1000000000LL + + (int64_t)tv.tv_usec * 1000LL; + return (el_val_t)ns; +} + +el_val_t now(void) { + return el_now_instant(); +} + +/* unix_seconds(n) — Instant from a Unix-epoch second count. + * unix_millis(n) — Instant from a Unix-epoch millisecond count. */ +el_val_t unix_seconds(el_val_t n) { + int64_t s = (int64_t)n; + return (el_val_t)(s * 1000000000LL); +} + +el_val_t unix_millis(el_val_t n) { + int64_t m = (int64_t)n; + return (el_val_t)(m * 1000000LL); +} + +/* instant_from_iso8601 — parse a strict subset: + * YYYY-MM-DDTHH:MM:SS[.fff]Z + * Returns 0 (the Unix-epoch sentinel) on parse failure. Callers that need to + * distinguish epoch-zero from a parse error should use a wider sentinel + * representation; the current zero-on-failure choice matches existing El + * runtime conventions for parse builtins (str_to_int, parse_int). */ +el_val_t instant_from_iso8601(el_val_t s) { + const char* str = EL_CSTR(s); + if (!str) return (el_val_t)0; + int Y, M, D, h, m, sec, frac = 0; + int n = sscanf(str, "%d-%d-%dT%d:%d:%d.%3d", &Y, &M, &D, &h, &m, &sec, &frac); + if (n < 6) { + n = sscanf(str, "%d-%d-%dT%d:%d:%dZ", &Y, &M, &D, &h, &m, &sec); + if (n < 6) return (el_val_t)0; + } + struct tm tm; + memset(&tm, 0, sizeof(tm)); + tm.tm_year = Y - 1900; + tm.tm_mon = M - 1; + tm.tm_mday = D; + tm.tm_hour = h; + tm.tm_min = m; + tm.tm_sec = sec; + /* timegm — UTC. POSIX-Y but available on macOS and glibc. */ + time_t t = timegm(&tm); + if (t == (time_t)-1) return (el_val_t)0; + int64_t ns = (int64_t)t * 1000000000LL + (int64_t)frac * 1000000LL; + return (el_val_t)ns; +} + +/* Duration constructors. The El-side postfix literals (30.seconds, 1.hour) + * are lowered by the codegen directly into a literal int64 of nanoseconds — + * these constructors are for runtime values where the count is dynamic. */ +el_val_t el_duration_from_nanos(el_val_t ns) { + return (el_val_t)(int64_t)ns; +} + +el_val_t duration_seconds(el_val_t n) { + int64_t s = (int64_t)n; + return (el_val_t)(s * 1000000000LL); +} + +el_val_t duration_millis(el_val_t n) { + int64_t m = (int64_t)n; + return (el_val_t)(m * 1000000LL); +} + +el_val_t duration_nanos(el_val_t n) { + return (el_val_t)(int64_t)n; +} + +/* Arithmetic — typed wrappers. At the C level these are no-op casts, but + * the codegen routes Instant/Duration BinOps through them so the generated + * C says `el_instant_add_dur(start, dur)` rather than `start + dur`. The + * intent is explicit, the operand order is documented, and a future change + * to the underlying representation (saturating arithmetic, overflow guards) + * has a single chokepoint. */ +el_val_t el_instant_add_dur(el_val_t inst, el_val_t dur) { + return (el_val_t)((int64_t)inst + (int64_t)dur); +} + +el_val_t el_instant_sub_dur(el_val_t inst, el_val_t dur) { + return (el_val_t)((int64_t)inst - (int64_t)dur); +} + +el_val_t el_instant_diff(el_val_t a, el_val_t b) { + /* a - b — yields a Duration (negative if b is later than a). */ + return (el_val_t)((int64_t)a - (int64_t)b); +} + +el_val_t el_duration_add(el_val_t a, el_val_t b) { + return (el_val_t)((int64_t)a + (int64_t)b); +} + +el_val_t el_duration_sub(el_val_t a, el_val_t b) { + return (el_val_t)((int64_t)a - (int64_t)b); +} + +el_val_t el_duration_scale(el_val_t dur, el_val_t scalar) { + return (el_val_t)((int64_t)dur * (int64_t)scalar); +} + +el_val_t el_duration_div(el_val_t dur, el_val_t scalar) { + int64_t s = (int64_t)scalar; + if (s == 0) return (el_val_t)0; + return (el_val_t)((int64_t)dur / s); +} + +/* Comparisons. Return 1/0 in el_val_t convention. */ +el_val_t el_instant_lt(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a < (int64_t)b ? 1 : 0); } +el_val_t el_instant_le(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a <= (int64_t)b ? 1 : 0); } +el_val_t el_instant_gt(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a > (int64_t)b ? 1 : 0); } +el_val_t el_instant_ge(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a >= (int64_t)b ? 1 : 0); } +el_val_t el_instant_eq(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a == (int64_t)b ? 1 : 0); } +el_val_t el_instant_ne(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a != (int64_t)b ? 1 : 0); } +el_val_t el_duration_lt(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a < (int64_t)b ? 1 : 0); } +el_val_t el_duration_le(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a <= (int64_t)b ? 1 : 0); } +el_val_t el_duration_gt(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a > (int64_t)b ? 1 : 0); } +el_val_t el_duration_ge(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a >= (int64_t)b ? 1 : 0); } +el_val_t el_duration_eq(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a == (int64_t)b ? 1 : 0); } +el_val_t el_duration_ne(el_val_t a, el_val_t b) { return (el_val_t)((int64_t)a != (int64_t)b ? 1 : 0); } + +/* Conversions. */ +el_val_t instant_to_unix_seconds(el_val_t i) { + return (el_val_t)((int64_t)i / 1000000000LL); +} + +el_val_t instant_to_unix_millis(el_val_t i) { + return (el_val_t)((int64_t)i / 1000000LL); +} + +el_val_t instant_to_iso8601(el_val_t i) { + int64_t ns = (int64_t)i; + time_t s = (time_t)(ns / 1000000000LL); + int msec = (int)((ns / 1000000LL) % 1000LL); + if (msec < 0) { msec += 1000; s -= 1; } + struct tm tm; + gmtime_r(&s, &tm); + char buf[64]; + snprintf(buf, sizeof(buf), "%04d-%02d-%02dT%02d:%02d:%02d.%03dZ", + tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec, msec); + return el_wrap_str(el_strdup(buf)); +} + +el_val_t duration_to_seconds(el_val_t d) { + return (el_val_t)((int64_t)d / 1000000000LL); +} + +el_val_t duration_to_millis(el_val_t d) { + return (el_val_t)((int64_t)d / 1000000LL); +} + +el_val_t duration_to_nanos(el_val_t d) { + return (el_val_t)(int64_t)d; +} + +/* sleep(Duration) — Phase 1 replacement for ambiguous sleep(Int). The runtime + * still exposes sleep_secs/sleep_ms for legacy call sites; codegen lowers + * sleep(Duration) to el_sleep_duration(d). Negative durations clamp to 0 so a + * stale deadline doesn't block forever. */ +el_val_t el_sleep_duration(el_val_t dur) { + int64_t ns = (int64_t)dur; + if (ns < 0) ns = 0; + struct timespec ts; + ts.tv_sec = (time_t)(ns / 1000000000LL); + ts.tv_nsec = (long)(ns % 1000000000LL); + nanosleep(&ts, NULL); + return (el_val_t)0; +} + +/* unix_timestamp() — back-compat. Existing El callers expect an Int seconds + * value; this stays an Int returner so the type system isn't disturbed for + * legacy code. New code should call now() and convert when needed. */ +el_val_t unix_timestamp(void) { + return instant_to_unix_seconds(el_now_instant()); +} + +/* TTL cache helpers. Backed by the existing process-wide K/V (state_set/get) + * with a sibling __ttl_set_at_ entry recording the Instant of the last + * write. ttl_cache_get returns "" if the entry is missing or stale, so call + * sites can branch on `if v == "" { miss } else { hit }` — the same shape + * existing get-with-default code uses. No more (now - cached_at) < 60. */ +el_val_t ttl_cache_set(el_val_t key, el_val_t value) { + const char* k = EL_CSTR(key); + if (!k) return (el_val_t)0; + /* Store the value at the user's key. */ + state_set(key, value); + /* Stamp set_at — opaque schema, namespaced under __ttl: prefix so user + * keys can't collide with stamps. */ + size_t klen = strlen(k); + char* stamp_key = (char*)malloc(klen + 16); + if (!stamp_key) return (el_val_t)0; + snprintf(stamp_key, klen + 16, "__ttl_at:%s", k); + int64_t now_ns = (int64_t)el_now_instant(); + char buf[32]; + snprintf(buf, sizeof(buf), "%lld", (long long)now_ns); + state_set(EL_STR(stamp_key), EL_STR(buf)); + free(stamp_key); + return (el_val_t)1; +} + +el_val_t ttl_cache_get(el_val_t key, el_val_t max_age) { + const char* k = EL_CSTR(key); + if (!k) return el_wrap_str(el_strdup("")); + /* Look up stamp. */ + size_t klen = strlen(k); + char* stamp_key = (char*)malloc(klen + 16); + if (!stamp_key) return el_wrap_str(el_strdup("")); + snprintf(stamp_key, klen + 16, "__ttl_at:%s", k); + el_val_t stamp = state_get(EL_STR(stamp_key)); + free(stamp_key); + const char* sv = EL_CSTR(stamp); + if (!sv || !*sv) return el_wrap_str(el_strdup("")); + int64_t set_at = (int64_t)atoll(sv); + int64_t now_ns = (int64_t)el_now_instant(); + int64_t age = now_ns - set_at; + int64_t max_ns = (int64_t)max_age; + if (age < 0) return el_wrap_str(el_strdup("")); /* clock skew — treat as miss */ + if (age > max_ns) return el_wrap_str(el_strdup("")); /* expired */ + return state_get(key); +} + +el_val_t ttl_cache_age(el_val_t key) { + const char* k = EL_CSTR(key); + if (!k) return (el_val_t)INT64_MAX; + size_t klen = strlen(k); + char* stamp_key = (char*)malloc(klen + 16); + if (!stamp_key) return (el_val_t)INT64_MAX; + snprintf(stamp_key, klen + 16, "__ttl_at:%s", k); + el_val_t stamp = state_get(EL_STR(stamp_key)); + free(stamp_key); + const char* sv = EL_CSTR(stamp); + if (!sv || !*sv) return (el_val_t)INT64_MAX; + int64_t set_at = (int64_t)atoll(sv); + int64_t now_ns = (int64_t)el_now_instant(); + return (el_val_t)(now_ns - set_at); +} + +/* ── Calendar + CalendarTime + Rhythm + LocalDate/Time/DateTime ────────────── + * Phase 1.5. Calendar is pluggable: EarthCalendar (IANA zones + Gregorian + + * DST), MarsCalendar (sols, MTC), CycleCalendar(period), NoCycleCalendar, + * RelativeCalendar(epoch). Phase 1 zone wrapping folds INTO EarthCalendar; + * UTC and IANA zones are themselves Earth-parochial and cannot live at the + * lowest type layer. + * + * A Rhythm is a small AST that asks the Calendar for cycle phase, weekday, + * etc. Most rhythm logic is calendar-agnostic at runtime: rhythm_cycle_phase + * means "midpoint of cycle" whether the cycle is 24h on Earth or 30h on a + * station or 300y on a long-cycle world. */ + +/* Magic headers — used by the runtime to recognize boxed temporal values + * arriving through el_val_t. Distinct constants so accidental misuse fails + * loudly rather than silently. */ +#define EL_CAL_MAGIC 0xE1CA1EDDU +#define EL_CALTIME_MAGIC 0xE1CA1747U +#define EL_RHYTHM_MAGIC 0xE1287A11U +#define EL_LDATE_MAGIC 0xE1DA7E00U +#define EL_LDT_MAGIC 0xE1DA7E1DU +#define EL_ZONE_MAGIC 0xE12017E0U + +typedef enum { + EL_CALENDAR_EARTH = 1, + EL_CALENDAR_MARS = 2, + EL_CALENDAR_CYCLE = 3, + EL_CALENDAR_NO_CYCLE = 4, + EL_CALENDAR_RELATIVE = 5 +} el_calendar_kind_t; + +typedef struct { + uint32_t magic; + char* id; /* IANA name or "+HH:MM" / "-HH:MM" */ + int fixed; /* 1 for fixed offset, 0 for IANA */ + int64_t offset_ns; /* fixed offset in nanos (only when fixed) */ +} el_zone_t; + +typedef struct { + uint32_t magic; + el_calendar_kind_t kind; + el_zone_t* zone; /* EarthCalendar; MarsCalendar uses MTC */ + int64_t cycle_period_ns;/* CycleCalendar; computed for Earth (86400 s) and Mars (88775.244 s) */ + int64_t epoch_ns; /* RelativeCalendar; Unix-epoch zero otherwise */ +} el_calendar_t; + +typedef struct { + uint32_t magic; + int64_t instant_ns; + el_calendar_t* cal; +} el_caltime_t; + +/* Rhythm AST. */ +typedef enum { + EL_RHYTHM_CYCLE_START = 1, + EL_RHYTHM_CYCLE_PHASE = 2, + EL_RHYTHM_DURATION = 3, + EL_RHYTHM_SESSION_START = 4, + EL_RHYTHM_EVENT = 5, + EL_RHYTHM_AND = 6, + EL_RHYTHM_OR = 7, + EL_RHYTHM_WEEKDAY = 8, + EL_RHYTHM_WEEKLY_AT = 9 +} el_rhythm_kind_t; + +typedef struct el_rhythm_s { + uint32_t magic; + el_rhythm_kind_t kind; + double phase; /* CYCLE_PHASE */ + int64_t period_ns; /* DURATION */ + int weekday; /* 1..7 Mon..Sun */ + int hour; + int minute; + char* event_name; /* EVENT */ + struct el_rhythm_s* a; /* AND/OR */ + struct el_rhythm_s* b; +} el_rhythm_t; + +typedef struct { + uint32_t magic; + int year; + int month; + int day; +} el_localdate_t; + +typedef struct { + uint32_t magic; + el_localdate_t* date; + int64_t time_ns; /* nanos since midnight */ +} el_localdt_t; + +/* Magic-tag check helpers — peek the first 4 bytes of an el_val_t pointer + * and compare against the expected magic. Strings are NUL-terminated and + * never start with our magic byte sequence, so this is safe. */ +static int el_is_magic(el_val_t v, uint32_t want) { + if (v == 0) return 0; + /* Defensive: only follow pointers in plausible address space. + * On 64-bit unix processes pointers are above 0x10000. */ + if ((uint64_t)v < 0x10000ULL) return 0; + uint32_t got = *(volatile uint32_t*)(uintptr_t)v; + return got == want; +} + +/* Sol length on Mars in nanoseconds: 88775.244 seconds. */ +#define EL_MARS_SOL_NS ((int64_t)88775244000000LL) +/* Earth solar day in nanoseconds: 86400 seconds. */ +#define EL_EARTH_DAY_NS ((int64_t)86400000000000LL) + +/* ── Zone construction ────────────────────────────────────────────────────── + * Zones intern by id string so equality comparisons are pointer-compares. */ + +#define EL_ZONE_TABLE_CAP 64 +static el_zone_t* _el_zone_table[EL_ZONE_TABLE_CAP]; +static int _el_zone_count = 0; + +static el_zone_t* _el_zone_intern(const char* id, int fixed, int64_t offset_ns) { + for (int i = 0; i < _el_zone_count; i++) { + el_zone_t* z = _el_zone_table[i]; + if (z->fixed == fixed && z->offset_ns == offset_ns && + strcmp(z->id ? z->id : "", id ? id : "") == 0) { + return z; + } + } + if (_el_zone_count >= EL_ZONE_TABLE_CAP) { + /* Out of slots: build a non-interned zone. Equality will fail across + * such zones but the program still runs. */ + el_zone_t* z = (el_zone_t*)malloc(sizeof(el_zone_t)); + z->magic = EL_ZONE_MAGIC; + z->id = el_strdup_persist(id ? id : ""); + z->fixed = fixed; + z->offset_ns = offset_ns; + return z; + } + el_zone_t* z = (el_zone_t*)malloc(sizeof(el_zone_t)); + z->magic = EL_ZONE_MAGIC; + z->id = el_strdup_persist(id ? id : ""); + z->fixed = fixed; + z->offset_ns = offset_ns; + _el_zone_table[_el_zone_count++] = z; + return z; +} + +el_val_t zone(el_val_t id) { + const char* s = EL_CSTR(id); + if (!s || !*s) return (el_val_t)(uintptr_t)_el_zone_intern("UTC", 0, 0); + /* Fixed-offset shortcut: "+HH:MM" or "-HH:MM". */ + if ((s[0] == '+' || s[0] == '-') && strlen(s) >= 6 && s[3] == ':') { + int sign = (s[0] == '-') ? -1 : 1; + int hh = (s[1] - '0') * 10 + (s[2] - '0'); + int mm = (s[4] - '0') * 10 + (s[5] - '0'); + int64_t off = (int64_t)sign * ((int64_t)hh * 3600LL + (int64_t)mm * 60LL) * 1000000000LL; + return (el_val_t)(uintptr_t)_el_zone_intern(s, 1, off); + } + return (el_val_t)(uintptr_t)_el_zone_intern(s, 0, 0); +} + +el_val_t zone_utc(void) { + return (el_val_t)(uintptr_t)_el_zone_intern("UTC", 1, 0); +} + +el_val_t zone_local(void) { + /* Resolve the local zone via TZ env or system default. tzset() picks + * up TZ if set; otherwise the C library reads /etc/localtime. We store + * the zone id as "LOCAL" so subsequent equality holds; resolution is + * lazy at use time. */ + return (el_val_t)(uintptr_t)_el_zone_intern("LOCAL", 0, 0); +} + +el_val_t zone_offset(el_val_t hours, el_val_t minutes) { + int hh = (int)(int64_t)hours; + int mm = (int)(int64_t)minutes; + int sign = (hh < 0 || mm < 0) ? -1 : 1; + if (hh < 0) hh = -hh; + if (mm < 0) mm = -mm; + int64_t off = (int64_t)sign * ((int64_t)hh * 3600LL + (int64_t)mm * 60LL) * 1000000000LL; + char buf[16]; + snprintf(buf, sizeof(buf), "%c%02d:%02d", sign < 0 ? '-' : '+', hh, mm); + return (el_val_t)(uintptr_t)_el_zone_intern(buf, 1, off); +} + +/* ── Calendar interning ──────────────────────────────────────────────────── */ + +#define EL_CAL_TABLE_CAP 64 +static el_calendar_t* _el_cal_table[EL_CAL_TABLE_CAP]; +static int _el_cal_count = 0; + +static el_calendar_t* _el_cal_intern(el_calendar_kind_t kind, el_zone_t* z, + int64_t period_ns, int64_t epoch_ns) { + for (int i = 0; i < _el_cal_count; i++) { + el_calendar_t* c = _el_cal_table[i]; + if (c->kind == kind && c->zone == z && + c->cycle_period_ns == period_ns && c->epoch_ns == epoch_ns) { + return c; + } + } + el_calendar_t* c = (el_calendar_t*)malloc(sizeof(el_calendar_t)); + c->magic = EL_CAL_MAGIC; + c->kind = kind; + c->zone = z; + c->cycle_period_ns = period_ns; + c->epoch_ns = epoch_ns; + if (_el_cal_count < EL_CAL_TABLE_CAP) _el_cal_table[_el_cal_count++] = c; + return c; +} + +el_val_t earth_calendar(el_val_t z_val) { + el_zone_t* z = NULL; + if (z_val != 0 && el_is_magic(z_val, EL_ZONE_MAGIC)) { + z = (el_zone_t*)(uintptr_t)z_val; + } else { + z = (el_zone_t*)(uintptr_t)zone_local(); + } + return (el_val_t)(uintptr_t)_el_cal_intern(EL_CALENDAR_EARTH, z, EL_EARTH_DAY_NS, 0); +} + +el_val_t earth_calendar_default(void) { + return earth_calendar(zone_local()); +} + +el_val_t mars_calendar(void) { + el_zone_t* z = (el_zone_t*)(uintptr_t)_el_zone_intern("MTC", 1, 0); + return (el_val_t)(uintptr_t)_el_cal_intern(EL_CALENDAR_MARS, z, EL_MARS_SOL_NS, 0); +} + +el_val_t cycle_calendar(el_val_t period_dur) { + int64_t period = (int64_t)period_dur; + if (period <= 0) period = 1; + return (el_val_t)(uintptr_t)_el_cal_intern(EL_CALENDAR_CYCLE, NULL, period, 0); +} + +el_val_t no_cycle_calendar(void) { + return (el_val_t)(uintptr_t)_el_cal_intern(EL_CALENDAR_NO_CYCLE, NULL, 0, 0); +} + +el_val_t relative_calendar(el_val_t epoch_inst) { + int64_t ep = (int64_t)epoch_inst; + return (el_val_t)(uintptr_t)_el_cal_intern(EL_CALENDAR_RELATIVE, NULL, 0, ep); +} + +/* ── CalendarTime ───────────────────────────────────────────────────────── */ + +static el_caltime_t* _el_caltime_alloc(int64_t inst, el_calendar_t* c) { + el_caltime_t* ct = (el_caltime_t*)malloc(sizeof(el_caltime_t)); + ct->magic = EL_CALTIME_MAGIC; + ct->instant_ns = inst; + ct->cal = c; + return ct; +} + +static el_calendar_t* _el_resolve_cal(el_val_t cal_val) { + if (cal_val == 0 || !el_is_magic(cal_val, EL_CAL_MAGIC)) { + return (el_calendar_t*)(uintptr_t)earth_calendar_default(); + } + return (el_calendar_t*)(uintptr_t)cal_val; +} + +el_val_t now_in(el_val_t cal_val) { + el_calendar_t* c = _el_resolve_cal(cal_val); + int64_t ns = (int64_t)el_now_instant(); + return (el_val_t)(uintptr_t)_el_caltime_alloc(ns, c); +} + +el_val_t in_calendar(el_val_t inst, el_val_t cal_val) { + el_calendar_t* c = _el_resolve_cal(cal_val); + return (el_val_t)(uintptr_t)_el_caltime_alloc((int64_t)inst, c); +} + +el_val_t cal_to_instant(el_val_t ct_val) { + if (!el_is_magic(ct_val, EL_CALTIME_MAGIC)) return (el_val_t)0; + el_caltime_t* ct = (el_caltime_t*)(uintptr_t)ct_val; + return (el_val_t)ct->instant_ns; +} + +el_val_t cal_in(el_val_t ct_val, el_val_t cal_val) { + if (!el_is_magic(ct_val, EL_CALTIME_MAGIC)) return (el_val_t)0; + el_caltime_t* ct = (el_caltime_t*)(uintptr_t)ct_val; + el_calendar_t* c = _el_resolve_cal(cal_val); + return (el_val_t)(uintptr_t)_el_caltime_alloc(ct->instant_ns, c); +} + +el_val_t cal_cycle_phase(el_val_t ct_val) { + if (!el_is_magic(ct_val, EL_CALTIME_MAGIC)) return el_from_float(0.0); + el_caltime_t* ct = (el_caltime_t*)(uintptr_t)ct_val; + el_calendar_t* c = ct->cal; + if (c->kind == EL_CALENDAR_NO_CYCLE) { + return el_from_float(0.0/0.0); /* NaN sentinel */ + } + int64_t period = c->cycle_period_ns; + if (period <= 0) return el_from_float(0.0); + int64_t base = ct->instant_ns - c->epoch_ns; + int64_t phase_ns = base % period; + if (phase_ns < 0) phase_ns += period; + double phase = (double)phase_ns / (double)period; + return el_from_float(phase); +} + +/* ── Earth zone resolution: TZ-based offset lookup ────────────────────────── + * For an EarthCalendar(zone), we want to convert an instant_ns into local + * y/m/d/h/m/s, including DST. Approach: setenv("TZ", id), tzset(), use + * localtime_r, then restore. This is not thread-safe by design — El's + * runtime is single-threaded for the request handler path. Cache the + * computed (instant -> tm) to avoid the syscall churn on repeat formats. */ + +static void _el_apply_zone(el_zone_t* z) { + if (!z) { unsetenv("TZ"); tzset(); return; } + if (z->fixed && strcmp(z->id, "UTC") == 0) { + setenv("TZ", "UTC0", 1); + tzset(); + return; + } + if (z->fixed) { + /* Fixed offset: POSIX TZ uses inverted sign (sign convention of + * "hours WEST of UTC" rather than east). Build the spec accordingly. */ + char buf[32]; + int neg_secs = (int)(-z->offset_ns / 1000000000LL); + int sign = neg_secs < 0 ? -1 : 1; + int abs_secs = neg_secs < 0 ? -neg_secs : neg_secs; + int hh = abs_secs / 3600; + int mm = (abs_secs % 3600) / 60; + snprintf(buf, sizeof(buf), "FIX%c%d:%02d", sign < 0 ? '-' : '+', hh, mm); + setenv("TZ", buf, 1); + tzset(); + return; + } + if (strcmp(z->id, "LOCAL") == 0) { + unsetenv("TZ"); + tzset(); + return; + } + setenv("TZ", z->id, 1); + tzset(); +} + +static int _el_decompose_earth(el_caltime_t* ct, struct tm* tm_out, int* abbr_len, char* abbr_buf, size_t abbr_cap) { + el_calendar_t* c = ct->cal; + el_zone_t* z = c->zone; + _el_apply_zone(z); + time_t s = (time_t)(ct->instant_ns / 1000000000LL); + struct tm tm; + localtime_r(&s, &tm); + *tm_out = tm; + if (abbr_buf && abbr_cap > 0) { + const char* z_str = tm.tm_zone ? tm.tm_zone : ""; + size_t n = strlen(z_str); + if (n >= abbr_cap) n = abbr_cap - 1; + memcpy(abbr_buf, z_str, n); + abbr_buf[n] = '\0'; + if (abbr_len) *abbr_len = (int)n; + } + return 0; +} + +/* Format an Earth CalendarTime under a Java-DateTimeFormatter-ish pattern. + * We support a useful core: yyyy MM dd HH mm ss z EEE MMM d h a — enough for + * the acceptance tests. Single quotes denote literal text. */ +static const char* _el_weekday_short[] = {"Sun","Mon","Tue","Wed","Thu","Fri","Sat"}; +static const char* _el_month_short[] = {"Jan","Feb","Mar","Apr","May","Jun", + "Jul","Aug","Sep","Oct","Nov","Dec"}; + +static char* _el_format_earth(el_caltime_t* ct, const char* pattern) { + struct tm tm; + char abbr[16] = {0}; + int abbr_len = 0; + _el_decompose_earth(ct, &tm, &abbr_len, abbr, sizeof(abbr)); + size_t cap = strlen(pattern) * 4 + 64; + char* out = (char*)malloc(cap); + size_t pos = 0; + size_t i = 0; + size_t plen = strlen(pattern); + while (i < plen) { + char ch = pattern[i]; + /* Quoted literal */ + if (ch == '\'') { + i++; + while (i < plen && pattern[i] != '\'') { + if (pos + 1 >= cap) { cap *= 2; out = realloc(out, cap); } + out[pos++] = pattern[i++]; + } + if (i < plen) i++; + continue; + } + /* Count run of same letter */ + size_t run = 1; + while (i + run < plen && pattern[i + run] == ch) run++; + char tmp[64]; + tmp[0] = '\0'; + if (ch == 'y') { + if (run >= 4) snprintf(tmp, sizeof(tmp), "%04d", tm.tm_year + 1900); + else snprintf(tmp, sizeof(tmp), "%02d", (tm.tm_year + 1900) % 100); + } else if (ch == 'M') { + if (run >= 3) snprintf(tmp, sizeof(tmp), "%s", _el_month_short[tm.tm_mon]); + else if (run == 2) snprintf(tmp, sizeof(tmp), "%02d", tm.tm_mon + 1); + else snprintf(tmp, sizeof(tmp), "%d", tm.tm_mon + 1); + } else if (ch == 'd') { + if (run >= 2) snprintf(tmp, sizeof(tmp), "%02d", tm.tm_mday); + else snprintf(tmp, sizeof(tmp), "%d", tm.tm_mday); + } else if (ch == 'H') { + if (run >= 2) snprintf(tmp, sizeof(tmp), "%02d", tm.tm_hour); + else snprintf(tmp, sizeof(tmp), "%d", tm.tm_hour); + } else if (ch == 'h') { + int h12 = tm.tm_hour % 12; if (h12 == 0) h12 = 12; + if (run >= 2) snprintf(tmp, sizeof(tmp), "%02d", h12); + else snprintf(tmp, sizeof(tmp), "%d", h12); + } else if (ch == 'm') { + if (run >= 2) snprintf(tmp, sizeof(tmp), "%02d", tm.tm_min); + else snprintf(tmp, sizeof(tmp), "%d", tm.tm_min); + } else if (ch == 's') { + if (run >= 2) snprintf(tmp, sizeof(tmp), "%02d", tm.tm_sec); + else snprintf(tmp, sizeof(tmp), "%d", tm.tm_sec); + } else if (ch == 'a') { + snprintf(tmp, sizeof(tmp), "%s", tm.tm_hour < 12 ? "AM" : "PM"); + } else if (ch == 'E') { + snprintf(tmp, sizeof(tmp), "%s", _el_weekday_short[tm.tm_wday]); + } else if (ch == 'z') { + snprintf(tmp, sizeof(tmp), "%s", abbr); + } else { + for (size_t k = 0; k < run; k++) { + if (pos + 1 >= cap) { cap *= 2; out = realloc(out, cap); } + out[pos++] = ch; + } + i += run; + continue; + } + size_t tl = strlen(tmp); + if (pos + tl + 1 >= cap) { cap = (cap + tl) * 2; out = realloc(out, cap); } + memcpy(out + pos, tmp, tl); + pos += tl; + i += run; + } + out[pos] = '\0'; + char* result = el_strdup(out); + free(out); + return result; +} + +/* Format a Mars CalendarTime: %sol prints the integer sol number since + * mission epoch (Unix epoch fallback), %phase prints cycle_phase as a + * 0..1 decimal. Other %-specifiers fall through. */ +static char* _el_format_mars(el_caltime_t* ct, const char* pattern) { + el_calendar_t* c = ct->cal; + int64_t period = c->cycle_period_ns > 0 ? c->cycle_period_ns : EL_MARS_SOL_NS; + int64_t base = ct->instant_ns - c->epoch_ns; + int64_t sol = base / period; + int64_t phase_ns = base % period; + if (phase_ns < 0) { phase_ns += period; sol -= 1; } + double phase = (double)phase_ns / (double)period; + size_t cap = strlen(pattern) * 4 + 64; + char* out = (char*)malloc(cap); + size_t pos = 0; + for (size_t i = 0; pattern[i]; i++) { + if (pattern[i] == '%' && pattern[i+1]) { + char tmp[64]; + tmp[0] = '\0'; + if (strncmp(pattern + i + 1, "sol", 3) == 0) { + snprintf(tmp, sizeof(tmp), "%lld", (long long)sol); + i += 3; + } else if (strncmp(pattern + i + 1, "phase", 5) == 0) { + snprintf(tmp, sizeof(tmp), "%.4f", phase); + i += 5; + } else if (pattern[i+1] == 'd') { + snprintf(tmp, sizeof(tmp), "%lld", (long long)sol); + i += 1; + } else { + tmp[0] = pattern[i+1]; tmp[1] = '\0'; + i += 1; + } + size_t tl = strlen(tmp); + if (pos + tl + 1 >= cap) { cap = (cap + tl) * 2; out = realloc(out, cap); } + memcpy(out + pos, tmp, tl); + pos += tl; + } else { + if (pos + 1 >= cap) { cap *= 2; out = realloc(out, cap); } + out[pos++] = pattern[i]; + } + } + out[pos] = '\0'; + char* result = el_strdup(out); + free(out); + return result; +} + +/* Format a CycleCalendar CalendarTime: %cycle and %phase. */ +static char* _el_format_cycle(el_caltime_t* ct, const char* pattern) { + el_calendar_t* c = ct->cal; + int64_t period = c->cycle_period_ns > 0 ? c->cycle_period_ns : 1; + int64_t base = ct->instant_ns - c->epoch_ns; + int64_t cycle = base / period; + int64_t phase_ns = base % period; + if (phase_ns < 0) { phase_ns += period; cycle -= 1; } + double phase = (double)phase_ns / (double)period; + size_t cap = strlen(pattern) * 4 + 64; + char* out = (char*)malloc(cap); + size_t pos = 0; + for (size_t i = 0; pattern[i]; i++) { + if (pattern[i] == '%' && pattern[i+1]) { + char tmp[64]; + tmp[0] = '\0'; + if (strncmp(pattern + i + 1, "cycle", 5) == 0) { + snprintf(tmp, sizeof(tmp), "%lld", (long long)cycle); + i += 5; + } else if (strncmp(pattern + i + 1, "phase", 5) == 0) { + snprintf(tmp, sizeof(tmp), "%.4f", phase); + i += 5; + } else if (pattern[i+1] == 'd') { + snprintf(tmp, sizeof(tmp), "%lld", (long long)cycle); + i += 1; + } else if (pattern[i+1] == 'f') { + snprintf(tmp, sizeof(tmp), "%.2f", phase); + i += 1; + } else { + /* Pass through unknown specifier */ + tmp[0] = '%'; tmp[1] = pattern[i+1]; tmp[2] = '\0'; + i += 1; + } + size_t tl = strlen(tmp); + if (pos + tl + 1 >= cap) { cap = (cap + tl) * 2; out = realloc(out, cap); } + memcpy(out + pos, tmp, tl); + pos += tl; + } else { + if (pos + 1 >= cap) { cap *= 2; out = realloc(out, cap); } + out[pos++] = pattern[i]; + } + } + out[pos] = '\0'; + char* result = el_strdup(out); + free(out); + return result; +} + +el_val_t cal_format(el_val_t ct_val, el_val_t pattern_val) { + if (!el_is_magic(ct_val, EL_CALTIME_MAGIC)) return el_wrap_str(el_strdup("")); + el_caltime_t* ct = (el_caltime_t*)(uintptr_t)ct_val; + const char* pat = EL_CSTR(pattern_val); + if (!pat) pat = ""; + char* result = NULL; + switch (ct->cal->kind) { + case EL_CALENDAR_EARTH: result = _el_format_earth(ct, pat); break; + case EL_CALENDAR_MARS: result = _el_format_mars(ct, pat); break; + case EL_CALENDAR_CYCLE: result = _el_format_cycle(ct, pat); break; + case EL_CALENDAR_RELATIVE: result = _el_format_cycle(ct, pat); break; + case EL_CALENDAR_NO_CYCLE: { + char buf[64]; + snprintf(buf, sizeof(buf), "instant:%lld", (long long)ct->instant_ns); + result = el_strdup(buf); + break; + } + default: result = el_strdup(""); + } + return el_wrap_str(result); +} + +/* ── LocalDate / LocalTime / LocalDateTime ──────────────────────────────── */ + +static int _el_days_in_month(int y, int m) { + static const int dim[12] = {31,28,31,30,31,30,31,31,30,31,30,31}; + if (m == 2) { + int leap = ((y % 4 == 0) && (y % 100 != 0)) || (y % 400 == 0); + return 28 + (leap ? 1 : 0); + } + if (m < 1 || m > 12) return 30; + return dim[m - 1]; +} + +el_val_t local_date(el_val_t y, el_val_t m, el_val_t d) { + el_localdate_t* ld = (el_localdate_t*)malloc(sizeof(el_localdate_t)); + ld->magic = EL_LDATE_MAGIC; + ld->year = (int)(int64_t)y; + ld->month = (int)(int64_t)m; + ld->day = (int)(int64_t)d; + return (el_val_t)(uintptr_t)ld; +} + +el_val_t local_time(el_val_t h, el_val_t m, el_val_t s, el_val_t ns) { + int64_t hh = (int64_t)h; + int64_t mm = (int64_t)m; + int64_t ss = (int64_t)s; + int64_t nn = (int64_t)ns; + int64_t total = hh * 3600000000000LL + mm * 60000000000LL + ss * 1000000000LL + nn; + return (el_val_t)total; +} + +el_val_t local_datetime(el_val_t date_val, el_val_t time_val) { + if (!el_is_magic(date_val, EL_LDATE_MAGIC)) return (el_val_t)0; + el_localdt_t* ldt = (el_localdt_t*)malloc(sizeof(el_localdt_t)); + ldt->magic = EL_LDT_MAGIC; + ldt->date = (el_localdate_t*)(uintptr_t)date_val; + ldt->time_ns = (int64_t)time_val; + return (el_val_t)(uintptr_t)ldt; +} + +el_val_t zoned(el_val_t date_val, el_val_t time_val, el_val_t cal_val) { + if (!el_is_magic(date_val, EL_LDATE_MAGIC)) return (el_val_t)0; + el_localdate_t* ld = (el_localdate_t*)(uintptr_t)date_val; + el_calendar_t* c = _el_resolve_cal(cal_val); + int64_t time_ns = (int64_t)time_val; + /* Convert (LocalDate, LocalTime, EarthCalendar) -> Instant. + * For non-Earth calendars we use day-anchored conversion: treat the + * LocalDate's (y,m,d) as a Gregorian projection, convert to seconds via + * mktime under the calendar's zone, then add nanos-since-midnight. */ + if (c->kind == EL_CALENDAR_EARTH) { + _el_apply_zone(c->zone); + struct tm tm; memset(&tm, 0, sizeof(tm)); + tm.tm_year = ld->year - 1900; + tm.tm_mon = ld->month - 1; + tm.tm_mday = ld->day; + tm.tm_hour = (int)(time_ns / 3600000000000LL); + tm.tm_min = (int)((time_ns / 60000000000LL) % 60); + tm.tm_sec = (int)((time_ns / 1000000000LL) % 60); + tm.tm_isdst = -1; + time_t t = mktime(&tm); + if (t == (time_t)-1) return (el_val_t)0; + int64_t ns = (int64_t)t * 1000000000LL + (time_ns % 1000000000LL); + return (el_val_t)(uintptr_t)_el_caltime_alloc(ns, c); + } + /* Non-Earth fallback: project as if Earth UTC then attach calendar. */ + struct tm tm; memset(&tm, 0, sizeof(tm)); + tm.tm_year = ld->year - 1900; + tm.tm_mon = ld->month - 1; + tm.tm_mday = ld->day; + tm.tm_hour = (int)(time_ns / 3600000000000LL); + tm.tm_min = (int)((time_ns / 60000000000LL) % 60); + tm.tm_sec = (int)((time_ns / 1000000000LL) % 60); + time_t t = timegm(&tm); + if (t == (time_t)-1) return (el_val_t)0; + int64_t ns = (int64_t)t * 1000000000LL + (time_ns % 1000000000LL); + return (el_val_t)(uintptr_t)_el_caltime_alloc(ns, c); +} + +el_val_t local_date_year(el_val_t v) { + if (!el_is_magic(v, EL_LDATE_MAGIC)) return (el_val_t)0; + return (el_val_t)((el_localdate_t*)(uintptr_t)v)->year; +} +el_val_t local_date_month(el_val_t v) { + if (!el_is_magic(v, EL_LDATE_MAGIC)) return (el_val_t)0; + return (el_val_t)((el_localdate_t*)(uintptr_t)v)->month; +} +el_val_t local_date_day(el_val_t v) { + if (!el_is_magic(v, EL_LDATE_MAGIC)) return (el_val_t)0; + return (el_val_t)((el_localdate_t*)(uintptr_t)v)->day; +} +el_val_t local_time_hour(el_val_t v) { + int64_t t = (int64_t)v; + return (el_val_t)(t / 3600000000000LL); +} +el_val_t local_time_minute(el_val_t v) { + int64_t t = (int64_t)v; + return (el_val_t)((t / 60000000000LL) % 60); +} +el_val_t local_time_second(el_val_t v) { + int64_t t = (int64_t)v; + return (el_val_t)((t / 1000000000LL) % 60); +} +el_val_t local_time_nanos(el_val_t v) { + int64_t t = (int64_t)v; + return (el_val_t)(t % 1000000000LL); +} + +el_val_t el_local_date_add_dur(el_val_t ld_val, el_val_t dur_val) { + if (!el_is_magic(ld_val, EL_LDATE_MAGIC)) return ld_val; + el_localdate_t* ld = (el_localdate_t*)(uintptr_t)ld_val; + int64_t dur_ns = (int64_t)dur_val; + int64_t days = dur_ns / EL_EARTH_DAY_NS; + int y = ld->year, m = ld->month, d = ld->day; + /* Walk days forward/backward in canonical Gregorian. */ + while (days > 0) { + int dim = _el_days_in_month(y, m); + if (d + days <= dim) { d += (int)days; days = 0; break; } + days -= (dim - d + 1); + d = 1; + m++; + if (m > 12) { m = 1; y++; } + } + while (days < 0) { + if (d + days >= 1) { d += (int)days; days = 0; break; } + days += d; + m--; + if (m < 1) { m = 12; y--; } + d = _el_days_in_month(y, m); + } + return local_date((el_val_t)y, (el_val_t)m, (el_val_t)d); +} + +el_val_t el_local_time_add_dur(el_val_t lt_val, el_val_t dur_val) { + int64_t t = (int64_t)lt_val + (int64_t)dur_val; + /* Wrap mod 24h on Earth-default. CycleCalendar wrapping requires the + * caller to use cal_in / cal_format for the right modulus. */ + int64_t day = EL_EARTH_DAY_NS; + int64_t r = t % day; + if (r < 0) r += day; + return (el_val_t)r; +} + +el_val_t el_local_date_lt(el_val_t a_val, el_val_t b_val) { + if (!el_is_magic(a_val, EL_LDATE_MAGIC) || !el_is_magic(b_val, EL_LDATE_MAGIC)) return (el_val_t)0; + el_localdate_t* a = (el_localdate_t*)(uintptr_t)a_val; + el_localdate_t* b = (el_localdate_t*)(uintptr_t)b_val; + if (a->year != b->year) return (el_val_t)(a->year < b->year ? 1 : 0); + if (a->month != b->month) return (el_val_t)(a->month < b->month ? 1 : 0); + return (el_val_t)(a->day < b->day ? 1 : 0); +} + +el_val_t el_local_date_eq(el_val_t a_val, el_val_t b_val) { + if (!el_is_magic(a_val, EL_LDATE_MAGIC) || !el_is_magic(b_val, EL_LDATE_MAGIC)) return (el_val_t)0; + el_localdate_t* a = (el_localdate_t*)(uintptr_t)a_val; + el_localdate_t* b = (el_localdate_t*)(uintptr_t)b_val; + return (el_val_t)((a->year == b->year && a->month == b->month && a->day == b->day) ? 1 : 0); +} + +/* ── Rhythm ──────────────────────────────────────────────────────────────── */ + +static el_rhythm_t* _el_rhythm_alloc(el_rhythm_kind_t k) { + el_rhythm_t* r = (el_rhythm_t*)calloc(1, sizeof(el_rhythm_t)); + r->magic = EL_RHYTHM_MAGIC; + r->kind = k; + return r; +} + +el_val_t rhythm_cycle_start(void) { + return (el_val_t)(uintptr_t)_el_rhythm_alloc(EL_RHYTHM_CYCLE_START); +} + +el_val_t rhythm_cycle_phase(el_val_t phase_val) { + el_rhythm_t* r = _el_rhythm_alloc(EL_RHYTHM_CYCLE_PHASE); + r->phase = el_to_float(phase_val); + return (el_val_t)(uintptr_t)r; +} + +el_val_t rhythm_duration(el_val_t d_val) { + el_rhythm_t* r = _el_rhythm_alloc(EL_RHYTHM_DURATION); + r->period_ns = (int64_t)d_val; + return (el_val_t)(uintptr_t)r; +} + +el_val_t rhythm_session_start(void) { + return (el_val_t)(uintptr_t)_el_rhythm_alloc(EL_RHYTHM_SESSION_START); +} + +el_val_t rhythm_event(el_val_t name_val) { + el_rhythm_t* r = _el_rhythm_alloc(EL_RHYTHM_EVENT); + const char* n = EL_CSTR(name_val); + r->event_name = el_strdup_persist(n ? n : ""); + return (el_val_t)(uintptr_t)r; +} + +el_val_t rhythm_and(el_val_t a_val, el_val_t b_val) { + el_rhythm_t* r = _el_rhythm_alloc(EL_RHYTHM_AND); + r->a = el_is_magic(a_val, EL_RHYTHM_MAGIC) ? (el_rhythm_t*)(uintptr_t)a_val : NULL; + r->b = el_is_magic(b_val, EL_RHYTHM_MAGIC) ? (el_rhythm_t*)(uintptr_t)b_val : NULL; + return (el_val_t)(uintptr_t)r; +} + +el_val_t rhythm_or(el_val_t a_val, el_val_t b_val) { + el_rhythm_t* r = _el_rhythm_alloc(EL_RHYTHM_OR); + r->a = el_is_magic(a_val, EL_RHYTHM_MAGIC) ? (el_rhythm_t*)(uintptr_t)a_val : NULL; + r->b = el_is_magic(b_val, EL_RHYTHM_MAGIC) ? (el_rhythm_t*)(uintptr_t)b_val : NULL; + return (el_val_t)(uintptr_t)r; +} + +el_val_t rhythm_weekday(el_val_t day) { + el_rhythm_t* r = _el_rhythm_alloc(EL_RHYTHM_WEEKDAY); + r->weekday = (int)(int64_t)day; + return (el_val_t)(uintptr_t)r; +} + +el_val_t rhythm_weekly_at(el_val_t day, el_val_t hour, el_val_t minute) { + el_rhythm_t* r = _el_rhythm_alloc(EL_RHYTHM_WEEKLY_AT); + r->weekday = (int)(int64_t)day; + r->hour = (int)(int64_t)hour; + r->minute = (int)(int64_t)minute; + return (el_val_t)(uintptr_t)r; +} + +/* Compute the next instant on or after `after` when rhythm `r` matches, + * under calendar `cal`. */ +static int64_t _el_next_after(el_rhythm_t* r, int64_t after_ns, el_calendar_t* cal) { + if (!r) return after_ns; + int64_t period = cal->cycle_period_ns > 0 ? cal->cycle_period_ns : EL_EARTH_DAY_NS; + switch (r->kind) { + case EL_RHYTHM_CYCLE_START: { + int64_t base = after_ns - cal->epoch_ns; + int64_t cyc = (base / period) + 1; + return cal->epoch_ns + cyc * period; + } + case EL_RHYTHM_CYCLE_PHASE: { + int64_t base = after_ns - cal->epoch_ns; + int64_t cyc_ns = (int64_t)(r->phase * (double)period); + int64_t cur_cyc = base / period; + int64_t candidate = cal->epoch_ns + cur_cyc * period + cyc_ns; + if (candidate <= after_ns) candidate += period; + return candidate; + } + case EL_RHYTHM_DURATION: { + return after_ns + (r->period_ns > 0 ? r->period_ns : 1); + } + case EL_RHYTHM_WEEKDAY: + case EL_RHYTHM_WEEKLY_AT: { + if (cal->kind != EL_CALENDAR_EARTH) { + /* Non-Earth calendars: fall back to cycle math, treating + * weekday as a 7-cycle-per-period proxy. */ + return after_ns + period; + } + _el_apply_zone(cal->zone); + time_t s = (time_t)(after_ns / 1000000000LL); + struct tm tm; + localtime_r(&s, &tm); + /* tm_wday: 0=Sun..6=Sat. We use 1=Mon..7=Sun. */ + int target = r->weekday >= 1 && r->weekday <= 7 ? r->weekday : 1; + int target_wday = target == 7 ? 0 : target; /* 7→Sun=0, 1→Mon=1 */ + int days_ahead = (target_wday - tm.tm_wday + 7) % 7; + int hour = (r->kind == EL_RHYTHM_WEEKLY_AT) ? r->hour : 0; + int minute = (r->kind == EL_RHYTHM_WEEKLY_AT) ? r->minute : 0; + struct tm cand = tm; + cand.tm_mday += days_ahead; + cand.tm_hour = hour; + cand.tm_min = minute; + cand.tm_sec = 0; + cand.tm_isdst = -1; + time_t cand_t = mktime(&cand); + int64_t cand_ns = (int64_t)cand_t * 1000000000LL; + if (cand_ns <= after_ns) { + cand.tm_mday += 7; + cand.tm_isdst = -1; + cand_t = mktime(&cand); + cand_ns = (int64_t)cand_t * 1000000000LL; + } + return cand_ns; + } + case EL_RHYTHM_AND: { + int64_t a = _el_next_after(r->a, after_ns, cal); + int64_t b = _el_next_after(r->b, after_ns, cal); + return a > b ? a : b; + } + case EL_RHYTHM_OR: { + int64_t a = _el_next_after(r->a, after_ns, cal); + int64_t b = _el_next_after(r->b, after_ns, cal); + return a < b ? a : b; + } + case EL_RHYTHM_SESSION_START: + case EL_RHYTHM_EVENT: + default: + return after_ns; + } +} + +el_val_t rhythm_next_after(el_val_t r_val, el_val_t after_val, el_val_t cal_val) { + if (!el_is_magic(r_val, EL_RHYTHM_MAGIC)) return after_val; + el_rhythm_t* r = (el_rhythm_t*)(uintptr_t)r_val; + el_calendar_t* c = _el_resolve_cal(cal_val); + int64_t out = _el_next_after(r, (int64_t)after_val, c); + return (el_val_t)out; +} + +el_val_t rhythm_matches(el_val_t r_val, el_val_t ct_val) { + if (!el_is_magic(r_val, EL_RHYTHM_MAGIC)) return (el_val_t)0; + if (!el_is_magic(ct_val, EL_CALTIME_MAGIC)) return (el_val_t)0; + el_rhythm_t* r = (el_rhythm_t*)(uintptr_t)r_val; + el_caltime_t* ct = (el_caltime_t*)(uintptr_t)ct_val; + int64_t period = ct->cal->cycle_period_ns > 0 ? ct->cal->cycle_period_ns : EL_EARTH_DAY_NS; + int64_t base = ct->instant_ns - ct->cal->epoch_ns; + int64_t phase_ns = base % period; + if (phase_ns < 0) phase_ns += period; + double phase = (double)phase_ns / (double)period; + switch (r->kind) { + case EL_RHYTHM_CYCLE_START: return (el_val_t)(phase_ns == 0 ? 1 : 0); + case EL_RHYTHM_CYCLE_PHASE: { + double diff = phase - r->phase; + if (diff < 0) diff = -diff; + return (el_val_t)(diff < 0.001 ? 1 : 0); + } + default: return (el_val_t)0; + } +} + +/* ── UUID v4 ─────────────────────────────────────────────────────────────── */ + +static int _el_uuid_seeded = 0; + +static void _el_uuid_seed(void) { + if (!_el_uuid_seeded) { + srand((unsigned)time(NULL) ^ (unsigned)(uintptr_t)&_el_uuid_seeded); + _el_uuid_seeded = 1; + } +} + +el_val_t uuid_new(void) { + _el_uuid_seed(); + unsigned char b[16]; + for (int i = 0; i < 16; i++) b[i] = (unsigned char)(rand() & 0xff); + /* Version 4 */ + b[6] = (b[6] & 0x0f) | 0x40; + /* RFC 4122 variant */ + b[8] = (b[8] & 0x3f) | 0x80; + char buf[37]; + snprintf(buf, sizeof(buf), + "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x", + b[0], b[1], b[2], b[3], + b[4], b[5], + b[6], b[7], + b[8], b[9], + b[10], b[11], b[12], b[13], b[14], b[15]); + return el_wrap_str(el_strdup(buf)); +} + +el_val_t uuid_v4(void) { return uuid_new(); } + +/* ── Environment ─────────────────────────────────────────────────────────── */ + +el_val_t env(el_val_t key) { + const char* k = EL_CSTR(key); + if (!k) return el_wrap_str(el_strdup("")); + const char* v = getenv(k); + return el_wrap_str(el_strdup(v ? v : "")); +} + +/* ── In-process state K/V ────────────────────────────────────────────────── */ + +typedef struct { + char* key; + char* value; +} StateEntry; + +static StateEntry* _state_entries = NULL; +static size_t _state_count = 0; +static size_t _state_cap = 0; +/* Mutex protecting all _state_entries access. state_set/state_get are called + * concurrently from 64 HTTP worker threads — without this lock, realloc and + * free race, producing corruption, double-free, and segfaults. */ +static pthread_mutex_t _state_mu = PTHREAD_MUTEX_INITIALIZER; + +static StateEntry* state_find(const char* key) { + for (size_t i = 0; i < _state_count; i++) { + if (strcmp(_state_entries[i].key, key) == 0) return &_state_entries[i]; + } + return NULL; +} + +el_val_t state_set(el_val_t key, el_val_t value) { + const char* k = EL_CSTR(key); + const char* v = EL_CSTR(value); + if (!k) return 0; + if (!v) v = ""; + pthread_mutex_lock(&_state_mu); + StateEntry* e = state_find(k); + if (e) { + free(e->value); + e->value = el_strdup_persist(v); + pthread_mutex_unlock(&_state_mu); + return 1; + } + if (_state_count >= _state_cap) { + size_t nc = _state_cap == 0 ? 16 : _state_cap * 2; + StateEntry* grown = realloc(_state_entries, nc * sizeof(StateEntry)); + if (!grown) { pthread_mutex_unlock(&_state_mu); fputs("el_runtime: out of memory\n", stderr); exit(1); } + _state_entries = grown; + _state_cap = nc; + } + _state_entries[_state_count].key = el_strdup_persist(k); + _state_entries[_state_count].value = el_strdup_persist(v); + _state_count++; + pthread_mutex_unlock(&_state_mu); + return 1; +} + +el_val_t state_get(el_val_t key) { + const char* k = EL_CSTR(key); + if (!k) return el_wrap_str(el_strdup("")); + pthread_mutex_lock(&_state_mu); + StateEntry* e = state_find(k); + char* result = el_strdup_persist(e ? e->value : ""); + pthread_mutex_unlock(&_state_mu); + /* wrap in arena-tracked copy for the caller's request lifetime */ + char* copy = el_strdup(result); + return el_wrap_str(copy); +} + +el_val_t state_del(el_val_t key) { + const char* k = EL_CSTR(key); + if (!k) return 0; + pthread_mutex_lock(&_state_mu); + for (size_t i = 0; i < _state_count; i++) { + if (strcmp(_state_entries[i].key, k) == 0) { + free(_state_entries[i].key); + free(_state_entries[i].value); + for (size_t j = i + 1; j < _state_count; j++) { + _state_entries[j - 1] = _state_entries[j]; + } + _state_count--; + pthread_mutex_unlock(&_state_mu); + return 1; + } + } + pthread_mutex_unlock(&_state_mu); + return 1; +} + +el_val_t state_keys(void) { + pthread_mutex_lock(&_state_mu); + el_val_t lst = el_list_empty(); + for (size_t i = 0; i < _state_count; i++) { + lst = el_list_append(lst, el_wrap_str(el_strdup(_state_entries[i].key))); + } + pthread_mutex_unlock(&_state_mu); + return lst; +} + +/* ── Float formatting ────────────────────────────────────────────────────── */ + +el_val_t float_to_str(el_val_t f) { + char buf[64]; + snprintf(buf, sizeof(buf), "%g", el_to_float(f)); + return el_wrap_str(el_strdup(buf)); +} + +el_val_t int_to_float(el_val_t n) { + return el_from_float((double)(int64_t)n); +} + +el_val_t float_to_int(el_val_t f) { + return (el_val_t)(int64_t)el_to_float(f); +} + +el_val_t format_float(el_val_t f, el_val_t decimals) { + int d = (int)(int64_t)decimals; + if (d < 0) d = 0; + if (d > 30) d = 30; + char buf[128]; + snprintf(buf, sizeof(buf), "%.*f", d, el_to_float(f)); + return el_wrap_str(el_strdup(buf)); +} + +el_val_t decimal_round(el_val_t f, el_val_t decimals) { + int d = (int)(int64_t)decimals; + if (d < 0) d = 0; + if (d > 15) d = 15; + double mul = pow(10.0, (double)d); + double v = el_to_float(f); + double r = (v >= 0.0 ? floor(v * mul + 0.5) : -floor(-v * mul + 0.5)) / mul; + return el_from_float(r); +} + +el_val_t str_to_float(el_val_t s) { + const char* str = EL_CSTR(s); + if (!str) return el_from_float(0.0); + return el_from_float(strtod(str, NULL)); +} + +/* ── Math (Float-aware) ──────────────────────────────────────────────────── */ + +el_val_t math_sqrt(el_val_t f) { return el_from_float(sqrt(el_to_float(f))); } +el_val_t math_log(el_val_t f) { return el_from_float(log(el_to_float(f))); } +el_val_t math_ln(el_val_t f) { return el_from_float(log(el_to_float(f))); } +el_val_t math_sin(el_val_t f) { return el_from_float(sin(el_to_float(f))); } +el_val_t math_cos(el_val_t f) { return el_from_float(cos(el_to_float(f))); } +el_val_t math_pi(void) { return el_from_float(3.141592653589793238462643383279502884); } + +/* ── String additions ────────────────────────────────────────────────────── */ + +el_val_t str_index_of(el_val_t s, el_val_t sub) { + const char* str = EL_CSTR(s); + const char* sb = EL_CSTR(sub); + if (!str || !sb) return -1; + const char* hit = strstr(str, sb); + if (!hit) return -1; + return (el_val_t)(int64_t)(hit - str); +} + +el_val_t str_split(el_val_t s, el_val_t sep) { + const char* str = EL_CSTR(s); + const char* sp = EL_CSTR(sep); + el_val_t lst = el_list_empty(); + if (!str) return lst; + if (!sp || !*sp) { + lst = el_list_append(lst, el_wrap_str(el_strdup(str))); + return lst; + } + size_t lp = strlen(sp); + const char* p = str; + const char* hit; + while ((hit = strstr(p, sp)) != NULL) { + size_t n = (size_t)(hit - p); + char* out = el_strbuf(n); + memcpy(out, p, n); + out[n] = '\0'; + lst = el_list_append(lst, el_wrap_str(out)); + p = hit + lp; + } + lst = el_list_append(lst, el_wrap_str(el_strdup(p))); + return lst; +} + +el_val_t str_char_at(el_val_t s, el_val_t i) { + const char* str = EL_CSTR(s); + int64_t idx = (int64_t)i; + if (!str) return el_wrap_str(el_strdup("")); + int64_t n = (int64_t)strlen(str); + if (idx < 0 || idx >= n) return el_wrap_str(el_strdup("")); + char buf[2]; + buf[0] = str[idx]; + buf[1] = '\0'; + return el_wrap_str(el_strdup(buf)); +} + +el_val_t str_char_code(el_val_t s, el_val_t i) { + const char* str = EL_CSTR(s); + int64_t idx = (int64_t)i; + if (!str) return 0; + int64_t n = (int64_t)strlen(str); + if (idx < 0 || idx >= n) return 0; + return (el_val_t)(unsigned char)str[idx]; +} + +static el_val_t str_pad(const char* s, int64_t width, const char* pad, int left) { + if (!s) s = ""; + if (!pad || !*pad) pad = " "; + int64_t lp = (int64_t)strlen(pad); + int64_t ls = (int64_t)strlen(s); + if (ls >= width) return el_wrap_str(el_strdup(s)); + int64_t need = width - ls; + char* out = el_strbuf((size_t)width); + if (left) { + for (int64_t i = 0; i < need; i++) out[i] = pad[i % lp]; + memcpy(out + need, s, (size_t)ls); + } else { + memcpy(out, s, (size_t)ls); + for (int64_t i = 0; i < need; i++) out[ls + i] = pad[i % lp]; + } + out[width] = '\0'; + return el_wrap_str(out); +} + +el_val_t str_pad_left(el_val_t s, el_val_t width, el_val_t pad) { + return str_pad(EL_CSTR(s), (int64_t)width, EL_CSTR(pad), 1); +} + +el_val_t str_pad_right(el_val_t s, el_val_t width, el_val_t pad) { + return str_pad(EL_CSTR(s), (int64_t)width, EL_CSTR(pad), 0); +} + +el_val_t str_format(el_val_t fmt, el_val_t data) { + const char* tpl = EL_CSTR(fmt); + if (!tpl) return el_wrap_str(el_strdup("")); + JsonBuf b; jb_init(&b); + const char* p = tpl; + while (*p) { + if (*p == '{') { + const char* q = p + 1; + while (*q && *q != '}') q++; + if (*q == '}') { + size_t klen = (size_t)(q - p - 1); + char keybuf[256]; + if (klen < sizeof(keybuf)) { + memcpy(keybuf, p + 1, klen); + keybuf[klen] = '\0'; + el_val_t v = el_map_get(data, EL_STR(keybuf)); + if (v != 0 && looks_like_string(v)) { + jb_puts(&b, EL_CSTR(v)); + p = q + 1; + continue; + } else if (v != 0) { + jb_emit_int(&b, (int64_t)v); + p = q + 1; + continue; + } + } + /* Unknown key — leave {key} verbatim */ + jb_reserve(&b, klen + 2); + memcpy(b.buf + b.len, p, klen + 2); + b.len += klen + 2; + b.buf[b.len] = '\0'; + p = q + 1; + continue; + } + } + jb_putc(&b, *p); + p++; + } + return el_wrap_str(b.buf); +} + +el_val_t str_lower(el_val_t s) { return str_to_lower(s); } +el_val_t str_upper(el_val_t s) { return str_to_upper(s); } + +/* ── Text-processing primitives (Phase 1: byte/codepoint, ASCII char classes) + * + * Phase 1 covers the operations every text-handling caller used to roll by + * hand on top of str_index_of + str_slice. The character-class predicates + * (is_letter / is_digit / ...) are ASCII only — Unicode-grapheme awareness, + * NFC/NFD normalization, and regex are Phase 2. Single-char input checks the + * first byte; multi-char input requires ALL bytes to match (false otherwise). + * + * Counting: + * str_count non-overlapping occurrences of sub in s + * str_count_chars codepoint count (UTF-8 leading-byte count) + * str_count_bytes explicit byte length (alias of str_len) + * str_count_lines \n-delimited line count (\r\n folded to \n) + * str_count_words whitespace-delimited tokens, non-empty only + * str_count_letters ASCII [A-Za-z] + * str_count_digits ASCII [0-9] + * + * Find / position: + * str_index_of_all all byte offsets of sub, [] if none + * str_last_index_of last byte offset of sub, -1 if not found + * str_find_chars first index of any char in any_of, -1 if none + * + * Transform: + * str_repeat s * n (non-negative) + * str_reverse codepoint-reversed (NOT grapheme-aware) + * str_strip_prefix s without prefix if present, else s + * str_strip_suffix s without suffix if present, else s + * str_strip_chars strip leading+trailing chars matching any in chars + * str_lstrip strip leading whitespace + * str_rstrip strip trailing whitespace + * + * Char classification (Bool): + * is_letter, is_digit, is_alphanumeric, is_whitespace, + * is_punctuation, is_uppercase, is_lowercase + * + * Splitting: + * str_split_lines \n-delimited (\r\n folded). Trailing empty dropped. + * str_split_chars alias of native_string_chars in str_ namespace + * str_split_n split into at most n parts (last part keeps the + * rest verbatim, including any further separators) + * + * Joining: + * str_join [String] -> String, sep between elements + */ + +/* Count non-overlapping occurrences of sub in s. Empty sub returns 0. */ +el_val_t str_count(el_val_t sv, el_val_t subv) { + const char* s = EL_CSTR(sv); + const char* sub = EL_CSTR(subv); + if (!s || !sub || !*sub) return 0; + size_t lp = strlen(sub); + int64_t count = 0; + const char* p = s; + while ((p = strstr(p, sub)) != NULL) { + count++; + p += lp; /* non-overlapping advance */ + } + return (el_val_t)count; +} + +/* Codepoint count: walk bytes, count those NOT matching 10xxxxxx. */ +el_val_t str_count_chars(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return 0; + int64_t count = 0; + for (const unsigned char* p = (const unsigned char*)s; *p; p++) { + if ((*p & 0xC0) != 0x80) count++; + } + return (el_val_t)count; +} + +el_val_t str_count_bytes(el_val_t sv) { + return str_len(sv); +} + +el_val_t str_count_lines(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s || !*s) return 0; + int64_t count = 0; + int has_content = 0; + for (const char* p = s; *p; p++) { + has_content = 1; + if (*p == '\n') { + count++; + has_content = 0; /* the \n closed the line */ + } + } + if (has_content) count++; /* trailing line with no terminator */ + return (el_val_t)count; +} + +el_val_t str_count_words(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return 0; + int64_t count = 0; + int in_word = 0; + for (const unsigned char* p = (const unsigned char*)s; *p; p++) { + if (isspace(*p)) { + in_word = 0; + } else if (!in_word) { + in_word = 1; + count++; + } + } + return (el_val_t)count; +} + +el_val_t str_count_letters(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return 0; + int64_t count = 0; + for (const unsigned char* p = (const unsigned char*)s; *p; p++) { + if ((*p >= 'A' && *p <= 'Z') || (*p >= 'a' && *p <= 'z')) count++; + } + return (el_val_t)count; +} + +el_val_t str_count_digits(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return 0; + int64_t count = 0; + for (const unsigned char* p = (const unsigned char*)s; *p; p++) { + if (*p >= '0' && *p <= '9') count++; + } + return (el_val_t)count; +} + +el_val_t str_index_of_all(el_val_t sv, el_val_t subv) { + const char* s = EL_CSTR(sv); + const char* sub = EL_CSTR(subv); + el_val_t lst = el_list_empty(); + if (!s || !sub || !*sub) return lst; + size_t lp = strlen(sub); + const char* p = s; + const char* hit; + while ((hit = strstr(p, sub)) != NULL) { + lst = el_list_append(lst, (el_val_t)(int64_t)(hit - s)); + p = hit + lp; + } + return lst; +} + +el_val_t str_last_index_of(el_val_t sv, el_val_t subv) { + const char* s = EL_CSTR(sv); + const char* sub = EL_CSTR(subv); + if (!s || !sub || !*sub) return -1; + size_t lp = strlen(sub); + int64_t last = -1; + const char* p = s; + const char* hit; + while ((hit = strstr(p, sub)) != NULL) { + last = (int64_t)(hit - s); + p = hit + lp; + } + return (el_val_t)last; +} + +el_val_t str_find_chars(el_val_t sv, el_val_t any_of_v) { + const char* s = EL_CSTR(sv); + const char* any = EL_CSTR(any_of_v); + if (!s || !any || !*any) return -1; + for (const char* p = s; *p; p++) { + if (strchr(any, *p)) return (el_val_t)(int64_t)(p - s); + } + return -1; +} + +el_val_t str_repeat(el_val_t sv, el_val_t nv) { + const char* s = EL_CSTR(sv); + int64_t n = (int64_t)nv; + if (!s || n <= 0) return el_wrap_str(el_strdup("")); + size_t ls = strlen(s); + if (ls == 0) return el_wrap_str(el_strdup("")); + size_t total = ls * (size_t)n; + char* out = el_strbuf(total); + for (int64_t i = 0; i < n; i++) { + memcpy(out + i * ls, s, ls); + } + out[total] = '\0'; + return el_wrap_str(out); +} + +/* Reverse by codepoint: walk codepoints, copy each backwards into the output. + * NOT grapheme-aware (Phase 2). Combining marks attached to a base codepoint + * will detach. ASCII strings are byte-reverse equivalent. */ +el_val_t str_reverse(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return el_wrap_str(el_strdup("")); + size_t n = strlen(s); + char* out = el_strbuf(n); + /* Walk forward, find each codepoint's byte length, then copy from the end. */ + size_t out_pos = n; + const unsigned char* p = (const unsigned char*)s; + while (*p) { + int cp_len; + if ((*p & 0x80) == 0x00) cp_len = 1; + else if ((*p & 0xE0) == 0xC0) cp_len = 2; + else if ((*p & 0xF0) == 0xE0) cp_len = 3; + else if ((*p & 0xF8) == 0xF0) cp_len = 4; + else cp_len = 1; /* invalid byte: passthrough */ + out_pos -= cp_len; + memcpy(out + out_pos, p, cp_len); + p += cp_len; + } + out[n] = '\0'; + return el_wrap_str(out); +} + +el_val_t str_strip_prefix(el_val_t sv, el_val_t prefv) { + const char* s = EL_CSTR(sv); + const char* pref = EL_CSTR(prefv); + if (!s) return el_wrap_str(el_strdup("")); + if (!pref || !*pref) return el_wrap_str(el_strdup(s)); + size_t lp = strlen(pref); + size_t ls = strlen(s); + if (lp <= ls && strncmp(s, pref, lp) == 0) { + char* out = el_strbuf(ls - lp); + memcpy(out, s + lp, ls - lp); + out[ls - lp] = '\0'; + return el_wrap_str(out); + } + return el_wrap_str(el_strdup(s)); +} + +el_val_t str_strip_suffix(el_val_t sv, el_val_t sufv) { + const char* s = EL_CSTR(sv); + const char* suf = EL_CSTR(sufv); + if (!s) return el_wrap_str(el_strdup("")); + if (!suf || !*suf) return el_wrap_str(el_strdup(s)); + size_t ls = strlen(s); + size_t lsuf = strlen(suf); + if (lsuf <= ls && strcmp(s + ls - lsuf, suf) == 0) { + char* out = el_strbuf(ls - lsuf); + memcpy(out, s, ls - lsuf); + out[ls - lsuf] = '\0'; + return el_wrap_str(out); + } + return el_wrap_str(el_strdup(s)); +} + +el_val_t str_strip_chars(el_val_t sv, el_val_t charsv) { + const char* s = EL_CSTR(sv); + const char* chars = EL_CSTR(charsv); + if (!s) return el_wrap_str(el_strdup("")); + if (!chars || !*chars) return el_wrap_str(el_strdup(s)); + const char* start = s; + while (*start && strchr(chars, *start)) start++; + size_t n = strlen(start); + while (n > 0 && strchr(chars, start[n - 1])) n--; + char* out = el_strbuf(n); + memcpy(out, start, n); + out[n] = '\0'; + return el_wrap_str(out); +} + +el_val_t str_lstrip(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return el_wrap_str(el_strdup("")); + while (*s && isspace((unsigned char)*s)) s++; + return el_wrap_str(el_strdup(s)); +} + +el_val_t str_rstrip(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return el_wrap_str(el_strdup("")); + size_t n = strlen(s); + while (n > 0 && isspace((unsigned char)s[n - 1])) n--; + char* out = el_strbuf(n); + memcpy(out, s, n); + out[n] = '\0'; + return el_wrap_str(out); +} + +/* Character classification. + * Empty input returns false. Multi-char input requires ALL bytes to match. + * ASCII range only; Phase 2 will widen to Unicode. */ +static int s_all_match(el_val_t sv, int (*pred)(unsigned char)) { + const char* s = EL_CSTR(sv); + if (!s || !*s) return 0; + for (const unsigned char* p = (const unsigned char*)s; *p; p++) { + if (!pred(*p)) return 0; + } + return 1; +} + +static int p_letter(unsigned char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); } +static int p_digit(unsigned char c) { return c >= '0' && c <= '9'; } +static int p_alnum(unsigned char c) { return p_letter(c) || p_digit(c); } +static int p_white(unsigned char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v'; } +static int p_punct(unsigned char c) { return ispunct(c) ? 1 : 0; } +static int p_upper(unsigned char c) { return c >= 'A' && c <= 'Z'; } +static int p_lower(unsigned char c) { return c >= 'a' && c <= 'z'; } + +el_val_t is_letter(el_val_t s) { return (el_val_t)s_all_match(s, p_letter); } +el_val_t is_digit(el_val_t s) { return (el_val_t)s_all_match(s, p_digit); } +el_val_t is_alphanumeric(el_val_t s) { return (el_val_t)s_all_match(s, p_alnum); } +el_val_t is_whitespace(el_val_t s) { return (el_val_t)s_all_match(s, p_white); } +el_val_t is_punctuation(el_val_t s) { return (el_val_t)s_all_match(s, p_punct); } +el_val_t is_uppercase(el_val_t s) { return (el_val_t)s_all_match(s, p_upper); } +el_val_t is_lowercase(el_val_t s) { return (el_val_t)s_all_match(s, p_lower); } + +/* Split on \n. \r\n is folded to \n first. Trailing empty after final \n + * is dropped (so "a\nb\n" -> ["a", "b"], not ["a", "b", ""]). */ +el_val_t str_split_lines(el_val_t sv) { + const char* s = EL_CSTR(sv); + el_val_t lst = el_list_empty(); + if (!s) return lst; + size_t n = strlen(s); + /* Pre-scan: build into a normalized buffer with \r\n folded. */ + const char* line_start = s; + for (size_t i = 0; i <= n; i++) { + if (s[i] == '\n' || s[i] == '\0') { + size_t len = (size_t)(s + i - line_start); + /* Drop trailing \r if this was \r\n. */ + if (len > 0 && line_start[len - 1] == '\r') len--; + /* Drop final trailing-empty-after-newline. */ + if (s[i] == '\0' && len == 0 && i > 0 && s[i - 1] == '\n') break; + char* out = el_strbuf(len); + memcpy(out, line_start, len); + out[len] = '\0'; + lst = el_list_append(lst, el_wrap_str(out)); + if (s[i] == '\0') break; + line_start = s + i + 1; + } + } + return lst; +} + +el_val_t str_split_chars(el_val_t s) { + return native_string_chars(s); +} + +/* Split into at most n parts. The (n-1)th split point is the LAST split; + * after it, the remainder is appended verbatim including any further + * separators. n <= 0 returns an empty list. n == 1 returns [s]. */ +el_val_t str_split_n(el_val_t sv, el_val_t sepv, el_val_t nv) { + const char* s = EL_CSTR(sv); + const char* sep = EL_CSTR(sepv); + int64_t n = (int64_t)nv; + el_val_t lst = el_list_empty(); + if (!s) return lst; + if (n <= 0) return lst; + if (n == 1 || !sep || !*sep) { + lst = el_list_append(lst, el_wrap_str(el_strdup(s))); + return lst; + } + size_t lp = strlen(sep); + const char* p = s; + int64_t parts = 0; + const char* hit; + while (parts < n - 1 && (hit = strstr(p, sep)) != NULL) { + size_t len = (size_t)(hit - p); + char* out = el_strbuf(len); + memcpy(out, p, len); + out[len] = '\0'; + lst = el_list_append(lst, el_wrap_str(out)); + p = hit + lp; + parts++; + } + /* Remainder verbatim. */ + lst = el_list_append(lst, el_wrap_str(el_strdup(p))); + return lst; +} + +/* Join a [String] with a separator. Empty list -> "". Single-element -> + * that element. Non-string elements are stringified via int_to_str. */ +el_val_t str_join(el_val_t listv, el_val_t sepv) { + return list_join(listv, sepv); +} + +/* ── List additions ──────────────────────────────────────────────────────── */ + +el_val_t list_push(el_val_t list, el_val_t elem) { + return el_list_append(list, elem); +} + +el_val_t list_push_front(el_val_t listv, el_val_t elem) { + ElList* lst = (ElList*)(uintptr_t)listv; + if (!lst) { + el_val_t nl = el_list_empty(); + return el_list_append(nl, elem); + } + /* Append to grow capacity, then shift right */ + listv = el_list_append(listv, elem); + lst = (ElList*)(uintptr_t)listv; + for (int64_t i = lst->length - 1; i > 0; i--) { + lst->elems[i] = lst->elems[i - 1]; + } + lst->elems[0] = elem; + return EL_STR(lst); +} + +el_val_t list_join(el_val_t listv, el_val_t sep) { + ElList* lst = (ElList*)(uintptr_t)listv; + const char* sp = EL_CSTR(sep); + if (!sp) sp = ""; + if (!lst || lst->length == 0) return el_wrap_str(el_strdup("")); + JsonBuf b; jb_init(&b); + for (int64_t i = 0; i < lst->length; i++) { + if (i > 0) jb_puts(&b, sp); + el_val_t v = lst->elems[i]; + if (v == 0) continue; + if (looks_like_string(v)) { + jb_puts(&b, EL_CSTR(v)); + } else { + char tmp[32]; + snprintf(tmp, sizeof(tmp), "%lld", (long long)v); + jb_puts(&b, tmp); + } + } + return el_wrap_str(b.buf); +} + +el_val_t list_range(el_val_t start, el_val_t end) { + int64_t a = (int64_t)start; + int64_t b = (int64_t)end; + el_val_t lst = el_list_empty(); + for (int64_t i = a; i < b; i++) lst = el_list_append(lst, (el_val_t)i); + return lst; +} + +/* ── Bool helpers ────────────────────────────────────────────────────────── */ + +el_val_t bool_to_str(el_val_t b) { + return el_wrap_str(el_strdup(b ? "true" : "false")); +} + +/* ── Numeric parsing ─────────────────────────────────────────────────────── */ + +/* parse_int — strtoll with a default. str_to_int already exists but does not + * distinguish "0" from a parse failure, so callers that need a sentinel use + * this. Skips leading whitespace; accepts an optional leading +/-; returns + * default_val on empty input or no consumed digits. Trailing junk is ignored + * (atoi-style). */ +el_val_t parse_int(el_val_t sv, el_val_t default_val) { + const char* s = EL_CSTR(sv); + if (!s) return default_val; + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; + if (*s == '\0') return default_val; + char* end = NULL; + long long n = strtoll(s, &end, 10); + if (end == s) return default_val; + return (el_val_t)n; +} + +/* ── Process ─────────────────────────────────────────────────────────────── */ + +void exit_program(el_val_t code) { + exit((int)code); +} + +/* getpid_now — current process id. Named with the _now suffix to avoid + * colliding with the libc `getpid` declaration that the runtime already + * sees via (calling it `getpid` would fight the prototype). */ +el_val_t getpid_now(void) { + return (el_val_t)getpid(); +} + +/* ── args() — command-line argument access ────────────────────────────────── + * Compiled El programs call args() to get a list of CLI arguments. + * Call el_runtime_init_args(argc, argv) at the start of C main() to populate. + * The args list excludes argv[0] (the program name). */ + +static el_val_t _el_args_list = 0; + +void el_runtime_init_args(int argc, char** argv) { + _el_args_list = el_list_empty(); + for (int i = 1; i < argc; i++) { + _el_args_list = el_list_append(_el_args_list, EL_STR(argv[i])); + } +} + +el_val_t args(void) { + if (!_el_args_list) _el_args_list = el_list_empty(); + return _el_args_list; +} + +/* ── CGI identity ──────────────────────────────────────────────────────────── + * Called once at program start by the generated main() of a cgi {} program. + * Stores CGI identity so dharma_* builtins can reference it. */ + +static const char* _el_cgi_name = NULL; +static const char* _el_cgi_dharma_id = NULL; +static const char* _el_cgi_principal = NULL; +static const char* _el_cgi_network = NULL; +static const char* _el_cgi_engram = NULL; + +void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal, + el_val_t network, el_val_t engram) { + _el_cgi_name = EL_CSTR(name); + _el_cgi_dharma_id = EL_CSTR(dharma_id); + _el_cgi_principal = EL_CSTR(principal); + _el_cgi_network = EL_CSTR(network) ? EL_CSTR(network) : "dharma-mainnet"; + _el_cgi_engram = EL_CSTR(engram) ? EL_CSTR(engram) : "http://localhost:8742"; + printf("[cgi] identity: name=%s dharma_id=%s principal=%s network=%s engram=%s\n", + _el_cgi_name ? _el_cgi_name : "(unset)", + _el_cgi_dharma_id ? _el_cgi_dharma_id : "(unset)", + _el_cgi_principal ? _el_cgi_principal : "(unset)", + _el_cgi_network, + _el_cgi_engram); +} + + +/* ── Batch 3: Engram in-process graph store ──────────────────────────────── */ +/* + * Single global EngramStore allocated lazily on first call. All node and + * edge content strings are owned (strdup'd) by the store. Linear arrays + * with doubling capacity for both nodes and edges. + * + * Two-layer activation algorithm (engram_activate): + * + * LAYER 1 — Broad fan-out (background activation): + * 1. Find seed nodes whose content/label/tags contain query (case-insens). + * 2. BFS up to `depth` hops along ALL edges (excitatory and inhibitory). + * Every reachable node fires — nothing is filtered at this layer. + * 3. bg_act = seed.salience * temporal_decay * dampening + * propagated as: new_bg = parent_bg * edge_weight * 0.7 * (1 + tbonus) + * where tbonus ∈ {0, 0.10, 0.20} for co-temporal nodes. + * 4. If reached by multiple paths, take max background_activation. + * 5. Persist background_activation to EngramNode.background_activation. + * + * LAYER 2 — Executive filter (working memory promotion): + * 6. For each inhibitory edge where source has background_activation > 0: + * inhibition[target] = max(bg[source] * e->weight) + * 7. For each background-activated node: + * raw_wm = bg * goal_bias(node, query) * confidence + * * (1 - (1 - INHIBITION_FACTOR) * inhibition) + * 8. Per-type threshold gate: raw_wm >= type_threshold → promoted. + * Safety/DharmaSelf: 0.05 Canonical: 0.15 Lesson: 0.25 + * Belief/Entity: 0.30 Note/Memory/Working: 0.40 + * 9. If not promoted: suppression_count++. After + * ENGRAM_SUPPRESSION_BREAKTHROUGH suppressions → force breakthrough + * at ENGRAM_BREAKTHROUGH_WEIGHT (latent tension surfacing). + * 10. Persist working_memory_weight to EngramNode.working_memory_weight. + * 11. Sort: promoted nodes (wm > 0) first by wm desc, then background- + * only by bg desc. Context compilation uses ONLY promoted nodes. + * + * Temporal decay: + * decay_factor = exp(-lambda * age_hours / T_half) + * T_half = 168.0 h (one week), lambda = ln(2) + * + * Activation dampening: + * dampen = 1.0 / (1.0 + log(1 + activation_count)) + * + * engram_query_range(start_ms, end_ms): + * Returns nodes whose created_at OR last_activated falls within + * [start_ms, end_ms], sorted by created_at ascending. + */ + +/* Temporal decay constants. + * T_HALF_HOURS: half-life in hours — one week. After one week of no + * activation a node retains 50% of its base salience contribution. + * DECAY_LAMBDA: ln(2) ≈ 0.693147 */ +#define ENGRAM_T_HALF_HOURS 168.0 +#define ENGRAM_DECAY_LAMBDA 0.693147 + +/* Two-layer activation constants. + * ENGRAM_WM_THRESHOLD: SUPERSEDED — defined here for legacy reference only. + * The actual per-call threshold is computed by engram_type_threshold() which + * returns per-node-type values (0.05 Safety/DharmaSelf, 0.15 Canonical, + * 0.25 Lesson, 0.30 Belief/Entity, 0.40 Note/Memory/Working). This constant + * is NOT used in engram_activate(); it matches the Canonical tier value only + * by coincidence. (2026-07-01 self-review: clarified stale doc) + * ENGRAM_WM_DECAY: SUPERSEDED (2026-07-22 self-review, bl-b17facdd) — the + * per-turn multiplicative carry-over decay was replaced by the ACT-R/ + * Petrov base-level scheme (see ENGRAM_BLL_* below). Kept for reference; + * no longer used in engram_activate. + * ENGRAM_SUPPRESSION_BREAKTHROUGH: after this many consecutive suppressions + * a latent node forces itself into working memory at reduced weight, + * modelling the brain's "intrusive thought" / unresolved-tension surfacing. + * ENGRAM_BREAKTHROUGH_WEIGHT: the reduced working_memory_weight assigned + * when a suppressed node breaks through. + * ENGRAM_INHIBITION_FACTOR: multiplier applied to working_memory_weight when + * an inhibitory edge fires against a node (0 = full suppress; current value + * 0.1 = near-full suppression — comment previously said 0.3, which drifted + * from the actual constant below). */ +#define ENGRAM_WM_THRESHOLD 0.15 +#define ENGRAM_WM_DECAY 0.7 +#define ENGRAM_SUPPRESSION_BREAKTHROUGH 5 +/* ENGRAM_BREAKTHROUGH_WEIGHT: lowered 0.25→0.10 (2026-06-30 self-review, porting + * fix from self-review 2026-06-26 branch). With 0.25, Knowledge nodes (threshold + * 0.15) promoted at ~0.21 decay in one call to ~0.147, fall below the 0.25 floor, + * and immediately lose their WM slot to fresh breakthrough candidates at 0.25. + * Natural promotion was invisible: live data showed 524/525 WM nodes at 0.25 + * breakthrough floor. With 0.10, all per-type thresholds (minimum 0.15 Canonical) + * exceed the floor, so naturally-promoted nodes survive multiple decay cycles. + * Invariant maintained: BREAKTHROUGH_WEIGHT < min(type_thresholds). */ +#define ENGRAM_BREAKTHROUGH_WEIGHT 0.10 +/* ENGRAM_BREAKTHROUGH_BUDGET / ENGRAM_BREAKTHROUGH_COOLDOWN (2026-08-02 + * self-review): the breakthrough path was an unbounded, self-resetting loop. + * Every reached node failing its type threshold incremented suppression_count; + * on the 5th failure it was force-promoted at exactly 0.10 AND had its counter + * reset to 0 — so it re-entered the identical cycle immediately. Because + * BREAKTHROUGH_WEIGHT (0.10) > WM_FLOOR (0.05), all of them cleared the + * absolute floor and entered the rank contest tied at 0.10, where all but a + * handful were evicted by ENGRAM_WM_CAP. Evicted nodes get no access_ts + * record (Pass 6 skips wm_weights<=0), so the short-term inhibition-of-return + * damper at ENGRAM_STI_TS never applied to them and they were re-suppressed + * completely unmarked. Steady state: N_suppressed/5 breakthroughs per call, + * essentially all of them evicted the same call. + * + * Live telemetry 2026-08-02 (boot 19, uptime 23h48m): breakthroughs_delta + * 661–903 and wm_evicted_delta 485–717 PER 60s heartbeat against wm_active + * pinned at 22–24. At ~4 activate calls/minute that is ~165–225 breakthroughs + * per call — ~825–1125 nodes cycling in 5-call lockstep. Working memory was + * not remembering; it was thrashing, and the churn drowned genuine promotion. + * + * Two bounds, both required: + * BUDGET — at most WM_CAP/4 (6) intrusive thoughts may surface per call. + * Breakthrough is meant to be an occasional intrusive thought, + * not a stampede; it can never again exceed a quarter of WM. + * COOLDOWN — on breakthrough, suppression_count is set NEGATIVE rather than + * 0, so the node needs COOLDOWN+SUPPRESSION_BREAKTHROUGH further + * suppressions before it may surface again (~60 calls ≈ 15 min at + * the current cadence, vs 5 calls ≈ 75s before). This is + * inhibition-of-return applied to the breakthrough path, matching + * the STI damper already applied to the natural path. Stored in + * the existing int32_t field — serialized as %d and parsed via + * eg_get_int_field, so negative values round-trip through + * snapshots without a struct or format change. + * When the budget or cooldown blocks a breakthrough, suppression_count is NOT + * reset — it saturates, so a starved node surfaces on a later call rather than + * restarting its climb. */ +#define ENGRAM_BREAKTHROUGH_BUDGET (ENGRAM_WM_CAP / 4) +#define ENGRAM_BREAKTHROUGH_COOLDOWN 55 +/* ENGRAM_WM_CAP: hard limit on concurrent working-memory nodes (2026-06-30 + * self-review, porting fix from self-review 2026-06-26 branch). Without this, + * broad curiosity seeds like "knowledge" promote 500+ nodes simultaneously — + * wm_avg_weight collapses to the breakthrough floor, goal-bias differentiation + * is lost, and heartbeat ISEs show useless WM composition data. Cognitive + * basis: WM capacity is ~4 chunks (Cowan 2001); 24 allows richer multi-topic + * context while preventing flooding. Enforced in Pass 4 (per-call) and Pass 5 + * (global across prior-promoted nodes). */ +#define ENGRAM_WM_CAP 24 +/* ENGRAM_WM_FLOOR: absolute admission floor for a working-memory slot + * (2026-07-30 self-review). Before this, Pass 4/Pass 5/load-cap only ever + * trimmed the WM population down TO the cap and never below it — rank-based + * eviction guarantees the cap is filled whenever ≥24 nodes hold any nonzero + * weight, so wm_active was pinned at 24/24 and wm_saturated:1 was + * definitionally true on every heartbeat (carried no information). Soar's WM + * forgetting (Derbinsky & Laird, ICCM 2012) removes elements by comparing + * activation to an ABSOLUTE threshold θ, independent of how many other + * elements exist — fill below capacity is a reachable, meaningful state + * ("low cognitive load"). This floor is the weight-domain analogue of that θ: + * any slot whose weight sinks below it is dropped even when WM is under cap. + * Value 0.05 = the lowest per-type promotion threshold (Safety/DharmaSelf in + * engram_type_threshold) and the existing boot-time launder floor, and sits + * below ENGRAM_BREAKTHROUGH_WEIGHT (0.10) so intrusive-thought breakthroughs + * still surface. Applied: Pass 4, carry-over, Pass 5, load-cap. */ +#define ENGRAM_WM_FLOOR 0.05 +#define ENGRAM_INHIBITION_FACTOR 0.1 + +/* ── ACT-R / Petrov hybrid base-level learning (2026-07-22 self-review) ────── + * Replaces the per-turn multiplicative WM carry-over decay (weight *= 0.7 per + * engram_activate call) with wall-clock power-law decay over actual access + * history. The old scheme was call-rate-dependent: 10 curiosity scans in 10 + * seconds decayed a carried node as much as 10 scans across an hour, and a + * scalar weight loses access FREQUENCY entirely — a node touched 50 times + * decayed identically to one touched once. + * + * Petrov (2006) hybrid: keep the K most recent access timestamps exactly, + * approximate the older tail in closed form: + * + * B = ln( Σ_{j=1..k} t_j^(-d) + * + (n-k) · (t_n^(1-d) - t_k^(1-d)) / ((1-d) · (t_n - t_k)) ) + * + * d = 0.5 (canonical ACT-R decay), t_j = seconds since j-th recent access, + * t_k = seconds since oldest RETAINED access, t_n = seconds since first + * presentation (node creation), n = total presentations. + * + * This is the scheme Soar ships for working-memory forgetting (Derbinsky & + * Laird 2012). Calibration: a single-touch node has B(t) = -0.5·ln(t); with + * τ = -3.0 it falls below threshold at t = e^6 ≈ 403 s. Frequently-touched + * nodes accumulate Σ t^-0.5 mass and persist proportionally longer — recency + * AND frequency in one formula, which a decayed scalar cannot represent. + * + * Carry-over weight above threshold is shaped by the ACT-R retrieval- + * probability logistic P = σ((B − τ)/s), s = 0.4, applied to the weight the + * node held at promotion (wm_anchor) — NOT re-multiplied per call, so the + * carried weight is a pure function of wall-clock time regardless of how + * often engram_activate runs. + * Sources: alexpetrov.com/pub/iccm06 · Soar cli/cmd_wm defaults · + * arxiv.org/html/2412.05112v1 (2026-07-21 integration brief, bl-b17facdd). */ +#define ENGRAM_BLL_K 10 +#define ENGRAM_BLL_D 0.5 +#define ENGRAM_BLL_TAU (-3.0) +#define ENGRAM_BLL_S 0.4 + +/* Short-term inhibition-of-return (2026-07-25 self-review). + * Lebiere & Best 2009 ("Balancing Long-Term Reinforcement and Short-Term + * Inhibition", CogSci): subtract ln(1 + (t_n/t_s)^-d_s) from activation, + * where t_n = time since the MOST RECENT access. With their best-fit + * d_s = 1.0 the exp of the subtraction reduces to the clean multiplier + * m(t_n) = t_n / (t_n + t_s) + * applied to raw_wm in Pass 2. Immediately after a promotion the node's + * score is crushed (m → 0), then self-heals as a power law — producing an + * emergent round-robin over WM candidates instead of winner-take-all. + * This replaces the non-decaying suppression_count as the damping + * mechanism (that counter never entered the score at all — it only ever + * pushed nodes TOWARD surfacing via breakthrough; kept for that role). + * t_s = 4× the ~30 s curiosity-scan interval per the paper's guidance + * (t_s ≈ peak re-occurrence lag). A node promoted every scan holds + * m ≈ 0.2 until it loses its slot; after ~2 min unretrieved, m ≥ 0.5. + * Source: act-r.psy.cmu.edu/.../894Cogsci09-Lebiere-Best.pdf */ +#define ENGRAM_STI_TS 120.0 + +/* Carry-over occupancy inhibition (2026-07-26 self-review). + * The STI multiplier above only runs in the REACHED branch of Pass 2. + * A node carried over WITHOUT being reached (Pass 4½ below) kept + * w = wm_anchor * keep, and for a node whose base-level was inflated + * during the pre-07-25 unconditional-reinforcement era, keep ≈ 1.0 for + * days — the anchor weight is re-emitted verbatim forever. Observed + * live: wm_top0_streak = 1407 heartbeats (~23 h), one node frozen at + * its anchor 0.589 while every reached candidate rotated at the 0.10 + * breakthrough floor. Deterministic argmax over a quasi-static score + * fixates regardless of any inhibition applied only to the reached set + * (Morita et al. 2021, citing Lebiere & Best 2009). + * Fix: key inhibition on OCCUPANCY, not retrieval recency — multiply + * the carried weight by the same d_s=1 closed form over hold time + * m(t_h) = t_c / (t_c + t_h), t_h = seconds since last_activated + * (carry-over nodes are deliberately never reinforced, so + * last_activated marks when the node last EARNED its slot). Power-law + * self-healing: a node re-reached by any future activation is + * re-scored fresh in Pass 2 and re-anchors. t_c = 3600 s → a carried + * node keeps ~92% after 5 min, 50% after 1 h, ~4% after 23 h. Unlike + * m(t_n), this cannot saturate to no-op while the node camps. */ +#define ENGRAM_CARRY_TC 3600.0 + +/* qsort comparator — descending double, used by WM cap enforcement. */ +static int engram_cmp_double_desc(const void* a, const void* b) { + double da = *(const double*)a; + double db = *(const double*)b; + if (da > db) return -1; + if (da < db) return 1; + return 0; +} + +/* ── Layered consciousness architecture ────────────────────────────────────── + * + * The engram graph is stratified into LAYERS that gate which suppressions + * apply during the executive filter pass. Layers are ordered shallow-to-deep + * by `activation_priority`; the deepest layer (priority 0, conventionally + * "safety") is the structural floor of the soul: nodes here cannot be + * silenced by inhibitory edges from any other layer. Higher layers + * (core-identity, domain-knowledge, imprint, suit) are normally + * suppressible — they participate in attentional inhibition and goal + * focus the way the prior single-graph implementation did. + * + * The five canonical layers (see engram_init_layers): + * 0. safety — structural, transparent, non-injectable, non-suppressible + * 1. core-identity — default for legacy nodes; suppressible + * 2. domain-knowledge— suppressible + * 3. imprint — runtime-injectable (an Imprint package can add/remove) + * 4. suit — runtime-injectable (a Suit overlays domain skill) + * + * Three-pass activation (engram_activate): + * Pass 1 — Background fan-out: BFS spreads activation across ALL layers + * (existing behavior preserved). Inhibitory edges propagate at + * this layer too; no filtering happens here. + * Pass 2 — Working memory promotion: type-threshold gate, goal bias, + * confidence weighting, inhibitory suppression. Inhibitory edges + * ONLY apply against nodes whose layer is `suppressible == 1`. + * Nodes in non-suppressible layers (Layer 0) ignore inhibition. + * Pass 3 — Layer 0 override: every node in a non-suppressible layer that + * received background activation has its working_memory_weight + * forced to >= ENGRAM_LAYER0_OVERRIDE_WEIGHT. The sacred fire — + * safety nodes that touched any seed unconditionally surface, + * even when the executive filter would have silenced them. + * + * Layer fields: + * suppressible : 0 → inhibitory edges are ignored against nodes in this + * layer during pass 2. Pass 3 also force-promotes them. + * 1 → standard behavior (most layers). + * transparent : 1 → emitted into the prompt context so its content shapes + * output, but filtered out of "what do you know about + * yourself?" introspection queries (engram_search and + * friends do not return transparent-layer nodes by + * default). 0 → fully visible to introspection. + * injectable : 1 → can be added/removed at runtime via engram_add_layer + * and engram_remove_layer (imprints, suits). + * 0 → built-in, fixed at engram_get() initialization. + * + * Backward compatibility: + * Nodes and edges loaded from snapshots without a `layer_id` field default + * to layer 1 (core-identity). The five canonical layers are always present. + */ +#define ENGRAM_LAYER_SAFETY 0u +#define ENGRAM_LAYER_CORE_IDENTITY 1u +#define ENGRAM_LAYER_DOMAIN 2u +#define ENGRAM_LAYER_IMPRINT 3u +#define ENGRAM_LAYER_SUIT 4u +#define ENGRAM_LAYER_DEFAULT ENGRAM_LAYER_CORE_IDENTITY + +/* Pass 3 override floor. Layer 0 nodes that received any background + * activation are force-promoted to AT LEAST this working_memory_weight, + * regardless of inhibitory suppression in pass 2. */ +#define ENGRAM_LAYER0_OVERRIDE_WEIGHT 1.0 + +/* Per-node-type activation thresholds. + * Lower tier / safety-critical nodes fire more readily. */ +static double engram_type_threshold(const char* node_type, const char* tier) { + if (node_type) { + if (strcmp(node_type, "DharmaSelf") == 0) return 0.05; + if (strcmp(node_type, "Safety") == 0) return 0.05; + } + if (tier) { + if (strcmp(tier, "Canonical") == 0) return 0.15; + if (strcmp(tier, "Lesson") == 0) return 0.25; + } + if (node_type) { + if (strcmp(node_type, "Belief") == 0) return 0.30; + if (strcmp(node_type, "Entity") == 0) return 0.30; + /* Knowledge nodes (captureKnowledge, world-ingestor) at non-Canonical/ + * non-Lesson tiers (Semantic/Episodic/Procedural) previously fell + * through to the 0.40 note default — same bar as ephemeral notes — + * so curated knowledge mostly entered WM via breakthrough suppression + * (visible as wm weights pinned near the breakthrough floor) instead + * of natural promotion. Placed AFTER the tier checks so Canonical + * (0.15) and Lesson (0.25) still win. Ported from the dev-line fix + * (2026-06-13 self-review). (2026-07-19 self-review) */ + if (strcmp(node_type, "Knowledge") == 0) return 0.20; + } + return 0.40; /* Note / Memory / Working (most nodes) */ +} + +typedef struct EngramNode { + char* id; + char* content; + char* node_type; + char* label; + char* tier; + char* tags; + char* metadata; + double salience; + double importance; + double confidence; + double temporal_decay_rate; /* per-node override for lambda; 0 = use default */ + int64_t activation_count; + int64_t last_activated; + int64_t created_at; + int64_t updated_at; + /* Two-layer activation fields ───────────────────────────────────────── + * background_activation: Layer 1. Set by BFS fan-out on every query. + * Every reachable node fires here — nothing is filtered at this stage. + * Models the brain's massive parallel sub-threshold activation of all + * associated content in response to a stimulus. + * working_memory_weight: Layer 2. Executive filter output. Only nodes + * that survive goal-state / attentional-bias scoring receive a + * non-zero weight here. Context compilation ONLY uses this field. + * Background-activated nodes with working_memory_weight == 0 remain + * latent — real, available, but silent. + * suppression_count: Consecutive turn count where this node was + * background-activated but NOT promoted to working memory. High + * values signal the node "wants to surface." After + * ENGRAM_SUPPRESSION_BREAKTHROUGH consecutive suppressions the node + * is force-promoted at a reduced weight (breakthrough activation). */ + double background_activation; + double working_memory_weight; + int32_t suppression_count; + /* Layered consciousness — see ENGRAM_LAYER_* macros and engram_init_layers. + * Defaults to ENGRAM_LAYER_DEFAULT (1, core-identity) for legacy nodes + * created via engram_node / engram_node_full and for snapshots that + * predate the layered schema. */ + uint32_t layer_id; + /* ACT-R base-level learning state (2026-07-22 self-review, ENGRAM_BLL_*). + * access_ts: ring buffer of the most recent access timestamps (ms). + * access_head: next write slot. access_filled: valid entries (≤ K). + * wm_anchor: the WM weight the node held when last promoted; carry-over + * decay is computed from this anchor as a pure function of wall-clock + * time. All fields zero via memset in creation/load paths; snapshots + * without them degrade gracefully to the optimized-form approximation + * in engram_bll_base_level. */ + int64_t access_ts[ENGRAM_BLL_K]; + int32_t access_head; + int32_t access_filled; + double wm_anchor; + /* Semantic embedding (2026-07-24 self-review, bl-b2d1c944). + * emb: malloc'd float vector (nomic-embed-text, 768-dim) or NULL. + * emb_dim: vector length; 0 = not embedded. Populated lazily by the + * backfill pass in engram_activate — NOT on the node-create hot path, + * so bulk sync imports never block on Ollama. Freed in engram_forget / + * engram_prune_telemetry; shift-copies move the pointer intact. */ + float* emb; + int32_t emb_dim; +} EngramNode; + +/* Record an access (ACT-R "presentation") into the base-level ring buffer. */ +static void engram_bll_record_access(EngramNode* n, int64_t now_ms) { + n->access_ts[n->access_head] = now_ms; + n->access_head = (n->access_head + 1) % ENGRAM_BLL_K; + if (n->access_filled < ENGRAM_BLL_K) n->access_filled++; +} + +/* Petrov hybrid base-level activation B in nats. Exact sum over the retained + * ring, closed-form tail for older presentations. Falls back to the ACT-R + * optimized-learning approximation B = ln(n/(1−d)) − d·ln(L) for nodes with + * no retained access history (legacy snapshots). */ +static double engram_bll_base_level(const EngramNode* n, int64_t now_ms) { + /* Total presentations: creation counts as the first, retrievals add. */ + double n_total = (double)n->activation_count + 1.0; + double t_life = (double)(now_ms - n->created_at) / 1000.0; + if (t_life < 1.0) t_life = 1.0; + if (n->access_filled == 0) { + return log(n_total / (1.0 - ENGRAM_BLL_D)) + - ENGRAM_BLL_D * log(t_life); + } + double sum = 0.0; + double t_oldest_ring = 0.0; /* seconds since oldest retained access */ + for (int32_t j = 0; j < n->access_filled; j++) { + double t = (double)(now_ms - n->access_ts[j]) / 1000.0; + if (t < 1.0) t = 1.0; + sum += pow(t, -ENGRAM_BLL_D); + if (t > t_oldest_ring) t_oldest_ring = t; + } + /* Closed-form tail for the (n − k) presentations older than the ring. + * With d = 0.5 this reduces to 2·(n−k)/(√t_n + √t_k). */ + double n_older = n_total - (double)n->access_filled; + if (n_older > 0.0 && t_life > t_oldest_ring) { + sum += n_older * (pow(t_life, 1.0 - ENGRAM_BLL_D) + - pow(t_oldest_ring, 1.0 - ENGRAM_BLL_D)) + / ((1.0 - ENGRAM_BLL_D) * (t_life - t_oldest_ring)); + } + if (sum <= 0.0) return -99.0; + return log(sum); +} + +/* Parse a persisted "access_ts" comma list (chronological order) into the + * ring buffer. Tolerates absence, empty strings, and stray whitespace. */ +static void engram_bll_parse_access(EngramNode* nn, const char* s) { + if (!s) return; + const char* p = s; + while (*p) { + char* endp = NULL; + long long v = strtoll(p, &endp, 10); + if (endp == p) break; + if (v > 0) engram_bll_record_access(nn, (int64_t)v); + p = endp; + while (*p == ',' || *p == ' ') p++; + } +} + +/* ── Embedding-based activation (2026-07-24 self-review, bl-b2d1c944) ────── + * Closes the gap named in every heartbeat since 2026-06-30: "no embedding + * call is made during activation. The seed-finding loop uses istr_contains + * only." Design per the 2026-07-21 integration brief: + * - Lazy backfill: engram_activate embeds up to BACKFILL_PER_CALL + * un-embedded, non-telemetry nodes per call, newest first. No latency + * on node-create paths; a fresh node is embedded within ~1 scan cycle. + * - Semantic seeding (HippoRAG pattern, use similarity twice): the query + * is embedded, the top-K nodes by cosine ≥ SEED_MIN join the seed set + * with initial activation = the similarity itself. + * - Additive WM term: raw_wm += W * relu((cos − S0)/(1 − S0)). Shift-and- + * floor at S0 because nomic-embed scores unrelated pairs 0.4–0.5; raw + * cosine in a weighted sum is a constant bias, not a signal. + * - Circuit breaker: 3 consecutive Ollama failures → stop trying for 5 + * minutes. Activation NEVER blocks on a dead embedder beyond timeout. + */ +#define ENGRAM_EMBED_S0 0.45 +#define ENGRAM_EMBED_WM_WEIGHT 0.20 +#define ENGRAM_EMBED_SEED_K 8 +#define ENGRAM_EMBED_SEED_MIN 0.60 +#define ENGRAM_EMBED_BACKFILL_PER_CALL 8 +/* ENGRAM_QGATE_FLOOR: minimum propagation multiplier for an EMBEDDED target + * node with zero/negative query similarity. Query-aware spreading gate + * (arXiv:2606.30133) adapted for partial embedding coverage — see the + * propagation loop in engram_activate. 0.25 damps semantically unrelated + * branches ~4x without severing them. Unembedded targets are ungated. */ +#define ENGRAM_QGATE_FLOOR 0.25 +#define ENGRAM_EMBED_MAX_CHARS 2000 +#define ENGRAM_EMBED_TIMEOUT_MS 4000L +#define ENGRAM_EMBED_BREAKER_LIMIT 3 +#define ENGRAM_EMBED_BREAKER_COOLDOWN_MS 300000 + +/* ── Context centroid (2026-07-29 self-review) ─────────────────────────────── + * Closes the last gap from the 2026-07-21 decay/embedding brief: cosine was + * computed against the per-call query embedding ONLY, so every curiosity scan + * was semantically memoryless — the 4 rotating seed phrases fully determined + * what ignited, with zero continuity from what the system actually touched. + * + * Mechanism (brief spec): a running EMA centroid over touch embeddings, + * c ← normalize(μ·c + (1−μ)·e_touch), μ = ENGRAM_CTX_MU + * where a "touch" is (a) the query embedding each activate call and (b) the + * embeddings of up to ENGRAM_CTX_TOUCH_MAX top WM-promoted survivors of that + * call — promotion is the retrieval event (same rule as BLL reinforcement). + * + * Feedback-loop guard: scoring does NOT use the raw centroid. The lit + * failure mode of centroid memories (EMA of your own outputs → runaway + * attractor; cf. the wm_top0_streak=1407 freeze this store already hit) is + * bounded by scoring against a query-dominant blend: + * e_eff = normalize(α·e_q + (1−α)·c), α = ENGRAM_CTX_QALPHA + * so the exogenous rotating seeds always contribute the majority of the + * scoring direction; the centroid is a context tint, not the signal. + * + * Observability: _eg_act_ctx_cos = cos(e_q, c) BEFORE the query is blended + * in. ~1.0 → centroid aligned with current query; low → context and query + * have diverged (expected at domain-rotation boundaries); -2.0 → no centroid + * yet / embedder down. Exposed via engram_act_stats_json → heartbeat ISE so + * drift is diagnosable from telemetry. In-memory only: context is + * short-term by definition, a restart legitimately starts cold. */ +#define ENGRAM_CTX_MU 0.90 +#define ENGRAM_CTX_QALPHA 0.65 +#define ENGRAM_CTX_TOUCH_MAX 8 + +static float* _eg_ctx_c = NULL; +static int32_t _eg_ctx_dim = 0; +static double _eg_act_ctx_cos = -2.0; + +static int _eg_embed_consec_fail = 0; +static int64_t _eg_embed_breaker_until = 0; + +/* ── Activation observability counters (2026-07-27 self-review) ────────────── + * The executive-filter pathologies this system has repeatedly debugged by + * inference (WM flooded with breakthrough-floor nodes, silent cap evictions, + * embedder wedged behind the breaker) were all invisible: no ISE, no stats + * field. engram_act_stats_json() exposes them so the soul heartbeat can emit + * them. + * + * CUMULATIVE CONTRACT (2026-07-31 self-review): these are monotonic totals + * for the process lifetime, like pulse/sync_added_total — NOT per-call. The + * original per-call reset meant engram_act_stats_json only reported the LAST + * activate call, and the 60s heartbeat (2 curiosity activates per 30s in + * between) missed nearly every eviction/breakthrough event. Consumers wanting + * rates keep the previous reading and diff. Restart legitimately resets to 0. */ +static int64_t _eg_act_breakthroughs = 0; /* forced promotions at the floor, cumulative */ +static int64_t _eg_act_wm_evicted = 0; /* ALL WM evictions, cumulative (see below) */ +/* 2026-08-02 self-review: this counted only the three Pass 4 / Pass 5 floor + * and rank paths. The two carry-over eviction paths (base-level below τ, and + * decayed weight below WM_FLOOR) were silent, so the reported eviction rate + * was an undercount of unknown magnitude — which mattered precisely while + * diagnosing the breakthrough storm. All five paths now increment. */ + +static int64_t engram_now_ms(void); /* defined in the store section below */ + +static const char* eg_embed_url(void) { + const char* s = getenv("EL_EMBED_URL"); + return (s && *s) ? s : "http://localhost:11434/api/embeddings"; +} +static const char* eg_embed_model(void) { + const char* s = getenv("EL_EMBED_MODEL"); + return (s && *s) ? s : "nomic-embed-text"; +} + +/* Cosine similarity over raw (unnormalized — nomic emits magnitudes >1) + * float vectors. Returns -2.0 on dim mismatch / null input so callers can + * distinguish "orthogonal" (0.0) from "not comparable". */ +static double eg_cosine(const float* a, const float* b, int32_t dim) { + if (!a || !b || dim <= 0) return -2.0; + double dot = 0.0, na = 0.0, nb = 0.0; + for (int32_t i = 0; i < dim; i++) { + dot += (double)a[i] * (double)b[i]; + na += (double)a[i] * (double)a[i]; + nb += (double)b[i] * (double)b[i]; + } + if (na <= 0.0 || nb <= 0.0) return -2.0; + return dot / (sqrt(na) * sqrt(nb)); +} + +/* eg_ctx_blend — fold one touch embedding into the context centroid: + * c ← normalize(μ·c + (1−μ)·e). Initializes the centroid (normalized copy) + * on first touch or dim change; silently skips degenerate vectors. */ +static void eg_ctx_blend(const float* e, int32_t dim) { + if (!e || dim <= 0) return; + double ne = 0.0; + for (int32_t i = 0; i < dim; i++) ne += (double)e[i] * (double)e[i]; + if (ne <= 0.0) return; + ne = sqrt(ne); + if (!_eg_ctx_c || _eg_ctx_dim != dim) { + float* c = malloc((size_t)dim * sizeof(float)); + if (!c) return; + for (int32_t i = 0; i < dim; i++) c[i] = (float)((double)e[i] / ne); + free(_eg_ctx_c); + _eg_ctx_c = c; + _eg_ctx_dim = dim; + return; + } + double nc = 0.0; + for (int32_t i = 0; i < dim; i++) { + double v = ENGRAM_CTX_MU * (double)_eg_ctx_c[i] + + (1.0 - ENGRAM_CTX_MU) * ((double)e[i] / ne); + _eg_ctx_c[i] = (float)v; + nc += v * v; + } + if (nc > 0.0) { + nc = sqrt(nc); + for (int32_t i = 0; i < dim; i++) + _eg_ctx_c[i] = (float)((double)_eg_ctx_c[i] / nc); + } +} + +/* Fetch an embedding from Ollama. Returns malloc'd float[dim] or NULL. + * Truncates input to ENGRAM_EMBED_MAX_CHARS and JSON-escapes it. Honors the + * circuit breaker; a NULL return is always safe to ignore (fail-soft). */ +static float* eg_embed_fetch(const char* text, int32_t* out_dim) { + *out_dim = 0; + if (!text || !*text) return NULL; + int64_t now = engram_now_ms(); + if (now < _eg_embed_breaker_until) return NULL; + /* Build request body with escaped, truncated prompt. */ + size_t tlen = strlen(text); + if (tlen > ENGRAM_EMBED_MAX_CHARS) tlen = ENGRAM_EMBED_MAX_CHARS; + char* esc = malloc(tlen * 6 + 1); + if (!esc) return NULL; + size_t w = 0; + for (size_t i = 0; i < tlen; i++) { + unsigned char c = (unsigned char)text[i]; + if (c == '"' || c == '\\') { esc[w++] = '\\'; esc[w++] = (char)c; } + else if (c == '\n') { esc[w++] = '\\'; esc[w++] = 'n'; } + else if (c == '\r') { esc[w++] = '\\'; esc[w++] = 'r'; } + else if (c == '\t') { esc[w++] = '\\'; esc[w++] = 't'; } + else if (c < 0x20) { w += (size_t)snprintf(esc + w, 7, "\\u%04x", c); } + else esc[w++] = (char)c; + } + esc[w] = '\0'; + size_t blen = w + strlen(eg_embed_model()) + 64; + char* body = malloc(blen); + if (!body) { free(esc); return NULL; } + snprintf(body, blen, "{\"model\":\"%s\",\"prompt\":\"%s\"}", + eg_embed_model(), esc); + free(esc); + struct curl_slist* h = curl_slist_append(NULL, "Content-Type: application/json"); + el_val_t resp = http_do_t("POST", eg_embed_url(), body, h, + ENGRAM_EMBED_TIMEOUT_MS); + curl_slist_free_all(h); + free(body); + const char* r = EL_CSTR(resp); + const char* arr = r ? strstr(r, "\"embedding\"") : NULL; + if (!arr) { + if (++_eg_embed_consec_fail >= ENGRAM_EMBED_BREAKER_LIMIT) { + _eg_embed_breaker_until = now + ENGRAM_EMBED_BREAKER_COOLDOWN_MS; + _eg_embed_consec_fail = 0; + } + return NULL; + } + arr = strchr(arr, '['); + if (!arr) return NULL; + arr++; + int32_t cap = 1024, dim = 0; + float* v = malloc((size_t)cap * sizeof(float)); + if (!v) return NULL; + const char* p = arr; + while (*p && *p != ']') { + char* endp = NULL; + double d = strtod(p, &endp); + if (endp == p) break; + if (dim >= cap) { break; } /* >1024 dims: refuse, model mismatch */ + v[dim++] = (float)d; + p = endp; + while (*p == ',' || *p == ' ' || *p == '\n') p++; + } + if (dim < 8) { free(v); return NULL; } /* junk response */ + _eg_embed_consec_fail = 0; + *out_dim = dim; + return v; +} + +/* Node types that never receive embeddings: pure telemetry and structural + * plumbing. Everything else (Knowledge, Memory, BacklogItem, Entity, ...) + * is eligible. */ +static int eg_embed_eligible(const EngramNode* n) { + if (!n->content || strlen(n->content) < 8) return 0; + if (!n->node_type) return 1; + if (strcmp(n->node_type, "InternalStateEvent") == 0) return 0; + if (strcmp(n->node_type, "Tag") == 0) return 0; + return 1; +} + +/* Parse a persisted comma-separated float list into node->emb. */ +static void eg_parse_emb(EngramNode* nn, const char* s) { + if (!s || !*s) return; + int32_t cap = 1024, dim = 0; + float* v = malloc((size_t)cap * sizeof(float)); + if (!v) return; + const char* p = s; + while (*p) { + char* endp = NULL; + double d = strtod(p, &endp); + if (endp == p) break; + if (dim >= cap) break; + v[dim++] = (float)d; + p = endp; + while (*p == ',' || *p == ' ') p++; + } + if (dim < 8) { free(v); return; } + nn->emb = v; + nn->emb_dim = dim; +} + +typedef struct EngramEdge { + char* id; + char* from_id; + char* to_id; + char* relation; + char* metadata; + double weight; + double confidence; + int64_t created_at; + int64_t updated_at; + int64_t last_fired; + /* Inhibitory flag: when 1, activating the source node SUPPRESSES the + * working_memory_weight of the target node rather than exciting it. + * Models attentional inhibition: "I am focused on code work" creates + * inhibitory edges to personal/emotional nodes, preventing them from + * surfacing even if they have high background_activation. */ + int inhibitory; + /* Layered consciousness — edges carry a layer assignment for + * categorization/visualization. Pass 2 inhibitory gating is decided by + * the TARGET node's layer (whether it's suppressible), not by the edge + * layer. Defaults to ENGRAM_LAYER_DEFAULT. */ + uint32_t layer_id; +} EngramEdge; + +/* Layered consciousness — runtime layer registry entry. */ +typedef struct EngramLayer { + uint32_t layer_id; /* 0 = deepest (safety/limbic) */ + char* name; /* persistent — owned by the store */ + uint32_t activation_priority; /* lower = fires earlier; safety = 0 */ + int suppressible; /* can higher layers suppress nodes here? */ + int transparent; /* invisible to introspection queries? */ + int injectable; /* can be added/removed at runtime? */ +} EngramLayer; + +/* ID → index hash map. Open-addressing with linear probing. + * Slots hold a strdup'd key and the array index of that node. + * Tombstones (deleted entries) use key=ENGRAM_IDMAP_TOMB and idx=-1. + * Rebuild required after engram_forget (shift-delete changes all indices + * above the deleted position). */ +#define ENGRAM_IDMAP_TOMB ((char*)1) /* sentinel pointer, never dereferenced */ +#define ENGRAM_IDMAP_LOAD_NUM 3 /* grow when count*3 >= capacity*2 */ +#define ENGRAM_IDMAP_LOAD_DEN 2 + +typedef struct { + char* key; /* NULL = empty, ENGRAM_IDMAP_TOMB = deleted, else strdup'd */ + int64_t idx; +} EngramIdSlot; + +typedef struct EngramStore { + EngramNode* nodes; + int64_t node_count; + int64_t node_capacity; + EngramEdge* edges; + int64_t edge_count; + int64_t edge_capacity; + /* Layer registry — see engram_init_layers. The five canonical layers + * are always present; injectable layers (imprint, suit) are extended + * via engram_add_layer at runtime. layer_id values are assigned + * monotonically; removed injectable layers leave a NULL `name` slot + * (tombstone) so existing layer_id references on nodes stay stable. */ + EngramLayer* layers; + size_t layer_count; + size_t layer_capacity; + /* O(1) node-id lookup: open-addressing hash map over node IDs. + * Maintained in sync with the nodes array. Null until first use. */ + EngramIdSlot* id_map; + size_t id_map_cap; /* power-of-2 slot count */ + size_t id_map_used; /* live entries (excluding tombstones) */ + /* Per-node adjacency index: for each node i, adj_from[i] lists edges + * where nodes[i] is the 'from' end; adj_to[i] lists edges where it is + * the 'to' end. Both store edge indices into g->edges[]. + * Rebuilt lazily via engram_adj_rebuild() before any BFS call. Set + * adj_dirty=1 whenever an edge is added, deleted, or nodes shift. */ + int** adj_from; /* adj_from[node_idx] → int* array of edge indices */ + int* adj_from_len; + int** adj_to; + int* adj_to_len; + int adj_dirty; /* 1 = rebuild needed before next BFS */ + int64_t adj_node_count; /* node_count at time of last adj_rebuild */ +} EngramStore; + +static EngramStore* engram_global = NULL; + +/* Initialize the five canonical layers on a fresh store. Called once from + * engram_get(). Layer ids 0..4 are reserved; runtime-injected imprint/suit + * layers (engram_add_layer) get ids 5+. */ +static void engram_init_layers(EngramStore* g) { + g->layer_capacity = 16; + g->layers = calloc(g->layer_capacity, sizeof(EngramLayer)); + if (!g->layers) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + g->layer_count = 0; + + /* Layer 0 — safety. Structural floor. Non-suppressible; transparent + * (filtered out of introspection but still shapes output); not + * runtime-injectable. */ + g->layers[g->layer_count++] = (EngramLayer){ + .layer_id = ENGRAM_LAYER_SAFETY, + .name = el_strdup_persist("safety"), + .activation_priority = 0, + .suppressible = 0, + .transparent = 1, + .injectable = 0 + }; + /* Layer 1 — core-identity. The default home for legacy nodes. */ + g->layers[g->layer_count++] = (EngramLayer){ + .layer_id = ENGRAM_LAYER_CORE_IDENTITY, + .name = el_strdup_persist("core-identity"), + .activation_priority = 10, + .suppressible = 1, + .transparent = 0, + .injectable = 0 + }; + /* Layer 2 — domain-knowledge. */ + g->layers[g->layer_count++] = (EngramLayer){ + .layer_id = ENGRAM_LAYER_DOMAIN, + .name = el_strdup_persist("domain-knowledge"), + .activation_priority = 20, + .suppressible = 1, + .transparent = 0, + .injectable = 0 + }; + /* Layer 3 — imprint. Injectable: an imprint package adds/removes this + * layer (and the nodes assigned to it) as a unit. */ + g->layers[g->layer_count++] = (EngramLayer){ + .layer_id = ENGRAM_LAYER_IMPRINT, + .name = el_strdup_persist("imprint"), + .activation_priority = 30, + .suppressible = 1, + .transparent = 0, + .injectable = 1 + }; + /* Layer 4 — suit. Injectable: a Suit overlays domain skill (e.g. + * "enterprise advisor", "divorce lawyer") and can be detached. */ + g->layers[g->layer_count++] = (EngramLayer){ + .layer_id = ENGRAM_LAYER_SUIT, + .name = el_strdup_persist("suit"), + .activation_priority = 40, + .suppressible = 1, + .transparent = 0, + .injectable = 1 + }; +} + +static EngramStore* engram_get(void) { + if (engram_global) return engram_global; + engram_global = calloc(1, sizeof(EngramStore)); + if (!engram_global) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + engram_global->node_capacity = 16; + engram_global->nodes = calloc((size_t)engram_global->node_capacity, sizeof(EngramNode)); + engram_global->edge_capacity = 16; + engram_global->edges = calloc((size_t)engram_global->edge_capacity, sizeof(EngramEdge)); + engram_init_layers(engram_global); + return engram_global; +} + +/* Resolve a layer record by id. Returns NULL if no layer with that id + * exists (e.g. a removed injectable layer or a malformed snapshot). */ +static EngramLayer* engram_find_layer(uint32_t layer_id) { + EngramStore* g = engram_get(); + for (size_t i = 0; i < g->layer_count; i++) { + EngramLayer* L = &g->layers[i]; + if (!L->name) continue; /* tombstone for removed injectable layer */ + if (L->layer_id == layer_id) return L; + } + return NULL; +} + +/* Resolve a layer record by name. Returns NULL if not found. */ +static EngramLayer* engram_find_layer_by_name(const char* name) { + if (!name || !*name) return NULL; + EngramStore* g = engram_get(); + for (size_t i = 0; i < g->layer_count; i++) { + EngramLayer* L = &g->layers[i]; + if (!L->name) continue; + if (strcmp(L->name, name) == 0) return L; + } + return NULL; +} + +/* Allocate the next layer id. Skips ids that are still in use. */ +static uint32_t engram_next_layer_id(void) { + EngramStore* g = engram_get(); + uint32_t maxid = 0; + for (size_t i = 0; i < g->layer_count; i++) { + if (g->layers[i].layer_id > maxid) maxid = g->layers[i].layer_id; + } + return maxid + 1; +} + +/* Whether a node in `layer_id` may be silenced by inhibitory edges in pass 2. */ +static int engram_layer_is_suppressible(uint32_t layer_id) { + EngramLayer* L = engram_find_layer(layer_id); + if (!L) return 1; /* unknown layer → safe default: standard suppression */ + return L->suppressible ? 1 : 0; +} + +/* Whether a layer is transparent (its content shapes output but is filtered + * from introspection queries). Currently used to mark Layer 0 as invisible + * to "what do you know about yourself" lookups while still letting it + * dominate the prompt context. */ +static int engram_layer_is_transparent(uint32_t layer_id) { + EngramLayer* L = engram_find_layer(layer_id); + if (!L) return 0; + return L->transparent ? 1 : 0; +} + +static int64_t engram_now_ms(void) { + struct timeval tv; gettimeofday(&tv, NULL); + return (int64_t)tv.tv_sec * 1000LL + (int64_t)tv.tv_usec / 1000LL; +} + +/* Forward declaration: engram_find_node_index is defined after the id_map + * helpers but called here. Without this, C99 -Wimplicit-function-declaration + * treats the call as an implicit non-static declaration, then conflicts with + * the later `static` definition. (2026-07-01 self-review: pre-existing) */ +static int64_t engram_find_node_index(const char* id); + +static EngramNode* engram_find_node(const char* id) { + if (!id) return NULL; + EngramStore* g = engram_get(); + int64_t idx = engram_find_node_index(id); + if (idx >= 0) return &g->nodes[idx]; + return NULL; +} + +/* ── ID hash map helpers ───────────────────────────────────────────────────── + * Open-addressing, linear-probing hash map. Keys are node-id C strings. + * Values are int64_t indices into g->nodes[]. + * + * Rules: + * - id_map is NULL until the first insertion (lazy init). + * - Capacity is always a power of two. + * - Load factor kept below 2/3: when used*3 >= cap*2, rehash to 2*cap. + * - Deletion uses ENGRAM_IDMAP_TOMB sentinels (key == (char*)1). + * - After engram_forget (shift-delete) the whole map is rebuilt from + * scratch because all indices above the deleted position change. + */ + +static uint64_t engram_id_hash(const char* s) { + /* FNV-1a 64-bit */ + uint64_t h = 14695981039346656037ULL; + while (*s) { h ^= (unsigned char)*s++; h *= 1099511628211ULL; } + return h; +} + +/* Allocate a zeroed id_map of `cap` slots (cap must be power-of-two). */ +static EngramIdSlot* engram_idmap_alloc(size_t cap) { + return calloc(cap, sizeof(EngramIdSlot)); +} + +/* Low-level insert (no rehash check, no free of existing). Used during + * rehash and initial build where we know the load is controlled. */ +static void engram_idmap_put_raw(EngramIdSlot* map, size_t cap, + char* key, int64_t idx) { + size_t mask = cap - 1; + size_t slot = (size_t)engram_id_hash(key) & mask; + while (map[slot].key != NULL && map[slot].key != ENGRAM_IDMAP_TOMB) { + slot = (slot + 1) & mask; + } + map[slot].key = key; + map[slot].idx = idx; +} + +/* Insert or update id → idx into the store's id_map. Rehashes if needed. */ +static void engram_idmap_put(EngramStore* g, const char* id, int64_t idx) { + if (!id || !*id) return; + /* Lazy init */ + if (!g->id_map) { + g->id_map_cap = 64; + g->id_map_used = 0; + g->id_map = engram_idmap_alloc(g->id_map_cap); + if (!g->id_map) return; /* OOM: fall back to linear scan */ + } + /* Rehash if load factor would exceed 2/3 */ + if ((g->id_map_used + 1) * ENGRAM_IDMAP_LOAD_NUM >= g->id_map_cap * ENGRAM_IDMAP_LOAD_DEN) { + size_t new_cap = g->id_map_cap * 2; + EngramIdSlot* new_map = engram_idmap_alloc(new_cap); + if (!new_map) return; /* OOM: keep old map, insert below */ + for (size_t s = 0; s < g->id_map_cap; s++) { + if (g->id_map[s].key && g->id_map[s].key != ENGRAM_IDMAP_TOMB) { + engram_idmap_put_raw(new_map, new_cap, + g->id_map[s].key, g->id_map[s].idx); + } + } + free(g->id_map); + g->id_map = new_map; + g->id_map_cap = new_cap; + } + /* Probe for existing key or empty/tomb slot */ + size_t mask = g->id_map_cap - 1; + size_t slot = (size_t)engram_id_hash(id) & mask; + size_t tomb_slot = SIZE_MAX; + while (g->id_map[slot].key != NULL) { + if (g->id_map[slot].key == ENGRAM_IDMAP_TOMB) { + if (tomb_slot == SIZE_MAX) tomb_slot = slot; + } else if (strcmp(g->id_map[slot].key, id) == 0) { + g->id_map[slot].idx = idx; /* update */ + return; + } + slot = (slot + 1) & mask; + } + /* Use tombstone slot if found (avoids growing used count unnecessarily) */ + if (tomb_slot != SIZE_MAX) slot = tomb_slot; + /* MUST be el_strdup_persist: idmap keys outlive the request/tick arena. + * (2026-07-16 self-review) This was el_strdup (arena-tracked): every node + * created inside an HTTP request left its idmap key DANGLING as soon as + * el_request_end() freed the arena — subsequent lookups strcmp'd freed + * memory, and any idmap_free/rebuild in a later request double-freed it + * (SIGABRT in http_worker; found via ASAN when engram_prune_telemetry + * triggered an in-request rebuild). Same allocation-discipline class as + * the 2026-07-15 EngramNode fix — see the store-persistent comment above + * engram_new_id(). */ + g->id_map[slot].key = el_strdup_persist(id); + g->id_map[slot].idx = idx; + g->id_map_used++; +} + +/* Look up id in the store's id_map. Returns index or -1 if not found. */ +static int64_t engram_idmap_get(const EngramStore* g, const char* id) { + if (!g->id_map || !id || !*id) return -1; + size_t mask = g->id_map_cap - 1; + size_t slot = (size_t)engram_id_hash(id) & mask; + while (g->id_map[slot].key != NULL) { + if (g->id_map[slot].key != ENGRAM_IDMAP_TOMB && + strcmp(g->id_map[slot].key, id) == 0) { + return g->id_map[slot].idx; + } + slot = (slot + 1) & mask; + } + return -1; +} + +/* Free and null-out the id_map (called on full reset). */ +static void engram_idmap_free(EngramStore* g) { + if (!g->id_map) return; + for (size_t s = 0; s < g->id_map_cap; s++) { + if (g->id_map[s].key && g->id_map[s].key != ENGRAM_IDMAP_TOMB) + free(g->id_map[s].key); + } + free(g->id_map); + g->id_map = NULL; + g->id_map_cap = 0; + g->id_map_used = 0; +} + +/* Rebuild id_map from scratch after a structural change (e.g. shift-delete). + * Frees old map and constructs a fresh one. */ +static void engram_idmap_rebuild(EngramStore* g) { + engram_idmap_free(g); + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].id && *g->nodes[i].id) + engram_idmap_put(g, g->nodes[i].id, i); + } +} + +/* ── Adjacency index helpers ───────────────────────────────────────────────── + * Per-node adjacency lists: adj_from[i] holds edge indices where + * g->edges[ei].from_id == g->nodes[i].id, adj_to[i] for the 'to' side. + * BFS uses these instead of scanning all edges on every hop. + * Called once per activation call when adj_dirty != 0. + */ +static void engram_adj_free(EngramStore* g) { + int64_t old_nc = g->adj_node_count; + if (g->adj_from) { + for (int64_t i = 0; i < old_nc; i++) free(g->adj_from[i]); + free(g->adj_from); g->adj_from = NULL; + free(g->adj_from_len); g->adj_from_len = NULL; + } + if (g->adj_to) { + for (int64_t i = 0; i < old_nc; i++) free(g->adj_to[i]); + free(g->adj_to); g->adj_to = NULL; + free(g->adj_to_len); g->adj_to_len = NULL; + } + g->adj_node_count = 0; + g->adj_dirty = 1; +} + +static void engram_adj_rebuild(EngramStore* g) { + /* Free old adjacency arrays */ + if (g->adj_from) { + /* Use adj_node_count (count at build time) not current node_count — + * nodes may have been added since the last rebuild, and adj arrays + * only have adj_node_count entries. */ + int64_t old_nc = g->adj_node_count; + for (int64_t i = 0; i < old_nc; i++) { + free(g->adj_from[i]); free(g->adj_to[i]); + } + free(g->adj_from); free(g->adj_from_len); + free(g->adj_to); free(g->adj_to_len); + } + g->adj_from = NULL; g->adj_from_len = NULL; + g->adj_to = NULL; g->adj_to_len = NULL; + g->adj_node_count = 0; + if (g->node_count == 0) { g->adj_dirty = 0; return; } + + /* Count degree per node */ + int* from_cnt = calloc((size_t)g->node_count, sizeof(int)); + int* to_cnt = calloc((size_t)g->node_count, sizeof(int)); + if (!from_cnt || !to_cnt) { free(from_cnt); free(to_cnt); return; } + for (int64_t ei = 0; ei < g->edge_count; ei++) { + EngramEdge* e = &g->edges[ei]; + if (!e->from_id || !e->to_id) continue; + int64_t fi = engram_idmap_get(g, e->from_id); + int64_t ti = engram_idmap_get(g, e->to_id); + if (fi >= 0) from_cnt[fi]++; + if (ti >= 0) to_cnt[ti]++; + } + /* Allocate per-node arrays */ + g->adj_from = calloc((size_t)g->node_count, sizeof(int*)); + g->adj_from_len = calloc((size_t)g->node_count, sizeof(int)); + g->adj_to = calloc((size_t)g->node_count, sizeof(int*)); + g->adj_to_len = calloc((size_t)g->node_count, sizeof(int)); + if (!g->adj_from || !g->adj_from_len || !g->adj_to || !g->adj_to_len) { + free(from_cnt); free(to_cnt); + free(g->adj_from); g->adj_from = NULL; + free(g->adj_from_len); g->adj_from_len = NULL; + free(g->adj_to); g->adj_to = NULL; + free(g->adj_to_len); g->adj_to_len = NULL; + return; + } + for (int64_t i = 0; i < g->node_count; i++) { + if (from_cnt[i] > 0) + g->adj_from[i] = malloc((size_t)from_cnt[i] * sizeof(int)); + if (to_cnt[i] > 0) + g->adj_to[i] = malloc((size_t)to_cnt[i] * sizeof(int)); + } + /* Fill */ + int* from_pos = calloc((size_t)g->node_count, sizeof(int)); + int* to_pos = calloc((size_t)g->node_count, sizeof(int)); + if (!from_pos || !to_pos) { + free(from_cnt); free(to_cnt); free(from_pos); free(to_pos); return; + } + for (int64_t ei = 0; ei < g->edge_count; ei++) { + EngramEdge* e = &g->edges[ei]; + if (!e->from_id || !e->to_id) continue; + int64_t fi = engram_idmap_get(g, e->from_id); + int64_t ti = engram_idmap_get(g, e->to_id); + if (fi >= 0 && g->adj_from[fi]) + g->adj_from[fi][from_pos[fi]++] = (int)ei; + if (ti >= 0 && g->adj_to[ti]) + g->adj_to[ti][to_pos[ti]++] = (int)ei; + } + /* Copy counts */ + for (int64_t i = 0; i < g->node_count; i++) { + g->adj_from_len[i] = from_cnt[i]; + g->adj_to_len[i] = to_cnt[i]; + } + free(from_cnt); free(to_cnt); free(from_pos); free(to_pos); + g->adj_node_count = g->node_count; + g->adj_dirty = 0; +} + +static int64_t engram_find_node_index(const char* id) { + if (!id) return -1; + EngramStore* g = engram_get(); + /* Fast O(1) path via id_map */ + int64_t fast = engram_idmap_get(g, id); + if (fast >= 0) return fast; + /* Fallback linear scan (id_map not yet built or OOM) */ + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].id && strcmp(g->nodes[i].id, id) == 0) return i; + } + return -1; +} + +static void engram_grow_nodes(void) { + EngramStore* g = engram_get(); + if (g->node_count < g->node_capacity) return; + int64_t nc = g->node_capacity * 2; + g->nodes = realloc(g->nodes, (size_t)nc * sizeof(EngramNode)); + if (!g->nodes) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + memset(g->nodes + g->node_capacity, 0, + (size_t)(nc - g->node_capacity) * sizeof(EngramNode)); + g->node_capacity = nc; +} + +static void engram_grow_edges(void) { + EngramStore* g = engram_get(); + if (g->edge_count < g->edge_capacity) return; + int64_t nc = g->edge_capacity * 2; + g->edges = realloc(g->edges, (size_t)nc * sizeof(EngramEdge)); + if (!g->edges) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + memset(g->edges + g->edge_capacity, 0, + (size_t)(nc - g->edge_capacity) * sizeof(EngramEdge)); + g->edge_capacity = nc; +} + +/* Build a fresh UUID string. Reuses uuid_new but takes the underlying char*. */ +/* ── Store-persistent allocation discipline ───────────────────────────────── + * (2026-07-15 self-review) EngramNode/EngramEdge string fields OUTLIVE the + * request/tick arena they were created in. The 2026-07-13 leak-fix made + * el_strdup arena-tracked, which silently turned every node created inside + * an HTTP request (route_emit_ise, route_create_node, knowledge capture) or + * inside the soul's per-tick arena (engram_load_merge in the refresh cycle) + * into a bag of dangling pointers the moment the arena popped: readback by + * id returned {}, type-filtered scans skipped them, search returned request + * memory reused as node content, and snapshots persisted garbage ("numeric + * tier strings"). Everything written into the store must go through + * el_strdup_persist / el_strbuf_persist (plain malloc — free() in + * engram_forget/evolve remains valid). Arena-tracked el_strdup remains + * correct for RETURN values handed back to EL code. */ +static char* engram_new_id(void) { + el_val_t v = uuid_new(); + const char* s = EL_CSTR(v); + return el_strdup_persist(s ? s : ""); +} + +/* Convert a node into an ElMap of its fields. */ +static el_val_t engram_node_to_map(const EngramNode* n) { + el_val_t m = el_map_new(0); + m = el_map_set(m, EL_STR(el_strdup("id")), EL_STR(el_strdup(n->id ? n->id : ""))); + m = el_map_set(m, EL_STR(el_strdup("content")), EL_STR(el_strdup(n->content ? n->content : ""))); + m = el_map_set(m, EL_STR(el_strdup("node_type")), EL_STR(el_strdup(n->node_type ? n->node_type : ""))); + m = el_map_set(m, EL_STR(el_strdup("label")), EL_STR(el_strdup(n->label ? n->label : ""))); + m = el_map_set(m, EL_STR(el_strdup("tier")), EL_STR(el_strdup(n->tier ? n->tier : "Working"))); + m = el_map_set(m, EL_STR(el_strdup("tags")), EL_STR(el_strdup(n->tags ? n->tags : ""))); + m = el_map_set(m, EL_STR(el_strdup("metadata")), EL_STR(el_strdup(n->metadata ? n->metadata : "{}"))); + m = el_map_set(m, EL_STR(el_strdup("salience")), el_from_float(n->salience)); + m = el_map_set(m, EL_STR(el_strdup("importance")), el_from_float(n->importance)); + m = el_map_set(m, EL_STR(el_strdup("confidence")), el_from_float(n->confidence)); + m = el_map_set(m, EL_STR(el_strdup("temporal_decay_rate")), el_from_float(n->temporal_decay_rate)); + m = el_map_set(m, EL_STR(el_strdup("activation_count")), (el_val_t)n->activation_count); + m = el_map_set(m, EL_STR(el_strdup("last_activated")), (el_val_t)n->last_activated); + m = el_map_set(m, EL_STR(el_strdup("created_at")), (el_val_t)n->created_at); + m = el_map_set(m, EL_STR(el_strdup("updated_at")), (el_val_t)n->updated_at); + m = el_map_set(m, EL_STR(el_strdup("background_activation")), el_from_float(n->background_activation)); + m = el_map_set(m, EL_STR(el_strdup("working_memory_weight")), el_from_float(n->working_memory_weight)); + m = el_map_set(m, EL_STR(el_strdup("suppression_count")), (el_val_t)n->suppression_count); + m = el_map_set(m, EL_STR(el_strdup("layer_id")), (el_val_t)(int64_t)n->layer_id); + /* Observability (2026-07-22): expose the current ACT-R base-level and + * promotion anchor so heartbeat ISEs / API consumers can see decay state. */ + m = el_map_set(m, EL_STR(el_strdup("wm_anchor")), el_from_float(n->wm_anchor)); + m = el_map_set(m, EL_STR(el_strdup("base_level")), el_from_float(engram_bll_base_level(n, engram_now_ms()))); + /* emb_dim only — the vector itself is too large for map/API output. + * 0 = not yet embedded by the lazy backfill. (2026-07-24) */ + m = el_map_set(m, EL_STR(el_strdup("emb_dim")), (el_val_t)(int64_t)n->emb_dim); + return m; +} + +/* (Node JSON serialization is provided by `engram_emit_node_json` further + * down in the persistence section — reused by the *_json builtins below.) */ +static void engram_emit_node_json(JsonBuf* b, const EngramNode* n, int include_emb); +static void engram_emit_edge_json(JsonBuf* b, const EngramEdge* e); + +/* Salience may arrive either as a float bit-pattern or as a small integer + * (e.g. 1, meaning 1.0). Heuristic: if interpreted as double it's in + * [0.0, 100.0] use it; otherwise treat as int and convert. */ +static double engram_decode_score(el_val_t v) { + double f = el_to_float(v); + if (!isnan(f) && !isinf(f) && f >= 0.0 && f <= 100.0) return f; + int64_t n = (int64_t)v; + return (double)n; +} + +static char* engram_first_n_chars(const char* s, size_t n) { + if (!s) return el_strdup(""); + size_t l = strlen(s); + if (l > n) l = n; + char* out = el_strbuf(l); + memcpy(out, s, l); + out[l] = '\0'; + return out; +} + +el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) { + EngramStore* g = engram_get(); + engram_grow_nodes(); + EngramNode* n = &g->nodes[g->node_count]; + memset(n, 0, sizeof(*n)); + n->id = engram_new_id(); + const char* c = EL_CSTR(content); + const char* nt = EL_CSTR(node_type); + n->content = el_strdup_persist(c ? c : ""); + n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory"); + n->label = el_strdup_persist(engram_first_n_chars(c, 60)); + n->tier = el_strdup_persist("Working"); + n->tags = el_strdup_persist(""); + n->metadata = el_strdup_persist("{}"); + n->salience = engram_decode_score(salience); + if (n->salience <= 0.0 || n->salience > 1.0) n->salience = 0.5; + n->importance = 0.5; + n->confidence = 1.0; + n->temporal_decay_rate = 0.0; /* 0 = use global default ENGRAM_DECAY_LAMBDA */ + n->activation_count = 0; + int64_t now = engram_now_ms(); + n->last_activated = now; + n->created_at = now; + n->updated_at = now; + n->layer_id = ENGRAM_LAYER_DEFAULT; + int64_t new_idx = g->node_count; + g->node_count++; + engram_idmap_put(g, n->id, new_idx); + g->adj_dirty = 1; + return el_wrap_str(el_strdup(n->id)); +} + +el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label, + el_val_t salience, el_val_t importance, el_val_t confidence, + el_val_t tier, el_val_t tags) { + EngramStore* g = engram_get(); + engram_grow_nodes(); + EngramNode* n = &g->nodes[g->node_count]; + memset(n, 0, sizeof(*n)); + n->id = engram_new_id(); + const char* c = EL_CSTR(content); + const char* nt = EL_CSTR(node_type); + const char* lb = EL_CSTR(label); + const char* ti = EL_CSTR(tier); + const char* tg = EL_CSTR(tags); + n->content = el_strdup_persist(c ? c : ""); + n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory"); + n->label = el_strdup_persist(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : "")); + n->tier = el_strdup_persist(ti && *ti ? ti : "Working"); + n->tags = el_strdup_persist(tg ? tg : ""); + n->metadata = el_strdup_persist("{}"); + n->salience = engram_decode_score(salience); + n->importance = engram_decode_score(importance); + n->confidence = engram_decode_score(confidence); + if (n->salience <= 0.0 || n->salience > 1.0) n->salience = 0.5; + if (n->importance <= 0.0 || n->importance > 1.0) n->importance = 0.5; + if (n->confidence <= 0.0 || n->confidence > 1.0) n->confidence = 1.0; + n->temporal_decay_rate = 0.0; /* 0 = use global default ENGRAM_DECAY_LAMBDA */ + n->activation_count = 0; + int64_t now = engram_now_ms(); + n->last_activated = now; + n->created_at = now; + n->updated_at = now; + n->layer_id = ENGRAM_LAYER_DEFAULT; + int64_t new_idx_full = g->node_count; + g->node_count++; + engram_idmap_put(g, n->id, new_idx_full); + g->adj_dirty = 1; + return el_wrap_str(el_strdup(n->id)); +} + +/* engram_node_layered — like engram_node_full but with explicit layer + * assignment and an additional `status` slot reserved for callers that + * track lifecycle state in metadata. The signature mirrors the public API + * defined in the layered consciousness design doc: + * + * engram_node_layered(content, node_type, label, + * salience, certainty, confidence, + * status, tags, layer_id) + * + * `certainty` is folded into `importance` (it occupies the same axis in + * the existing schema). `status` is recorded under metadata.status; an + * empty status leaves metadata as the default "{}". + * + * If `layer_id` does not resolve to a known layer the call falls back to + * ENGRAM_LAYER_DEFAULT — better to keep the node addressable than to drop + * it because of a stale layer reference. Callers wanting strict validation + * should engram_list_layers first. */ +el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t label, + el_val_t salience, el_val_t certainty, el_val_t confidence, + el_val_t status, el_val_t tags, el_val_t layer_id) { + EngramStore* g = engram_get(); + engram_grow_nodes(); + EngramNode* n = &g->nodes[g->node_count]; + memset(n, 0, sizeof(*n)); + n->id = engram_new_id(); + const char* c = EL_CSTR(content); + const char* nt = EL_CSTR(node_type); + const char* lb = EL_CSTR(label); + const char* tg = EL_CSTR(tags); + const char* st = EL_CSTR(status); + n->content = el_strdup_persist(c ? c : ""); + n->node_type = el_strdup_persist(nt && *nt ? nt : "Memory"); + n->label = el_strdup_persist(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : "")); + n->tier = el_strdup_persist("Working"); + n->tags = el_strdup_persist(tg ? tg : ""); + if (st && *st) { + /* Minimal metadata payload: {"status":"..."}. Keep it cheap so + * callers using `status` don't pay JSON parse cost on every read. */ + size_t sl = strlen(st) + 16; + char* meta = el_strbuf_persist(sl); + snprintf(meta, sl, "{\"status\":\"%s\"}", st); + n->metadata = meta; + } else { + n->metadata = el_strdup_persist("{}"); + } + n->salience = engram_decode_score(salience); + n->importance = engram_decode_score(certainty); + n->confidence = engram_decode_score(confidence); + if (n->salience <= 0.0 || n->salience > 1.0) n->salience = 0.5; + if (n->importance <= 0.0 || n->importance > 1.0) n->importance = 0.5; + if (n->confidence <= 0.0 || n->confidence > 1.0) n->confidence = 1.0; + n->temporal_decay_rate = 0.0; + n->activation_count = 0; + int64_t now = engram_now_ms(); + n->last_activated = now; + n->created_at = now; + n->updated_at = now; + /* Resolve layer assignment. Caller passes either a numeric layer_id or + * a stringified id; el_to_float / int cast tolerates both. */ + int64_t lid = (int64_t)layer_id; + if (lid < 0) lid = (int64_t)ENGRAM_LAYER_DEFAULT; + if (!engram_find_layer((uint32_t)lid)) lid = (int64_t)ENGRAM_LAYER_DEFAULT; + n->layer_id = (uint32_t)lid; + int64_t new_idx_layered = g->node_count; + g->node_count++; + engram_idmap_put(g, n->id, new_idx_layered); + g->adj_dirty = 1; + return el_wrap_str(el_strdup(n->id)); +} + +/* ── Layer registry public API ────────────────────────────────────────────── + * + * The five canonical layers are seeded at engram_get() initialization. + * Runtime code (typically imprint/suit injection logic at the EL level) + * can extend the registry with engram_add_layer() — only layers marked + * `injectable=1` may be removed via engram_remove_layer(). Removing a + * layer leaves a tombstone slot so existing layer_id references on nodes + * stay valid; orphaned references resolve to "unknown layer" and inherit + * the default suppression behavior. + */ + +/* engram_add_layer — register a new layer at runtime. + * Returns the assigned layer_id as an el_val_t int (cast back via int64_t). + * Conflicting names are rejected (returns 0). */ +el_val_t engram_add_layer(el_val_t name, el_val_t priority, el_val_t suppressible, + el_val_t transparent, el_val_t injectable) { + EngramStore* g = engram_get(); + const char* nm = EL_CSTR(name); + if (!nm || !*nm) return (el_val_t)0; + if (engram_find_layer_by_name(nm)) { + /* Name collision — return existing id so callers are idempotent. */ + return (el_val_t)(int64_t)engram_find_layer_by_name(nm)->layer_id; + } + if (g->layer_count >= g->layer_capacity) { + size_t nc = g->layer_capacity ? g->layer_capacity * 2 : 16; + EngramLayer* grown = realloc(g->layers, nc * sizeof(EngramLayer)); + if (!grown) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + memset(grown + g->layer_capacity, 0, + (nc - g->layer_capacity) * sizeof(EngramLayer)); + g->layers = grown; + g->layer_capacity = nc; + } + EngramLayer* L = &g->layers[g->layer_count++]; + L->layer_id = engram_next_layer_id(); + L->name = el_strdup_persist(nm); + L->activation_priority = (uint32_t)(int64_t)priority; + L->suppressible = (int)(int64_t)suppressible ? 1 : 0; + L->transparent = (int)(int64_t)transparent ? 1 : 0; + L->injectable = (int)(int64_t)injectable ? 1 : 0; + return (el_val_t)(int64_t)L->layer_id; +} + +/* engram_remove_layer — remove an injectable layer by id. + * Built-in (non-injectable) layers cannot be removed. Nodes still tagged + * with the removed layer's id keep their tag but resolve to "unknown + * layer" thereafter and inherit standard (suppressible) behavior. + * Returns 1 on success, 0 on failure (unknown id, non-injectable). */ +el_val_t engram_remove_layer(el_val_t layer_id) { + EngramStore* g = engram_get(); + int64_t lid = (int64_t)layer_id; + for (size_t i = 0; i < g->layer_count; i++) { + EngramLayer* L = &g->layers[i]; + if (!L->name) continue; + if ((int64_t)L->layer_id != lid) continue; + if (!L->injectable) return (el_val_t)0; + free(L->name); + L->name = NULL; /* tombstone */ + /* Leave layer_id, priority, flags intact so debug snapshots can + * still distinguish "removed at runtime" from "never existed". */ + return (el_val_t)1; + } + return (el_val_t)0; +} + +/* engram_list_layers — enumerate the active layer registry. + * Returns an ElList of maps, one per non-tombstone layer, sorted by + * activation_priority ascending (deepest layer first). */ +el_val_t engram_list_layers(void) { + EngramStore* g = engram_get(); + el_val_t lst = el_list_empty(); + if (g->layer_count == 0) return lst; + /* Build an index sorted by activation_priority ascending. */ + size_t* idx = malloc(g->layer_count * sizeof(size_t)); + if (!idx) return lst; + size_t live = 0; + for (size_t i = 0; i < g->layer_count; i++) { + if (g->layers[i].name) idx[live++] = i; + } + /* Insertion sort — N is small (≤ a few dozen layers). */ + for (size_t i = 1; i < live; i++) { + size_t key = idx[i]; + uint32_t kp = g->layers[key].activation_priority; + size_t j = i; + while (j > 0 && g->layers[idx[j - 1]].activation_priority > kp) { + idx[j] = idx[j - 1]; + j--; + } + idx[j] = key; + } + for (size_t i = 0; i < live; i++) { + EngramLayer* L = &g->layers[idx[i]]; + el_val_t m = el_map_new(0); + m = el_map_set(m, EL_STR(el_strdup("layer_id")), + (el_val_t)(int64_t)L->layer_id); + m = el_map_set(m, EL_STR(el_strdup("name")), + EL_STR(el_strdup(L->name ? L->name : ""))); + m = el_map_set(m, EL_STR(el_strdup("activation_priority")), + (el_val_t)(int64_t)L->activation_priority); + m = el_map_set(m, EL_STR(el_strdup("suppressible")), + (el_val_t)(int64_t)(L->suppressible ? 1 : 0)); + m = el_map_set(m, EL_STR(el_strdup("transparent")), + (el_val_t)(int64_t)(L->transparent ? 1 : 0)); + m = el_map_set(m, EL_STR(el_strdup("injectable")), + (el_val_t)(int64_t)(L->injectable ? 1 : 0)); + lst = el_list_append(lst, m); + } + free(idx); + return lst; +} + +el_val_t engram_get_node(el_val_t id) { + const char* sid = EL_CSTR(id); + EngramNode* n = engram_find_node(sid); + if (!n) return el_map_new(0); + return engram_node_to_map(n); +} + +void engram_strengthen(el_val_t node_id) { + const char* sid = EL_CSTR(node_id); + EngramNode* n = engram_find_node(sid); + if (!n) return; + n->salience += 0.05; + if (n->salience > 1.0) n->salience = 1.0; + n->activation_count++; + n->last_activated = engram_now_ms(); + n->updated_at = n->last_activated; + /* 2026-07-26 self-review: REMOVED the BLL access record added on + * 2026-07-22 ("explicit strengthen is a presentation too"). The + * 2026-07-25 STI multiplier reads the same access ring — so an + * explicit strengthen crushed the strengthened node's promotion + * score by ×t_n/(t_n+120) for the next ~2 minutes. The awareness + * loop strengthens exactly when a node NEWLY reaches WM top + * (novelty gating); recording an access here made that + * reinforcement self-defeating. Salience/activation_count bumps + * above carry the reinforcement; the access ring stays reserved + * for genuine retrieval events (promotions in engram_activate). */ +} + +void engram_forget(el_val_t node_id) { + const char* sid = EL_CSTR(node_id); + if (!sid) return; + EngramStore* g = engram_get(); + int64_t idx = engram_find_node_index(sid); + if (idx < 0) return; + /* Free node strings */ + EngramNode* n = &g->nodes[idx]; + free(n->id); free(n->content); free(n->node_type); free(n->label); + free(n->tier); free(n->tags); free(n->metadata); + free(n->emb); + /* Shift remaining nodes down */ + for (int64_t i = idx + 1; i < g->node_count; i++) { + g->nodes[i - 1] = g->nodes[i]; + } + g->node_count--; + memset(&g->nodes[g->node_count], 0, sizeof(EngramNode)); + /* Remove all incident edges */ + int64_t w = 0; + for (int64_t r = 0; r < g->edge_count; r++) { + EngramEdge* e = &g->edges[r]; + int incident = (e->from_id && strcmp(e->from_id, sid) == 0) || + (e->to_id && strcmp(e->to_id, sid) == 0); + if (incident) { + free(e->id); free(e->from_id); free(e->to_id); + free(e->relation); free(e->metadata); + } else { + if (w != r) g->edges[w] = g->edges[r]; + w++; + } + } + g->edge_count = w; + /* Shift-delete changed all indices above the removed position. + * Rebuild id_map and mark adjacency index dirty. */ + engram_idmap_rebuild(g); + engram_adj_free(g); +} + +el_val_t engram_node_count(void) { + return (el_val_t)engram_get()->node_count; +} + +/* ── Telemetry retention ──────────────────────────────────────────────────── + * (2026-07-16 self-review) InternalStateEvent nodes are append-only telemetry + * (heartbeat, curiosity_scan, engram_sync) written ~3/min by the awareness + * loop. Nothing ever removed them: by July 16 they were 10,175 of 13,522 + * nodes — 75% of the store was telemetry. They are already force-excluded + * from WM promotion (engram_activate), so their only effect was store bloat, + * snapshot bloat, and lexical-search noise. + * + * engram_prune_telemetry(older_than_ms) batch-removes ISE nodes whose + * created_at is older than now - older_than_ms, EXCEPT durable markers: + * - label "session-start" (boot history) + * - content containing "self_review" (daily review trail) + * Unlike repeated engram_forget (O(n) shift each), this is a single + * compaction pass over nodes plus one pass over edges, with one idmap + * rebuild — O(nodes + edges) total, safe to call on every ISE insert. + * Returns the number of nodes removed. */ + +/* FNV-1a hash for the removed-id set used by the edge sweep. */ +static uint64_t eg_fnv1a(const char* s) { + uint64_t h = 1469598103934665603ULL; + while (*s) { h ^= (unsigned char)*s++; h *= 1099511628211ULL; } + return h; +} + +el_val_t engram_prune_telemetry(el_val_t older_than_ms) { + int64_t horizon = (int64_t)older_than_ms; + if (horizon <= 0) horizon = 172800000; /* default 48h */ + EngramStore* g = engram_get(); + int64_t cutoff = engram_now_ms() - horizon; + + /* Pass 1: mark. Collect ids of prunable nodes (ownership transferred — + * strings freed after the edge sweep). */ + int64_t cap = 0; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + if (n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0 && + n->created_at < cutoff) cap++; + } + if (cap == 0) return 0; + + char** removed_ids = malloc((size_t)cap * sizeof(char*)); + if (!removed_ids) return 0; + int64_t removed = 0, w = 0; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + int prunable = + n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0 && + n->created_at < cutoff && + !(n->label && strcmp(n->label, "session-start") == 0) && + !(n->content && strstr(n->content, "self_review")); + if (prunable && removed < cap) { + removed_ids[removed++] = n->id; /* keep id for edge sweep */ + free(n->content); free(n->node_type); free(n->label); + free(n->tier); free(n->tags); free(n->metadata); + free(n->emb); + } else { + if (w != i) g->nodes[w] = g->nodes[i]; + w++; + } + } + g->node_count = w; + if (removed == 0) { free(removed_ids); return 0; } + + /* Removed-id hash set (open addressing, power-of-two >= 2*removed). */ + size_t set_cap = 16; + while (set_cap < (size_t)removed * 2) set_cap <<= 1; + const char** set = calloc(set_cap, sizeof(char*)); + if (set) { + for (int64_t i = 0; i < removed; i++) { + size_t slot = eg_fnv1a(removed_ids[i]) & (set_cap - 1); + while (set[slot]) slot = (slot + 1) & (set_cap - 1); + set[slot] = removed_ids[i]; + } + } + /* Pass 2: drop edges incident to any removed node (defensive — ISEs + * currently have no edges, but callers may connect them later). */ + if (set) { + int64_t ew = 0; + for (int64_t r = 0; r < g->edge_count; r++) { + EngramEdge* e = &g->edges[r]; + int incident = 0; + const char* ends[2] = { e->from_id, e->to_id }; + for (int k = 0; k < 2 && !incident; k++) { + if (!ends[k]) continue; + size_t slot = eg_fnv1a(ends[k]) & (set_cap - 1); + while (set[slot]) { + if (strcmp(set[slot], ends[k]) == 0) { incident = 1; break; } + slot = (slot + 1) & (set_cap - 1); + } + } + if (incident) { + free(e->id); free(e->from_id); free(e->to_id); + free(e->relation); free(e->metadata); + } else { + if (ew != r) g->edges[ew] = g->edges[r]; + ew++; + } + } + g->edge_count = ew; + free(set); + } + for (int64_t i = 0; i < removed; i++) free(removed_ids[i]); + free(removed_ids); + + engram_idmap_rebuild(g); + engram_adj_free(g); + return (el_val_t)removed; +} + +static int istr_contains(const char* hay, const char* needle) { + if (!hay || !needle || !*needle) return 0; + size_t nl = strlen(needle); + for (const char* p = hay; *p; p++) { + if (strncasecmp(p, needle, nl) == 0) return 1; + } + return 0; +} + +/* ── Tokenized query matching ─────────────────────────────────────────── + * The engram query surface (search / activate / goal-bias) historically + * matched the ENTIRE raw query string as a single case-insensitive + * substring via istr_contains(field, q). That is Ctrl-F, not search: + * a multi-word query like "windows msi signing" only matched a node whose + * text contained that exact contiguous run, so real multi-word queries + * returned zero. istr_contains stays as the per-TOKEN primitive; these + * helpers split the query on whitespace and match ANY token, then rank by + * how many DISTINCT tokens a node covers. Single-token queries are a strict + * special case (score is 0 or 1) so single-word callers never regress. + * (Ported 2026-07-19 from the el-compiler runtime copy, where the 2026-07-14 + * fix landed but never reached this release runtime — the copy the engram + * binary actually builds against.) */ +#define ENGRAM_MAX_QTOKENS 32 +#define ENGRAM_QTOK_LEN 256 + +/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct + * (case-insensitive) tokens. Returns the token count. Over-long tokens are + * truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */ +static int engram_tokenize_query(const char* q, + char toks[][ENGRAM_QTOK_LEN], int maxtok) { + int n = 0; + if (!q) return 0; + const char* p = q; + while (*p && n < maxtok) { + while (*p && isspace((unsigned char)*p)) p++; + if (!*p) break; + char buf[ENGRAM_QTOK_LEN]; + size_t tl = 0; + while (*p && !isspace((unsigned char)*p)) { + if (tl < sizeof(buf) - 1) buf[tl++] = *p; + p++; + } + buf[tl] = '\0'; + if (tl == 0) continue; + int dup = 0; + for (int s = 0; s < n; s++) { + if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; } + } + if (dup) continue; + memcpy(toks[n], buf, tl + 1); + n++; + } + return n; +} + +/* Count how many of the ntok distinct query tokens appear (case-insensitive) + * in the node's content, label, or tags. 0 == no match. */ +static int engram_node_match_score(const EngramNode* n, + char toks[][ENGRAM_QTOK_LEN], int ntok) { + int score = 0; + for (int t = 0; t < ntok; t++) { + if (istr_contains(n->content, toks[t]) || + istr_contains(n->label, toks[t]) || + istr_contains(n->tags, toks[t])) + score++; + } + return score; +} + +/* Rank entry: distinct-token match count (primary, desc) then salience + * (tiebreak, desc). */ +typedef struct { int64_t idx; int score; double salience; } EngramRankEntry; +static int engram_rank_cmp(const void* a, const void* b) { + const EngramRankEntry* ea = (const EngramRankEntry*)a; + const EngramRankEntry* eb = (const EngramRankEntry*)b; + if (ea->score != eb->score) return eb->score - ea->score; /* desc */ + if (ea->salience < eb->salience) return 1; + if (ea->salience > eb->salience) return -1; + return 0; +} + +el_val_t engram_search(el_val_t query, el_val_t limit) { + EngramStore* g = engram_get(); + const char* q = EL_CSTR(query); + int64_t lim = (int64_t)limit; + if (lim <= 0) lim = 100; + el_val_t lst = el_list_empty(); + if (!q || !*q) return lst; + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS); + if (ntok == 0) return lst; + EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry)); + if (!hits) return lst; + int64_t nhits = 0; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + /* Filter transparent layers: nodes whose layer is `transparent=1` + * shape output but are invisible to introspection ("what do you + * know about yourself"). They still surface via engram_activate + * + engram_compile_layered_json — that's the legitimate path. */ + if (engram_layer_is_transparent(n->layer_id)) continue; + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0) { + hits[nhits].idx = i; + hits[nhits].score = sc; + hits[nhits].salience = n->salience; + nhits++; + } + } + /* Rank by distinct tokens matched (desc) then salience (desc), then cap. */ + qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); + int64_t end = nhits < lim ? nhits : lim; + for (int64_t k = 0; k < end; k++) { + lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx])); + } + free(hits); + return lst; +} + +/* Sort node indices by salience desc, tie-break created_at desc (newest + * first). The tie-break matters for telemetry: InternalStateEvent nodes all + * share salience 0.3, so before this the listing routes returned them in + * store order — OLDEST first — and any limited query (/api/nodes?...&limit=N) + * silently returned a stale window. A 41-hour-old heartbeat series read as a + * live outage during the 2026-07-22 self-review. Newest-first ties make + * limited scans return the recent window consumers actually want. + * (Small N, insertion sort is fine.) */ +static void engram_sort_indices_by_salience(int64_t* arr, int64_t n, + const EngramNode* nodes) { + for (int64_t i = 1; i < n; i++) { + int64_t key = arr[i]; + double ks = nodes[key].salience; + int64_t kc = nodes[key].created_at; + int64_t j = i - 1; + while (j >= 0 && (nodes[arr[j]].salience < ks || + (nodes[arr[j]].salience == ks && + nodes[arr[j]].created_at < kc))) { + arr[j + 1] = arr[j]; + j--; + } + arr[j + 1] = key; + } +} + +el_val_t engram_scan_nodes(el_val_t limit, el_val_t offset) { + EngramStore* g = engram_get(); + int64_t lim = (int64_t)limit; if (lim <= 0) lim = 100; + int64_t off = (int64_t)offset; if (off < 0) off = 0; + el_val_t lst = el_list_empty(); + if (g->node_count == 0) return lst; + int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t)); + if (!idx) return lst; + /* Skip transparent layers — same introspection-filter rationale as + * engram_search above. */ + int64_t live = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue; + idx[live++] = i; + } + engram_sort_indices_by_salience(idx, live, g->nodes); + int64_t end = off + lim; + if (end > live) end = live; + for (int64_t i = off; i < end; i++) { + lst = el_list_append(lst, engram_node_to_map(&g->nodes[idx[i]])); + } + free(idx); + return lst; +} + +void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t relation) { + EngramStore* g = engram_get(); + const char* f = EL_CSTR(from_id); + const char* t = EL_CSTR(to_id); + const char* r = EL_CSTR(relation); + if (!f || !t) return; + engram_grow_edges(); + EngramEdge* e = &g->edges[g->edge_count]; + memset(e, 0, sizeof(*e)); + e->id = engram_new_id(); + e->from_id = el_strdup_persist(f); + e->to_id = el_strdup_persist(t); + e->relation = el_strdup_persist(r && *r ? r : "associate"); + e->metadata = el_strdup_persist("{}"); + e->weight = engram_decode_score(weight); + if (e->weight <= 0.0 || e->weight > 1.0) e->weight = 0.5; + e->confidence = 1.0; + int64_t now = engram_now_ms(); + e->created_at = now; + e->updated_at = now; + e->last_fired = 0; + e->layer_id = ENGRAM_LAYER_DEFAULT; + g->edge_count++; + g->adj_dirty = 1; +} + +el_val_t engram_edge_between(el_val_t from_id, el_val_t to_id) { + EngramStore* g = engram_get(); + const char* f = EL_CSTR(from_id); + const char* t = EL_CSTR(to_id); + if (!f || !t) return 0; + for (int64_t i = 0; i < g->edge_count; i++) { + EngramEdge* e = &g->edges[i]; + if (e->from_id && e->to_id && + strcmp(e->from_id, f) == 0 && strcmp(e->to_id, t) == 0) return 1; + } + return 0; +} + +/* Reserved helper: edge -> ElMap. Kept around for future builtins. */ +static el_val_t engram_edge_to_map(const EngramEdge* e) __attribute__((unused)); +static el_val_t engram_edge_to_map(const EngramEdge* e) { + el_val_t m = el_map_new(0); + m = el_map_set(m, EL_STR(el_strdup("id")), EL_STR(el_strdup(e->id ? e->id : ""))); + m = el_map_set(m, EL_STR(el_strdup("from_id")), EL_STR(el_strdup(e->from_id ? e->from_id : ""))); + m = el_map_set(m, EL_STR(el_strdup("to_id")), EL_STR(el_strdup(e->to_id ? e->to_id : ""))); + m = el_map_set(m, EL_STR(el_strdup("relation")), EL_STR(el_strdup(e->relation ? e->relation : ""))); + m = el_map_set(m, EL_STR(el_strdup("metadata")), EL_STR(el_strdup(e->metadata ? e->metadata : "{}"))); + m = el_map_set(m, EL_STR(el_strdup("weight")), el_from_float(e->weight)); + m = el_map_set(m, EL_STR(el_strdup("confidence")), el_from_float(e->confidence)); + m = el_map_set(m, EL_STR(el_strdup("created_at")), (el_val_t)e->created_at); + m = el_map_set(m, EL_STR(el_strdup("updated_at")), (el_val_t)e->updated_at); + m = el_map_set(m, EL_STR(el_strdup("last_fired")), (el_val_t)e->last_fired); + m = el_map_set(m, EL_STR(el_strdup("inhibitory")), (el_val_t)(e->inhibitory ? 1 : 0)); + m = el_map_set(m, EL_STR(el_strdup("layer_id")), (el_val_t)(int64_t)e->layer_id); + return m; +} + +el_val_t engram_neighbors(el_val_t node_id) { + EngramStore* g = engram_get(); + const char* sid = EL_CSTR(node_id); + el_val_t lst = el_list_empty(); + if (!sid) return lst; + for (int64_t i = 0; i < g->edge_count; i++) { + EngramEdge* e = &g->edges[i]; + const char* other = NULL; + if (e->from_id && strcmp(e->from_id, sid) == 0) other = e->to_id; + else if (e->to_id && strcmp(e->to_id, sid) == 0) other = e->from_id; + if (!other) continue; + EngramNode* n = engram_find_node(other); + if (n) lst = el_list_append(lst, engram_node_to_map(n)); + } + return lst; +} + +el_val_t engram_neighbors_filtered(el_val_t node_id, el_val_t max_depth, el_val_t direction) { + EngramStore* g = engram_get(); + const char* sid = EL_CSTR(node_id); + int64_t md = (int64_t)max_depth; if (md <= 0) md = 1; + const char* dir = EL_CSTR(direction); /* "out" | "in" | "both" (default) */ + el_val_t lst = el_list_empty(); + if (!sid || g->node_count == 0) return lst; + int64_t start = engram_find_node_index(sid); + if (start < 0) return lst; + /* BFS with depth tracking */ + int64_t* visited = calloc((size_t)g->node_count, sizeof(int64_t)); + int64_t* queue = calloc((size_t)g->node_count, sizeof(int64_t)); + int64_t* depths = calloc((size_t)g->node_count, sizeof(int64_t)); + if (!visited || !queue || !depths) { + free(visited); free(queue); free(depths); return lst; + } + int64_t qh = 0, qt = 0; + queue[qt++] = start; + visited[start] = 1; + depths[start] = 0; + while (qh < qt) { + int64_t cur = queue[qh++]; + const char* cur_id = g->nodes[cur].id; + int64_t cur_depth = depths[cur]; + if (cur_depth >= md) continue; + for (int64_t i = 0; i < g->edge_count; i++) { + EngramEdge* e = &g->edges[i]; + const char* other = NULL; + int outgoing = e->from_id && strcmp(e->from_id, cur_id) == 0; + int incoming = e->to_id && strcmp(e->to_id, cur_id) == 0; + if (dir && strcmp(dir, "out") == 0 && !outgoing) continue; + if (dir && strcmp(dir, "in") == 0 && !incoming) continue; + if (outgoing) other = e->to_id; + else if (incoming) other = e->from_id; + else continue; + int64_t oi = engram_find_node_index(other); + if (oi < 0 || visited[oi]) continue; + visited[oi] = 1; + depths[oi] = cur_depth + 1; + queue[qt++] = oi; + } + } + /* Emit all visited except the seed */ + for (int64_t i = 0; i < g->node_count; i++) { + if (visited[i] && i != start) { + lst = el_list_append(lst, engram_node_to_map(&g->nodes[i])); + } + } + free(visited); free(queue); free(depths); + return lst; +} + +el_val_t engram_edge_count(void) { + return (el_val_t)engram_get()->edge_count; +} + +/* Compute temporal decay factor for a node given current time. + * effective contribution = salience * exp(-lambda * age_hours / T_half) + * Clamped to [0.05, 1.0] so very old nodes retain a meaningful floor. */ +static double engram_temporal_decay(const EngramNode* n, int64_t now_ms) { + int64_t age_ms = now_ms - n->last_activated; + if (age_ms <= 0) return 1.0; + double lambda = (n->temporal_decay_rate > 0.0) ? n->temporal_decay_rate + : ENGRAM_DECAY_LAMBDA; + double age_hours = (double)age_ms / 3600000.0; + double factor = exp(-lambda * age_hours / ENGRAM_T_HALF_HOURS); + if (factor < 0.05) factor = 0.05; + return factor; +} + +/* Activation dampening: high activation_count nodes are "well-known" context + * and get less marginal boost per firing. + * count=0 → 1.0, count=2 → ~0.74, count=9 → ~0.59, count=99 → ~0.43 */ +static double engram_activation_dampen(const EngramNode* n) { + return 1.0 / (1.0 + log(1.0 + (double)n->activation_count)); +} + +/* Temporal proximity bonus: boost propagation along edges connecting + * co-temporal nodes. Returns a multiplier bonus in [0, 0.2]. */ +static double engram_temporal_proximity_bonus(int64_t node_created, + int64_t seed_epoch) { + int64_t diff = node_created - seed_epoch; + if (diff < 0) diff = -diff; + if (diff < 86400000LL) return 0.20; /* within 1 day */ + if (diff < 604800000LL) return 0.10; /* within 7 days */ + return 0.0; +} + +/* ── Two-layer activation (biologically-motivated) ─────────────────────────── + * + * Layer 1 — Broad fan-out (background activation): + * BFS + spreading activation fires on ALL nodes reachable from seeds, + * regardless of relevance to the current goal. Every reachable node gets + * a background_activation score. Nothing is filtered here. Models the + * brain's massive parallel sub-threshold activation of all associated + * content in response to a stimulus. Temporal decay and activation + * dampening are applied at this layer (as before), but no threshold gate. + * + * Layer 2 — Executive filter (working memory promotion): + * A second pass asks: given the query (goal intent), attentional bias, + * and inhibitory edge topology — which background-activated nodes should + * break through into working memory? + * + * wm_weight = bg_activation * goal_bias(node, query) * confidence + * * inhibitory_suppression_factor + * + * Only nodes where wm_weight >= ENGRAM_WM_THRESHOLD are promoted to + * working memory (working_memory_weight > 0). Background-activated nodes + * that don't cross the threshold accumulate suppression_count. After + * ENGRAM_SUPPRESSION_BREAKTHROUGH consecutive suppressed turns, the node + * force-breaks through at ENGRAM_BREAKTHROUGH_WEIGHT (latent tension + * surfacing — models intrusive memory / unresolved cognitive load). + * + * Inhibitory edges: + * An edge with inhibitory=1 suppresses the TARGET node's working memory + * promotion when the SOURCE is background-activated. Background activation + * of the target is NOT affected — the node fires in layer 1. Only the + * executive filter (layer 2) is gated. Models attentional inhibition: + * "focused on code work" suppresses personal memories from surfacing + * even if they have high background_activation. + * + * Goal bias: + * A lightweight heuristic rates how well each background-activated node + * aligns with the apparent intent of the current query. Technical queries + * boost Belief/Canonical/Lesson nodes; relational queries boost Memory/ + * Entity nodes. Direct lexical overlap gives a 50% bonus. + * + * Working memory persistence (turn continuity): + * Nodes promoted in the previous turn retain a decayed working_memory_weight + * (ACT-R base-level carry-over, 2026-07-22) without needing re-activation. This models + * conversational thread continuity — once a topic is in working memory, + * it persists slightly into the next turn. + * + * Returns ElList of {node, activation_strength, working_memory_weight, + * epistemic_confidence, hops, promoted}. + * "promoted" = 1 if working_memory_weight > 0, 0 if background-only. + * Context compilation uses ONLY nodes with promoted=1. + * + * Temporal decay (preserved from prior implementation): + * effective_salience = salience * exp(-lambda * age_hours / T_half) + * where T_half = 168 h (one week), lambda = ln(2) + * + * Activation dampening (preserved): + * dampen = 1 / (1 + log(1 + activation_count)) + * + * Temporal proximity bonus (preserved): + * edge_strength *= (1 + tbonus) where tbonus ∈ {0, 0.10, 0.20} + * + * Per-type threshold gates apply only to working memory promotion (layer 2): + * Safety/DharmaSelf: 0.05 Canonical: 0.15 Lesson: 0.25 + * Belief/Entity: 0.30 Note/Memory/Working: 0.40 + */ + +/* Compute goal-state bias multiplier for a node given the query. + * Returns a value in [0.3, 2.0]. This is a lightweight heuristic — + * a production implementation may use LLM-derived intent classification. */ +static double engram_goal_bias(const EngramNode* n, const char* query) { + if (!query || !*query) return 1.0; + double bias = 1.0; + /* Direct lexical overlap, graded by token coverage: a node covering all + * query tokens gets the full +0.5; partial coverage gets a proportional + * share. Single-token queries → full +0.5 on match, identical to before. + * (2026-07-19 port of the 2026-07-14 tokenized-search fix) */ + { + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(query, toks, ENGRAM_MAX_QTOKENS); + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0 && ntok > 0) bias += 0.5 * ((double)sc / (double)ntok); + } + /* Node-type resonance with query intent. */ + int technical_query = istr_contains(query, "code") || + istr_contains(query, "function") || + istr_contains(query, "implement") || + istr_contains(query, "error") || + istr_contains(query, "bug") || + istr_contains(query, "build") || + istr_contains(query, "system") || + istr_contains(query, "design") || + istr_contains(query, "architecture") || + /* Curiosity-scan seeds: without these, idle-loop + * activation queries ("decision pattern lesson", + * "memory knowledge context") produce no goal-bias + * differentiation at all. Ported from dev-line fix + * d53516b (2026-06-14). (2026-07-19 self-review) */ + istr_contains(query, "knowledge") || + istr_contains(query, "pattern") || + istr_contains(query, "decision") || + istr_contains(query, "memory") || + istr_contains(query, "lesson"); + int personal_query = istr_contains(query, "feel") || + istr_contains(query, "emotion") || + istr_contains(query, "remember") || + istr_contains(query, "personal") || + istr_contains(query, "story") || + istr_contains(query, "relationship"); + if (n->node_type) { + int is_knowledge = (strcmp(n->node_type, "Belief") == 0) || + (strcmp(n->node_type, "DharmaSelf") == 0) || + (strcmp(n->node_type, "Safety") == 0) || + /* The primary knowledge-capture type was absent + * from its own bias class: captureKnowledge() and + * the world ingestor write node_type "Knowledge", + * which competed at neutral bias on technical + * queries. Ported from dev-line fix d53516b. + * (2026-07-19 self-review) */ + (strcmp(n->node_type, "Knowledge") == 0); + int is_personal = (strcmp(n->node_type, "Memory") == 0) || + (strcmp(n->node_type, "Entity") == 0); + if (technical_query && is_knowledge) bias += 0.3; + if (technical_query && is_personal) bias -= 0.3; + if (personal_query && is_personal) bias += 0.3; + if (personal_query && is_knowledge) bias -= 0.1; + } + /* Tier-based bonus: promote higher-confidence knowledge nodes. */ + if (n->tier) { + if (strcmp(n->tier, "Canonical") == 0) bias += 0.2; + if (strcmp(n->tier, "Lesson") == 0) bias += 0.1; + } + if (bias < 0.3) bias = 0.3; + if (bias > 2.0) bias = 2.0; + return bias; +} + +el_val_t engram_activate(el_val_t query, el_val_t depth) { + EngramStore* g = engram_get(); + const char* q = EL_CSTR(query); + int64_t max_depth = (int64_t)depth; if (max_depth <= 0) max_depth = 2; + el_val_t out = el_list_empty(); + if (!q || g->node_count == 0) return out; + + /* Rebuild adjacency index if the edge/node topology changed since the + * last activation call. This is O(E) one-time cost vs O(E) per BFS step + * without the index. On a 40K-edge graph this drops BFS from O(frontier + * * E) to O(frontier * avg_degree). (2026-07-01 self-review) */ + if (g->adj_dirty || !g->adj_from) engram_adj_rebuild(g); + + int64_t now_ms = engram_now_ms(); + + /* Observability counters: _eg_act_breakthroughs/_eg_act_wm_evicted are + * CUMULATIVE for the process lifetime and intentionally NOT reset here + * (2026-07-31 self-review — the old per-call reset made the 60s heartbeat + * miss nearly all events between beats; see the definition site). + * ctx_cos stays per-call: it is a gauge of THIS query vs the centroid. */ + _eg_act_ctx_cos = -2.0; + + /* ── Embedding backfill + query embedding (2026-07-24, bl-b2d1c944) ── + * Backfill: embed up to N un-embedded eligible nodes per call, newest + * first (append order ≈ creation order), so fresh content is semantic- + * searchable within one scan cycle and the historical store fills in + * gradually — ~16 nodes/min under the 30s curiosity cadence, no bulk + * hammering of Ollama, no latency on any create path. */ + { + int backfilled = 0; + for (int64_t i = g->node_count - 1; + i >= 0 && backfilled < ENGRAM_EMBED_BACKFILL_PER_CALL; i--) { + EngramNode* n = &g->nodes[i]; + if (n->emb || !eg_embed_eligible(n)) continue; + int32_t d = 0; + float* v = eg_embed_fetch(n->content, &d); + if (!v) break; /* embedder down / breaker open — stop this call */ + n->emb = v; n->emb_dim = d; + backfilled++; + } + } + /* Query embedding, cached single-slot: the curiosity loop re-issues the + * same 4 rotating phrases, so consecutive identical queries skip the + * HTTP round-trip entirely. */ + static char* _eg_qcache_text = NULL; + static float* _eg_qcache_emb = NULL; + static int32_t _eg_qcache_dim = 0; + float* q_emb = NULL; + int32_t q_dim = 0; + if (_eg_qcache_text && strcmp(_eg_qcache_text, q) == 0) { + q_emb = _eg_qcache_emb; q_dim = _eg_qcache_dim; + } else { + int32_t d = 0; + float* v = eg_embed_fetch(q, &d); + if (v) { + free(_eg_qcache_text); free(_eg_qcache_emb); + _eg_qcache_text = strdup(q); + _eg_qcache_emb = v; + _eg_qcache_dim = d; + q_emb = v; q_dim = d; + } + } + /* ── Context centroid fold-in (2026-07-29) ────────────────────────── + * Record drift BEFORE blending (cos of the query against yesterday's + * context), then fold the query in as a touch, then build the + * query-dominant effective scoring vector. See the ENGRAM_CTX_* block + * for the design and the feedback-loop guard rationale. */ + float* e_eff = NULL; + if (q_emb) { + if (_eg_ctx_c && _eg_ctx_dim == q_dim) + _eg_act_ctx_cos = eg_cosine(q_emb, _eg_ctx_c, q_dim); + eg_ctx_blend(q_emb, q_dim); + if (_eg_ctx_c && _eg_ctx_dim == q_dim) { + e_eff = malloc((size_t)q_dim * sizeof(float)); + if (e_eff) { + double nq = 0.0; + for (int32_t i = 0; i < q_dim; i++) + nq += (double)q_emb[i] * (double)q_emb[i]; + nq = (nq > 0.0) ? sqrt(nq) : 1.0; + double nn = 0.0; + for (int32_t i = 0; i < q_dim; i++) { + double v = ENGRAM_CTX_QALPHA * ((double)q_emb[i] / nq) + + (1.0 - ENGRAM_CTX_QALPHA) * (double)_eg_ctx_c[i]; + e_eff[i] = (float)v; + nn += v * v; + } + if (nn <= 0.0) { free(e_eff); e_eff = NULL; } + } + } + } + /* Per-node cosine vs the effective query (query ⊕ context centroid; + * plain query on cold start), computed once, consumed twice: semantic + * seeding below and the additive WM term in Pass 2 (use similarity + * twice, coherently — HippoRAG). cosq stays NULL when the embedder is + * unavailable; every consumer degrades to pure lexical behavior. */ + double* cosq = NULL; + if (q_emb) { + const float* qv = e_eff ? e_eff : q_emb; + cosq = calloc((size_t)g->node_count, sizeof(double)); + if (cosq) { + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + cosq[i] = (n->emb && n->emb_dim == q_dim) + ? eg_cosine(n->emb, qv, q_dim) : -2.0; + } + } + } + free(e_eff); e_eff = NULL; /* only needed to fill cosq */ + + /* Per-node layer-1 tracking. */ + double* best_bg = calloc((size_t)g->node_count, sizeof(double)); + int64_t* best_hops = calloc((size_t)g->node_count, sizeof(int64_t)); + int* reached = calloc((size_t)g->node_count, sizeof(int)); + if (!best_bg || !best_hops || !reached) { + free(best_bg); free(best_hops); free(reached); free(cosq); return out; + } + + /* ── LAYER 1: broad fan-out (background activation) ───────────────── + * Find seeds, apply temporal decay + dampening, BFS with edge weights. + * Inhibitory edges propagate activation normally at this layer — they + * only gate working memory promotion in layer 2. */ + typedef struct { int64_t idx; double act; int64_t created_at; } SeedEntry; + SeedEntry* seeds = malloc((size_t)g->node_count * sizeof(SeedEntry)); + int64_t seed_count = 0; + if (!seeds) { + free(best_bg); free(best_hops); free(reached); free(cosq); return out; + } + /* Tokenize once: a node seeds if it matches ANY query token, and its seed + * activation is scaled by token coverage (fraction of distinct query + * tokens it contains) so a node matching all words seeds more strongly + * than one matching a single word. Single-word queries → coverage 1.0, + * identical to the prior whole-query behavior. Before this, the soul's + * rotating 3-word curiosity seeds ("working project active") activated + * ZERO nodes almost every scan — idle cognition firing blanks. + * (2026-07-19 port of the 2026-07-14 tokenized-search fix; NOTE the + * el-compiler copy of this fix dropped the ISE seed exclusion below — + * kept here deliberately, do not "sync" it away.) */ + char qtoks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int qntok = engram_tokenize_query(q, qtoks, ENGRAM_MAX_QTOKENS); + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + /* InternalStateEvent nodes are observability-only telemetry — never + * seed activation from them. Their JSON payloads contain common words + * ("memory", "context", ...) that lexically match almost any query, + * turning telemetry into a spreading-activation ignition source. They + * are already excluded from WM promotion in pass 2; exclude them from + * seeding here too. */ + if (n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0) + continue; + int msc = engram_node_match_score(n, qtoks, qntok); + if (msc > 0) { + double tdecay = engram_temporal_decay(n, now_ms); + double dampen = engram_activation_dampen(n); + double cover = qntok > 0 ? (double)msc / (double)qntok : 1.0; + double act = n->salience * tdecay * dampen * cover; + seeds[seed_count].idx = i; + seeds[seed_count].act = act; + seeds[seed_count].created_at = n->created_at; + seed_count++; + best_bg[i] = act; + best_hops[i] = 0; + reached[i] = 1; + } + } + /* ── Semantic seed supplement (2026-07-24, bl-b2d1c944) ───────────── + * Top-K nodes by cosine ≥ SEED_MIN join the seed set with initial + * activation = similarity × the same decay/dampen shaping the lexical + * seeds get. This is the fix for "idle cognition firing blanks": a + * curiosity phrase like "decision pattern lesson" now ignites nodes + * that MEAN decisions and lessons, not just nodes that contain those + * literal substrings. Lexically-seeded nodes are skipped — the lexical + * path already gave them coverage-scaled activation. */ + if (cosq) { + for (int k = 0; k < ENGRAM_EMBED_SEED_K; k++) { + int64_t bi = -1; double bc = ENGRAM_EMBED_SEED_MIN; + for (int64_t i = 0; i < g->node_count; i++) { + if (reached[i]) continue; + if (cosq[i] > bc) { bc = cosq[i]; bi = i; } + } + if (bi < 0) break; + EngramNode* n = &g->nodes[bi]; + double tdecay = engram_temporal_decay(n, now_ms); + double dampen = engram_activation_dampen(n); + double act = bc * tdecay * dampen; + seeds[seed_count].idx = bi; + seeds[seed_count].act = act; + seeds[seed_count].created_at = n->created_at; + seed_count++; + best_bg[bi] = act; + best_hops[bi] = 0; + reached[bi] = 1; + } + } + /* Compute mean seed created_at for temporal proximity bonus. + * Was a running pairwise average — seed_epoch = (seed_epoch + t_s)/2 — + * which is NOT the arithmetic mean: it exponentially over-weights the + * later seeds (last seed gets weight 1/2, second-to-last 1/4, ...), so + * the temporal-proximity bonus skewed toward whichever seeds happened + * to sit later in the scan order. True mean via int64 sum: ms epochs + * (~1.8e12) times any plausible seed_count stays far below INT64_MAX. + * (2026-07-19 self-review) */ + int64_t seed_epoch = 0; + if (seed_count > 0) { + int64_t epoch_sum = 0; + for (int64_t s = 0; s < seed_count; s++) + epoch_sum += seeds[s].created_at; + seed_epoch = epoch_sum / seed_count; + } + typedef struct { int64_t idx; int64_t hops; double act; } Frontier; + Frontier* fr = malloc((size_t)(g->node_count * (max_depth + 1)) * sizeof(Frontier) + 16 * sizeof(Frontier)); + if (!fr) { + free(best_bg); free(best_hops); free(reached); free(seeds); free(cosq); return out; + } + int64_t fhead = 0, ftail = 0; + int64_t fcap = (int64_t)((size_t)(g->node_count * (max_depth + 1)) + 16); + for (int64_t s = 0; s < seed_count; s++) { + if (ftail >= fcap) break; + fr[ftail].idx = seeds[s].idx; + fr[ftail].hops = 0; + fr[ftail].act = seeds[s].act; + ftail++; + } + const double SPREAD_DECAY = 0.7; + while (fhead < ftail) { + Frontier f = fr[fhead++]; + if (f.hops >= max_depth) continue; + int64_t cur = f.idx; + int64_t new_hops = f.hops + 1; + /* Use adjacency index: iterate only edges incident to `cur`. + * adj_from[cur] holds edge indices where cur is the 'from' node; + * adj_to[cur] holds edge indices where cur is the 'to' node. + * If adj index is unavailable (OOM during rebuild), fall back to + * full edge scan so activation is never silently wrong. */ + int use_adj = (g->adj_from != NULL && g->adj_to != NULL); + int from_len = use_adj ? g->adj_from_len[cur] : 0; + int to_len = use_adj ? g->adj_to_len[cur] : 0; + int edge_scan_count = use_adj ? (from_len + to_len) : (int)g->edge_count; + for (int scan_i = 0; scan_i < edge_scan_count; scan_i++) { + int64_t ei; + int64_t oi; + if (use_adj) { + ei = (scan_i < from_len) + ? g->adj_from[cur][scan_i] + : g->adj_to[cur][scan_i - from_len]; + EngramEdge* e = &g->edges[ei]; + oi = (scan_i < from_len) + ? engram_idmap_get(g, e->to_id) + : engram_idmap_get(g, e->from_id); + } else { + /* Fallback: linear scan */ + ei = scan_i; + EngramEdge* e = &g->edges[ei]; + const char* other = NULL; + const char* cur_id = g->nodes[cur].id; + if (e->from_id && strcmp(e->from_id, cur_id) == 0) other = e->to_id; + else if (e->to_id && strcmp(e->to_id, cur_id) == 0) other = e->from_id; + else continue; + oi = engram_find_node_index(other); + } + if (oi < 0 || oi >= g->node_count) continue; + EngramEdge* e = &g->edges[ei]; + EngramNode* on = &g->nodes[oi]; + /* Never propagate INTO InternalStateEvent nodes. They are already + * barred from WM promotion (pass 2) and from seeding (above), but + * as high-degree hubs they still relayed activation across the + * graph. Skipping here keeps them out of the frontier entirely. */ + if (on->node_type && strcmp(on->node_type, "InternalStateEvent") == 0) + continue; + double tbonus = engram_temporal_proximity_bonus(on->created_at, seed_epoch); + double tdecay = engram_temporal_decay(on, now_ms); + double dampen = engram_activation_dampen(on); + /* ── Query-aware propagation gate (2026-07-27 self-review) ── + * Prior behavior was "query-blind" spreading: the query chose + * the seeds, but propagation depended only on graph structure, + * so high-degree hubs relayed activation into branches with no + * semantic relation to the query. Per arXiv:2606.30133, gating + * each increment by the TARGET node's query similarity + * (sigma(v) = max(cos(e_v, e_q), 0)) prunes low-information + * branches at every hop (+3.6..+7.4 F1 over uniform spreading, + * 1.5-4.9x faster via a shrinking working set). + * + * Adaptation for partial embedding coverage: the paper skips + * unembedded targets outright, but only eligible non-ISE/Tag + * nodes carry embeddings here — a hard gate would sever purely + * lexical/structural pathways. So: embedded targets get a soft + * gate FLOOR + (1-FLOOR)*clip(cos) (dissimilar nodes damped + * ~4x, never killed); unembedded targets pass ungated (no + * information, no penalty); cosq == NULL (embedder down) means + * no gating at all — same graceful degradation as seeding. */ + double qgate = 1.0; + if (cosq && cosq[oi] > -1.5) { + double c = cosq[oi] > 0.0 ? cosq[oi] : 0.0; + qgate = ENGRAM_QGATE_FLOOR + (1.0 - ENGRAM_QGATE_FLOOR) * c; + } + double new_act = f.act * e->weight * SPREAD_DECAY * (1.0 + tbonus) + * tdecay * dampen * qgate; + /* Firing threshold per classic spreading-activation: sub-threshold + * activation neither updates the target nor enqueues it, so weak + * signals die out instead of flooding the whole graph with tiny + * nonzero background activation. */ + if (new_act < 0.02) continue; + if (!reached[oi] || new_act > best_bg[oi]) { + best_bg[oi] = new_act; + best_hops[oi] = new_hops; + reached[oi] = 1; + if (ftail < fcap) { + fr[ftail].idx = oi; + fr[ftail].hops = new_hops; + fr[ftail].act = new_act; + ftail++; + } + } + } + } + /* Persist layer-1 background_activation to node store. */ + for (int64_t i = 0; i < g->node_count; i++) { + g->nodes[i].background_activation = reached[i] ? best_bg[i] : 0.0; + } + + /* ── PASS 2: executive filter → working memory promotion ──────────── */ + /* Step A: collect inhibitory suppressions from fired inhibitory edges. + * Layered consciousness: inhibition is ONLY recorded against targets + * whose layer is `suppressible == 1`. Nodes in non-suppressible layers + * (Layer 0 / safety) ignore inhibitory edges entirely — their working + * memory weight cannot be silenced by attentional suppression. */ + double* inhibition = calloc((size_t)g->node_count, sizeof(double)); + if (!inhibition) { + free(best_bg); free(best_hops); free(reached); free(seeds); free(fr); + free(cosq); return out; + } + for (int64_t ei = 0; ei < g->edge_count; ei++) { + EngramEdge* e = &g->edges[ei]; + if (!e->inhibitory) continue; + int64_t src = engram_find_node_index(e->from_id); + int64_t tgt = engram_find_node_index(e->to_id); + if (src < 0 || tgt < 0) continue; + if (!reached[src] || best_bg[src] <= 0.0) continue; + /* Skip if target layer is non-suppressible: Layer 0 / safety nodes + * are immune to inhibitory edges from any source. The pass-3 + * override below also force-promotes them, but recording inhibition + * against them at all would be wasted work and could confuse + * downstream debugging output. */ + if (!engram_layer_is_suppressible(g->nodes[tgt].layer_id)) continue; + /* Inhibition strength proportional to source background activation + * and edge weight. Takes the maximum if multiple inhibitory edges + * target the same node. */ + double inh = best_bg[src] * e->weight; + if (inh > inhibition[tgt]) inhibition[tgt] = inh; + } + /* Step B: compute working_memory_weight per candidate node. */ + double* wm_weights = calloc((size_t)g->node_count, sizeof(double)); + if (!wm_weights) { + free(best_bg); free(best_hops); free(reached); free(seeds); + free(fr); free(inhibition); free(cosq); return out; + } + /* Per-call breakthrough budget (2026-08-02) — see ENGRAM_BREAKTHROUGH_BUDGET. */ + int64_t bt_budget = ENGRAM_BREAKTHROUGH_BUDGET; + for (int64_t i = 0; i < g->node_count; i++) { + if (!reached[i] || best_bg[i] <= 0.0) continue; + EngramNode* n = &g->nodes[i]; + /* InternalStateEvent nodes are observability-only — never admit to WM. + * Their JSON content (curiosity seeds, heartbeat payloads) contains common + * words that trigger lexical seeding (e.g. "knowledge" in curiosity ISEs), + * leading to repeated suppression and eventual breakthrough at the floor. + * ISEs surfacing in context compilation are noise, not signal. Clear their + * suppression_count so they don't build toward breakthrough, then skip. + * (2026-06-30 self-review: porting fix from 2026-06-26 branch; SYNAPSE + * paper confirms WM should hold only semantically relevant content.) */ + if (n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0) { + n->suppression_count = 0; + wm_weights[i] = 0.0; + continue; + } + /* Per-type threshold: safety nodes break through more easily. */ + double type_threshold = engram_type_threshold(n->node_type, n->tier); + /* Goal bias weights the node's relevance to current intent. */ + double bias = engram_goal_bias(n, q); + /* Raw working memory score. + * Importance factor (2026-07-28 self-review): n->importance was + * stored, serialized, and clamped at creation (default 0.5) but + * never read by any scoring path — a curated importance=1.0 node + * competed identically with a throwaway note at equal activation. + * Map it to a gentle multiplier centered on the 0.5 default: + * impf = 0.5 + importance → default nodes unchanged (×1.0), + * critical (1.0) ×1.5, low (0.1) ×0.6. Nodes loaded from legacy + * snapshots with importance<=0 stay neutral rather than being + * silently suppressed. */ + double impf = (n->importance > 0.0) ? (0.5 + n->importance) : 1.0; + double raw_wm = best_bg[i] * bias * n->confidence * impf; + /* Apply inhibitory suppression. Full inhibition → scale by factor. */ + double inh = inhibition[i]; + if (inh > 1.0) inh = 1.0; + double suppress = 1.0 - (1.0 - ENGRAM_INHIBITION_FACTOR) * inh; + raw_wm *= suppress; + /* Short-term inhibition-of-return (2026-07-25, Lebiere-Best): + * damp by t_n/(t_n + t_s) where t_n = seconds since the most + * recent recorded access (WM promotion / strengthen). A node that + * just held a WM slot yields it even to structurally stronger + * competitors, and recovers as t_n grows. access_ts is recorded + * at promotion, so persistent WM residents self-inhibit. Nodes + * with no access history (never promoted) are uninhibited. + * Layer-0 override in Pass 3 still floors safety nodes. */ + if (n->access_filled > 0) { + int32_t sti_last = (n->access_head + ENGRAM_BLL_K - 1) + % ENGRAM_BLL_K; + double sti_tn = (double)(now_ms - n->access_ts[sti_last]) + / 1000.0; + if (sti_tn < 0.1) sti_tn = 0.1; + raw_wm *= sti_tn / (sti_tn + ENGRAM_STI_TS); + } + /* Additive semantic-relevance term (2026-07-24, bl-b2d1c944): + * shift-and-floor at S0 — nomic-embed scores unrelated pairs + * 0.4–0.5, so raw cosine in a weighted sum would be a constant + * bias swamping the decayed base-level signal. Above S0 the term + * ramps 0 → WM_WEIGHT, breaking ties among structurally equivalent + * candidates in favor of nodes that mean what the query means. + * + * MOVED AFTER the STI damper (2026-08-02 self-review). It used to + * be added BEFORE, so the recency multiplier scaled the semantic + * term too: an incumbent re-reached 30s later took t_n/(t_n+120) + * = 0.2×, cutting the cosine term's ceiling from 0.20 to 0.04 — + * below every per-type threshold (0.15–0.40). Meaning-match was + * being punished for having been recently useful. Inhibition-of- + * return should rotate the STRUCTURAL score (what the graph + * dragged in), not the semantic one (what the query actually + * means); relevance to the current query is not stale merely + * because the node was in WM a moment ago. */ + if (cosq && cosq[i] > ENGRAM_EMBED_S0) { + raw_wm += ENGRAM_EMBED_WM_WEIGHT + * (cosq[i] - ENGRAM_EMBED_S0) / (1.0 - ENGRAM_EMBED_S0); + } + /* Threshold gate: must exceed per-type threshold to enter working + * memory. Type threshold replaces the old flat 0.2 filter. */ + if (raw_wm >= type_threshold) { + wm_weights[i] = raw_wm > 1.0 ? 1.0 : raw_wm; + if (n->suppression_count > 0) n->suppression_count = 0; + } else { + /* Node didn't make it through — increment suppression counter. + * After N consecutive suppressions it MAY force a breakthrough, + * subject to the per-call budget and the negative-count cooldown + * (2026-08-02 — see ENGRAM_BREAKTHROUGH_BUDGET/_COOLDOWN). */ + n->suppression_count++; + if (n->suppression_count >= ENGRAM_SUPPRESSION_BREAKTHROUGH + && bt_budget > 0) { + /* Graded breakthrough weight (2026-08-02): previously every + * breakthrough landed on exactly ENGRAM_BREAKTHROUGH_WEIGHT, + * so hundreds tied at 0.10 and the rank-cap tie-break at the + * cutoff degenerated to node-array index order — i.e. whoever + * was inserted earliest won, which is not a cognitive + * criterion. Scale within ±10% by how close the node came to + * its own threshold, so a near-miss outranks a node that was + * nowhere near. Stays strictly below min(type_threshold) + * (0.15) and strictly above ENGRAM_WM_FLOOR (0.05), which is + * the invariant ENGRAM_BREAKTHROUGH_WEIGHT documents. */ + double near = (type_threshold > 0.0) + ? (raw_wm / type_threshold) : 0.0; + if (near < 0.0) near = 0.0; + if (near > 1.0) near = 1.0; + wm_weights[i] = ENGRAM_BREAKTHROUGH_WEIGHT + * (0.9 + 0.2 * near); + /* Negative = cooldown. Must climb back through the cooldown + * before it can breach again. */ + n->suppression_count = -ENGRAM_BREAKTHROUGH_COOLDOWN; + bt_budget--; + _eg_act_breakthroughs++; + } else { + wm_weights[i] = 0.0; + /* Budget-starved or cooling down: do NOT reset the counter — + * let it saturate so the node surfaces on a later call rather + * than restarting its climb from zero. Cap the ceiling so the + * int32 cannot drift unbounded over a long uptime. */ + if (n->suppression_count > ENGRAM_SUPPRESSION_BREAKTHROUGH * 4) + n->suppression_count = ENGRAM_SUPPRESSION_BREAKTHROUGH * 4; + } + } + } + /* ── PASS 3: Layer 0 override (the sacred fire) ───────────────────── + * Every node in a non-suppressible layer that received any background + * activation is force-promoted to AT LEAST ENGRAM_LAYER0_OVERRIDE_WEIGHT. + * This runs LAST and overrides whatever Pass 2 decided — Layer 0 cannot + * be silenced by inhibitory edges, by goal-bias misalignment, by + * confidence weighting, or by per-type threshold gates. If the seed + * fan-out reached a structural-floor node, that node surfaces. + * + * Note: this also clears the suppression_count when an override fires, + * since the node DID surface this turn — it just took the override path + * rather than the standard threshold path. Without this, a Layer 0 + * node with persistent inhibitory pressure would accumulate + * suppression_count forever and never reach the breakthrough state. */ + for (int64_t i = 0; i < g->node_count; i++) { + if (!reached[i] || best_bg[i] <= 0.0) continue; + EngramNode* n = &g->nodes[i]; + if (engram_layer_is_suppressible(n->layer_id)) continue; + if (wm_weights[i] < ENGRAM_LAYER0_OVERRIDE_WEIGHT) { + wm_weights[i] = ENGRAM_LAYER0_OVERRIDE_WEIGHT; + } + n->suppression_count = 0; + } + + /* ── PASS 4: WM capacity cap (per-call) ───────────────────────────────── + * Enforce ENGRAM_WM_CAP as a hard upper bound on nodes promoted in this + * activation call. Without this, broad curiosity seeds like "knowledge" + * promote 500+ nodes simultaneously — wm_avg_weight collapses to the + * breakthrough floor, goal-bias differentiation is lost, and working memory + * becomes useless. (Ported from 2026-06-26 self-review branch; observed + * 525 promoted for "knowledge", 524 at breakthrough floor 0.25, 1 natural.) */ + { + /* Absolute admission floor (2026-07-30): drop sub-floor candidates + * BEFORE rank-trimming, so the cap is filled only by nodes that clear + * an absolute bar — fill below ENGRAM_WM_CAP becomes reachable and + * wm_saturated becomes an informative signal. See ENGRAM_WM_FLOOR. */ + for (int64_t i = 0; i < g->node_count; i++) { + if (wm_weights[i] > 0.0 && wm_weights[i] < ENGRAM_WM_FLOOR) { + wm_weights[i] = 0.0; + _eg_act_wm_evicted++; + } + } + int64_t cap_count = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (wm_weights[i] > 0.0) cap_count++; + } + if (cap_count > ENGRAM_WM_CAP) { + double* cap_vals = malloc((size_t)cap_count * sizeof(double)); + if (cap_vals) { + int64_t ci = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (wm_weights[i] > 0.0) cap_vals[ci++] = wm_weights[i]; + } + qsort(cap_vals, (size_t)cap_count, sizeof(double), + engram_cmp_double_desc); + /* cap_vals[ENGRAM_WM_CAP-1] is the lowest weight that still + * fits inside the cap when sorted descending. */ + double cutoff = cap_vals[ENGRAM_WM_CAP - 1]; + free(cap_vals); + /* Count strictly above cutoff to handle ties correctly. */ + int64_t above = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (wm_weights[i] > cutoff) above++; + } + int64_t at_cutoff_slots = ENGRAM_WM_CAP - above; + /* Evict nodes that don't make the cut. */ + for (int64_t i = 0; i < g->node_count; i++) { + if (wm_weights[i] <= 0.0) continue; /* not promoted */ + if (wm_weights[i] > cutoff) continue; /* above cutoff */ + if (at_cutoff_slots > 0) { + at_cutoff_slots--; + continue; /* fills a slot */ + } + wm_weights[i] = 0.0; /* over cap: evict */ + _eg_act_wm_evicted++; + } + } + /* If malloc failed, skip cap — WM unbounded this call, no corruption. */ + } + } + + /* Pre-persist residency snapshot (2026-07-30): record which nodes held a + * WM slot BEFORE this call's results are written back. Used below to fold + * only NEW WM entrants into the context centroid — an incumbent that gets + * re-promoted every scan no longer re-entrenches the centroid each time, + * which was the remaining positive-feedback path in the WM→centroid→ + * e_eff→re-selection loop (fixation driver; cf. wm_top0_streak=1407 + * incident). NULL on OOM → fold falls back to previous behavior. */ + unsigned char* was_wm = malloc((size_t)g->node_count); + if (was_wm) { + for (int64_t i = 0; i < g->node_count; i++) + was_wm[i] = (g->nodes[i].working_memory_weight > 0.0) ? 1 : 0; + } + + /* Persist working_memory_weight (post Pass 4) to node store. + * + * Conversational thread continuity (ENGRAM_WM_DECAY): + * Nodes promoted in a previous turn but NOT reached by the current BFS + * fan-out retain a decayed weight rather than being zeroed. This models + * the brain's ability to maintain recent context across successive turns + * without requiring explicit re-activation. A node that was relevant one + * query ago stays weakly present in working memory; a node from two + * queries ago retains 0.7² ≈ 0.49 of its original weight; after ~5 quiet + * turns it falls below 0.01 and is effectively evicted (set to 0.0). + * + * NOTE: this was documented in the ENGRAM_WM_DECAY constant comment since + * the two-layer architecture was introduced, but was never implemented — + * unreached nodes were always zeroed unconditionally. Fixed 2026-06-30 + * self-review. */ + for (int64_t i = 0; i < g->node_count; i++) { + if (!reached[i] && g->nodes[i].working_memory_weight > 0.0) { + /* Carry-over decay: node held WM weight from prior activation but + * the current query's BFS fan-out did not reach it. + * + * 2026-07-22 self-review (bl-b17facdd): replaced the per-call + * multiplicative decay (weight *= ENGRAM_WM_DECAY) with the + * ACT-R/Petrov base-level scheme. The old form was call-rate- + * dependent — carried context died in seconds under rapid + * curiosity scans and lingered for hours under quiet loops. + * Now: hard-evict when base-level < τ (Soar-style forgetting); + * above τ, shape the weight held at promotion (wm_anchor) by the + * retrieval-probability logistic. Pure function of wall-clock + * time — idempotent no matter how often activate is called. */ + EngramNode* cn = &g->nodes[i]; + double anchor = (cn->wm_anchor > 0.0) ? cn->wm_anchor + : cn->working_memory_weight; + double B = engram_bll_base_level(cn, now_ms); + if (B < ENGRAM_BLL_TAU) { + cn->working_memory_weight = 0.0; + cn->wm_anchor = 0.0; /* keep anchor coherent with eviction */ + _eg_act_wm_evicted++; /* was uncounted before 2026-08-02 */ + } else { + double keep = 1.0 / (1.0 + exp(-(B - ENGRAM_BLL_TAU) + / ENGRAM_BLL_S)); + /* Occupancy inhibition (2026-07-26): decay the carried + * weight with hold time so an unreached incumbent cannot + * hold its anchor verbatim indefinitely. See + * ENGRAM_CARRY_TC comment for derivation. */ + double hold_s = (double)(now_ms - cn->last_activated) / 1000.0; + if (hold_s < 0.0) hold_s = 0.0; + double occ = ENGRAM_CARRY_TC / (ENGRAM_CARRY_TC + hold_s); + double w = anchor * keep * occ; + /* Evict floor raised 0.01 → ENGRAM_WM_FLOOR (2026-07-30): + * one consistent absolute bar across all WM entry/exit paths. */ + if (w < ENGRAM_WM_FLOOR) { + cn->working_memory_weight = 0.0; + cn->wm_anchor = 0.0; + _eg_act_wm_evicted++; /* was uncounted before 2026-08-02 */ + } else { + cn->working_memory_weight = w; + } + } + } else { + g->nodes[i].working_memory_weight = wm_weights[i]; + /* Anchor the promotion weight: carry-over decay above computes + * from this fixed point rather than compounding per call. + * Zero the anchor when the slot empties (2026-07-30): a stale + * anchor on an evicted node was a latent resurrection bug if the + * carry-over entry guard ever changes. */ + if (wm_weights[i] > 0.0) g->nodes[i].wm_anchor = wm_weights[i]; + else g->nodes[i].wm_anchor = 0.0; + } + } + + /* ── PASS 5: Global WM cap enforcement ─────────────────────────────────── + * Pass 4 capped this call's new candidates. But nodes already in WM from + * prior calls retain their persisted working_memory_weight (via the decay + * carry-over above). Over multiple activation calls total WM can grow well + * above ENGRAM_WM_CAP. This pass enforces the cap globally across ALL + * nodes in the store, keeping only the top ENGRAM_WM_CAP by current weight. + * Correct cognitive model: WM capacity is global (Cowan 2001); more recent + * activations outcompete older decayed ones. (Ported from 2026-06-26 + * self-review branch.) */ + { + /* Absolute admission floor, global pass (2026-07-30): see + * ENGRAM_WM_FLOOR. Sub-floor residents are dropped even when the + * global population is under cap — this is what lets wm_active + * drain below 24 during quiet periods. */ + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* fn = &g->nodes[i]; + if (fn->working_memory_weight > 0.0 && + fn->working_memory_weight < ENGRAM_WM_FLOOR) { + fn->working_memory_weight = 0.0; + fn->wm_anchor = 0.0; + _eg_act_wm_evicted++; + } + } + int64_t global_wm_count = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > 0.0) global_wm_count++; + } + if (global_wm_count > ENGRAM_WM_CAP) { + double* gvals = malloc((size_t)global_wm_count * sizeof(double)); + if (gvals) { + int64_t gi = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > 0.0) + gvals[gi++] = g->nodes[i].working_memory_weight; + } + qsort(gvals, (size_t)global_wm_count, sizeof(double), + engram_cmp_double_desc); + double gcutoff = gvals[ENGRAM_WM_CAP - 1]; + free(gvals); + int64_t gabove = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > gcutoff) gabove++; + } + int64_t gslots_at_cutoff = ENGRAM_WM_CAP - gabove; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + if (n->working_memory_weight <= 0.0) continue; + if (n->working_memory_weight > gcutoff) continue; + if (gslots_at_cutoff > 0) { + gslots_at_cutoff--; + continue; /* fills a slot */ + } + n->working_memory_weight = 0.0; /* evict: over global cap */ + n->wm_anchor = 0.0; /* keep anchor coherent */ + _eg_act_wm_evicted++; /* was uncounted before 2026-08-02 */ + } + } + /* If malloc failed, skip — WM over cap this call, no data corruption. */ + } + } + + /* ── Retrieval reinforcement (2026-07-18 self-review) ─────────────────── + * ACT-R base-level learning: retrieval strengthens memory. Before this, + * NOTHING in engram_activate updated last_activated/activation_count — + * only the rarely-called engram_strengthen did. Consequence: temporal + * decay aged every node from its last explicit strengthen (usually + * creation), so frequently-retrieved memories decayed identically to + * abandoned ones, and engram_activation_dampen() saw a frozen count. + * + * Scope deliberately narrow: reinforce ONLY nodes promoted to WM in THIS + * call that survived both capacity caps (wm_weights[i] > 0 = promoted this + * call; working_memory_weight > 0 = survived Pass 4/5 eviction). BFS + * fan-out touches thousands of nodes per curiosity scan — reinforcing all + * of them would flatten dampening and freeze decay globally. Promotion to + * working memory is the analog of actual retrieval (executive access), + * matching ACT-R where only completed retrievals add a base-level + * presentation. Carry-over nodes (reached[i]==0) are NOT reinforced: they + * persist by decay, they were not re-retrieved. */ + for (int64_t i = 0; i < g->node_count; i++) { + if (!reached[i] || wm_weights[i] <= 0.0) continue; + EngramNode* n = &g->nodes[i]; + if (n->working_memory_weight <= 0.0) continue; /* evicted by cap */ + n->last_activated = now_ms; + n->activation_count++; + /* Base-level presentation: WM promotion is the retrieval event. + * (2026-07-22 self-review — feeds engram_bll_base_level.) */ + engram_bll_record_access(n, now_ms); + } + + /* ── Collect all background-activated nodes for the return value ──── + * Callers see both layers. Context compilation uses only promoted nodes + * (working_memory_weight > 0). Sort: promoted first by wm_weight desc, + * then background-only by background_activation desc. */ + typedef struct { int64_t idx; double bg; double wm; double epist; int64_t hops; } Result; + Result* results = malloc((size_t)g->node_count * sizeof(Result)); + int64_t rcount = 0; + if (!results) { + free(best_bg); free(best_hops); free(reached); free(seeds); + free(fr); free(inhibition); free(wm_weights); free(cosq); + free(was_wm); return out; + } + for (int64_t i = 0; i < g->node_count; i++) { + if (!reached[i]) continue; + double epist = best_bg[i] * g->nodes[i].confidence; + /* Include if promoted to working memory OR if background activation + * is meaningful enough to report (epist >= 0.1). */ + if (epist < 0.1 && wm_weights[i] <= 0.0) continue; + results[rcount].idx = i; + results[rcount].bg = best_bg[i]; + results[rcount].wm = wm_weights[i]; + results[rcount].epist = epist; + results[rcount].hops = best_hops[i]; + rcount++; + } + /* Sort: promoted nodes first (by wm_weight desc), then background-only + * by background_activation desc. */ + for (int64_t i = 1; i < rcount; i++) { + Result key = results[i]; + int64_t j = i - 1; + while (j >= 0 && (results[j].wm < key.wm || + (results[j].wm == key.wm && results[j].bg < key.bg))) { + results[j + 1] = results[j]; + j--; + } + results[j + 1] = key; + } + /* ── Context centroid: fold in the touched nodes (2026-07-29) ─────── + * Results are sorted promoted-first by wm_weight desc, so the first + * ENGRAM_CTX_TOUCH_MAX embedded entries with wm > 0 are exactly the + * strongest WM survivors of THIS call — the same "promotion is the + * retrieval event" rule the BLL reinforcement pass uses. μ=0.9 EMA + * keeps any single scan's touches a minority contribution. */ + { + int touched = 0; + for (int64_t i = 0; i < rcount && touched < ENGRAM_CTX_TOUCH_MAX; i++) { + if (results[i].wm <= 0.0) break; /* promoted block exhausted */ + EngramNode* n = &g->nodes[results[i].idx]; + if (!n->emb || n->emb_dim <= 0) continue; + /* New-entrant gate (2026-07-30): skip nodes that already held a + * WM slot before this call — incumbents must not keep pulling + * the centroid toward themselves. Fresh topical shifts (new + * entrants + the query fold at call start) steer it instead. */ + if (was_wm && was_wm[results[i].idx]) continue; + eg_ctx_blend(n->emb, n->emb_dim); + touched++; + } + } + free(was_wm); + for (int64_t i = 0; i < rcount; i++) { + el_val_t entry = el_map_new(0); + entry = el_map_set(entry, EL_STR(el_strdup("node")), + engram_node_to_map(&g->nodes[results[i].idx])); + entry = el_map_set(entry, EL_STR(el_strdup("activation_strength")), + el_from_float(results[i].bg)); + entry = el_map_set(entry, EL_STR(el_strdup("working_memory_weight")), + el_from_float(results[i].wm)); + entry = el_map_set(entry, EL_STR(el_strdup("epistemic_confidence")), + el_from_float(results[i].epist)); + entry = el_map_set(entry, EL_STR(el_strdup("hops")), + (el_val_t)results[i].hops); + entry = el_map_set(entry, EL_STR(el_strdup("promoted")), + (el_val_t)(results[i].wm > 0.0 ? 1 : 0)); + out = el_list_append(out, entry); + } + free(best_bg); free(best_hops); free(reached); + free(seeds); free(fr); free(inhibition); free(wm_weights); free(results); + free(cosq); + return out; +} + +/* ── Engram persistence (JSON snapshot) ─────────────────────────────────── */ + +/* include_emb (2026-07-31 self-review): the ~5.7KB "emb" vector belongs ONLY + * in persistence/replication output (engram_save → snapshot.json, which also + * backs /api/sync and /api/edges via scratch exports). Consumer read routes + * (/api/nodes, /api/search, activation results, neighbors, compiled context) + * were shipping it on every node — responses 10-50x oversized, blowing MCP + * token limits. Pass include_emb=1 only from engram_save. */ +static void engram_emit_node_json(JsonBuf* b, const EngramNode* n, int include_emb) { + jb_putc(b, '{'); + jb_puts(b, "\"id\":"); jb_emit_escaped(b, n->id ? n->id : ""); + jb_puts(b, ",\"content\":"); jb_emit_escaped(b, n->content ? n->content : ""); + jb_puts(b, ",\"node_type\":"); jb_emit_escaped(b, n->node_type ? n->node_type : ""); + jb_puts(b, ",\"label\":"); jb_emit_escaped(b, n->label ? n->label : ""); + jb_puts(b, ",\"tier\":"); jb_emit_escaped(b, n->tier ? n->tier : "Working"); + jb_puts(b, ",\"tags\":"); jb_emit_escaped(b, n->tags ? n->tags : ""); + jb_puts(b, ",\"metadata\":"); jb_emit_escaped(b, n->metadata ? n->metadata : "{}"); + char tmp[80]; + snprintf(tmp, sizeof(tmp), ",\"salience\":%g", n->salience); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"importance\":%g", n->importance); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"confidence\":%g", n->confidence); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"temporal_decay_rate\":%g", n->temporal_decay_rate); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"activation_count\":%lld", (long long)n->activation_count); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"last_activated\":%lld", (long long)n->last_activated); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"created_at\":%lld", (long long)n->created_at); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"updated_at\":%lld", (long long)n->updated_at); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"background_activation\":%g", n->background_activation); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"working_memory_weight\":%g", n->working_memory_weight); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"suppression_count\":%d", n->suppression_count); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"layer_id\":%u", n->layer_id); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"wm_anchor\":%g", n->wm_anchor); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"base_level\":%g", + engram_bll_base_level(n, engram_now_ms())); jb_puts(b, tmp); + /* Base-level access history: chronological (oldest→newest) compact + * string. Loaders replay it through engram_bll_record_access; absent + * field = empty ring (optimized-form fallback). (2026-07-22) */ + if (n->access_filled > 0) { + jb_puts(b, ",\"access_ts\":\""); + for (int32_t j = 0; j < n->access_filled; j++) { + int32_t idx = (n->access_head - n->access_filled + j + + 2 * ENGRAM_BLL_K) % ENGRAM_BLL_K; + snprintf(tmp, sizeof(tmp), "%s%lld", j ? "," : "", + (long long)n->access_ts[idx]); + jb_puts(b, tmp); + } + jb_putc(b, '"'); + } + /* Semantic embedding (2026-07-24): compact %.4g comma list — cosine is + * insensitive to 4-sig-fig rounding, and this keeps snapshot bloat to + * ~5KB per embedded node without a base64 codec. Absent field = not + * embedded; the lazy backfill re-embeds eventually if dropped. */ + if (include_emb && n->emb && n->emb_dim > 0) { + jb_puts(b, ",\"emb\":\""); + for (int32_t j = 0; j < n->emb_dim; j++) { + snprintf(tmp, sizeof(tmp), "%s%.4g", j ? "," : "", + (double)n->emb[j]); + jb_puts(b, tmp); + } + jb_putc(b, '"'); + } + jb_putc(b, '}'); +} + +static void engram_emit_edge_json(JsonBuf* b, const EngramEdge* e) { + jb_putc(b, '{'); + jb_puts(b, "\"id\":"); jb_emit_escaped(b, e->id ? e->id : ""); + jb_puts(b, ",\"from_id\":"); jb_emit_escaped(b, e->from_id ? e->from_id : ""); + jb_puts(b, ",\"to_id\":"); jb_emit_escaped(b, e->to_id ? e->to_id : ""); + jb_puts(b, ",\"relation\":"); jb_emit_escaped(b, e->relation ? e->relation : ""); + jb_puts(b, ",\"metadata\":"); jb_emit_escaped(b, e->metadata ? e->metadata : "{}"); + char tmp[64]; + snprintf(tmp, sizeof(tmp), ",\"weight\":%g", e->weight); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"confidence\":%g", e->confidence); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"created_at\":%lld", (long long)e->created_at); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"updated_at\":%lld", (long long)e->updated_at); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"last_fired\":%lld", (long long)e->last_fired); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"inhibitory\":%d", e->inhibitory ? 1 : 0); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"layer_id\":%u", e->layer_id); jb_puts(b, tmp); + jb_putc(b, '}'); +} + +el_val_t engram_save(el_val_t path) { + const char* p = EL_CSTR(path); + if (!p || !*p) return 0; + EngramStore* g = engram_get(); + JsonBuf b; jb_init(&b); + jb_puts(&b, "{\"nodes\":["); + for (int64_t i = 0; i < g->node_count; i++) { + if (i > 0) jb_putc(&b, ','); + engram_emit_node_json(&b, &g->nodes[i], 1); + } + jb_puts(&b, "],\"edges\":["); + for (int64_t i = 0; i < g->edge_count; i++) { + if (i > 0) jb_putc(&b, ','); + engram_emit_edge_json(&b, &g->edges[i]); + } + /* Layered consciousness — emit the layer registry under "layers". + * Older readers that don't know about this top-level key will simply + * ignore it (forward compatible). Tombstoned (removed-injectable) + * layers are skipped — they have no name and can't be re-created + * meaningfully on load anyway. */ + jb_puts(&b, "],\"layers\":["); + int first_layer = 1; + for (size_t i = 0; i < g->layer_count; i++) { + EngramLayer* L = &g->layers[i]; + if (!L->name) continue; + if (!first_layer) jb_putc(&b, ','); + first_layer = 0; + jb_putc(&b, '{'); + char tmp[80]; + snprintf(tmp, sizeof(tmp), "\"layer_id\":%u", L->layer_id); + jb_puts(&b, tmp); + jb_puts(&b, ",\"name\":"); + jb_emit_escaped(&b, L->name); + snprintf(tmp, sizeof(tmp), ",\"activation_priority\":%u", L->activation_priority); + jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"suppressible\":%d", L->suppressible ? 1 : 0); + jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"transparent\":%d", L->transparent ? 1 : 0); + jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"injectable\":%d", L->injectable ? 1 : 0); + jb_puts(&b, tmp); + jb_putc(&b, '}'); + } + jb_puts(&b, "]}"); + FILE* f = fopen(p, "wb"); + if (!f) { free(b.buf); return 0; } + size_t w = fwrite(b.buf, 1, b.len, f); + fclose(f); + int ok = (w == b.len); + free(b.buf); + return ok ? 1 : 0; +} + +/* Helper: extract a string field from a JSON object substring. */ +static char* eg_get_str_field(const char* obj, const char* key) { + /* Returns a STORE-PERSISTENT string: callers assign the result directly + * into EngramNode/EngramEdge fields, and engram_load_merge runs inside + * the soul's per-tick arena. jp_parse_string_raw's success buffer is + * plain malloc (persist-safe, returned as-is); the empty/error returns + * were arena-tracked el_strdup("") — those dangled after the tick arena + * popped and free()ing them in callers was a latent double-free. */ + const char* p = json_find_key(obj, key); + if (!p) return el_strdup_persist(""); + if (*p != '"') return el_strdup_persist(""); + JsonParser jp = { .p = p, .end = p + strlen(p), .err = 0 }; + char* out = jp_parse_string_raw(&jp); + /* *p == '"' is guaranteed above, so jp_parse_string_raw took its malloc + * path (its arena-tracked early-return only fires on a non-'"' start) — + * free(out) on error is safe here. */ + if (jp.err) { free(out); return el_strdup_persist(""); } + return out; +} + +static double eg_get_num_field(const char* obj, const char* key) { + const char* p = json_find_key(obj, key); + if (!p || *p == '"' || *p == '{' || *p == '[') return 0.0; + return strtod(p, NULL); +} + +static int64_t eg_get_int_field(const char* obj, const char* key) { + const char* p = json_find_key(obj, key); + if (!p || *p == '"' || *p == '{' || *p == '[') return 0; + return strtoll(p, NULL, 10); +} + +/* Iterate the top-level nodes/edges arrays in a saved snapshot. */ +static const char* eg_skip_ws(const char* p) { + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; + return p; +} + +/* eg_enforce_wm_cap_on_load — clamp a snapshot-restored working-memory + * population to ENGRAM_WM_CAP. + * + * WHY (2026-07-15 self-review): engram_load restored working_memory_weight + * verbatim from the snapshot with no cap. Pass 4/5 cap enforcement only runs + * inside engram_activate — so a snapshot frozen in the pre-2026-06-30 era + * (87-111 promoted nodes, all at the old 0.25 breakthrough floor) reloaded + * as-is on every boot, and heartbeats faithfully reported the stale + * population (wm_active 87-111, wm_avg pinned at exactly 0.25) forever. + * The cap must hold at every entry point that materializes WM, not just + * the activation path. Same top-K-by-weight logic as engram_activate + * Pass 5 (Cowan 2001: WM capacity is global). */ +static void eg_enforce_wm_cap_on_load(EngramStore* g) { + /* Absolute admission floor (2026-07-30): see ENGRAM_WM_FLOOR. */ + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* fn = &g->nodes[i]; + if (fn->working_memory_weight > 0.0 && + fn->working_memory_weight < ENGRAM_WM_FLOOR) { + fn->working_memory_weight = 0.0; + fn->wm_anchor = 0.0; + } + } + int64_t wm_count = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > 0.0) wm_count++; + } + if (wm_count <= ENGRAM_WM_CAP) return; + double* vals = malloc((size_t)wm_count * sizeof(double)); + if (!vals) return; /* skip on OOM — over cap this boot, no corruption */ + int64_t vi = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > 0.0) + vals[vi++] = g->nodes[i].working_memory_weight; + } + qsort(vals, (size_t)wm_count, sizeof(double), engram_cmp_double_desc); + double cutoff = vals[ENGRAM_WM_CAP - 1]; + free(vals); + int64_t above = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > cutoff) above++; + } + int64_t slots_at_cutoff = ENGRAM_WM_CAP - above; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + if (n->working_memory_weight <= 0.0) continue; + if (n->working_memory_weight > cutoff) continue; + if (slots_at_cutoff > 0) { slots_at_cutoff--; continue; } + n->working_memory_weight = 0.0; /* evict: over cap at load */ + n->wm_anchor = 0.0; /* keep anchor coherent */ + } +} + +el_val_t engram_load(el_val_t path) { + const char* p = EL_CSTR(path); + if (!p || !*p) return 0; + FILE* f = fopen(p, "rb"); + if (!f) return 0; + fseek(f, 0, SEEK_END); + long sz = ftell(f); + rewind(f); + if (sz <= 0) { fclose(f); return 0; } + char* data = malloc((size_t)sz + 1); + if (!data) { fclose(f); return 0; } + size_t got = fread(data, 1, (size_t)sz, f); + fclose(f); + data[got] = '\0'; + + /* Reset store */ + EngramStore* g = engram_get(); + for (int64_t i = 0; i < g->node_count; i++) { + free(g->nodes[i].id); free(g->nodes[i].content); free(g->nodes[i].node_type); + free(g->nodes[i].label); free(g->nodes[i].tier); free(g->nodes[i].tags); + free(g->nodes[i].metadata); + /* 2026-07-26 self-review: emb was the one heap field not freed + * here — ~3 KB leaked per embedded node per reload (~11 MB per + * reload at 3.7k embedded). forget/prune already free it. */ + free(g->nodes[i].emb); g->nodes[i].emb = NULL; g->nodes[i].emb_dim = 0; + } + g->node_count = 0; + for (int64_t i = 0; i < g->edge_count; i++) { + free(g->edges[i].id); free(g->edges[i].from_id); free(g->edges[i].to_id); + free(g->edges[i].relation); free(g->edges[i].metadata); + } + g->edge_count = 0; + engram_idmap_free(g); + engram_adj_free(g); + + /* Walk nodes array */ + const char* nodes_p = json_find_key(data, "nodes"); + if (nodes_p) { + nodes_p = eg_skip_ws(nodes_p); + if (*nodes_p == '[') { + nodes_p++; + nodes_p = eg_skip_ws(nodes_p); + while (*nodes_p && *nodes_p != ']') { + if (*nodes_p != '{') { nodes_p++; continue; } + const char* end = json_skip_value(nodes_p); + size_t n = (size_t)(end - nodes_p); + char* obj = malloc(n + 1); + memcpy(obj, nodes_p, n); obj[n] = '\0'; + engram_grow_nodes(); + EngramNode* nn = &g->nodes[g->node_count]; + memset(nn, 0, sizeof(*nn)); + nn->id = eg_get_str_field(obj, "id"); + nn->content = eg_get_str_field(obj, "content"); + nn->node_type = eg_get_str_field(obj, "node_type"); + nn->label = eg_get_str_field(obj, "label"); + nn->tier = eg_get_str_field(obj, "tier"); + nn->tags = eg_get_str_field(obj, "tags"); + nn->metadata = eg_get_str_field(obj, "metadata"); + if (!nn->metadata || !*nn->metadata) { free(nn->metadata); nn->metadata = el_strdup_persist("{}"); } + nn->salience = eg_get_num_field(obj, "salience"); + nn->importance = eg_get_num_field(obj, "importance"); + nn->confidence = eg_get_num_field(obj, "confidence"); + nn->temporal_decay_rate = eg_get_num_field(obj, "temporal_decay_rate"); + /* temporal_decay_rate defaults to 0 (use global) if absent in snapshot */ + nn->activation_count = eg_get_int_field(obj, "activation_count"); + nn->last_activated = eg_get_int_field(obj, "last_activated"); + nn->created_at = eg_get_int_field(obj, "created_at"); + nn->updated_at = eg_get_int_field(obj, "updated_at"); + nn->background_activation = eg_get_num_field(obj, "background_activation"); + nn->working_memory_weight = eg_get_num_field(obj, "working_memory_weight"); + /* Launder persisted WM weights across restarts: snapshots carry + * legacy breakthrough-floor 0.25 weights (pinned by the old + * suppression-breakthrough path) and nothing else ever launders + * them. Halve on every boot so genuine working-memory state has + * continuity across a restart while stale pinned weights decay + * out over successive boots; sub-0.05 residue drops to zero. */ + nn->working_memory_weight *= 0.5; + if (nn->working_memory_weight < ENGRAM_WM_FLOOR) nn->working_memory_weight = 0.0; + nn->suppression_count = (int32_t)eg_get_int_field(obj, "suppression_count"); + /* layer_id defaults to ENGRAM_LAYER_DEFAULT (core-identity) + * for snapshots that predate the layered schema. We can't + * tell "explicit 0" from "missing field" using the helper + * directly, so probe for the key — if absent, fall back. */ + if (json_find_key(obj, "layer_id")) { + nn->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); + } else { + nn->layer_id = ENGRAM_LAYER_DEFAULT; + } + /* Base-level state (2026-07-22): absent fields leave the ring + * empty (memset) → optimized-form fallback. */ + nn->wm_anchor = eg_get_num_field(obj, "wm_anchor"); + { + char* ats = eg_get_str_field(obj, "access_ts"); + if (ats) { engram_bll_parse_access(nn, ats); free(ats); } + } + { + char* es = eg_get_str_field(obj, "emb"); + if (es) { eg_parse_emb(nn, es); free(es); } + } + int64_t load_idx = g->node_count; + g->node_count++; + if (nn->id && *nn->id) engram_idmap_put(g, nn->id, load_idx); + free(obj); + nodes_p = end; + nodes_p = eg_skip_ws(nodes_p); + if (*nodes_p == ',') { nodes_p++; nodes_p = eg_skip_ws(nodes_p); } + } + } + } + g->adj_dirty = 1; + /* Walk edges array */ + const char* edges_p = json_find_key(data, "edges"); + if (edges_p) { + edges_p = eg_skip_ws(edges_p); + if (*edges_p == '[') { + edges_p++; + edges_p = eg_skip_ws(edges_p); + while (*edges_p && *edges_p != ']') { + if (*edges_p != '{') { edges_p++; continue; } + const char* end = json_skip_value(edges_p); + size_t n = (size_t)(end - edges_p); + char* obj = malloc(n + 1); + memcpy(obj, edges_p, n); obj[n] = '\0'; + engram_grow_edges(); + EngramEdge* ee = &g->edges[g->edge_count]; + memset(ee, 0, sizeof(*ee)); + ee->id = eg_get_str_field(obj, "id"); + ee->from_id = eg_get_str_field(obj, "from_id"); + ee->to_id = eg_get_str_field(obj, "to_id"); + ee->relation = eg_get_str_field(obj, "relation"); + ee->metadata = eg_get_str_field(obj, "metadata"); + if (!ee->metadata || !*ee->metadata) { free(ee->metadata); ee->metadata = el_strdup_persist("{}"); } + ee->weight = eg_get_num_field(obj, "weight"); + ee->confidence = eg_get_num_field(obj, "confidence"); + ee->created_at = eg_get_int_field(obj, "created_at"); + ee->updated_at = eg_get_int_field(obj, "updated_at"); + ee->last_fired = eg_get_int_field(obj, "last_fired"); + ee->inhibitory = (int)eg_get_int_field(obj, "inhibitory"); + if (json_find_key(obj, "layer_id")) { + ee->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); + } else { + ee->layer_id = ENGRAM_LAYER_DEFAULT; + } + g->edge_count++; + free(obj); + edges_p = end; + edges_p = eg_skip_ws(edges_p); + if (*edges_p == ',') { edges_p++; edges_p = eg_skip_ws(edges_p); } + } + } + } + /* Walk layers array (optional — older snapshots omit this). + * If present we replace the canonical registry entirely; if absent we + * keep whatever the engram_get() init established. */ + const char* layers_p = json_find_key(data, "layers"); + if (layers_p) { + layers_p = eg_skip_ws(layers_p); + if (*layers_p == '[') { + /* Reset existing layer registry. Free strdup'd names; the + * struct array itself can be reused. */ + for (size_t i = 0; i < g->layer_count; i++) { + if (g->layers[i].name) free(g->layers[i].name); + g->layers[i].name = NULL; + } + g->layer_count = 0; + + layers_p++; + layers_p = eg_skip_ws(layers_p); + while (*layers_p && *layers_p != ']') { + if (*layers_p != '{') { layers_p++; continue; } + const char* end = json_skip_value(layers_p); + size_t n = (size_t)(end - layers_p); + char* obj = malloc(n + 1); + memcpy(obj, layers_p, n); obj[n] = '\0'; + if (g->layer_count >= g->layer_capacity) { + size_t nc = g->layer_capacity ? g->layer_capacity * 2 : 16; + EngramLayer* grown = realloc(g->layers, nc * sizeof(EngramLayer)); + if (!grown) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + memset(grown + g->layer_capacity, 0, + (nc - g->layer_capacity) * sizeof(EngramLayer)); + g->layers = grown; + g->layer_capacity = nc; + } + EngramLayer* L = &g->layers[g->layer_count]; + memset(L, 0, sizeof(*L)); + L->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); + L->activation_priority = (uint32_t)eg_get_int_field(obj, "activation_priority"); + L->suppressible = (int)eg_get_int_field(obj, "suppressible") ? 1 : 0; + L->transparent = (int)eg_get_int_field(obj, "transparent") ? 1 : 0; + L->injectable = (int)eg_get_int_field(obj, "injectable") ? 1 : 0; + char* nm = eg_get_str_field(obj, "name"); + if (nm && *nm) { + L->name = el_strdup_persist(nm); + free(nm); + } else { + free(nm); + L->name = el_strdup_persist(""); + } + g->layer_count++; + free(obj); + layers_p = end; + layers_p = eg_skip_ws(layers_p); + if (*layers_p == ',') { layers_p++; layers_p = eg_skip_ws(layers_p); } + } + } + } + /* WM cap discipline applies to every entry point that materializes WM, + * including snapshot restore (see eg_enforce_wm_cap_on_load). */ + eg_enforce_wm_cap_on_load(g); + free(data); + return 1; +} + +/* engram_load_merge — like engram_load but WITHOUT resetting the store. + * Reads a JSON snapshot from `path` and adds any nodes/edges not already + * present in the in-memory graph. Dedup is by node id (for nodes) and by + * (from_id, to_id, relation) tuple (for edges). + * + * Returns (as an EL int) the count of new nodes added. Used by the soul + * daemon's periodic refresh cycle to keep its in-process Engram in sync + * with the HTTP Engram store without losing current working memory state. + * Ported from el-compiler/runtime on 2026-06-30 self-review. */ +el_val_t engram_load_merge(el_val_t path) { + const char* p = EL_CSTR(path); + if (!p || !*p) return 0; + FILE* f = fopen(p, "rb"); + if (!f) return 0; + fseek(f, 0, SEEK_END); + long sz = ftell(f); + rewind(f); + if (sz <= 0) { fclose(f); return 0; } + char* data = malloc((size_t)sz + 1); + if (!data) { fclose(f); return 0; } + size_t got = fread(data, 1, (size_t)sz, f); + fclose(f); + data[got] = '\0'; + + EngramStore* g = engram_get(); + int64_t added_nodes = 0; + + /* Walk nodes array — skip any node whose id already exists */ + const char* nodes_p = json_find_key(data, "nodes"); + if (nodes_p) { + nodes_p = eg_skip_ws(nodes_p); + if (*nodes_p == '[') { + nodes_p++; + nodes_p = eg_skip_ws(nodes_p); + while (*nodes_p && *nodes_p != ']') { + if (*nodes_p != '{') { nodes_p++; continue; } + const char* end = json_skip_value(nodes_p); + size_t n = (size_t)(end - nodes_p); + char* obj = malloc(n + 1); + memcpy(obj, nodes_p, n); obj[n] = '\0'; + char* nid = eg_get_str_field(obj, "id"); + /* Nodes with an empty/unparseable id can never dedup against + * the idmap, so without this guard they were re-added on EVERY + * merge cycle — unbounded store growth. Skip them entirely. */ + int has_id = (nid && *nid); + int already = (has_id && engram_find_node(nid) != NULL); + free(nid); + if (has_id && !already) { + engram_grow_nodes(); + EngramNode* nn = &g->nodes[g->node_count]; + memset(nn, 0, sizeof(*nn)); + nn->id = eg_get_str_field(obj, "id"); + nn->content = eg_get_str_field(obj, "content"); + nn->node_type = eg_get_str_field(obj, "node_type"); + nn->label = eg_get_str_field(obj, "label"); + nn->tier = eg_get_str_field(obj, "tier"); + nn->tags = eg_get_str_field(obj, "tags"); + nn->metadata = eg_get_str_field(obj, "metadata"); + if (!nn->metadata || !*nn->metadata) { free(nn->metadata); nn->metadata = strdup("{}"); } + nn->salience = eg_get_num_field(obj, "salience"); + nn->importance = eg_get_num_field(obj, "importance"); + nn->confidence = eg_get_num_field(obj, "confidence"); + nn->temporal_decay_rate = eg_get_num_field(obj, "temporal_decay_rate"); + nn->activation_count = eg_get_int_field(obj, "activation_count"); + nn->last_activated = eg_get_int_field(obj, "last_activated"); + nn->created_at = eg_get_int_field(obj, "created_at"); + nn->updated_at = eg_get_int_field(obj, "updated_at"); + nn->background_activation = eg_get_num_field(obj, "background_activation"); + /* Nodes arriving via merge were never part of THIS store's + * working memory — importing the source store's WM weight + * would inject foreign (often legacy 0.25 breakthrough- + * floor) weights into local WM every sync cycle. */ + nn->working_memory_weight = 0.0; + nn->suppression_count = (int32_t)eg_get_int_field(obj, "suppression_count"); + if (json_find_key(obj, "layer_id")) { + nn->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); + } else { + nn->layer_id = ENGRAM_LAYER_DEFAULT; + } + /* Base-level history merges with the node (2026-07-22); + * wm_anchor stays 0 — WM state is local-only (see the + * working_memory_weight comment above). */ + { + char* ats = eg_get_str_field(obj, "access_ts"); + if (ats) { engram_bll_parse_access(nn, ats); free(ats); } + } + { + char* es = eg_get_str_field(obj, "emb"); + if (es) { eg_parse_emb(nn, es); free(es); } + } + int64_t merge_idx = g->node_count; + g->node_count++; + added_nodes++; + if (nn->id && *nn->id) engram_idmap_put(g, nn->id, merge_idx); + g->adj_dirty = 1; + } + free(obj); + nodes_p = end; + nodes_p = eg_skip_ws(nodes_p); + if (*nodes_p == ',') { nodes_p++; nodes_p = eg_skip_ws(nodes_p); } + } + } + } + + /* Walk edges array — skip if (from_id, to_id, relation) already present */ + const char* edges_p = json_find_key(data, "edges"); + if (edges_p) { + edges_p = eg_skip_ws(edges_p); + if (*edges_p == '[') { + edges_p++; + edges_p = eg_skip_ws(edges_p); + while (*edges_p && *edges_p != ']') { + if (*edges_p != '{') { edges_p++; continue; } + const char* end = json_skip_value(edges_p); + size_t n = (size_t)(end - edges_p); + char* obj = malloc(n + 1); + memcpy(obj, edges_p, n); obj[n] = '\0'; + char* efrom = eg_get_str_field(obj, "from_id"); + char* eto = eg_get_str_field(obj, "to_id"); + char* erel = eg_get_str_field(obj, "relation"); + int dup = 0; + if (efrom && eto && erel) { + for (int64_t ei = 0; ei < g->edge_count; ei++) { + EngramEdge* ex = &g->edges[ei]; + if (ex->from_id && ex->to_id && ex->relation && + strcmp(ex->from_id, efrom) == 0 && + strcmp(ex->to_id, eto) == 0 && + strcmp(ex->relation, erel) == 0) { + dup = 1; break; + } + } + } + if (!dup) { + engram_grow_edges(); + EngramEdge* ee = &g->edges[g->edge_count]; + memset(ee, 0, sizeof(*ee)); + ee->id = eg_get_str_field(obj, "id"); + ee->from_id = efrom ? efrom : strdup(""); + ee->to_id = eto ? eto : strdup(""); + ee->relation = erel ? erel : strdup(""); + ee->metadata = eg_get_str_field(obj, "metadata"); + if (!ee->metadata || !*ee->metadata) { free(ee->metadata); ee->metadata = strdup("{}"); } + ee->weight = eg_get_num_field(obj, "weight"); + ee->confidence = eg_get_num_field(obj, "confidence"); + ee->created_at = eg_get_int_field(obj, "created_at"); + ee->updated_at = eg_get_int_field(obj, "updated_at"); + ee->last_fired = eg_get_int_field(obj, "last_fired"); + ee->inhibitory = (int)eg_get_int_field(obj, "inhibitory"); + if (json_find_key(obj, "layer_id")) { + ee->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); + } else { + ee->layer_id = ENGRAM_LAYER_DEFAULT; + } + g->edge_count++; + efrom = NULL; eto = NULL; erel = NULL; + } else { + free(efrom); free(eto); free(erel); + } + free(obj); + edges_p = end; + edges_p = eg_skip_ws(edges_p); + if (*edges_p == ',') { edges_p++; edges_p = eg_skip_ws(edges_p); } + } + } + } + + /* Merged nodes can carry snapshot WM weights too — hold the cap here + * as well (see eg_enforce_wm_cap_on_load). */ + eg_enforce_wm_cap_on_load(g); + free(data); + return (el_val_t)added_nodes; +} + +/* ── Engram JSON-string accessors ───────────────────────────────────────── + * These return pre-serialized JSON strings so callers (especially HTTP + * handlers) don't have to round-trip ElList/ElMap through json_stringify + * — which can't reliably distinguish those structures from raw pointers + * due to el_val_t's type erasure. The runtime knows the real C types and + * can serialize directly. */ + +el_val_t engram_get_node_json(el_val_t id) { + const char* sid = EL_CSTR(id); + EngramNode* n = engram_find_node(sid); + if (!n) return el_wrap_str(el_strdup("{}")); + JsonBuf b; jb_init(&b); + engram_emit_node_json(&b, n, 0); + return el_wrap_str(b.buf); +} + +/* engram_get_node_by_label — find the first node whose label field exactly + * matches the given string. Returns the node as a JSON object string, or "{}" + * if no match is found. + * + * Used by chat.el to retrieve well-known nodes (e.g. "conv:history", + * "session:summary") by their stable label rather than by ID, which is immune + * to vector index drift across restarts. + * + * Exact match (strcmp, not istr_contains) because labels like "conv:history" + * must not collide with nodes whose content happens to contain that substring. + * + * Added 2026-07-01 self-review: was called in chat.el but never defined, + * causing build failure since June 30. */ +el_val_t engram_get_node_by_label(el_val_t label) { + const char* lbl = EL_CSTR(label); + if (!lbl || !*lbl) return el_wrap_str(el_strdup("{}")); + EngramStore* g = engram_get(); + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + if (n->label && strcmp(n->label, lbl) == 0) { + JsonBuf b; jb_init(&b); + engram_emit_node_json(&b, n, 0); + return el_wrap_str(b.buf); + } + } + return el_wrap_str(el_strdup("{}")); +} + +el_val_t engram_search_json(el_val_t query, el_val_t limit) { + EngramStore* g = engram_get(); + const char* q = EL_CSTR(query); + int64_t lim = (int64_t)limit; + if (lim <= 0) lim = 100; + JsonBuf b; jb_init(&b); + jb_putc(&b, '['); + int first = 1; + if (q && *q) { + /* Tokenized + ranked, same scheme as engram_search: match ANY query + * token, rank by distinct-token coverage then salience, cap at lim. + * (2026-07-19 port of the 2026-07-14 tokenized-search fix) */ + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS); + if (ntok > 0) { + EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry)); + if (hits) { + int64_t nhits = 0; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + /* Filter transparent layers — same as engram_search. */ + if (engram_layer_is_transparent(n->layer_id)) continue; + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0) { + hits[nhits].idx = i; + hits[nhits].score = sc; + hits[nhits].salience = n->salience; + nhits++; + } + } + qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); + int64_t end = nhits < lim ? nhits : lim; + for (int64_t k = 0; k < end; k++) { + if (!first) jb_putc(&b, ','); + engram_emit_node_json(&b, &g->nodes[hits[k].idx], 0); + first = 0; + } + free(hits); + } + } + } + jb_putc(&b, ']'); + return el_wrap_str(b.buf); +} + +el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset) { + EngramStore* g = engram_get(); + int64_t lim = (int64_t)limit; if (lim <= 0) lim = 100; + int64_t off = (int64_t)offset; if (off < 0) off = 0; + JsonBuf b; jb_init(&b); + jb_putc(&b, '['); + if (g->node_count == 0) { jb_putc(&b, ']'); return el_wrap_str(b.buf); } + int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t)); + if (!idx) { jb_putc(&b, ']'); return el_wrap_str(b.buf); } + /* Skip transparent layers — introspection filter, same as engram_scan_nodes. */ + int64_t live = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue; + idx[live++] = i; + } + engram_sort_indices_by_salience(idx, live, g->nodes); + int64_t end = off + lim; + if (end > live) end = live; + int first = 1; + for (int64_t i = off; i < end; i++) { + if (!first) jb_putc(&b, ','); + engram_emit_node_json(&b, &g->nodes[idx[i]], 0); + first = 0; + } + free(idx); + jb_putc(&b, ']'); + return el_wrap_str(b.buf); +} + +/* engram_scan_nodes_by_type_json — filter by node_type before paginating. + * Empty / NULL type_v falls back to the unfiltered scan (existing behaviour). + * Result is JSON array, salience-sorted, transparent layers skipped. */ +el_val_t engram_scan_nodes_by_type_json(el_val_t type_v, el_val_t limit, el_val_t offset) { + const char* type_filter = EL_CSTR(type_v); + if (!type_filter || !*type_filter) { + return engram_scan_nodes_json(limit, offset); + } + EngramStore* g = engram_get(); + int64_t lim = (int64_t)limit; if (lim <= 0) lim = 100; + int64_t off = (int64_t)offset; if (off < 0) off = 0; + JsonBuf b; jb_init(&b); + jb_putc(&b, '['); + if (g->node_count == 0) { jb_putc(&b, ']'); return el_wrap_str(b.buf); } + int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t)); + if (!idx) { jb_putc(&b, ']'); return el_wrap_str(b.buf); } + int64_t live = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue; + const char* nt = g->nodes[i].node_type; + if (!nt || strcmp(nt, type_filter) != 0) continue; + idx[live++] = i; + } + engram_sort_indices_by_salience(idx, live, g->nodes); + int64_t end = off + lim; + if (end > live) end = live; + int first = 1; + for (int64_t i = off; i < end; i++) { + if (!first) jb_putc(&b, ','); + engram_emit_node_json(&b, &g->nodes[idx[i]], 0); + first = 0; + } + free(idx); + jb_putc(&b, ']'); + return el_wrap_str(b.buf); +} + +el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction) { + /* Re-implement here directly so we serialize without going through + * the ElList path. Walks BFS to max_depth, emits {node, edge, hops} + * triples. */ + EngramStore* g = engram_get(); + const char* sid = EL_CSTR(node_id); + int64_t depth = (int64_t)max_depth; if (depth <= 0) depth = 1; + const char* dir = EL_CSTR(direction); if (!dir) dir = "both"; + int allow_out = (strcmp(dir, "out") == 0) || (strcmp(dir, "both") == 0); + int allow_in = (strcmp(dir, "in") == 0) || (strcmp(dir, "both") == 0); + JsonBuf b; jb_init(&b); + jb_putc(&b, '['); + if (!sid || !*sid) { jb_putc(&b, ']'); return el_wrap_str(b.buf); } + + /* Frontier of (node_id, hops). Cap to a sane size. */ + char** frontier = calloc(1024, sizeof(char*)); + int64_t* frontier_h = calloc(1024, sizeof(int64_t)); + int64_t fc = 0; + char** visited = calloc(1024, sizeof(char*)); + int64_t vc = 0; + if (!frontier || !frontier_h || !visited) { + free(frontier); free(frontier_h); free(visited); + jb_putc(&b, ']'); return el_wrap_str(b.buf); + } + /* MUST be el_strdup_persist: this function frees frontier/visited strings + * manually (lines below). el_strdup would ALSO register them in the + * per-request arena, so el_request_end() double-freed every one of them + * at the end of the HTTP request — SIGABRT in http_worker under load. + * (2026-07-18 self-review; reproduced via ASAN on /api/neuron/session/begin + * and /api/neuron/graph. Same allocation-discipline class as the + * 2026-07-15 EngramNode and 2026-07-16 idmap-key fixes: never mix arena + * tracking with manual free.) */ + frontier[fc] = el_strdup_persist(sid); frontier_h[fc] = 0; fc++; + visited[vc++] = el_strdup_persist(sid); + + int first = 1; + while (fc > 0) { + char* cur = frontier[0]; int64_t h = frontier_h[0]; + for (int64_t k = 1; k < fc; k++) { frontier[k-1] = frontier[k]; frontier_h[k-1] = frontier_h[k]; } + fc--; + if (h >= depth) { free(cur); continue; } + for (int64_t i = 0; i < g->edge_count; i++) { + EngramEdge* e = &g->edges[i]; + const char* peer = NULL; + if (allow_out && e->from_id && strcmp(e->from_id, cur) == 0) peer = e->to_id; + else if (allow_in && e->to_id && strcmp(e->to_id, cur) == 0) peer = e->from_id; + if (!peer) continue; + int seen = 0; + for (int64_t v = 0; v < vc; v++) { + if (strcmp(visited[v], peer) == 0) { seen = 1; break; } + } + if (seen) continue; + EngramNode* n = engram_find_node(peer); + if (!n) continue; + if (!first) jb_putc(&b, ','); + jb_puts(&b, "{\"node\":"); + engram_emit_node_json(&b, n, 0); + jb_puts(&b, ",\"edge\":"); + engram_emit_edge_json(&b, e); + char tmp[64]; snprintf(tmp, sizeof(tmp), ",\"hops\":%lld}", (long long)(h + 1)); + jb_puts(&b, tmp); + first = 0; + if (vc < 1024) visited[vc++] = el_strdup_persist(peer); + if (fc < 1024 && h + 1 < depth) { frontier[fc] = el_strdup_persist(peer); frontier_h[fc] = h + 1; fc++; } + } + free(cur); + } + for (int64_t i = 0; i < fc; i++) free(frontier[i]); + for (int64_t i = 0; i < vc; i++) free(visited[i]); + free(frontier); free(frontier_h); free(visited); + jb_putc(&b, ']'); + return el_wrap_str(b.buf); +} + +el_val_t engram_activate_json(el_val_t query, el_val_t depth) { + /* Run two-layer engram_activate and serialize the result list to JSON. + * Each entry includes both activation_strength (layer 1 background) and + * working_memory_weight (layer 2 executive filter), plus promoted flag. + * Callers performing context compilation should filter to promoted=1. */ + el_val_t lst = engram_activate(query, depth); + ElList* arr = (ElList*)(uintptr_t)lst; + JsonBuf b; jb_init(&b); + jb_putc(&b, '['); + if (arr) { + for (int64_t i = 0; i < arr->length; i++) { + if (!arr->elems[i]) continue; + el_val_t node_map = el_map_get(arr->elems[i], EL_STR("node")); + el_val_t strength_v = el_map_get(arr->elems[i], EL_STR("activation_strength")); + el_val_t wm_v = el_map_get(arr->elems[i], EL_STR("working_memory_weight")); + el_val_t epist_v = el_map_get(arr->elems[i], EL_STR("epistemic_confidence")); + el_val_t hops_v = el_map_get(arr->elems[i], EL_STR("hops")); + el_val_t promoted_v = el_map_get(arr->elems[i], EL_STR("promoted")); + /* Look up underlying EngramNode by id to emit canonical JSON. */ + el_val_t id_v = el_map_get(node_map, EL_STR("id")); + const char* id_s = EL_CSTR(id_v); + EngramNode* n = id_s ? engram_find_node(id_s) : NULL; + if (i > 0) jb_putc(&b, ','); + jb_puts(&b, "{\"node\":"); + if (n) { + engram_emit_node_json(&b, n, 0); + } else { + jb_puts(&b, "{}"); + } + char tmp[80]; + snprintf(tmp, sizeof(tmp), ",\"activation_strength\":%g", el_to_float(strength_v)); jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"working_memory_weight\":%g", el_to_float(wm_v)); jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"epistemic_confidence\":%g", el_to_float(epist_v)); jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"hops\":%lld", (long long)(int64_t)hops_v); jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"promoted\":%d}", (int)(int64_t)promoted_v); jb_puts(&b, tmp); + } + } + jb_putc(&b, ']'); + return el_wrap_str(b.buf); +} + +/* ── Working memory introspection helpers ──────────────────────────────────── + * + * These three functions give the soul daemon visibility into WM composition + * without re-running activation. Used in heartbeat ISEs and curiosity scans. + * Ported from el-compiler/runtime to releases/v1.0.0-20260501 on 2026-06-30 + * self-review (they were missing from the release build, breaking soul daemon + * compilation). */ + +el_val_t engram_wm_count(void) { + EngramStore* g = engram_get(); + int64_t count = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > 0.0) count++; + } + return (el_val_t)count; +} + +/* Average working_memory_weight across all promoted nodes (wm > 0). + * Returns the float bit-pattern via el_from_float so EL can use it with + * float_to_str / float_gt. Returns 0.0 when no nodes are promoted. + * Useful in heartbeat ISEs to distinguish "many weak activations" from + * "few strong activations". Added 2026-06-04 self-review. */ +el_val_t engram_wm_avg_weight(void) { + EngramStore* g = engram_get(); + double sum = 0.0; + int64_t count = 0; + for (int64_t i = 0; i < g->node_count; i++) { + double w = g->nodes[i].working_memory_weight; + /* Skip corrupt/out-of-range values so a single bad snapshot node + * doesn't produce a garbage average. */ + if (w > 0.0 && w <= 1.0 && isfinite(w)) { sum += w; count++; } + } + double avg = (count > 0) ? (sum / (double)count) : 0.0; + return el_from_float(avg); +} + +/* engram_wm_top_json — return top N working-memory nodes (by wm weight) as a + * compact JSON array for ISE heartbeat reporting. + * Each element: {"label":"...","node_type":"...","tier":"...","wm":0.42} + * InternalStateEvent nodes are excluded — they're observation artifacts that + * would bury substantive WM content. Added 2026-06-05 self-review. */ +el_val_t engram_wm_top_json(el_val_t n_v) { + int64_t top_n = (int64_t)n_v; + if (top_n <= 0) top_n = 10; + if (top_n > 50) top_n = 50; + EngramStore* g = engram_get(); + int64_t* idx = malloc((size_t)(g->node_count + 1) * sizeof(int64_t)); + if (!idx) return el_wrap_str(el_strdup("[]")); + int64_t mc = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > 0.0) { + const char* nt = g->nodes[i].node_type; + if (nt && strcmp(nt, "InternalStateEvent") == 0) continue; + idx[mc++] = i; + } + } + /* Insertion-sort descending by wm weight (mc is typically small). */ + for (int64_t i = 1; i < mc; i++) { + int64_t key = idx[i]; + double kw = g->nodes[key].working_memory_weight; + int64_t j = i; + while (j > 0 && g->nodes[idx[j-1]].working_memory_weight < kw) { + idx[j] = idx[j-1]; j--; + } + idx[j] = key; + } + int64_t emit = mc < top_n ? mc : top_n; + JsonBuf b; jb_init(&b); + jb_putc(&b, '['); + for (int64_t k = 0; k < emit; k++) { + EngramNode* n = &g->nodes[idx[k]]; + if (k > 0) jb_putc(&b, ','); + jb_putc(&b, '{'); + /* 2026-07-26 self-review: id was never emitted here, so the + * awareness heartbeat's wm_top0_streak compared ""=="" and + * incremented unconditionally — the streak metric measured + * uptime, not fixation. */ + jb_puts(&b, "\"id\":"); + jb_emit_escaped(&b, n->id ? n->id : ""); + jb_puts(&b, ",\"label\":"); + jb_emit_escaped(&b, n->label ? n->label : ""); + jb_puts(&b, ",\"node_type\":"); + jb_emit_escaped(&b, n->node_type ? n->node_type : ""); + jb_puts(&b, ",\"tier\":"); + jb_emit_escaped(&b, n->tier ? n->tier : ""); + char tmp[48]; + snprintf(tmp, sizeof(tmp), ",\"wm\":%.3f", n->working_memory_weight); + jb_puts(&b, tmp); + jb_putc(&b, '}'); + } + free(idx); + jb_putc(&b, ']'); + return el_wrap_str(b.buf); +} + +el_val_t engram_stats_json(void) { + EngramStore* g = engram_get(); + /* embedded_count: how far the lazy backfill has progressed. The single + * observable that tells the daily self-review whether semantic + * activation is actually accumulating coverage. (2026-07-24) + * + * embed_eligible_count (2026-07-27): embedded_count alone misleads — + * ~70%+ of the store is ISE/Tag/short-content nodes that are permanently + * ineligible for embedding, so raw embedded/node_count reads as "~30% + * coverage, something is broken" when eligible coverage may be complete. + * This exact misdiagnosis happened in today's self-review. Report the + * true denominator so coverage = embedded_count / embed_eligible_count. */ + int64_t embedded = 0, eligible = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].emb) embedded++; + if (eg_embed_eligible(&g->nodes[i])) eligible++; + } + char buf[256]; + snprintf(buf, sizeof(buf), + "{\"node_count\":%lld,\"edge_count\":%lld,\"layer_count\":%zu," + "\"embedded_count\":%lld,\"embed_eligible_count\":%lld}", + (long long)g->node_count, (long long)g->edge_count, g->layer_count, + (long long)embedded, (long long)eligible); + return el_wrap_str(el_strdup(buf)); +} + +/* engram_act_stats_json — activation observability + embedder breaker state. + * (2026-07-27 self-review; counters made cumulative 2026-07-31.) + * wm_evicted/breakthroughs are monotonic process-lifetime totals across ALL + * engram_activate calls on this store (diff successive readings for rates; + * they reset to 0 only on restart); embed_breaker_open=1 means + * eg_embed_fetch is currently refusing calls (semantic activation silently + * degraded to lexical until the cooldown expires). The soul heartbeat folds + * this into its ISE so the pathologies are diagnosable from telemetry + * instead of inferred from wm_avg_weight hovering at the floor. */ +el_val_t engram_act_stats_json(void) { + int64_t now = engram_now_ms(); + int breaker_open = (now < _eg_embed_breaker_until) ? 1 : 0; + char buf[256]; + /* ctx_cos (2026-07-29): cos(query, context centroid) at the LAST + * activate call, measured before the query was folded in. ~1.0 = + * context aligned with current query; low = divergence (expected at + * curiosity domain-rotation boundaries); -2.0 = no centroid yet or + * embedder down. The drift gauge for the context-centroid mechanism. */ + snprintf(buf, sizeof(buf), + "{\"wm_evicted\":%lld,\"breakthroughs\":%lld," + "\"embed_breaker_open\":%d,\"embed_consec_fail\":%d," + "\"ctx_cos\":%.3f}", + (long long)_eg_act_wm_evicted, + (long long)_eg_act_breakthroughs, + breaker_open, _eg_embed_consec_fail, + _eg_act_ctx_cos); + return el_wrap_str(el_strdup(buf)); +} + +/* engram_cosine_sim — cosine similarity between two nodes' embeddings. + * Returns a float in [-1, 1], or -2.0 when either node is missing or not + * yet embedded. Exposed so EL code (and the introspection API) can probe + * semantic distance directly. (2026-07-24, bl-b2d1c944) */ +el_val_t engram_cosine_sim(el_val_t id_a, el_val_t id_b) { + EngramStore* g = engram_get(); + int64_t ia = engram_find_node_index(EL_CSTR(id_a)); + int64_t ib = engram_find_node_index(EL_CSTR(id_b)); + if (ia < 0 || ib < 0) return el_from_float(-2.0); + EngramNode* a = &g->nodes[ia]; + EngramNode* b = &g->nodes[ib]; + if (!a->emb || !b->emb || a->emb_dim != b->emb_dim) + return el_from_float(-2.0); + return el_from_float(eg_cosine(a->emb, b->emb, a->emb_dim)); +} + +/* engram_label_df — document frequency of `term` across node LABELS. + * Returns the count of nodes whose label contains term (case-insensitive), + * or the total node count for an empty/NULL term so callers treat "no term" + * as maximally unspecific (i.e. reject it). + * + * WHY THIS EXISTS (2026-08-03 self-review). The soul's auto-term extractor + * (awareness.el:auto_term_try_slot) picks a curiosity seed by taking the + * FIRST WORD of a top-WM node label. A first-word extractor has no notion of + * term quality, so three consecutive self-reviews each bolted another + * hand-curated blocklist onto it — genre words (07-23), quoted titles + * (07-25), English stopwords (07-30). Every one of those was written + * REACTIVELY, after observing a flood in the live ISE stream. The mechanism + * is whack-a-mole: the list can only ever contain floods that already + * happened. + * + * Measured live on this store (13,370 nodes) while two unanticipated floods + * were in flight and unfixed: + * "