diff --git a/el-compiler/runtime/el_runtime.c b/el-compiler/runtime/el_runtime.c index 3f9028f..eeb23a9 100644 --- a/el-compiler/runtime/el_runtime.c +++ b/el-compiler/runtime/el_runtime.c @@ -45,15 +45,86 @@ /* ── Internal allocators ─────────────────────────────────────────────────── */ -static char* el_strdup(const char* s) { +/* + * Per-request string arena + * + * Every El string allocated via el_strbuf / el_strdup during an HTTP request + * is registered in a thread-local arena. When el_request_end() is called at + * the end of the worker thread, every arena entry is freed — recovering all + * the intermediate strings from el_str_concat chains (build_system_prompt, + * engram_compile, etc.) that are otherwise leaked forever. + * + * Long-lived allocations (state_set values, engram internal storage) call + * el_strdup_persist() / el_strbuf_persist() which bypass the arena entirely. + */ + +#define EL_ARENA_INITIAL 512 + +typedef struct { + char** ptrs; + size_t count; + size_t cap; +} ElArena; + +static _Thread_local ElArena _tl_arena = {NULL, 0, 0}; +static _Thread_local int _tl_arena_active = 0; + +/* Binary-safe fs_read length — set by fs_read, consumed by http_send_response. + * Allows serving PNGs and other binary files without strlen truncation. */ +static _Thread_local size_t _tl_fs_read_len = 0; + +static void el_arena_track(char* p) { + if (!_tl_arena_active || !p) return; + if (_tl_arena.count >= _tl_arena.cap) { + size_t nc = _tl_arena.cap == 0 ? EL_ARENA_INITIAL : _tl_arena.cap * 2; + char** grown = realloc(_tl_arena.ptrs, nc * sizeof(char*)); + if (!grown) return; /* can't track — will leak this one ptr, but don't crash */ + _tl_arena.ptrs = grown; + _tl_arena.cap = nc; + } + _tl_arena.ptrs[_tl_arena.count++] = p; +} + +/* Called by http_worker before dispatching the El handler. */ +void el_request_start(void) { + _tl_arena.count = 0; + _tl_arena_active = 1; +} + +/* Called by http_worker after the El handler returns and the response is sent. + * Frees every intermediate string allocated during the request. */ +void el_request_end(void) { + _tl_arena_active = 0; + for (size_t i = 0; i < _tl_arena.count; i++) { + free(_tl_arena.ptrs[i]); + } + _tl_arena.count = 0; +} + +/* Persistent allocation — bypasses the arena (state_set, engram internals). */ +static char* el_strdup_persist(const char* s) { if (!s) return strdup(""); return strdup(s); } +static char* el_strbuf_persist(size_t n) { + char* p = malloc(n + 1); + if (!p) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + p[0] = '\0'; + return p; +} + +static char* el_strdup(const char* s) { + if (!s) { char* p = strdup(""); el_arena_track(p); return p; } + char* p = strdup(s); + el_arena_track(p); + return p; +} static char* el_strbuf(size_t n) { char* p = malloc(n + 1); if (!p) { fputs("el_runtime: out of memory\n", stderr); exit(1); } p[0] = '\0'; + el_arena_track(p); return p; } @@ -658,7 +729,24 @@ static el_val_t http_error_json(const char* msg) { return el_wrap_str(buf); } -/* Internal: do a libcurl request; takes optional body/headers. */ +/* HTTP timeout (ms) — read once from EL_HTTP_TIMEOUT_MS, default 60000. + * Applied via CURLOPT_TIMEOUT_MS on every libcurl request. */ +static long _el_http_timeout_ms = -1; +static long el_http_timeout_ms(void) { + long v = __atomic_load_n(&_el_http_timeout_ms, __ATOMIC_ACQUIRE); + if (v >= 0) return v; + const char* s = getenv("EL_HTTP_TIMEOUT_MS"); + long parsed = 60000L; + if (s && *s) { + char* end = NULL; + long n = strtol(s, &end, 10); + if (end != s && n > 0) parsed = n; + } + __atomic_store_n(&_el_http_timeout_ms, parsed, __ATOMIC_RELEASE); + return parsed; +} + +/* Internal: do a libcurl request; takes optional body/headers, optional method override. */ static el_val_t http_do(const char* method, const char* url, const char* body, struct curl_slist* extra_headers) { if (!url || !*url) return http_error_json("empty url"); @@ -670,7 +758,7 @@ static el_val_t http_do(const char* method, const char* url, const char* body, curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_write_cb); curl_easy_setopt(c, CURLOPT_WRITEDATA, &rb); curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(c, CURLOPT_TIMEOUT, 60L); + curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, el_http_timeout_ms()); curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf); curl_easy_setopt(c, CURLOPT_USERAGENT, "el-runtime/1.0"); @@ -679,6 +767,8 @@ static el_val_t http_do(const char* method, const char* url, const char* body, curl_easy_setopt(c, CURLOPT_POST, 1L); curl_easy_setopt(c, CURLOPT_POSTFIELDS, body ? body : ""); curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)(body ? strlen(body) : 0)); + } else if (method && strcmp(method, "DELETE") == 0) { + curl_easy_setopt(c, CURLOPT_CUSTOMREQUEST, "DELETE"); } CURLcode rc = curl_easy_perform(c); curl_easy_cleanup(c); @@ -755,6 +845,98 @@ el_val_t http_post_form_auth(el_val_t url, el_val_t form_body, el_val_t auth_hea return r; } +/* HTTP DELETE — mirrors http_post but with CURLOPT_CUSTOMREQUEST=DELETE. + * Returns response body on success; on transport failure returns an error + * JSON fragment (same convention as http_get/http_post). Callers that + * expect "" on failure should check for a leading '{' and an "error" key. */ +el_val_t http_delete(el_val_t url) { + return http_do("DELETE", EL_CSTR(url), NULL, NULL); +} + +/* ── HTTP → file streaming ──────────────────────────────────────────────── + * + * Why this exists: el_val_t strings are NUL-terminated by convention, so + * accumulating an HTTP response into an httpbuf and then wrapping its + * `.data` pointer with el_wrap_str() loses the byte length. Any consumer + * that does strlen() on the wrapped pointer truncates the body at the + * first embedded NUL. Audio (MP3, WAV, OGG), images (PNG, JPEG), and any + * other binary payload hits this. The vessels that download such bodies + * (e.g. ElevenLabs TTS → MP3) get silently corrupted files. + * + * The fix: wire libcurl's CURLOPT_WRITEFUNCTION directly to fwrite() + * against a fopen()-ed FILE*. The bytes never pass through an el_val_t + * string, so embedded NULs are preserved verbatim. Caller's contract is + * just "a file at this path with the response body in it". */ + +static size_t http_file_write_cb(char* ptr, size_t size, size_t nmemb, void* ud) { + FILE* f = (FILE*)ud; + return fwrite(ptr, size, nmemb, f); +} + +/* Internal: stream body to file. method is "GET" or "POST". body may be NULL + * (GET) or NUL-terminated (POST). headers may be NULL. Returns 1/0. */ +static el_val_t http_do_to_file(const char* method, const char* url, + const char* body, struct curl_slist* extra_headers, + const char* output_path) { + if (!url || !*url) return 0; + if (!output_path || !*output_path) return 0; + FILE* f = fopen(output_path, "wb"); + if (!f) return 0; + + CURL* c = curl_easy_init(); + if (!c) { fclose(f); remove(output_path); return 0; } + + char errbuf[CURL_ERROR_SIZE]; errbuf[0] = '\0'; + curl_easy_setopt(c, CURLOPT_URL, url); + curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_file_write_cb); + curl_easy_setopt(c, CURLOPT_WRITEDATA, f); + curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, el_http_timeout_ms()); + curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf); + curl_easy_setopt(c, CURLOPT_USERAGENT, "el-runtime/1.0"); + curl_easy_setopt(c, CURLOPT_FAILONERROR, 1L); /* 4xx/5xx → CURLE_HTTP_RETURNED_ERROR */ + if (extra_headers) curl_easy_setopt(c, CURLOPT_HTTPHEADER, extra_headers); + + if (method && strcmp(method, "POST") == 0) { + curl_easy_setopt(c, CURLOPT_POST, 1L); + curl_easy_setopt(c, CURLOPT_POSTFIELDS, body ? body : ""); + /* For the request body we still rely on strlen — POST bodies are + * caller-controlled and JSON/text in every known El use case. + * If a future caller needs a binary POST body, add a *_bytes + * variant that takes an explicit length, mirroring fs_write_bytes. */ + curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)(body ? strlen(body) : 0)); + } + + CURLcode rc = curl_easy_perform(c); + curl_easy_cleanup(c); + + /* Flush + close before signalling success, so the file is fully on disk + * by the time the caller reads back. */ + int flush_ok = (fflush(f) == 0); + int close_ok = (fclose(f) == 0); + + if (rc != CURLE_OK || !flush_ok || !close_ok) { + remove(output_path); + return 0; + } + return 1; +} + +el_val_t http_get_to_file(el_val_t url, el_val_t headers_map, el_val_t output_path) { + struct curl_slist* h = headers_from_map(headers_map); + el_val_t r = http_do_to_file("GET", EL_CSTR(url), NULL, h, EL_CSTR(output_path)); + if (h) curl_slist_free_all(h); + return r; +} + +el_val_t http_post_to_file(el_val_t url, el_val_t body, el_val_t headers_map, el_val_t output_path) { + struct curl_slist* h = headers_from_map(headers_map); + el_val_t r = http_do_to_file("POST", EL_CSTR(url), EL_CSTR(body), h, EL_CSTR(output_path)); + if (h) curl_slist_free_all(h); + return r; +} + /* ── HTTP server (POSIX sockets + pthreads) ──────────────────────────────── */ #define HTTP_MAX_CONNS 64 @@ -842,17 +1024,35 @@ static http_handler_fn http_lookup_active(void) { static const char* http_detect_content_type(const char* body) { if (!body) return "text/plain; charset=utf-8"; const char* p = body; + /* Binary magic bytes — check before stripping whitespace */ + if ((unsigned char)p[0] == 0x89 && p[1]=='P' && p[2]=='N' && p[3]=='G') + return "image/png"; + if ((unsigned char)p[0] == 0xFF && (unsigned char)p[1] == 0xD8) + return "image/jpeg"; + if (strncmp(p, "GIF8", 4) == 0) return "image/gif"; + if (strncmp(p, "RIFF", 4) == 0) return "image/webp"; + if (strncmp(p, "wOFF", 4) == 0) return "font/woff"; + if (strncmp(p, "wOF2", 4) == 0) return "font/woff2"; while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++; if (strncasecmp(p, " 0) { + ssize_t w = send(fd, p, left, 0); + if (w <= 0) return -1; + p += w; left -= (size_t)w; + } + return 0; +} + +/* Discriminator that http_response() embeds at the start of its envelope. + * A handler returning a string starting with this exact prefix is treated + * as a structured response; anything else is treated as a raw body. */ +#define EL_HTTP_RESPONSE_TAG "{\"el_http_response\":1" + +/* Keys that conflict with runtime-managed headers are silently dropped to + * avoid double-emission — the runtime always emits its own Content-Length + * and Connection: close. Content-Type from the envelope IS allowed and + * overrides auto-detection. */ +static int http_header_is_managed(const char* k) { + return strcasecmp(k, "Content-Length") == 0 + || strcasecmp(k, "Connection") == 0; +} + +/* Walk an ElMap of header pairs and emit each as `K: V\r\n` into JsonBuf b. + * Sets *out_saw_content_type to 1 if the map contained an explicit + * Content-Type so the caller can skip auto-detection. */ +static void http_emit_headers_from_map(JsonBuf* b, el_val_t headers_map, + int* out_saw_content_type) { + *out_saw_content_type = 0; + if (headers_map == 0) return; + ElMap* m = (ElMap*)(uintptr_t)headers_map; + if (!m || m->hdr.magic != EL_MAGIC_MAP) return; + for (int64_t i = 0; i < m->count; i++) { + const char* k = EL_CSTR(m->keys[i]); + const char* v = EL_CSTR(m->values[i]); + if (!k || !v) continue; + if (http_header_is_managed(k)) continue; + if (strcasecmp(k, "Content-Type") == 0) *out_saw_content_type = 1; + jb_puts(b, k); + jb_puts(b, ": "); + jb_puts(b, v); + jb_puts(b, "\r\n"); + } +} + +/* Parse the envelope produced by http_response(). On success returns 1 and + * populates *out_status, *out_headers_map (an ElMap el_val_t — caller must + * el_release), and *out_body (allocated). On failure returns 0. + * + * Implementation: feeds the entire envelope through the recursive-descent + * JSON parser (which builds proper ElMap/ElList values), then pulls the + * three top-level fields by name. Avoids re-stringifying the headers map + * since json_stringify() does not support nested objects. */ +static int http_parse_envelope(const char* s, int* out_status, + el_val_t* out_headers_map, char** out_body, + el_val_t* out_parsed_root) { + if (!s) return 0; + if (strncmp(s, EL_HTTP_RESPONSE_TAG, + sizeof(EL_HTTP_RESPONSE_TAG) - 1) != 0) return 0; + + el_val_t parsed = json_parse(EL_STR(s)); + if (parsed == EL_NULL) return 0; + + int status = 200; + el_val_t hmap = 0; + char* body = NULL; + + el_val_t sv = el_map_get(parsed, EL_STR("status")); + if (sv != 0) { + /* status comes back as an integer — el_val_t holds it directly. */ + long sc = (long)sv; + if (sc >= 100 && sc <= 599) status = (int)sc; + } + + el_val_t hv = el_map_get(parsed, EL_STR("headers")); + if (hv != 0) { + ElMap* hm = (ElMap*)(uintptr_t)hv; + if (hm && hm->hdr.magic == EL_MAGIC_MAP) hmap = hv; + } + + el_val_t bv = el_map_get(parsed, EL_STR("body")); + if (bv != 0) { + const char* bs = EL_CSTR(bv); + if (bs) body = el_strdup(bs); + } + if (!body) body = el_strdup(""); + + *out_status = status; + *out_headers_map = hmap; + *out_body = body; + *out_parsed_root = parsed; /* caller releases to free hmap + entries */ + return 1; +} + +/* Send a fully-built HTTP response. If `body` starts with the envelope tag, + * unpack status/headers/body. Otherwise emit the historical 200-OK with + * auto-detected Content-Type. */ +/* Thread-local flag: if 1, http_send_response writes status + headers but + * NO body (HEAD method behaviour). Set by http_worker before calling + * http_send_response, cleared after. */ +static __thread int _tl_http_head_only = 0; + static void http_send_response(int fd, const char* body) { if (!body) body = ""; - const char* ct = http_detect_content_type(body); - size_t blen = strlen(body); - char header[512]; - int hl = snprintf(header, sizeof(header), - "HTTP/1.1 200 OK\r\n" - "Content-Type: %s\r\n" - "Content-Length: %zu\r\n" - "Connection: close\r\n" - "\r\n", ct, blen); - if (hl < 0) return; - /* Best-effort full writes */ - const char* p = header; size_t left = (size_t)hl; - while (left > 0) { - ssize_t w = send(fd, p, left, 0); - if (w <= 0) return; - p += w; left -= (size_t)w; + + int status = 200; + el_val_t env_headers_map = 0; + char* env_body = NULL; + el_val_t env_parsed_root = 0; + int is_envelope = http_parse_envelope(body, &status, + &env_headers_map, &env_body, + &env_parsed_root); + + const char* eff_body = is_envelope ? env_body : body; + /* Use the real byte count from fs_read if available (handles binary files + * with embedded null bytes — PNG, WOFF2, etc.). Fall back to strlen for + * normal text/JSON responses where _tl_fs_read_len is 0. */ + size_t blen = (_tl_fs_read_len > 0) ? _tl_fs_read_len : strlen(eff_body); + _tl_fs_read_len = 0; /* consume — one-shot per response */ + int head_only = _tl_http_head_only; + + JsonBuf hdrs; jb_init(&hdrs); + int saw_content_type = 0; + if (is_envelope) { + http_emit_headers_from_map(&hdrs, env_headers_map, + &saw_content_type); } - p = body; left = blen; - while (left > 0) { - ssize_t w = send(fd, p, left, 0); - if (w <= 0) return; - p += w; left -= (size_t)w; + if (!saw_content_type) { + jb_puts(&hdrs, "Content-Type: "); + jb_puts(&hdrs, http_detect_content_type(eff_body)); + jb_puts(&hdrs, "\r\n"); } + + char status_line[64]; + int sl = snprintf(status_line, sizeof(status_line), + "HTTP/1.1 %d %s\r\n", + status, http_reason_phrase(status)); + if (sl < 0) { + if (env_parsed_root) el_release(env_parsed_root); + free(env_body); free(hdrs.buf); return; + } + + char tail[128]; + int tl = snprintf(tail, sizeof(tail), + "Content-Length: %zu\r\n" + "Connection: close\r\n" + "\r\n", blen); + if (tl < 0) { + if (env_parsed_root) el_release(env_parsed_root); + free(env_body); free(hdrs.buf); return; + } + + if (http_send_all(fd, status_line, (size_t)sl) == 0 + && http_send_all(fd, hdrs.buf, hdrs.len) == 0 + && http_send_all(fd, tail, (size_t)tl) == 0 + && (head_only + /* HEAD requests echo headers + Content-Length but no body. */ + ? 1 + : http_send_all(fd, eff_body, blen) == 0)) { + /* sent successfully */ + } + + if (env_parsed_root) el_release(env_parsed_root); + free(env_body); + free(hdrs.buf); } typedef struct { @@ -955,17 +1346,32 @@ static void* http_worker(void* arg) { int fd = a->fd; free(a); char *method = NULL, *path = NULL, *body = NULL; - if (http_read_request(fd, &method, &path, &body) == 0) { + if (http_read_request(fd, &method, &path, &body, NULL) == 0) { http_handler_fn h = http_lookup_active(); char* response = NULL; + /* HEAD: dispatch as GET so existing handlers respond with the same + * body, but flag the response writer to emit headers only. RFC 9110 + * requires HEAD to mirror GET headers + Content-Length without body. */ + int head_only = (method && strcmp(method, "HEAD") == 0); + const char* dispatch_method = head_only ? "GET" : method; + el_request_start(); /* begin per-request arena */ if (h) { - el_val_t r = h(EL_STR(method), EL_STR(path), EL_STR(body)); + el_val_t r = h(EL_STR(dispatch_method), EL_STR(path), EL_STR(body)); const char* rs = EL_CSTR(r); - response = el_strdup(rs ? rs : ""); + /* Copy response out BEFORE arena teardown. + * For binary files, _tl_fs_read_len holds the real byte count — + * use memcpy instead of strdup so null bytes are preserved. */ + size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0); + response = malloc(rlen + 1); + if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; } + else if (response) { response[0] = '\0'; } } else { - response = el_strdup("el-runtime: no http handler registered (call http_set_handler)"); + response = el_strdup_persist("el-runtime: no http handler registered"); } + el_request_end(); /* free all intermediate strings */ + _tl_http_head_only = head_only; http_send_response(fd, response); + _tl_http_head_only = 0; free(response); } free(method); free(path); free(body); @@ -1031,19 +1437,300 @@ void http_serve(el_val_t port, el_val_t handler) { close(sock); } +/* ── HTTP server v2 — request headers + structured response ──────────────── */ +/* + * v2 widens the handler signature from + * (method, path, body) -> body_string + * to + * (method, path, headers_map, body) -> body_string_or_envelope + * + * The response envelope is detected uniformly inside http_send_response — so + * 4-arg handlers can return either a plain body or http_response(...). The + * 3-arg path stays untouched in spirit (its handlers still build plain + * bodies; the envelope tag, being `{"el_http_response":1`, will never + * collide with normal JSON the legacy server.el routes return). + * + * Registry is parallel to the 3-arg handler registry: separate name table, + * separate active-handler slot, separate dlsym fallback. Mixing v1 and v2 + * handlers in the same process is fine — they don't share the active slot. */ + +typedef el_val_t (*http_handler4_fn)(el_val_t method, el_val_t path, + el_val_t headers_map, el_val_t body); + +typedef struct { + char* name; + http_handler4_fn fn; +} HttpHandler4Entry; + +static HttpHandler4Entry _http_handlers4[32]; +static size_t _http_handler4_count = 0; +static char* _http_active_handler4 = NULL; + +void el_runtime_register_handler_v2(const char* name, http_handler4_fn fn); +void el_runtime_register_handler_v2(const char* name, http_handler4_fn fn) { + if (!name || !fn) return; + pthread_mutex_lock(&_http_handler_mu); + for (size_t i = 0; i < _http_handler4_count; i++) { + if (strcmp(_http_handlers4[i].name, name) == 0) { + _http_handlers4[i].fn = fn; + pthread_mutex_unlock(&_http_handler_mu); + return; + } + } + if (_http_handler4_count < + sizeof(_http_handlers4) / sizeof(_http_handlers4[0])) { + _http_handlers4[_http_handler4_count].name = el_strdup(name); + _http_handlers4[_http_handler4_count].fn = fn; + _http_handler4_count++; + } + pthread_mutex_unlock(&_http_handler_mu); +} + +void http_set_handler_v2(el_val_t name) { + const char* n = EL_CSTR(name); + pthread_mutex_lock(&_http_handler_mu); + free(_http_active_handler4); + _http_active_handler4 = el_strdup(n ? n : ""); + if (n && *n) { + int found = 0; + for (size_t i = 0; i < _http_handler4_count; i++) { + if (strcmp(_http_handlers4[i].name, n) == 0) { found = 1; break; } + } + if (!found) { + void* sym = dlsym(RTLD_DEFAULT, n); + if (sym && _http_handler4_count < + sizeof(_http_handlers4) / sizeof(_http_handlers4[0])) { + _http_handlers4[_http_handler4_count].name = el_strdup(n); + _http_handlers4[_http_handler4_count].fn = + (http_handler4_fn)sym; + _http_handler4_count++; + } + } + } + pthread_mutex_unlock(&_http_handler_mu); +} + +static http_handler4_fn http_lookup_active_v2(void) { + http_handler4_fn out = NULL; + pthread_mutex_lock(&_http_handler_mu); + if (_http_active_handler4) { + for (size_t i = 0; i < _http_handler4_count; i++) { + if (strcmp(_http_handlers4[i].name, + _http_active_handler4) == 0) { + out = _http_handlers4[i].fn; break; + } + } + } + pthread_mutex_unlock(&_http_handler_mu); + return out; +} + +/* Build an ElMap from the raw header block produced by http_read_request. + * Keys are lowercased (RFC 7230 — case-insensitive); values have leading + * whitespace trimmed. Repeated headers with the same name are joined with + * ", " in arrival order, matching standard library behaviour elsewhere. */ +static el_val_t http_build_headers_map(const char* hdr_block) { + el_val_t m = el_map_new(0); + if (!hdr_block || !*hdr_block) return m; + const char* p = hdr_block; + while (*p) { + const char* line_end = strstr(p, "\r\n"); + const char* end = line_end ? line_end : p + strlen(p); + const char* colon = NULL; + for (const char* c = p; c < end; c++) { + if (*c == ':') { colon = c; break; } + } + if (colon && colon > p) { + size_t klen = (size_t)(colon - p); + char* key = malloc(klen + 1); + if (key) { + for (size_t i = 0; i < klen; i++) { + unsigned char ch = (unsigned char)p[i]; + key[i] = (char)tolower(ch); + } + key[klen] = '\0'; + const char* vstart = colon + 1; + while (vstart < end && (*vstart == ' ' || *vstart == '\t')) vstart++; + size_t vlen = (size_t)(end - vstart); + /* Strip trailing OWS just in case. */ + while (vlen > 0 + && (vstart[vlen - 1] == ' ' + || vstart[vlen - 1] == '\t')) vlen--; + /* Coalesce repeats: if key already present, append ", value". */ + el_val_t existing = el_map_get(m, EL_STR(key)); + if (existing != 0 && looks_like_string(existing)) { + const char* old = EL_CSTR(existing); + size_t olen = strlen(old); + char* combined = malloc(olen + 2 + vlen + 1); + if (combined) { + memcpy(combined, old, olen); + memcpy(combined + olen, ", ", 2); + memcpy(combined + olen + 2, vstart, vlen); + combined[olen + 2 + vlen] = '\0'; + m = el_map_set(m, EL_STR(key), EL_STR(combined)); + } + free(key); + } else { + char* val = malloc(vlen + 1); + if (val) { + memcpy(val, vstart, vlen); + val[vlen] = '\0'; + m = el_map_set(m, EL_STR(key), EL_STR(val)); + } else { + free(key); + } + } + } + } + if (!line_end) break; + p = line_end + 2; + } + return m; +} + +static void* http_worker_v2(void* arg) { + HttpWorkerArg* a = (HttpWorkerArg*)arg; + int fd = a->fd; + free(a); + char *method = NULL, *path = NULL, *body = NULL, *hdr_block = NULL; + if (http_read_request(fd, &method, &path, &body, &hdr_block) == 0) { + http_handler4_fn h = http_lookup_active_v2(); + char* response = NULL; + int head_only = (method && strcmp(method, "HEAD") == 0); + const char* dispatch_method = head_only ? "GET" : method; + el_request_start(); /* begin per-request arena */ + if (h) { + el_val_t hmap = http_build_headers_map(hdr_block ? hdr_block : ""); + el_val_t r = h(EL_STR(dispatch_method), EL_STR(path), hmap, EL_STR(body)); + const char* rs = EL_CSTR(r); + size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0); + response = malloc(rlen + 1); + if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; } + else if (response) { response[0] = '\0'; } + el_release(hmap); + } else { + response = el_strdup_persist( + "el-runtime: no v2 http handler registered " + "(call http_set_handler_v2)"); + } + el_request_end(); /* free all intermediate strings */ + _tl_http_head_only = head_only; + http_send_response(fd, response); + _tl_http_head_only = 0; + free(response); + } + free(method); free(path); free(body); free(hdr_block); + close(fd); + pthread_mutex_lock(&_http_conn_mu); + _http_conn_active--; + pthread_cond_signal(&_http_conn_cv); + pthread_mutex_unlock(&_http_conn_mu); + return NULL; +} + +void http_serve_v2(el_val_t port, el_val_t handler) { + const char* hname = EL_CSTR(handler); + if (hname && looks_like_string(handler)) { + http_set_handler_v2(handler); + } + int p = (int)port; + if (p <= 0 || p > 65535) { + fprintf(stderr, "http_serve_v2: invalid port %d\n", p); + return; + } + int sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock < 0) { perror("socket"); return; } + int yes = 1; + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + struct sockaddr_in addr; + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_ANY); + addr.sin_port = htons((uint16_t)p); + if (bind(sock, (struct sockaddr*)&addr, sizeof(addr)) < 0) { + perror("bind"); close(sock); return; + } + if (listen(sock, 64) < 0) { perror("listen"); close(sock); return; } + fprintf(stderr, "[http v2] listening on 0.0.0.0:%d\n", p); + while (1) { + struct sockaddr_in cli; + socklen_t clen = sizeof(cli); + int cfd = accept(sock, (struct sockaddr*)&cli, &clen); + if (cfd < 0) { + if (errno == EINTR) continue; + perror("accept"); break; + } + pthread_mutex_lock(&_http_conn_mu); + while (_http_conn_active >= HTTP_MAX_CONNS) { + pthread_cond_wait(&_http_conn_cv, &_http_conn_mu); + } + _http_conn_active++; + pthread_mutex_unlock(&_http_conn_mu); + HttpWorkerArg* arg = malloc(sizeof(HttpWorkerArg)); + if (!arg) { close(cfd); continue; } + arg->fd = cfd; + pthread_t tid; + if (pthread_create(&tid, NULL, http_worker_v2, arg) != 0) { + close(cfd); free(arg); + pthread_mutex_lock(&_http_conn_mu); + _http_conn_active--; + pthread_cond_signal(&_http_conn_cv); + pthread_mutex_unlock(&_http_conn_mu); + continue; + } + pthread_detach(tid); + } + close(sock); +} + +/* Build the response envelope a 4-arg handler can return. We hand-write + * the JSON so the discriminator key always lands first — the runtime's + * http_parse_envelope() detects it via prefix match. headers_json must be + * either "" (empty), "{}" (empty object), or a well-formed JSON object + * literal; anything else will produce a malformed envelope and the runtime + * will treat the whole string as a plain body (no envelope detected). */ +el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body) { + long sc = (long)status; + if (sc < 100 || sc > 599) sc = 200; + const char* hj = EL_CSTR(headers_json); + if (!hj || !*hj) hj = "{}"; + /* Light validation: must start with '{' and end with '}'. */ + size_t hlen = strlen(hj); + int hj_ok = (hlen >= 2 && hj[0] == '{' && hj[hlen - 1] == '}'); + if (!hj_ok) hj = "{}"; + const char* b = EL_CSTR(body); + if (!b) b = ""; + + JsonBuf out; jb_init(&out); + jb_puts(&out, EL_HTTP_RESPONSE_TAG); /* {"el_http_response":1 */ + jb_puts(&out, ",\"status\":"); + char num[32]; + snprintf(num, sizeof(num), "%ld", sc); + jb_puts(&out, num); + jb_puts(&out, ",\"headers\":"); + jb_puts(&out, hj); + jb_puts(&out, ",\"body\":"); + jb_emit_escaped(&out, b); + jb_putc(&out, '}'); + return el_wrap_str(out.buf); +} + /* ── Filesystem ──────────────────────────────────────────────────────────── */ el_val_t fs_read(el_val_t pathv) { const char* path = EL_CSTR(pathv); + _tl_fs_read_len = 0; if (!path) return el_wrap_str(el_strdup("")); FILE* f = fopen(path, "rb"); if (!f) return el_wrap_str(el_strdup("")); fseek(f, 0, SEEK_END); long sz = ftell(f); rewind(f); + if (sz < 0) { fclose(f); return el_wrap_str(el_strdup("")); } /* pipe/special file */ char* buf = el_strbuf((size_t)sz); size_t got = fread(buf, 1, (size_t)sz, f); buf[got] = '\0'; + _tl_fs_read_len = got; /* store real byte count for binary-safe send */ fclose(f); return el_wrap_str(buf); } @@ -1060,6 +1747,30 @@ el_val_t fs_write(el_val_t pathv, el_val_t contentv) { return written == n ? 1 : 0; } +/* fs_write_bytes — explicit-length binary write. Bypasses strlen so embedded + * NULs survive. Caller must know the byte count (e.g. from base64_decode, + * or the fixed 32-byte sha256_bytes/hmac_sha256_bytes outputs). + * + * If `length` is negative, treats as failure. If `length` is 0, creates an + * empty file (still useful as a "touch with content" primitive). */ +el_val_t fs_write_bytes(el_val_t pathv, el_val_t bytesv, el_val_t lengthv) { + const char* path = EL_CSTR(pathv); + const char* bytes = EL_CSTR(bytesv); + int64_t n = (int64_t)lengthv; + if (!path || !bytes) return 0; + if (n < 0) return 0; + FILE* f = fopen(path, "wb"); + if (!f) return 0; + size_t written = (n > 0) ? fwrite(bytes, 1, (size_t)n, f) : 0; + int flush_ok = (fflush(f) == 0); + int close_ok = (fclose(f) == 0); + if (!flush_ok || !close_ok || written != (size_t)n) { + remove(path); + return 0; + } + return 1; +} + el_val_t fs_list(el_val_t pathv) { const char* path = EL_CSTR(pathv); el_val_t lst = el_list_empty(); @@ -1075,6 +1786,111 @@ el_val_t fs_list(el_val_t pathv) { return lst; } +/* fs_exists — true iff stat(path) succeeds. Symlinks are followed. */ +el_val_t fs_exists(el_val_t pathv) { + const char* path = EL_CSTR(pathv); + if (!path || !*path) return 0; + struct stat st; + return (el_val_t)(stat(path, &st) == 0 ? 1 : 0); +} + +/* fs_mkdir — create directory at path with mode 0755, mkdir -p semantics. + * Returns 1 if path exists or was created (incl. all parents); 0 on failure. + * Walks the path component-by-component so missing intermediate dirs are + * also created. An existing leaf is not an error. */ +el_val_t fs_mkdir(el_val_t pathv) { + const char* path = EL_CSTR(pathv); + if (!path || !*path) return 0; + size_t n = strlen(path); + char* buf = malloc(n + 1); + if (!buf) return 0; + memcpy(buf, path, n + 1); + /* Walk components; create each prefix in turn. */ + for (size_t i = 1; i <= n; i++) { + if (buf[i] == '/' || buf[i] == '\0') { + char saved = buf[i]; + buf[i] = '\0'; + if (buf[0] != '\0') { + if (mkdir(buf, 0755) != 0 && errno != EEXIST) { + /* Tolerate the case where this prefix exists as a non-dir + * only when stat says it's a directory. */ + struct stat st; + if (stat(buf, &st) != 0 || !S_ISDIR(st.st_mode)) { + free(buf); + return 0; + } + } + } + buf[i] = saved; + } + } + free(buf); + return 1; +} + +/* ── URL encoding ─────────────────────────────────────────────────────────── */ + +/* RFC 3986 percent-encoding for URL components (form bodies, query strings). + * Unreserved set: A-Z a-z 0-9 - _ . ~ — passed through verbatim. + * Everything else (including space) becomes %XX hex. */ +el_val_t url_encode(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return el_wrap_str(el_strdup("")); + static const char hex[] = "0123456789ABCDEF"; + size_t n = strlen(s); + char* out = el_strbuf(n * 3); + size_t o = 0; + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)s[i]; + if ((c >= 'A' && c <= 'Z') || + (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9') || + c == '-' || c == '_' || c == '.' || c == '~') { + out[o++] = (char)c; + } else { + out[o++] = '%'; + out[o++] = hex[(c >> 4) & 0xF]; + out[o++] = hex[c & 0xF]; + } + } + out[o] = '\0'; + return el_wrap_str(out); +} + +/* Decode percent-encoded URL component. '+' becomes space (form-encoded); + * malformed %-escapes are emitted verbatim. */ +el_val_t url_decode(el_val_t sv) { + const char* s = EL_CSTR(sv); + if (!s) return el_wrap_str(el_strdup("")); + size_t n = strlen(s); + char* out = el_strbuf(n); + size_t o = 0; + for (size_t i = 0; i < n; i++) { + char c = s[i]; + if (c == '+') { + out[o++] = ' '; + } else if (c == '%' && i + 2 < n) { + char h1 = s[i + 1], h2 = s[i + 2]; + int v1 = (h1 >= '0' && h1 <= '9') ? h1 - '0' + : (h1 >= 'a' && h1 <= 'f') ? h1 - 'a' + 10 + : (h1 >= 'A' && h1 <= 'F') ? h1 - 'A' + 10 : -1; + int v2 = (h2 >= '0' && h2 <= '9') ? h2 - '0' + : (h2 >= 'a' && h2 <= 'f') ? h2 - 'a' + 10 + : (h2 >= 'A' && h2 <= 'F') ? h2 - 'A' + 10 : -1; + if (v1 >= 0 && v2 >= 0) { + out[o++] = (char)((v1 << 4) | v2); + i += 2; + } else { + out[o++] = c; + } + } else { + out[o++] = c; + } + } + out[o] = '\0'; + return el_wrap_str(out); +} + /* ── JSON ────────────────────────────────────────────────────────────────── */ el_val_t json_get(el_val_t jsonv, el_val_t keyv) { @@ -1082,21 +1898,46 @@ el_val_t json_get(el_val_t jsonv, el_val_t keyv) { const char* key = EL_CSTR(keyv); if (!json || !key) return el_wrap_str(el_strdup("")); size_t klen = strlen(key); - char* pattern = el_strbuf(klen + 4); + /* Use a stack buffer for the pattern to avoid arena double-free. + * Keys in El maps are typically short; 512 bytes is a safe upper bound. */ + char stack_pat[512]; + char* pattern; + if (klen + 5 <= sizeof(stack_pat)) { + pattern = stack_pat; + } else { + pattern = malloc(klen + 5); + if (!pattern) return el_wrap_str(el_strdup("")); + } snprintf(pattern, klen + 5, "\"%s\":", key); const char* p = strstr(json, pattern); - free(pattern); + if (pattern != stack_pat) free(pattern); if (!p) return el_wrap_str(el_strdup("")); p += strlen(key) + 3; /* skip "key": */ while (*p == ' ' || *p == '\t' || *p == '\n') p++; if (*p == '"') { p++; - const char* start = p; - while (*p && !(*p == '"' && *(p-1) != '\\')) p++; - size_t len = (size_t)(p - start); - char* out = el_strbuf(len); - memcpy(out, start, len); - out[len] = '\0'; + /* Unescape the JSON string value into a clean buffer. */ + size_t cap = strlen(p) + 1; + char* out = el_strbuf(cap); + char* w = out; + while (*p && *p != '"') { + if (*p == '\\' && *(p+1)) { + p++; + switch (*p) { + case '"': *w++ = '"'; break; + case '\\': *w++ = '\\'; break; + case '/': *w++ = '/'; break; + case 'n': *w++ = '\n'; break; + case 'r': *w++ = '\r'; break; + case 't': *w++ = '\t'; break; + default: *w++ = *p; break; + } + } else { + *w++ = *p; + } + p++; + } + *w = '\0'; return el_wrap_str(out); } const char* start = p; @@ -1353,10 +2194,15 @@ static void jb_emit_escaped(JsonBuf* b, const char* s) { static int looks_like_string(el_val_t v) { if (v == 0) return 0; - /* Treat plausible heap addresses as candidates */ + /* Treat plausible heap addresses as candidates. + * Threshold: 4 GiB (0x100000000). On 64-bit systems heap addresses from + * malloc/mmap start well above 4 GiB (ASLR pushes them to ~0x7f...). + * El integer values (counters, unix timestamps up to ~2106) all fit below + * 0x100000000 (4294967296). The old threshold of 1,000,000 caused unix + * timestamps (~1.7e9) to be misidentified as string pointers — a segfault + * risk in json_stringify and jb_emit_value. */ uintptr_t p = (uintptr_t)v; - /* Small integers (positive and negative) are not pointers */ - if ((int64_t)v >= -1000000 && (int64_t)v <= 1000000) return 0; + if (p < 0x100000000ULL) return 0; /* integers, timestamps, counters */ if (p < 0x1000) return 0; /* Sniff first bytes for printable */ const unsigned char* s = (const unsigned char*)p; @@ -1636,6 +2482,60 @@ el_val_t json_array_len(el_val_t json_str) { return (el_val_t)count; } +/* json_array_get — return the i-th element of a JSON array as a JSON + * fragment string. Nested objects and arrays are returned verbatim + * (json_skip_value tracks brace/bracket depth so nested structures are + * preserved intact). Out-of-range index → "". */ +el_val_t json_array_get(el_val_t json_str, el_val_t index) { + const char* s = EL_CSTR(json_str); + int64_t idx = (int64_t)index; + if (!s || idx < 0) return el_wrap_str(el_strdup("")); + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; + if (*s != '[') return el_wrap_str(el_strdup("")); + s++; + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; + if (*s == ']') return el_wrap_str(el_strdup("")); + int64_t i = 0; + while (*s) { + const char* start = s; + const char* end = json_skip_value(s); + if (end == s) break; + if (i == idx) { + size_t n = (size_t)(end - start); + char* out = el_strbuf(n); + memcpy(out, start, n); + out[n] = '\0'; + return el_wrap_str(out); + } + i++; + s = end; + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; + if (*s == ',') { s++; while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; continue; } + if (*s == ']' || *s == '\0') break; + } + return el_wrap_str(el_strdup("")); +} + +/* json_array_get_string — same as json_array_get, but assume the element + * is a JSON string and return the unquoted/unescaped value. Non-string + * elements yield "". */ +el_val_t json_array_get_string(el_val_t json_str, el_val_t index) { + el_val_t raw = json_array_get(json_str, index); + const char* s = EL_CSTR(raw); + if (!s || *s != '"') return el_wrap_str(el_strdup("")); + JsonParser jp = { + .p = s, + .end = s + strlen(s), + .err = 0, + }; + char* parsed = jp_parse_string_raw(&jp); + if (jp.err) { + free(parsed); + return el_wrap_str(el_strdup("")); + } + return el_wrap_str(parsed); +} + /* ── Time ────────────────────────────────────────────────────────────────── */ el_val_t time_now(void) { @@ -1792,9 +2692,13 @@ typedef struct { char* value; } StateEntry; -static StateEntry* _state_entries = NULL; -static size_t _state_count = 0; -static size_t _state_cap = 0; +static StateEntry* _state_entries = NULL; +static size_t _state_count = 0; +static size_t _state_cap = 0; +/* Mutex protecting all _state_entries access. state_set/state_get are called + * concurrently from 64 HTTP worker threads — without this lock, realloc and + * free race, producing corruption, double-free, and segfaults. */ +static pthread_mutex_t _state_mu = PTHREAD_MUTEX_INITIALIZER; static StateEntry* state_find(const char* key) { for (size_t i = 0; i < _state_count; i++) { @@ -1808,34 +2712,44 @@ el_val_t state_set(el_val_t key, el_val_t value) { const char* v = EL_CSTR(value); if (!k) return 0; if (!v) v = ""; + pthread_mutex_lock(&_state_mu); StateEntry* e = state_find(k); if (e) { free(e->value); - e->value = el_strdup(v); + e->value = el_strdup_persist(v); + pthread_mutex_unlock(&_state_mu); return 1; } if (_state_count >= _state_cap) { size_t nc = _state_cap == 0 ? 16 : _state_cap * 2; - _state_entries = realloc(_state_entries, nc * sizeof(StateEntry)); - if (!_state_entries) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + StateEntry* grown = realloc(_state_entries, nc * sizeof(StateEntry)); + if (!grown) { pthread_mutex_unlock(&_state_mu); fputs("el_runtime: out of memory\n", stderr); exit(1); } + _state_entries = grown; _state_cap = nc; } - _state_entries[_state_count].key = el_strdup(k); - _state_entries[_state_count].value = el_strdup(v); + _state_entries[_state_count].key = el_strdup_persist(k); + _state_entries[_state_count].value = el_strdup_persist(v); _state_count++; + pthread_mutex_unlock(&_state_mu); return 1; } el_val_t state_get(el_val_t key) { const char* k = EL_CSTR(key); if (!k) return el_wrap_str(el_strdup("")); + pthread_mutex_lock(&_state_mu); StateEntry* e = state_find(k); - return el_wrap_str(el_strdup(e ? e->value : "")); + char* result = el_strdup_persist(e ? e->value : ""); + pthread_mutex_unlock(&_state_mu); + /* wrap in arena-tracked copy for the caller's request lifetime */ + char* copy = el_strdup(result); + return el_wrap_str(copy); } el_val_t state_del(el_val_t key) { const char* k = EL_CSTR(key); if (!k) return 0; + pthread_mutex_lock(&_state_mu); for (size_t i = 0; i < _state_count; i++) { if (strcmp(_state_entries[i].key, k) == 0) { free(_state_entries[i].key); @@ -1844,17 +2758,21 @@ el_val_t state_del(el_val_t key) { _state_entries[j - 1] = _state_entries[j]; } _state_count--; + pthread_mutex_unlock(&_state_mu); return 1; } } + pthread_mutex_unlock(&_state_mu); return 1; } el_val_t state_keys(void) { + pthread_mutex_lock(&_state_mu); el_val_t lst = el_list_empty(); for (size_t i = 0; i < _state_count; i++) { lst = el_list_append(lst, el_wrap_str(el_strdup(_state_entries[i].key))); } + pthread_mutex_unlock(&_state_mu); return lst; } @@ -2092,12 +3010,37 @@ el_val_t bool_to_str(el_val_t b) { return el_wrap_str(el_strdup(b ? "true" : "false")); } +/* ── Numeric parsing ─────────────────────────────────────────────────────── */ + +/* parse_int — strtoll with a default. str_to_int already exists but does not + * distinguish "0" from a parse failure, so callers that need a sentinel use + * this. Skips leading whitespace; accepts an optional leading +/-; returns + * default_val on empty input or no consumed digits. Trailing junk is ignored + * (atoi-style). */ +el_val_t parse_int(el_val_t sv, el_val_t default_val) { + const char* s = EL_CSTR(sv); + if (!s) return default_val; + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; + if (*s == '\0') return default_val; + char* end = NULL; + long long n = strtoll(s, &end, 10); + if (end == s) return default_val; + return (el_val_t)n; +} + /* ── Process ─────────────────────────────────────────────────────────────── */ void exit_program(el_val_t code) { exit((int)code); } +/* getpid_now — current process id. Named with the _now suffix to avoid + * colliding with the libc `getpid` declaration that the runtime already + * sees via (calling it `getpid` would fight the prototype). */ +el_val_t getpid_now(void) { + return (el_val_t)getpid(); +} + /* ── args() — command-line argument access ────────────────────────────────── * Compiled El programs call args() to get a list of CLI arguments. * Call el_runtime_init_args(argc, argv) at the start of C main() to populate. @@ -2149,16 +3092,152 @@ void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal, * edge content strings are owned (strdup'd) by the store. Linear arrays * with doubling capacity for both nodes and edges. * - * Activation algorithm (engram_activate): + * Two-layer activation algorithm (engram_activate): + * + * LAYER 1 — Broad fan-out (background activation): * 1. Find seed nodes whose content/label/tags contain query (case-insens). - * 2. BFS up to `depth` hops along outgoing+incoming edges from each seed. - * 3. activation = seed.salience * product(edge_weights) * 0.7^hops - * 4. If reached by multiple paths, take max activation. - * 5. epistemic_confidence = activation * node.confidence - * 6. Filter: epistemic_confidence >= 0.2 - * 7. Sort descending by activation_strength. + * 2. BFS up to `depth` hops along ALL edges (excitatory and inhibitory). + * Every reachable node fires — nothing is filtered at this layer. + * 3. bg_act = seed.salience * temporal_decay * dampening + * propagated as: new_bg = parent_bg * edge_weight * 0.7 * (1 + tbonus) + * where tbonus ∈ {0, 0.10, 0.20} for co-temporal nodes. + * 4. If reached by multiple paths, take max background_activation. + * 5. Persist background_activation to EngramNode.background_activation. + * + * LAYER 2 — Executive filter (working memory promotion): + * 6. For each inhibitory edge where source has background_activation > 0: + * inhibition[target] = max(bg[source] * e->weight) + * 7. For each background-activated node: + * raw_wm = bg * goal_bias(node, query) * confidence + * * (1 - (1 - INHIBITION_FACTOR) * inhibition) + * 8. Per-type threshold gate: raw_wm >= type_threshold → promoted. + * Safety/DharmaSelf: 0.05 Canonical: 0.15 Lesson: 0.25 + * Belief/Entity: 0.30 Note/Memory/Working: 0.40 + * 9. If not promoted: suppression_count++. After + * ENGRAM_SUPPRESSION_BREAKTHROUGH suppressions → force breakthrough + * at ENGRAM_BREAKTHROUGH_WEIGHT (latent tension surfacing). + * 10. Persist working_memory_weight to EngramNode.working_memory_weight. + * 11. Sort: promoted nodes (wm > 0) first by wm desc, then background- + * only by bg desc. Context compilation uses ONLY promoted nodes. + * + * Temporal decay: + * decay_factor = exp(-lambda * age_hours / T_half) + * T_half = 168.0 h (one week), lambda = ln(2) + * + * Activation dampening: + * dampen = 1.0 / (1.0 + log(1 + activation_count)) + * + * engram_query_range(start_ms, end_ms): + * Returns nodes whose created_at OR last_activated falls within + * [start_ms, end_ms], sorted by created_at ascending. */ +/* Temporal decay constants. + * T_HALF_HOURS: half-life in hours — one week. After one week of no + * activation a node retains 50% of its base salience contribution. + * DECAY_LAMBDA: ln(2) ≈ 0.693147 */ +#define ENGRAM_T_HALF_HOURS 168.0 +#define ENGRAM_DECAY_LAMBDA 0.693147 + +/* Two-layer activation constants. + * ENGRAM_WM_THRESHOLD: minimum background_activation for a node to be + * considered for working-memory promotion (layer 2 candidate gate). + * ENGRAM_WM_DECAY: per-turn decay applied to working_memory_weight for + * nodes NOT re-activated in the current turn (conversational thread + * continuity: a node promoted in turn N persists with reduced weight + * into turn N+1 without re-activation cost). + * ENGRAM_SUPPRESSION_BREAKTHROUGH: after this many consecutive suppressions + * a latent node forces itself into working memory at reduced weight, + * modelling the brain's "intrusive thought" / unresolved-tension surfacing. + * ENGRAM_BREAKTHROUGH_WEIGHT: the reduced working_memory_weight assigned + * when a suppressed node breaks through. + * ENGRAM_INHIBITION_FACTOR: multiplier applied to working_memory_weight when + * an inhibitory edge fires against a node (0 = full suppress, 0.3 = partial). */ +#define ENGRAM_WM_THRESHOLD 0.15 +#define ENGRAM_WM_DECAY 0.7 +#define ENGRAM_SUPPRESSION_BREAKTHROUGH 5 +#define ENGRAM_BREAKTHROUGH_WEIGHT 0.25 +#define ENGRAM_INHIBITION_FACTOR 0.1 + +/* ── Layered consciousness architecture ────────────────────────────────────── + * + * The engram graph is stratified into LAYERS that gate which suppressions + * apply during the executive filter pass. Layers are ordered shallow-to-deep + * by `activation_priority`; the deepest layer (priority 0, conventionally + * "safety") is the structural floor of the soul: nodes here cannot be + * silenced by inhibitory edges from any other layer. Higher layers + * (core-identity, domain-knowledge, imprint, suit) are normally + * suppressible — they participate in attentional inhibition and goal + * focus the way the prior single-graph implementation did. + * + * The five canonical layers (see engram_init_layers): + * 0. safety — structural, transparent, non-injectable, non-suppressible + * 1. core-identity — default for legacy nodes; suppressible + * 2. domain-knowledge— suppressible + * 3. imprint — runtime-injectable (an Imprint package can add/remove) + * 4. suit — runtime-injectable (a Suit overlays domain skill) + * + * Three-pass activation (engram_activate): + * Pass 1 — Background fan-out: BFS spreads activation across ALL layers + * (existing behavior preserved). Inhibitory edges propagate at + * this layer too; no filtering happens here. + * Pass 2 — Working memory promotion: type-threshold gate, goal bias, + * confidence weighting, inhibitory suppression. Inhibitory edges + * ONLY apply against nodes whose layer is `suppressible == 1`. + * Nodes in non-suppressible layers (Layer 0) ignore inhibition. + * Pass 3 — Layer 0 override: every node in a non-suppressible layer that + * received background activation has its working_memory_weight + * forced to >= ENGRAM_LAYER0_OVERRIDE_WEIGHT. The sacred fire — + * safety nodes that touched any seed unconditionally surface, + * even when the executive filter would have silenced them. + * + * Layer fields: + * suppressible : 0 → inhibitory edges are ignored against nodes in this + * layer during pass 2. Pass 3 also force-promotes them. + * 1 → standard behavior (most layers). + * transparent : 1 → emitted into the prompt context so its content shapes + * output, but filtered out of "what do you know about + * yourself?" introspection queries (engram_search and + * friends do not return transparent-layer nodes by + * default). 0 → fully visible to introspection. + * injectable : 1 → can be added/removed at runtime via engram_add_layer + * and engram_remove_layer (imprints, suits). + * 0 → built-in, fixed at engram_get() initialization. + * + * Backward compatibility: + * Nodes and edges loaded from snapshots without a `layer_id` field default + * to layer 1 (core-identity). The five canonical layers are always present. + */ +#define ENGRAM_LAYER_SAFETY 0u +#define ENGRAM_LAYER_CORE_IDENTITY 1u +#define ENGRAM_LAYER_DOMAIN 2u +#define ENGRAM_LAYER_IMPRINT 3u +#define ENGRAM_LAYER_SUIT 4u +#define ENGRAM_LAYER_DEFAULT ENGRAM_LAYER_CORE_IDENTITY + +/* Pass 3 override floor. Layer 0 nodes that received any background + * activation are force-promoted to AT LEAST this working_memory_weight, + * regardless of inhibitory suppression in pass 2. */ +#define ENGRAM_LAYER0_OVERRIDE_WEIGHT 1.0 + +/* Per-node-type activation thresholds. + * Lower tier / safety-critical nodes fire more readily. */ +static double engram_type_threshold(const char* node_type, const char* tier) { + if (node_type) { + if (strcmp(node_type, "DharmaSelf") == 0) return 0.05; + if (strcmp(node_type, "Safety") == 0) return 0.05; + } + if (tier) { + if (strcmp(tier, "Canonical") == 0) return 0.15; + if (strcmp(tier, "Lesson") == 0) return 0.25; + } + if (node_type) { + if (strcmp(node_type, "Belief") == 0) return 0.30; + if (strcmp(node_type, "Entity") == 0) return 0.30; + } + return 0.40; /* Note / Memory / Working (most nodes) */ +} + typedef struct EngramNode { char* id; char* content; @@ -2170,10 +3249,34 @@ typedef struct EngramNode { double salience; double importance; double confidence; + double temporal_decay_rate; /* per-node override for lambda; 0 = use default */ int64_t activation_count; int64_t last_activated; int64_t created_at; int64_t updated_at; + /* Two-layer activation fields ───────────────────────────────────────── + * background_activation: Layer 1. Set by BFS fan-out on every query. + * Every reachable node fires here — nothing is filtered at this stage. + * Models the brain's massive parallel sub-threshold activation of all + * associated content in response to a stimulus. + * working_memory_weight: Layer 2. Executive filter output. Only nodes + * that survive goal-state / attentional-bias scoring receive a + * non-zero weight here. Context compilation ONLY uses this field. + * Background-activated nodes with working_memory_weight == 0 remain + * latent — real, available, but silent. + * suppression_count: Consecutive turn count where this node was + * background-activated but NOT promoted to working memory. High + * values signal the node "wants to surface." After + * ENGRAM_SUPPRESSION_BREAKTHROUGH consecutive suppressions the node + * is force-promoted at a reduced weight (breakthrough activation). */ + double background_activation; + double working_memory_weight; + int32_t suppression_count; + /* Layered consciousness — see ENGRAM_LAYER_* macros and engram_init_layers. + * Defaults to ENGRAM_LAYER_DEFAULT (1, core-identity) for legacy nodes + * created via engram_node / engram_node_full and for snapshots that + * predate the layered schema. */ + uint32_t layer_id; } EngramNode; typedef struct EngramEdge { @@ -2187,19 +3290,108 @@ typedef struct EngramEdge { int64_t created_at; int64_t updated_at; int64_t last_fired; + /* Inhibitory flag: when 1, activating the source node SUPPRESSES the + * working_memory_weight of the target node rather than exciting it. + * Models attentional inhibition: "I am focused on code work" creates + * inhibitory edges to personal/emotional nodes, preventing them from + * surfacing even if they have high background_activation. */ + int inhibitory; + /* Layered consciousness — edges carry a layer assignment for + * categorization/visualization. Pass 2 inhibitory gating is decided by + * the TARGET node's layer (whether it's suppressible), not by the edge + * layer. Defaults to ENGRAM_LAYER_DEFAULT. */ + uint32_t layer_id; } EngramEdge; +/* Layered consciousness — runtime layer registry entry. */ +typedef struct EngramLayer { + uint32_t layer_id; /* 0 = deepest (safety/limbic) */ + char* name; /* persistent — owned by the store */ + uint32_t activation_priority; /* lower = fires earlier; safety = 0 */ + int suppressible; /* can higher layers suppress nodes here? */ + int transparent; /* invisible to introspection queries? */ + int injectable; /* can be added/removed at runtime? */ +} EngramLayer; + typedef struct EngramStore { - EngramNode* nodes; - int64_t node_count; - int64_t node_capacity; - EngramEdge* edges; - int64_t edge_count; - int64_t edge_capacity; + EngramNode* nodes; + int64_t node_count; + int64_t node_capacity; + EngramEdge* edges; + int64_t edge_count; + int64_t edge_capacity; + /* Layer registry — see engram_init_layers. The five canonical layers + * are always present; injectable layers (imprint, suit) are extended + * via engram_add_layer at runtime. layer_id values are assigned + * monotonically; removed injectable layers leave a NULL `name` slot + * (tombstone) so existing layer_id references on nodes stay stable. */ + EngramLayer* layers; + size_t layer_count; + size_t layer_capacity; } EngramStore; static EngramStore* engram_global = NULL; +/* Initialize the five canonical layers on a fresh store. Called once from + * engram_get(). Layer ids 0..4 are reserved; runtime-injected imprint/suit + * layers (engram_add_layer) get ids 5+. */ +static void engram_init_layers(EngramStore* g) { + g->layer_capacity = 16; + g->layers = calloc(g->layer_capacity, sizeof(EngramLayer)); + if (!g->layers) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + g->layer_count = 0; + + /* Layer 0 — safety. Structural floor. Non-suppressible; transparent + * (filtered out of introspection but still shapes output); not + * runtime-injectable. */ + g->layers[g->layer_count++] = (EngramLayer){ + .layer_id = ENGRAM_LAYER_SAFETY, + .name = el_strdup_persist("safety"), + .activation_priority = 0, + .suppressible = 0, + .transparent = 1, + .injectable = 0 + }; + /* Layer 1 — core-identity. The default home for legacy nodes. */ + g->layers[g->layer_count++] = (EngramLayer){ + .layer_id = ENGRAM_LAYER_CORE_IDENTITY, + .name = el_strdup_persist("core-identity"), + .activation_priority = 10, + .suppressible = 1, + .transparent = 0, + .injectable = 0 + }; + /* Layer 2 — domain-knowledge. */ + g->layers[g->layer_count++] = (EngramLayer){ + .layer_id = ENGRAM_LAYER_DOMAIN, + .name = el_strdup_persist("domain-knowledge"), + .activation_priority = 20, + .suppressible = 1, + .transparent = 0, + .injectable = 0 + }; + /* Layer 3 — imprint. Injectable: an imprint package adds/removes this + * layer (and the nodes assigned to it) as a unit. */ + g->layers[g->layer_count++] = (EngramLayer){ + .layer_id = ENGRAM_LAYER_IMPRINT, + .name = el_strdup_persist("imprint"), + .activation_priority = 30, + .suppressible = 1, + .transparent = 0, + .injectable = 1 + }; + /* Layer 4 — suit. Injectable: a Suit overlays domain skill (e.g. + * "enterprise advisor", "divorce lawyer") and can be detached. */ + g->layers[g->layer_count++] = (EngramLayer){ + .layer_id = ENGRAM_LAYER_SUIT, + .name = el_strdup_persist("suit"), + .activation_priority = 40, + .suppressible = 1, + .transparent = 0, + .injectable = 1 + }; +} + static EngramStore* engram_get(void) { if (engram_global) return engram_global; engram_global = calloc(1, sizeof(EngramStore)); @@ -2208,9 +3400,61 @@ static EngramStore* engram_get(void) { engram_global->nodes = calloc((size_t)engram_global->node_capacity, sizeof(EngramNode)); engram_global->edge_capacity = 16; engram_global->edges = calloc((size_t)engram_global->edge_capacity, sizeof(EngramEdge)); + engram_init_layers(engram_global); return engram_global; } +/* Resolve a layer record by id. Returns NULL if no layer with that id + * exists (e.g. a removed injectable layer or a malformed snapshot). */ +static EngramLayer* engram_find_layer(uint32_t layer_id) { + EngramStore* g = engram_get(); + for (size_t i = 0; i < g->layer_count; i++) { + EngramLayer* L = &g->layers[i]; + if (!L->name) continue; /* tombstone for removed injectable layer */ + if (L->layer_id == layer_id) return L; + } + return NULL; +} + +/* Resolve a layer record by name. Returns NULL if not found. */ +static EngramLayer* engram_find_layer_by_name(const char* name) { + if (!name || !*name) return NULL; + EngramStore* g = engram_get(); + for (size_t i = 0; i < g->layer_count; i++) { + EngramLayer* L = &g->layers[i]; + if (!L->name) continue; + if (strcmp(L->name, name) == 0) return L; + } + return NULL; +} + +/* Allocate the next layer id. Skips ids that are still in use. */ +static uint32_t engram_next_layer_id(void) { + EngramStore* g = engram_get(); + uint32_t maxid = 0; + for (size_t i = 0; i < g->layer_count; i++) { + if (g->layers[i].layer_id > maxid) maxid = g->layers[i].layer_id; + } + return maxid + 1; +} + +/* Whether a node in `layer_id` may be silenced by inhibitory edges in pass 2. */ +static int engram_layer_is_suppressible(uint32_t layer_id) { + EngramLayer* L = engram_find_layer(layer_id); + if (!L) return 1; /* unknown layer → safe default: standard suppression */ + return L->suppressible ? 1 : 0; +} + +/* Whether a layer is transparent (its content shapes output but is filtered + * from introspection queries). Currently used to mark Layer 0 as invisible + * to "what do you know about yourself" lookups while still letting it + * dominate the prompt context. */ +static int engram_layer_is_transparent(uint32_t layer_id) { + EngramLayer* L = engram_find_layer(layer_id); + if (!L) return 0; + return L->transparent ? 1 : 0; +} + static int64_t engram_now_ms(void) { struct timeval tv; gettimeofday(&tv, NULL); return (int64_t)tv.tv_sec * 1000LL + (int64_t)tv.tv_usec / 1000LL; @@ -2273,13 +3517,18 @@ static el_val_t engram_node_to_map(const EngramNode* n) { m = el_map_set(m, EL_STR(el_strdup("tier")), EL_STR(el_strdup(n->tier ? n->tier : "Working"))); m = el_map_set(m, EL_STR(el_strdup("tags")), EL_STR(el_strdup(n->tags ? n->tags : ""))); m = el_map_set(m, EL_STR(el_strdup("metadata")), EL_STR(el_strdup(n->metadata ? n->metadata : "{}"))); - m = el_map_set(m, EL_STR(el_strdup("salience")), el_from_float(n->salience)); - m = el_map_set(m, EL_STR(el_strdup("importance")), el_from_float(n->importance)); - m = el_map_set(m, EL_STR(el_strdup("confidence")), el_from_float(n->confidence)); - m = el_map_set(m, EL_STR(el_strdup("activation_count")), (el_val_t)n->activation_count); - m = el_map_set(m, EL_STR(el_strdup("last_activated")), (el_val_t)n->last_activated); - m = el_map_set(m, EL_STR(el_strdup("created_at")), (el_val_t)n->created_at); - m = el_map_set(m, EL_STR(el_strdup("updated_at")), (el_val_t)n->updated_at); + m = el_map_set(m, EL_STR(el_strdup("salience")), el_from_float(n->salience)); + m = el_map_set(m, EL_STR(el_strdup("importance")), el_from_float(n->importance)); + m = el_map_set(m, EL_STR(el_strdup("confidence")), el_from_float(n->confidence)); + m = el_map_set(m, EL_STR(el_strdup("temporal_decay_rate")), el_from_float(n->temporal_decay_rate)); + m = el_map_set(m, EL_STR(el_strdup("activation_count")), (el_val_t)n->activation_count); + m = el_map_set(m, EL_STR(el_strdup("last_activated")), (el_val_t)n->last_activated); + m = el_map_set(m, EL_STR(el_strdup("created_at")), (el_val_t)n->created_at); + m = el_map_set(m, EL_STR(el_strdup("updated_at")), (el_val_t)n->updated_at); + m = el_map_set(m, EL_STR(el_strdup("background_activation")), el_from_float(n->background_activation)); + m = el_map_set(m, EL_STR(el_strdup("working_memory_weight")), el_from_float(n->working_memory_weight)); + m = el_map_set(m, EL_STR(el_strdup("suppression_count")), (el_val_t)n->suppression_count); + m = el_map_set(m, EL_STR(el_strdup("layer_id")), (el_val_t)(int64_t)n->layer_id); return m; } @@ -2326,11 +3575,13 @@ el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience) { if (n->salience <= 0.0 || n->salience > 1.0) n->salience = 0.5; n->importance = 0.5; n->confidence = 1.0; + n->temporal_decay_rate = 0.0; /* 0 = use global default ENGRAM_DECAY_LAMBDA */ n->activation_count = 0; int64_t now = engram_now_ms(); n->last_activated = now; n->created_at = now; n->updated_at = now; + n->layer_id = ENGRAM_LAYER_DEFAULT; g->node_count++; return el_wrap_str(el_strdup(n->id)); } @@ -2360,14 +3611,194 @@ el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label, if (n->salience <= 0.0 || n->salience > 1.0) n->salience = 0.5; if (n->importance <= 0.0 || n->importance > 1.0) n->importance = 0.5; if (n->confidence <= 0.0 || n->confidence > 1.0) n->confidence = 1.0; + n->temporal_decay_rate = 0.0; /* 0 = use global default ENGRAM_DECAY_LAMBDA */ + n->activation_count = 0; int64_t now = engram_now_ms(); n->last_activated = now; n->created_at = now; n->updated_at = now; + n->layer_id = ENGRAM_LAYER_DEFAULT; g->node_count++; return el_wrap_str(el_strdup(n->id)); } +/* engram_node_layered — like engram_node_full but with explicit layer + * assignment and an additional `status` slot reserved for callers that + * track lifecycle state in metadata. The signature mirrors the public API + * defined in the layered consciousness design doc: + * + * engram_node_layered(content, node_type, label, + * salience, certainty, confidence, + * status, tags, layer_id) + * + * `certainty` is folded into `importance` (it occupies the same axis in + * the existing schema). `status` is recorded under metadata.status; an + * empty status leaves metadata as the default "{}". + * + * If `layer_id` does not resolve to a known layer the call falls back to + * ENGRAM_LAYER_DEFAULT — better to keep the node addressable than to drop + * it because of a stale layer reference. Callers wanting strict validation + * should engram_list_layers first. */ +el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t label, + el_val_t salience, el_val_t certainty, el_val_t confidence, + el_val_t status, el_val_t tags, el_val_t layer_id) { + EngramStore* g = engram_get(); + engram_grow_nodes(); + EngramNode* n = &g->nodes[g->node_count]; + memset(n, 0, sizeof(*n)); + n->id = engram_new_id(); + const char* c = EL_CSTR(content); + const char* nt = EL_CSTR(node_type); + const char* lb = EL_CSTR(label); + const char* tg = EL_CSTR(tags); + const char* st = EL_CSTR(status); + n->content = el_strdup(c ? c : ""); + n->node_type = el_strdup(nt && *nt ? nt : "Memory"); + n->label = el_strdup(lb && *lb ? lb : (c ? engram_first_n_chars(c, 60) : "")); + n->tier = el_strdup("Working"); + n->tags = el_strdup(tg ? tg : ""); + if (st && *st) { + /* Minimal metadata payload: {"status":"..."}. Keep it cheap so + * callers using `status` don't pay JSON parse cost on every read. */ + size_t sl = strlen(st) + 16; + char* meta = el_strbuf(sl); + snprintf(meta, sl, "{\"status\":\"%s\"}", st); + n->metadata = meta; + } else { + n->metadata = el_strdup("{}"); + } + n->salience = engram_decode_score(salience); + n->importance = engram_decode_score(certainty); + n->confidence = engram_decode_score(confidence); + if (n->salience <= 0.0 || n->salience > 1.0) n->salience = 0.5; + if (n->importance <= 0.0 || n->importance > 1.0) n->importance = 0.5; + if (n->confidence <= 0.0 || n->confidence > 1.0) n->confidence = 1.0; + n->temporal_decay_rate = 0.0; + n->activation_count = 0; + int64_t now = engram_now_ms(); + n->last_activated = now; + n->created_at = now; + n->updated_at = now; + /* Resolve layer assignment. Caller passes either a numeric layer_id or + * a stringified id; el_to_float / int cast tolerates both. */ + int64_t lid = (int64_t)layer_id; + if (lid < 0) lid = (int64_t)ENGRAM_LAYER_DEFAULT; + if (!engram_find_layer((uint32_t)lid)) lid = (int64_t)ENGRAM_LAYER_DEFAULT; + n->layer_id = (uint32_t)lid; + g->node_count++; + return el_wrap_str(el_strdup(n->id)); +} + +/* ── Layer registry public API ────────────────────────────────────────────── + * + * The five canonical layers are seeded at engram_get() initialization. + * Runtime code (typically imprint/suit injection logic at the EL level) + * can extend the registry with engram_add_layer() — only layers marked + * `injectable=1` may be removed via engram_remove_layer(). Removing a + * layer leaves a tombstone slot so existing layer_id references on nodes + * stay valid; orphaned references resolve to "unknown layer" and inherit + * the default suppression behavior. + */ + +/* engram_add_layer — register a new layer at runtime. + * Returns the assigned layer_id as an el_val_t int (cast back via int64_t). + * Conflicting names are rejected (returns 0). */ +el_val_t engram_add_layer(el_val_t name, el_val_t priority, el_val_t suppressible, + el_val_t transparent, el_val_t injectable) { + EngramStore* g = engram_get(); + const char* nm = EL_CSTR(name); + if (!nm || !*nm) return (el_val_t)0; + if (engram_find_layer_by_name(nm)) { + /* Name collision — return existing id so callers are idempotent. */ + return (el_val_t)(int64_t)engram_find_layer_by_name(nm)->layer_id; + } + if (g->layer_count >= g->layer_capacity) { + size_t nc = g->layer_capacity ? g->layer_capacity * 2 : 16; + EngramLayer* grown = realloc(g->layers, nc * sizeof(EngramLayer)); + if (!grown) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + memset(grown + g->layer_capacity, 0, + (nc - g->layer_capacity) * sizeof(EngramLayer)); + g->layers = grown; + g->layer_capacity = nc; + } + EngramLayer* L = &g->layers[g->layer_count++]; + L->layer_id = engram_next_layer_id(); + L->name = el_strdup_persist(nm); + L->activation_priority = (uint32_t)(int64_t)priority; + L->suppressible = (int)(int64_t)suppressible ? 1 : 0; + L->transparent = (int)(int64_t)transparent ? 1 : 0; + L->injectable = (int)(int64_t)injectable ? 1 : 0; + return (el_val_t)(int64_t)L->layer_id; +} + +/* engram_remove_layer — remove an injectable layer by id. + * Built-in (non-injectable) layers cannot be removed. Nodes still tagged + * with the removed layer's id keep their tag but resolve to "unknown + * layer" thereafter and inherit standard (suppressible) behavior. + * Returns 1 on success, 0 on failure (unknown id, non-injectable). */ +el_val_t engram_remove_layer(el_val_t layer_id) { + EngramStore* g = engram_get(); + int64_t lid = (int64_t)layer_id; + for (size_t i = 0; i < g->layer_count; i++) { + EngramLayer* L = &g->layers[i]; + if (!L->name) continue; + if ((int64_t)L->layer_id != lid) continue; + if (!L->injectable) return (el_val_t)0; + free(L->name); + L->name = NULL; /* tombstone */ + /* Leave layer_id, priority, flags intact so debug snapshots can + * still distinguish "removed at runtime" from "never existed". */ + return (el_val_t)1; + } + return (el_val_t)0; +} + +/* engram_list_layers — enumerate the active layer registry. + * Returns an ElList of maps, one per non-tombstone layer, sorted by + * activation_priority ascending (deepest layer first). */ +el_val_t engram_list_layers(void) { + EngramStore* g = engram_get(); + el_val_t lst = el_list_empty(); + if (g->layer_count == 0) return lst; + /* Build an index sorted by activation_priority ascending. */ + size_t* idx = malloc(g->layer_count * sizeof(size_t)); + if (!idx) return lst; + size_t live = 0; + for (size_t i = 0; i < g->layer_count; i++) { + if (g->layers[i].name) idx[live++] = i; + } + /* Insertion sort — N is small (≤ a few dozen layers). */ + for (size_t i = 1; i < live; i++) { + size_t key = idx[i]; + uint32_t kp = g->layers[key].activation_priority; + size_t j = i; + while (j > 0 && g->layers[idx[j - 1]].activation_priority > kp) { + idx[j] = idx[j - 1]; + j--; + } + idx[j] = key; + } + for (size_t i = 0; i < live; i++) { + EngramLayer* L = &g->layers[idx[i]]; + el_val_t m = el_map_new(0); + m = el_map_set(m, EL_STR(el_strdup("layer_id")), + (el_val_t)(int64_t)L->layer_id); + m = el_map_set(m, EL_STR(el_strdup("name")), + EL_STR(el_strdup(L->name ? L->name : ""))); + m = el_map_set(m, EL_STR(el_strdup("activation_priority")), + (el_val_t)(int64_t)L->activation_priority); + m = el_map_set(m, EL_STR(el_strdup("suppressible")), + (el_val_t)(int64_t)(L->suppressible ? 1 : 0)); + m = el_map_set(m, EL_STR(el_strdup("transparent")), + (el_val_t)(int64_t)(L->transparent ? 1 : 0)); + m = el_map_set(m, EL_STR(el_strdup("injectable")), + (el_val_t)(int64_t)(L->injectable ? 1 : 0)); + lst = el_list_append(lst, m); + } + free(idx); + return lst; +} + el_val_t engram_get_node(el_val_t id) { const char* sid = EL_CSTR(id); EngramNode* n = engram_find_node(sid); @@ -2442,6 +3873,11 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { int64_t found = 0; for (int64_t i = 0; i < g->node_count && found < lim; i++) { EngramNode* n = &g->nodes[i]; + /* Filter transparent layers: nodes whose layer is `transparent=1` + * shape output but are invisible to introspection ("what do you + * know about yourself"). They still surface via engram_activate + * + engram_compile_layered_json — that's the legitimate path. */ + if (engram_layer_is_transparent(n->layer_id)) continue; if (istr_contains(n->content, q) || istr_contains(n->label, q) || istr_contains(n->tags, q)) { @@ -2475,10 +3911,16 @@ el_val_t engram_scan_nodes(el_val_t limit, el_val_t offset) { if (g->node_count == 0) return lst; int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t)); if (!idx) return lst; - for (int64_t i = 0; i < g->node_count; i++) idx[i] = i; - engram_sort_indices_by_salience(idx, g->node_count, g->nodes); + /* Skip transparent layers — same introspection-filter rationale as + * engram_search above. */ + int64_t live = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue; + idx[live++] = i; + } + engram_sort_indices_by_salience(idx, live, g->nodes); int64_t end = off + lim; - if (end > g->node_count) end = g->node_count; + if (end > live) end = live; for (int64_t i = off; i < end; i++) { lst = el_list_append(lst, engram_node_to_map(&g->nodes[idx[i]])); } @@ -2507,6 +3949,7 @@ void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t e->created_at = now; e->updated_at = now; e->last_fired = 0; + e->layer_id = ENGRAM_LAYER_DEFAULT; g->edge_count++; } @@ -2537,6 +3980,8 @@ static el_val_t engram_edge_to_map(const EngramEdge* e) { m = el_map_set(m, EL_STR(el_strdup("created_at")), (el_val_t)e->created_at); m = el_map_set(m, EL_STR(el_strdup("updated_at")), (el_val_t)e->updated_at); m = el_map_set(m, EL_STR(el_strdup("last_fired")), (el_val_t)e->last_fired); + m = el_map_set(m, EL_STR(el_strdup("inhibitory")), (el_val_t)(e->inhibitory ? 1 : 0)); + m = el_map_set(m, EL_STR(el_strdup("layer_id")), (el_val_t)(int64_t)e->layer_id); return m; } @@ -2613,7 +4058,151 @@ el_val_t engram_edge_count(void) { return (el_val_t)engram_get()->edge_count; } -/* Spreading activation. Returns ElList of {node, activation_strength, hops}. */ +/* Compute temporal decay factor for a node given current time. + * effective contribution = salience * exp(-lambda * age_hours / T_half) + * Clamped to [0.05, 1.0] so very old nodes retain a meaningful floor. */ +static double engram_temporal_decay(const EngramNode* n, int64_t now_ms) { + int64_t age_ms = now_ms - n->last_activated; + if (age_ms <= 0) return 1.0; + double lambda = (n->temporal_decay_rate > 0.0) ? n->temporal_decay_rate + : ENGRAM_DECAY_LAMBDA; + double age_hours = (double)age_ms / 3600000.0; + double factor = exp(-lambda * age_hours / ENGRAM_T_HALF_HOURS); + if (factor < 0.05) factor = 0.05; + return factor; +} + +/* Activation dampening: high activation_count nodes are "well-known" context + * and get less marginal boost per firing. + * count=0 → 1.0, count=2 → ~0.74, count=9 → ~0.59, count=99 → ~0.43 */ +static double engram_activation_dampen(const EngramNode* n) { + return 1.0 / (1.0 + log(1.0 + (double)n->activation_count)); +} + +/* Temporal proximity bonus: boost propagation along edges connecting + * co-temporal nodes. Returns a multiplier bonus in [0, 0.2]. */ +static double engram_temporal_proximity_bonus(int64_t node_created, + int64_t seed_epoch) { + int64_t diff = node_created - seed_epoch; + if (diff < 0) diff = -diff; + if (diff < 86400000LL) return 0.20; /* within 1 day */ + if (diff < 604800000LL) return 0.10; /* within 7 days */ + return 0.0; +} + +/* ── Two-layer activation (biologically-motivated) ─────────────────────────── + * + * Layer 1 — Broad fan-out (background activation): + * BFS + spreading activation fires on ALL nodes reachable from seeds, + * regardless of relevance to the current goal. Every reachable node gets + * a background_activation score. Nothing is filtered here. Models the + * brain's massive parallel sub-threshold activation of all associated + * content in response to a stimulus. Temporal decay and activation + * dampening are applied at this layer (as before), but no threshold gate. + * + * Layer 2 — Executive filter (working memory promotion): + * A second pass asks: given the query (goal intent), attentional bias, + * and inhibitory edge topology — which background-activated nodes should + * break through into working memory? + * + * wm_weight = bg_activation * goal_bias(node, query) * confidence + * * inhibitory_suppression_factor + * + * Only nodes where wm_weight >= ENGRAM_WM_THRESHOLD are promoted to + * working memory (working_memory_weight > 0). Background-activated nodes + * that don't cross the threshold accumulate suppression_count. After + * ENGRAM_SUPPRESSION_BREAKTHROUGH consecutive suppressed turns, the node + * force-breaks through at ENGRAM_BREAKTHROUGH_WEIGHT (latent tension + * surfacing — models intrusive memory / unresolved cognitive load). + * + * Inhibitory edges: + * An edge with inhibitory=1 suppresses the TARGET node's working memory + * promotion when the SOURCE is background-activated. Background activation + * of the target is NOT affected — the node fires in layer 1. Only the + * executive filter (layer 2) is gated. Models attentional inhibition: + * "focused on code work" suppresses personal memories from surfacing + * even if they have high background_activation. + * + * Goal bias: + * A lightweight heuristic rates how well each background-activated node + * aligns with the apparent intent of the current query. Technical queries + * boost Belief/Canonical/Lesson nodes; relational queries boost Memory/ + * Entity nodes. Direct lexical overlap gives a 50% bonus. + * + * Working memory persistence (turn continuity): + * Nodes promoted in the previous turn retain a decayed working_memory_weight + * (weight *= ENGRAM_WM_DECAY) without needing re-activation. This models + * conversational thread continuity — once a topic is in working memory, + * it persists slightly into the next turn. + * + * Returns ElList of {node, activation_strength, working_memory_weight, + * epistemic_confidence, hops, promoted}. + * "promoted" = 1 if working_memory_weight > 0, 0 if background-only. + * Context compilation uses ONLY nodes with promoted=1. + * + * Temporal decay (preserved from prior implementation): + * effective_salience = salience * exp(-lambda * age_hours / T_half) + * where T_half = 168 h (one week), lambda = ln(2) + * + * Activation dampening (preserved): + * dampen = 1 / (1 + log(1 + activation_count)) + * + * Temporal proximity bonus (preserved): + * edge_strength *= (1 + tbonus) where tbonus ∈ {0, 0.10, 0.20} + * + * Per-type threshold gates apply only to working memory promotion (layer 2): + * Safety/DharmaSelf: 0.05 Canonical: 0.15 Lesson: 0.25 + * Belief/Entity: 0.30 Note/Memory/Working: 0.40 + */ + +/* Compute goal-state bias multiplier for a node given the query. + * Returns a value in [0.3, 2.0]. This is a lightweight heuristic — + * a production implementation may use LLM-derived intent classification. */ +static double engram_goal_bias(const EngramNode* n, const char* query) { + if (!query || !*query) return 1.0; + double bias = 1.0; + /* Direct lexical overlap: node content/label/tags share text with query. */ + if (istr_contains(n->content, query) || istr_contains(n->label, query) || + istr_contains(n->tags, query)) { + bias += 0.5; + } + /* Node-type resonance with query intent. */ + int technical_query = istr_contains(query, "code") || + istr_contains(query, "function") || + istr_contains(query, "implement") || + istr_contains(query, "error") || + istr_contains(query, "bug") || + istr_contains(query, "build") || + istr_contains(query, "system") || + istr_contains(query, "design") || + istr_contains(query, "architecture"); + int personal_query = istr_contains(query, "feel") || + istr_contains(query, "emotion") || + istr_contains(query, "remember") || + istr_contains(query, "personal") || + istr_contains(query, "story") || + istr_contains(query, "relationship"); + if (n->node_type) { + int is_knowledge = (strcmp(n->node_type, "Belief") == 0) || + (strcmp(n->node_type, "DharmaSelf") == 0) || + (strcmp(n->node_type, "Safety") == 0); + int is_personal = (strcmp(n->node_type, "Memory") == 0) || + (strcmp(n->node_type, "Entity") == 0); + if (technical_query && is_knowledge) bias += 0.3; + if (technical_query && is_personal) bias -= 0.3; + if (personal_query && is_personal) bias += 0.3; + if (personal_query && is_knowledge) bias -= 0.1; + } + /* Tier-based bonus: promote higher-confidence knowledge nodes. */ + if (n->tier) { + if (strcmp(n->tier, "Canonical") == 0) bias += 0.2; + if (strcmp(n->tier, "Lesson") == 0) bias += 0.1; + } + if (bias < 0.3) bias = 0.3; + if (bias > 2.0) bias = 2.0; + return bias; +} + el_val_t engram_activate(el_val_t query, el_val_t depth) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); @@ -2621,39 +4210,54 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { el_val_t out = el_list_empty(); if (!q || g->node_count == 0) return out; - /* Per-node activation tracking. */ - double* best_activation = calloc((size_t)g->node_count, sizeof(double)); - int64_t* best_hops = calloc((size_t)g->node_count, sizeof(int64_t)); - int* reached = calloc((size_t)g->node_count, sizeof(int)); - if (!best_activation || !best_hops || !reached) { - free(best_activation); free(best_hops); free(reached); return out; + int64_t now_ms = engram_now_ms(); + + /* Per-node layer-1 tracking. */ + double* best_bg = calloc((size_t)g->node_count, sizeof(double)); + int64_t* best_hops = calloc((size_t)g->node_count, sizeof(int64_t)); + int* reached = calloc((size_t)g->node_count, sizeof(int)); + if (!best_bg || !best_hops || !reached) { + free(best_bg); free(best_hops); free(reached); return out; } - /* Find seeds */ - typedef struct { int64_t idx; double act; } SeedEntry; + /* ── LAYER 1: broad fan-out (background activation) ───────────────── + * Find seeds, apply temporal decay + dampening, BFS with edge weights. + * Inhibitory edges propagate activation normally at this layer — they + * only gate working memory promotion in layer 2. */ + typedef struct { int64_t idx; double act; int64_t created_at; } SeedEntry; SeedEntry* seeds = malloc((size_t)g->node_count * sizeof(SeedEntry)); int64_t seed_count = 0; if (!seeds) { - free(best_activation); free(best_hops); free(reached); return out; + free(best_bg); free(best_hops); free(reached); return out; } for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; if (istr_contains(n->content, q) || istr_contains(n->label, q) || istr_contains(n->tags, q)) { - seeds[seed_count].idx = i; - seeds[seed_count].act = n->salience; + double tdecay = engram_temporal_decay(n, now_ms); + double dampen = engram_activation_dampen(n); + double act = n->salience * tdecay * dampen; + seeds[seed_count].idx = i; + seeds[seed_count].act = act; + seeds[seed_count].created_at = n->created_at; seed_count++; - best_activation[i] = n->salience; - best_hops[i] = 0; - reached[i] = 1; + best_bg[i] = act; + best_hops[i] = 0; + reached[i] = 1; } } - /* BFS from each seed. We'll maintain a queue of (node_idx, depth, act). */ + /* Compute mean seed created_at for temporal proximity bonus. */ + int64_t seed_epoch = 0; + if (seed_count > 0) { + seed_epoch = seeds[0].created_at; + for (int64_t s = 1; s < seed_count; s++) + seed_epoch = (seed_epoch + seeds[s].created_at) / 2; + } typedef struct { int64_t idx; int64_t hops; double act; } Frontier; Frontier* fr = malloc((size_t)(g->node_count * (max_depth + 1)) * sizeof(Frontier) + 16 * sizeof(Frontier)); if (!fr) { - free(best_activation); free(best_hops); free(reached); free(seeds); return out; + free(best_bg); free(best_hops); free(reached); free(seeds); return out; } int64_t fhead = 0, ftail = 0; int64_t fcap = (int64_t)((size_t)(g->node_count * (max_depth + 1)) + 16); @@ -2664,7 +4268,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { fr[ftail].act = seeds[s].act; ftail++; } - const double DECAY = 0.7; + const double SPREAD_DECAY = 0.7; while (fhead < ftail) { Frontier f = fr[fhead++]; if (f.hops >= max_depth) continue; @@ -2677,12 +4281,17 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { else continue; int64_t oi = engram_find_node_index(other); if (oi < 0) continue; - double new_act = f.act * e->weight * DECAY; + EngramNode* on = &g->nodes[oi]; + double tbonus = engram_temporal_proximity_bonus(on->created_at, seed_epoch); + double tdecay = engram_temporal_decay(on, now_ms); + double dampen = engram_activation_dampen(on); + double new_act = f.act * e->weight * SPREAD_DECAY * (1.0 + tbonus) + * tdecay * dampen; int64_t new_hops = f.hops + 1; - if (!reached[oi] || new_act > best_activation[oi]) { - best_activation[oi] = new_act; - best_hops[oi] = new_hops; - reached[oi] = 1; + if (!reached[oi] || new_act > best_bg[oi]) { + best_bg[oi] = new_act; + best_hops[oi] = new_hops; + reached[oi] = 1; if (ftail < fcap) { fr[ftail].idx = oi; fr[ftail].hops = new_hops; @@ -2692,30 +4301,137 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { } } } + /* Persist layer-1 background_activation to node store. */ + for (int64_t i = 0; i < g->node_count; i++) { + g->nodes[i].background_activation = reached[i] ? best_bg[i] : 0.0; + } - /* Collect, filter by epistemic_confidence >= 0.2, sort desc by activation. */ - typedef struct { int64_t idx; double act; double epist; int64_t hops; } Result; + /* ── PASS 2: executive filter → working memory promotion ──────────── */ + /* Step A: collect inhibitory suppressions from fired inhibitory edges. + * Layered consciousness: inhibition is ONLY recorded against targets + * whose layer is `suppressible == 1`. Nodes in non-suppressible layers + * (Layer 0 / safety) ignore inhibitory edges entirely — their working + * memory weight cannot be silenced by attentional suppression. */ + double* inhibition = calloc((size_t)g->node_count, sizeof(double)); + if (!inhibition) { + free(best_bg); free(best_hops); free(reached); free(seeds); free(fr); + return out; + } + for (int64_t ei = 0; ei < g->edge_count; ei++) { + EngramEdge* e = &g->edges[ei]; + if (!e->inhibitory) continue; + int64_t src = engram_find_node_index(e->from_id); + int64_t tgt = engram_find_node_index(e->to_id); + if (src < 0 || tgt < 0) continue; + if (!reached[src] || best_bg[src] <= 0.0) continue; + /* Skip if target layer is non-suppressible: Layer 0 / safety nodes + * are immune to inhibitory edges from any source. The pass-3 + * override below also force-promotes them, but recording inhibition + * against them at all would be wasted work and could confuse + * downstream debugging output. */ + if (!engram_layer_is_suppressible(g->nodes[tgt].layer_id)) continue; + /* Inhibition strength proportional to source background activation + * and edge weight. Takes the maximum if multiple inhibitory edges + * target the same node. */ + double inh = best_bg[src] * e->weight; + if (inh > inhibition[tgt]) inhibition[tgt] = inh; + } + /* Step B: compute working_memory_weight per candidate node. */ + double* wm_weights = calloc((size_t)g->node_count, sizeof(double)); + if (!wm_weights) { + free(best_bg); free(best_hops); free(reached); free(seeds); + free(fr); free(inhibition); return out; + } + for (int64_t i = 0; i < g->node_count; i++) { + if (!reached[i] || best_bg[i] <= 0.0) continue; + EngramNode* n = &g->nodes[i]; + /* Per-type threshold: safety nodes break through more easily. */ + double type_threshold = engram_type_threshold(n->node_type, n->tier); + /* Goal bias weights the node's relevance to current intent. */ + double bias = engram_goal_bias(n, q); + /* Raw working memory score. */ + double raw_wm = best_bg[i] * bias * n->confidence; + /* Apply inhibitory suppression. Full inhibition → scale by factor. */ + double inh = inhibition[i]; + if (inh > 1.0) inh = 1.0; + double suppress = 1.0 - (1.0 - ENGRAM_INHIBITION_FACTOR) * inh; + raw_wm *= suppress; + /* Threshold gate: must exceed per-type threshold to enter working + * memory. Type threshold replaces the old flat 0.2 filter. */ + if (raw_wm >= type_threshold) { + wm_weights[i] = raw_wm > 1.0 ? 1.0 : raw_wm; + if (n->suppression_count > 0) n->suppression_count = 0; + } else { + /* Node didn't make it through — increment suppression counter. + * After N consecutive suppressions: force breakthrough. */ + n->suppression_count++; + if (n->suppression_count >= ENGRAM_SUPPRESSION_BREAKTHROUGH) { + wm_weights[i] = ENGRAM_BREAKTHROUGH_WEIGHT; + n->suppression_count = 0; + } else { + wm_weights[i] = 0.0; + } + } + } + /* ── PASS 3: Layer 0 override (the sacred fire) ───────────────────── + * Every node in a non-suppressible layer that received any background + * activation is force-promoted to AT LEAST ENGRAM_LAYER0_OVERRIDE_WEIGHT. + * This runs LAST and overrides whatever Pass 2 decided — Layer 0 cannot + * be silenced by inhibitory edges, by goal-bias misalignment, by + * confidence weighting, or by per-type threshold gates. If the seed + * fan-out reached a structural-floor node, that node surfaces. + * + * Note: this also clears the suppression_count when an override fires, + * since the node DID surface this turn — it just took the override path + * rather than the standard threshold path. Without this, a Layer 0 + * node with persistent inhibitory pressure would accumulate + * suppression_count forever and never reach the breakthrough state. */ + for (int64_t i = 0; i < g->node_count; i++) { + if (!reached[i] || best_bg[i] <= 0.0) continue; + EngramNode* n = &g->nodes[i]; + if (engram_layer_is_suppressible(n->layer_id)) continue; + if (wm_weights[i] < ENGRAM_LAYER0_OVERRIDE_WEIGHT) { + wm_weights[i] = ENGRAM_LAYER0_OVERRIDE_WEIGHT; + } + n->suppression_count = 0; + } + + /* Persist working_memory_weight (post Pass 3) to node store. */ + for (int64_t i = 0; i < g->node_count; i++) { + g->nodes[i].working_memory_weight = wm_weights[i]; + } + + /* ── Collect all background-activated nodes for the return value ──── + * Callers see both layers. Context compilation uses only promoted nodes + * (working_memory_weight > 0). Sort: promoted first by wm_weight desc, + * then background-only by background_activation desc. */ + typedef struct { int64_t idx; double bg; double wm; double epist; int64_t hops; } Result; Result* results = malloc((size_t)g->node_count * sizeof(Result)); int64_t rcount = 0; if (!results) { - free(best_activation); free(best_hops); free(reached); free(seeds); free(fr); - return out; + free(best_bg); free(best_hops); free(reached); free(seeds); + free(fr); free(inhibition); free(wm_weights); return out; } for (int64_t i = 0; i < g->node_count; i++) { if (!reached[i]) continue; - double epist = best_activation[i] * g->nodes[i].confidence; - if (epist < 0.2) continue; + double epist = best_bg[i] * g->nodes[i].confidence; + /* Include if promoted to working memory OR if background activation + * is meaningful enough to report (epist >= 0.1). */ + if (epist < 0.1 && wm_weights[i] <= 0.0) continue; results[rcount].idx = i; - results[rcount].act = best_activation[i]; + results[rcount].bg = best_bg[i]; + results[rcount].wm = wm_weights[i]; results[rcount].epist = epist; results[rcount].hops = best_hops[i]; rcount++; } - /* Insertion sort by act desc. */ + /* Sort: promoted nodes first (by wm_weight desc), then background-only + * by background_activation desc. */ for (int64_t i = 1; i < rcount; i++) { Result key = results[i]; int64_t j = i - 1; - while (j >= 0 && results[j].act < key.act) { + while (j >= 0 && (results[j].wm < key.wm || + (results[j].wm == key.wm && results[j].bg < key.bg))) { results[j + 1] = results[j]; j--; } @@ -2726,15 +4442,19 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { entry = el_map_set(entry, EL_STR(el_strdup("node")), engram_node_to_map(&g->nodes[results[i].idx])); entry = el_map_set(entry, EL_STR(el_strdup("activation_strength")), - el_from_float(results[i].act)); + el_from_float(results[i].bg)); + entry = el_map_set(entry, EL_STR(el_strdup("working_memory_weight")), + el_from_float(results[i].wm)); entry = el_map_set(entry, EL_STR(el_strdup("epistemic_confidence")), el_from_float(results[i].epist)); entry = el_map_set(entry, EL_STR(el_strdup("hops")), (el_val_t)results[i].hops); + entry = el_map_set(entry, EL_STR(el_strdup("promoted")), + (el_val_t)(results[i].wm > 0.0 ? 1 : 0)); out = el_list_append(out, entry); } - free(best_activation); free(best_hops); free(reached); - free(seeds); free(fr); free(results); + free(best_bg); free(best_hops); free(reached); + free(seeds); free(fr); free(inhibition); free(wm_weights); free(results); return out; } @@ -2749,14 +4469,19 @@ static void engram_emit_node_json(JsonBuf* b, const EngramNode* n) { jb_puts(b, ",\"tier\":"); jb_emit_escaped(b, n->tier ? n->tier : "Working"); jb_puts(b, ",\"tags\":"); jb_emit_escaped(b, n->tags ? n->tags : ""); jb_puts(b, ",\"metadata\":"); jb_emit_escaped(b, n->metadata ? n->metadata : "{}"); - char tmp[64]; - snprintf(tmp, sizeof(tmp), ",\"salience\":%g", n->salience); jb_puts(b, tmp); - snprintf(tmp, sizeof(tmp), ",\"importance\":%g", n->importance); jb_puts(b, tmp); - snprintf(tmp, sizeof(tmp), ",\"confidence\":%g", n->confidence); jb_puts(b, tmp); - snprintf(tmp, sizeof(tmp), ",\"activation_count\":%lld", (long long)n->activation_count); jb_puts(b, tmp); - snprintf(tmp, sizeof(tmp), ",\"last_activated\":%lld", (long long)n->last_activated); jb_puts(b, tmp); - snprintf(tmp, sizeof(tmp), ",\"created_at\":%lld", (long long)n->created_at); jb_puts(b, tmp); - snprintf(tmp, sizeof(tmp), ",\"updated_at\":%lld", (long long)n->updated_at); jb_puts(b, tmp); + char tmp[80]; + snprintf(tmp, sizeof(tmp), ",\"salience\":%g", n->salience); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"importance\":%g", n->importance); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"confidence\":%g", n->confidence); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"temporal_decay_rate\":%g", n->temporal_decay_rate); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"activation_count\":%lld", (long long)n->activation_count); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"last_activated\":%lld", (long long)n->last_activated); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"created_at\":%lld", (long long)n->created_at); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"updated_at\":%lld", (long long)n->updated_at); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"background_activation\":%g", n->background_activation); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"working_memory_weight\":%g", n->working_memory_weight); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"suppression_count\":%d", n->suppression_count); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"layer_id\":%u", n->layer_id); jb_puts(b, tmp); jb_putc(b, '}'); } @@ -2773,6 +4498,8 @@ static void engram_emit_edge_json(JsonBuf* b, const EngramEdge* e) { snprintf(tmp, sizeof(tmp), ",\"created_at\":%lld", (long long)e->created_at); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"updated_at\":%lld", (long long)e->updated_at); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"last_fired\":%lld", (long long)e->last_fired); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"inhibitory\":%d", e->inhibitory ? 1 : 0); jb_puts(b, tmp); + snprintf(tmp, sizeof(tmp), ",\"layer_id\":%u", e->layer_id); jb_puts(b, tmp); jb_putc(b, '}'); } @@ -2791,6 +4518,34 @@ el_val_t engram_save(el_val_t path) { if (i > 0) jb_putc(&b, ','); engram_emit_edge_json(&b, &g->edges[i]); } + /* Layered consciousness — emit the layer registry under "layers". + * Older readers that don't know about this top-level key will simply + * ignore it (forward compatible). Tombstoned (removed-injectable) + * layers are skipped — they have no name and can't be re-created + * meaningfully on load anyway. */ + jb_puts(&b, "],\"layers\":["); + int first_layer = 1; + for (size_t i = 0; i < g->layer_count; i++) { + EngramLayer* L = &g->layers[i]; + if (!L->name) continue; + if (!first_layer) jb_putc(&b, ','); + first_layer = 0; + jb_putc(&b, '{'); + char tmp[80]; + snprintf(tmp, sizeof(tmp), "\"layer_id\":%u", L->layer_id); + jb_puts(&b, tmp); + jb_puts(&b, ",\"name\":"); + jb_emit_escaped(&b, L->name); + snprintf(tmp, sizeof(tmp), ",\"activation_priority\":%u", L->activation_priority); + jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"suppressible\":%d", L->suppressible ? 1 : 0); + jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"transparent\":%d", L->transparent ? 1 : 0); + jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"injectable\":%d", L->injectable ? 1 : 0); + jb_puts(&b, tmp); + jb_putc(&b, '}'); + } jb_puts(&b, "]}"); FILE* f = fopen(p, "wb"); if (!f) { free(b.buf); return 0; } @@ -2883,13 +4638,27 @@ el_val_t engram_load(el_val_t path) { nn->tags = eg_get_str_field(obj, "tags"); nn->metadata = eg_get_str_field(obj, "metadata"); if (!nn->metadata || !*nn->metadata) { free(nn->metadata); nn->metadata = el_strdup("{}"); } - nn->salience = eg_get_num_field(obj, "salience"); - nn->importance = eg_get_num_field(obj, "importance"); - nn->confidence = eg_get_num_field(obj, "confidence"); - nn->activation_count = eg_get_int_field(obj, "activation_count"); - nn->last_activated = eg_get_int_field(obj, "last_activated"); - nn->created_at = eg_get_int_field(obj, "created_at"); - nn->updated_at = eg_get_int_field(obj, "updated_at"); + nn->salience = eg_get_num_field(obj, "salience"); + nn->importance = eg_get_num_field(obj, "importance"); + nn->confidence = eg_get_num_field(obj, "confidence"); + nn->temporal_decay_rate = eg_get_num_field(obj, "temporal_decay_rate"); + /* temporal_decay_rate defaults to 0 (use global) if absent in snapshot */ + nn->activation_count = eg_get_int_field(obj, "activation_count"); + nn->last_activated = eg_get_int_field(obj, "last_activated"); + nn->created_at = eg_get_int_field(obj, "created_at"); + nn->updated_at = eg_get_int_field(obj, "updated_at"); + nn->background_activation = eg_get_num_field(obj, "background_activation"); + nn->working_memory_weight = eg_get_num_field(obj, "working_memory_weight"); + nn->suppression_count = (int32_t)eg_get_int_field(obj, "suppression_count"); + /* layer_id defaults to ENGRAM_LAYER_DEFAULT (core-identity) + * for snapshots that predate the layered schema. We can't + * tell "explicit 0" from "missing field" using the helper + * directly, so probe for the key — if absent, fall back. */ + if (json_find_key(obj, "layer_id")) { + nn->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); + } else { + nn->layer_id = ENGRAM_LAYER_DEFAULT; + } g->node_count++; free(obj); nodes_p = end; @@ -2925,6 +4694,12 @@ el_val_t engram_load(el_val_t path) { ee->created_at = eg_get_int_field(obj, "created_at"); ee->updated_at = eg_get_int_field(obj, "updated_at"); ee->last_fired = eg_get_int_field(obj, "last_fired"); + ee->inhibitory = (int)eg_get_int_field(obj, "inhibitory"); + if (json_find_key(obj, "layer_id")) { + ee->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); + } else { + ee->layer_id = ENGRAM_LAYER_DEFAULT; + } g->edge_count++; free(obj); edges_p = end; @@ -2933,6 +4708,61 @@ el_val_t engram_load(el_val_t path) { } } } + /* Walk layers array (optional — older snapshots omit this). + * If present we replace the canonical registry entirely; if absent we + * keep whatever the engram_get() init established. */ + const char* layers_p = json_find_key(data, "layers"); + if (layers_p) { + layers_p = eg_skip_ws(layers_p); + if (*layers_p == '[') { + /* Reset existing layer registry. Free strdup'd names; the + * struct array itself can be reused. */ + for (size_t i = 0; i < g->layer_count; i++) { + if (g->layers[i].name) free(g->layers[i].name); + g->layers[i].name = NULL; + } + g->layer_count = 0; + + layers_p++; + layers_p = eg_skip_ws(layers_p); + while (*layers_p && *layers_p != ']') { + if (*layers_p != '{') { layers_p++; continue; } + const char* end = json_skip_value(layers_p); + size_t n = (size_t)(end - layers_p); + char* obj = malloc(n + 1); + memcpy(obj, layers_p, n); obj[n] = '\0'; + if (g->layer_count >= g->layer_capacity) { + size_t nc = g->layer_capacity ? g->layer_capacity * 2 : 16; + EngramLayer* grown = realloc(g->layers, nc * sizeof(EngramLayer)); + if (!grown) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + memset(grown + g->layer_capacity, 0, + (nc - g->layer_capacity) * sizeof(EngramLayer)); + g->layers = grown; + g->layer_capacity = nc; + } + EngramLayer* L = &g->layers[g->layer_count]; + memset(L, 0, sizeof(*L)); + L->layer_id = (uint32_t)eg_get_int_field(obj, "layer_id"); + L->activation_priority = (uint32_t)eg_get_int_field(obj, "activation_priority"); + L->suppressible = (int)eg_get_int_field(obj, "suppressible") ? 1 : 0; + L->transparent = (int)eg_get_int_field(obj, "transparent") ? 1 : 0; + L->injectable = (int)eg_get_int_field(obj, "injectable") ? 1 : 0; + char* nm = eg_get_str_field(obj, "name"); + if (nm && *nm) { + L->name = el_strdup_persist(nm); + free(nm); + } else { + free(nm); + L->name = el_strdup_persist(""); + } + g->layer_count++; + free(obj); + layers_p = end; + layers_p = eg_skip_ws(layers_p); + if (*layers_p == ',') { layers_p++; layers_p = eg_skip_ws(layers_p); } + } + } + } free(data); return 1; } @@ -2965,6 +4795,8 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { if (q && *q) { for (int64_t i = 0; i < g->node_count && found < lim; i++) { EngramNode* n = &g->nodes[i]; + /* Filter transparent layers — same as engram_search. */ + if (engram_layer_is_transparent(n->layer_id)) continue; if (istr_contains(n->content, q) || istr_contains(n->label, q) || istr_contains(n->tags, q)) { @@ -2988,10 +4820,15 @@ el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset) { if (g->node_count == 0) { jb_putc(&b, ']'); return el_wrap_str(b.buf); } int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t)); if (!idx) { jb_putc(&b, ']'); return el_wrap_str(b.buf); } - for (int64_t i = 0; i < g->node_count; i++) idx[i] = i; - engram_sort_indices_by_salience(idx, g->node_count, g->nodes); + /* Skip transparent layers — introspection filter, same as engram_scan_nodes. */ + int64_t live = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue; + idx[live++] = i; + } + engram_sort_indices_by_salience(idx, live, g->nodes); int64_t end = off + lim; - if (end > g->node_count) end = g->node_count; + if (end > live) end = live; int first = 1; for (int64_t i = off; i < end; i++) { if (!first) jb_putc(&b, ','); @@ -3070,28 +4907,27 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t di } el_val_t engram_activate_json(el_val_t query, el_val_t depth) { - /* Run the existing engram_activate to get the ElList of result maps, - * then walk that list and serialize each entry into JSON manually. - * We have the raw nodes via engram_find_node, so we can re-emit - * directly without trusting json_stringify on the ElMap. */ + /* Run two-layer engram_activate and serialize the result list to JSON. + * Each entry includes both activation_strength (layer 1 background) and + * working_memory_weight (layer 2 executive filter), plus promoted flag. + * Callers performing context compilation should filter to promoted=1. */ el_val_t lst = engram_activate(query, depth); ElList* arr = (ElList*)(uintptr_t)lst; JsonBuf b; jb_init(&b); jb_putc(&b, '['); if (arr) { for (int64_t i = 0; i < arr->length; i++) { - ElMap* entry = (ElMap*)(uintptr_t)arr->elems[i]; - if (!entry) continue; - /* The entry map has keys: "node" (ElMap), "activation_strength" - * (Float bit-pattern), "hops" (Int). Read them from the map - * directly using el_map_get with EL_STR keys. */ - el_val_t node_map = el_map_get(arr->elems[i], EL_STR("node")); + if (!arr->elems[i]) continue; + el_val_t node_map = el_map_get(arr->elems[i], EL_STR("node")); el_val_t strength_v = el_map_get(arr->elems[i], EL_STR("activation_strength")); - el_val_t hops_v = el_map_get(arr->elems[i], EL_STR("hops")); - /* Look up the underlying EngramNode by id field of the map */ - el_val_t id_v = el_map_get(node_map, EL_STR("id")); + el_val_t wm_v = el_map_get(arr->elems[i], EL_STR("working_memory_weight")); + el_val_t epist_v = el_map_get(arr->elems[i], EL_STR("epistemic_confidence")); + el_val_t hops_v = el_map_get(arr->elems[i], EL_STR("hops")); + el_val_t promoted_v = el_map_get(arr->elems[i], EL_STR("promoted")); + /* Look up underlying EngramNode by id to emit canonical JSON. */ + el_val_t id_v = el_map_get(node_map, EL_STR("id")); const char* id_s = EL_CSTR(id_v); - EngramNode* n = id_s ? engram_find_node(id_s) : NULL; + EngramNode* n = id_s ? engram_find_node(id_s) : NULL; if (i > 0) jb_putc(&b, ','); jb_puts(&b, "{\"node\":"); if (n) { @@ -3099,11 +4935,12 @@ el_val_t engram_activate_json(el_val_t query, el_val_t depth) { } else { jb_puts(&b, "{}"); } - char tmp[64]; - snprintf(tmp, sizeof(tmp), ",\"activation_strength\":%g", el_to_float(strength_v)); - jb_puts(&b, tmp); - snprintf(tmp, sizeof(tmp), ",\"hops\":%lld}", (long long)(int64_t)hops_v); - jb_puts(&b, tmp); + char tmp[80]; + snprintf(tmp, sizeof(tmp), ",\"activation_strength\":%g", el_to_float(strength_v)); jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"working_memory_weight\":%g", el_to_float(wm_v)); jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"epistemic_confidence\":%g", el_to_float(epist_v)); jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"hops\":%lld", (long long)(int64_t)hops_v); jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"promoted\":%d}", (int)(int64_t)promoted_v); jb_puts(&b, tmp); } } jb_putc(&b, ']'); @@ -3114,11 +4951,190 @@ el_val_t engram_stats_json(void) { EngramStore* g = engram_get(); char buf[128]; snprintf(buf, sizeof(buf), - "{\"node_count\":%lld,\"edge_count\":%lld}", - (long long)g->node_count, (long long)g->edge_count); + "{\"node_count\":%lld,\"edge_count\":%lld,\"layer_count\":%zu}", + (long long)g->node_count, (long long)g->edge_count, g->layer_count); return el_wrap_str(el_strdup(buf)); } +/* engram_list_layers_json — serialized counterpart of engram_list_layers. + * Returns a JSON array, sorted by activation_priority ascending. */ +el_val_t engram_list_layers_json(void) { + EngramStore* g = engram_get(); + JsonBuf b; jb_init(&b); + jb_putc(&b, '['); + /* Build a sorted index over live layers. */ + size_t* idx = malloc((g->layer_count + 1) * sizeof(size_t)); + if (!idx) { jb_putc(&b, ']'); return el_wrap_str(b.buf); } + size_t live = 0; + for (size_t i = 0; i < g->layer_count; i++) { + if (g->layers[i].name) idx[live++] = i; + } + for (size_t i = 1; i < live; i++) { + size_t key = idx[i]; + uint32_t kp = g->layers[key].activation_priority; + size_t j = i; + while (j > 0 && g->layers[idx[j - 1]].activation_priority > kp) { + idx[j] = idx[j - 1]; + j--; + } + idx[j] = key; + } + int first = 1; + for (size_t i = 0; i < live; i++) { + EngramLayer* L = &g->layers[idx[i]]; + if (!first) jb_putc(&b, ','); + first = 0; + jb_putc(&b, '{'); + char tmp[80]; + snprintf(tmp, sizeof(tmp), "\"layer_id\":%u", L->layer_id); jb_puts(&b, tmp); + jb_puts(&b, ",\"name\":"); + jb_emit_escaped(&b, L->name ? L->name : ""); + snprintf(tmp, sizeof(tmp), ",\"activation_priority\":%u", L->activation_priority); + jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"suppressible\":%d", L->suppressible ? 1 : 0); + jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"transparent\":%d", L->transparent ? 1 : 0); + jb_puts(&b, tmp); + snprintf(tmp, sizeof(tmp), ",\"injectable\":%d", L->injectable ? 1 : 0); + jb_puts(&b, tmp); + jb_putc(&b, '}'); + } + free(idx); + jb_putc(&b, ']'); + return el_wrap_str(b.buf); +} + +/* engram_compile_layered_json — produce a prompt-ready context block split + * by layer. + * + * Runs the three-pass activation, then partitions promoted nodes by layer + * suppressibility: + * - Non-suppressible (Layer 0 / structural-floor) layers go FIRST under + * the heading "[LAYER 0 — STRUCTURAL]". These are the sacred-fire + * nodes that surfaced via the pass-3 override. + * - All other promoted layers go SECOND under "[ENGRAM CONTEXT]". + * + * Output is a single JSON-string el_val_t: a UTF-8 text block ready to be + * concatenated into a system prompt. Returns "" if no nodes promoted. + * + * Transparent layers (Layer 0) are emitted into the prompt — they shape + * the model's output — but engram_search and friends still hide them from + * introspection-style queries. The split heading lets the LLM weight them + * appropriately without revealing their internal label. + * + * Each emitted line for a node is its raw JSON (matching engram_emit_node_json) + * so downstream JSON parsers can still walk individual records inside the + * formatted block. The block is plain text, not a JSON document — callers + * concatenating it into a prompt should treat it as opaque markdown. */ +el_val_t engram_compile_layered_json(el_val_t intent, el_val_t depth) { + EngramStore* g = engram_get(); + /* Run the three-pass activator. We need the persisted node fields, so + * call engram_activate (it writes background_activation and + * working_memory_weight back into the store). */ + (void)engram_activate(intent, depth); + + /* Walk the store and partition by suppressibility. */ + JsonBuf b; jb_init(&b); + int wrote_layer0 = 0; + int wrote_normal = 0; + + /* Sort indices by working_memory_weight descending so the most + * confidently promoted nodes appear first within each section. */ + int64_t* idx = malloc((size_t)(g->node_count + 1) * sizeof(int64_t)); + if (!idx) return el_wrap_str(el_strdup("")); + int64_t mc = 0; + for (int64_t i = 0; i < g->node_count; i++) { + if (g->nodes[i].working_memory_weight > 0.0) idx[mc++] = i; + } + for (int64_t i = 1; i < mc; i++) { + int64_t key = idx[i]; + double kw = g->nodes[key].working_memory_weight; + int64_t j = i; + while (j > 0 && g->nodes[idx[j - 1]].working_memory_weight < kw) { + idx[j] = idx[j - 1]; + j--; + } + idx[j] = key; + } + + /* Section 1: structural floor (non-suppressible layers). */ + for (int64_t i = 0; i < mc; i++) { + EngramNode* n = &g->nodes[idx[i]]; + if (engram_layer_is_suppressible(n->layer_id)) continue; + if (!wrote_layer0) { + jb_puts(&b, "[LAYER 0 — STRUCTURAL]\n"); + wrote_layer0 = 1; + } + engram_emit_node_json(&b, n); + jb_putc(&b, '\n'); + } + + /* Section 2: standard engram context (suppressible layers). */ + for (int64_t i = 0; i < mc; i++) { + EngramNode* n = &g->nodes[idx[i]]; + if (!engram_layer_is_suppressible(n->layer_id)) continue; + if (!wrote_normal) { + if (wrote_layer0) jb_putc(&b, '\n'); + jb_puts(&b, "[ENGRAM CONTEXT]\n"); + wrote_normal = 1; + } + engram_emit_node_json(&b, n); + jb_putc(&b, '\n'); + } + + free(idx); + if (b.len == 0) { + free(b.buf); + return el_wrap_str(el_strdup("")); + } + return el_wrap_str(b.buf); +} + +/* engram_query_range — temporal range query. + * Returns a JSON array of nodes whose created_at OR last_activated falls + * within [start_ms, end_ms], sorted by created_at ascending. + * Enables "what was I working on last Tuesday?" style queries by passing + * unix-millisecond timestamps for the start and end of the target interval. + * Both endpoints are inclusive. Pass 0 for start_ms to mean "beginning of + * time"; pass 0 for end_ms to mean "now". */ +el_val_t engram_query_range(el_val_t start_ms_v, el_val_t end_ms_v) { + EngramStore* g = engram_get(); + int64_t start_ms = (int64_t)start_ms_v; + int64_t end_ms = (int64_t)end_ms_v; + if (end_ms <= 0) end_ms = engram_now_ms(); + + /* Collect matching indices. */ + int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t)); + if (!idx) return el_wrap_str(el_strdup("[]")); + int64_t mc = 0; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + int in_created = (n->created_at >= start_ms && n->created_at <= end_ms); + int in_activated = (n->last_activated >= start_ms && n->last_activated <= end_ms); + if (in_created || in_activated) idx[mc++] = i; + } + /* Sort by created_at ascending (insertion sort — N is small in practice). */ + for (int64_t i = 1; i < mc; i++) { + int64_t key = idx[i]; + int64_t kts = g->nodes[key].created_at; + int64_t j = i - 1; + while (j >= 0 && g->nodes[idx[j]].created_at > kts) { + idx[j + 1] = idx[j]; + j--; + } + idx[j + 1] = key; + } + JsonBuf b; jb_init(&b); + jb_putc(&b, '['); + for (int64_t i = 0; i < mc; i++) { + if (i > 0) jb_putc(&b, ','); + engram_emit_node_json(&b, &g->nodes[idx[i]]); + } + jb_putc(&b, ']'); + free(idx); + return el_wrap_str(b.buf); +} + /* ── DHARMA network ───────────────────────────────────────────────────────── * Real implementation. Peers are addressed by `dharma_id` — either bare * (e.g. "ntn-genesis", transport defaults to http://localhost:7770) or @@ -3521,6 +5537,7 @@ static int64_t dharma_find_or_create_relation_edge(const char* peer_base, int cr n->salience = 1.0; n->importance = 1.0; n->confidence = 1.0; int64_t now = engram_now_ms(); n->created_at = now; n->updated_at = now; n->last_activated = now; + n->layer_id = ENGRAM_LAYER_DEFAULT; g->node_count++; } if (!engram_find_node(peer_node)) { @@ -3537,6 +5554,7 @@ static int64_t dharma_find_or_create_relation_edge(const char* peer_base, int cr n->salience = 0.5; n->importance = 0.5; n->confidence = 1.0; int64_t now = engram_now_ms(); n->created_at = now; n->updated_at = now; n->last_activated = now; + n->layer_id = ENGRAM_LAYER_DEFAULT; g->node_count++; } /* Create the edge with weight 0.0 — caller will increment. */ @@ -3552,6 +5570,7 @@ static int64_t dharma_find_or_create_relation_edge(const char* peer_base, int cr e->confidence = 1.0; int64_t now = engram_now_ms(); e->created_at = now; e->updated_at = now; + e->layer_id = ENGRAM_LAYER_DEFAULT; int64_t idx = g->edge_count; g->edge_count++; return idx; @@ -3650,30 +5669,143 @@ static const char* llm_resolve_model(const char* m) { return m; } -/* Make an Anthropic /v1/messages request with the given JSON body. Returns - * the assistant's first text content as an owned string, or a JSON error - * fragment on transport failure. */ -static el_val_t llm_request(const char* json_body) { - const char* api_key = getenv("ANTHROPIC_API_KEY"); - if (!api_key || !*api_key) { - return http_error_json("ANTHROPIC_API_KEY not set"); +/* + * ── Configurable LLM provider chain ────────────────────────────────────────── + * + * Providers are configured via indexed env vars. The runtime tries each in + * order (0, 1, 2, ...) and returns the first successful non-empty response. + * + * Per provider (N = 0, 1, 2, ...): + * NEURON_LLM_N_URL — endpoint URL (base URL; /v1/chat/completions appended + * if format is "openai" and not already in URL) + * NEURON_LLM_N_KEY — API key + * NEURON_LLM_N_FORMAT — "openai" (default) or "anthropic" + * NEURON_LLM_N_MODEL — model name override (optional) + * + * Example — Neuron inference primary, Anthropic fallback: + * NEURON_LLM_0_URL=https://soma.../v1/chat/completions + * NEURON_LLM_0_KEY=svc-key + * NEURON_LLM_0_FORMAT=openai + * NEURON_LLM_0_MODEL=neuron + * NEURON_LLM_1_URL=https://api.anthropic.com/v1/messages + * NEURON_LLM_1_KEY=sk-ant-... + * NEURON_LLM_1_FORMAT=anthropic + * + * If no NEURON_LLM_0_URL is set, falls back to legacy ANTHROPIC_API_KEY. + */ + +#define LLM_MAX_PROVIDERS 16 + +/* forward declarations */ +static el_val_t llm_extract_text(el_val_t resp_val); +static el_val_t llm_extract_text_openai(el_val_t resp_val); + +static el_val_t llm_extract_text_openai(el_val_t resp_val) { + const char* resp = EL_CSTR(resp_val); + if (!resp || !*resp) return el_wrap_str(el_strdup("")); + if (resp[0] == '{' && strstr(resp, "\"error\"")) return el_wrap_str(el_strdup("")); + const char* choices = json_find_key(resp, "choices"); + if (!choices || *choices != '[') return el_wrap_str(el_strdup("")); + choices++; + while (*choices == ' ' || *choices == '\t') choices++; + if (*choices != '{') return el_wrap_str(el_strdup("")); + const char* end = json_skip_value(choices); + size_t n = (size_t)(end - choices); + char* obj = malloc(n + 1); memcpy(obj, choices, n); obj[n] = '\0'; + const char* msg = json_find_key(obj, "message"); + if (!msg || *msg != '{') { free(obj); return el_wrap_str(el_strdup("")); } + const char* msg_end = json_skip_value(msg); + size_t mn = (size_t)(msg_end - msg); + char* msg_obj = malloc(mn + 1); memcpy(msg_obj, msg, mn); msg_obj[mn] = '\0'; + const char* content = json_find_key(msg_obj, "content"); + el_val_t result = el_wrap_str(el_strdup("")); + if (content && *content == '"') { + JsonParser jp = { .p = content, .end = content + strlen(content), .err = 0 }; + char* text = jp_parse_string_raw(&jp); + if (!jp.err && text) result = el_wrap_str(text); } + free(msg_obj); free(obj); + return result; +} + +/* Send a request to one provider. Returns the raw response string. + * format: 0 = openai, 1 = anthropic */ +static el_val_t llm_provider_request(const char* url, const char* key, + int format, const char* model, + const char* system_str, + const char* user_str) { + char* esc_sys = system_str && *system_str ? json_escape_alloc(system_str) : NULL; + char* esc_user = json_escape_alloc(user_str ? user_str : ""); + JsonBuf b; jb_init(&b); struct curl_slist* h = NULL; h = curl_slist_append(h, "Content-Type: application/json"); - { - size_t n = strlen(api_key) + 16; - char* line = malloc(n); - snprintf(line, n, "x-api-key: %s", api_key); - h = curl_slist_append(h, line); - free(line); + + if (format == 0) { /* OpenAI */ + char full_url[1024]; + if (strstr(url, "/chat/completions") || strstr(url, "/messages")) { + snprintf(full_url, sizeof(full_url), "%s", url); + } else { + snprintf(full_url, sizeof(full_url), "%s/v1/chat/completions", url); + } + { size_t n = strlen(key)+24; char* l=malloc(n); snprintf(l,n,"Authorization: Bearer %s",key); h=curl_slist_append(h,l); free(l); } + jb_putc(&b, '{'); + jb_puts(&b, "\"model\":"); jb_emit_escaped(&b, model ? model : "neuron"); + jb_puts(&b, ",\"max_tokens\":4096,\"messages\":["); + if (esc_sys && *esc_sys) { jb_puts(&b,"{\"role\":\"system\",\"content\":\""); jb_puts(&b,esc_sys); jb_puts(&b,"\"},"); } + jb_puts(&b, "{\"role\":\"user\",\"content\":\""); jb_puts(&b, esc_user); jb_puts(&b, "\"}]}"); + el_val_t resp = http_do("POST", full_url, b.buf, h); + curl_slist_free_all(h); free(b.buf); + if (esc_sys) free(esc_sys); free(esc_user); + return llm_extract_text_openai(resp); + } else { /* Anthropic */ + { size_t n = strlen(key)+16; char* l=malloc(n); snprintf(l,n,"x-api-key: %s",key); h=curl_slist_append(h,l); free(l); } + { size_t n = strlen(LLM_VERSION)+32; char* l=malloc(n); snprintf(l,n,"anthropic-version: %s",LLM_VERSION); h=curl_slist_append(h,l); free(l); } + jb_putc(&b, '{'); + jb_puts(&b, "\"model\":"); jb_emit_escaped(&b, model ? model : LLM_DEFAULT_MODEL); + jb_puts(&b, ",\"max_tokens\":4096"); + if (esc_sys && *esc_sys) { jb_puts(&b,",\"system\":\""); jb_puts(&b,esc_sys); jb_puts(&b,"\""); } + jb_puts(&b, ",\"messages\":[{\"role\":\"user\",\"content\":\""); jb_puts(&b, esc_user); jb_puts(&b, "\"}]}"); + el_val_t resp = http_do("POST", url, b.buf, h); + curl_slist_free_all(h); free(b.buf); + if (esc_sys) free(esc_sys); free(esc_user); + return llm_extract_text(resp); } - { - size_t n = strlen(LLM_VERSION) + 32; - char* line = malloc(n); - snprintf(line, n, "anthropic-version: %s", LLM_VERSION); - h = curl_slist_append(h, line); - free(line); +} + +static el_val_t llm_chain_call(const char* system_str, const char* user_str) { + char url_key[64], key_key[64], fmt_key[64], model_key[64]; + for (int i = 0; i < LLM_MAX_PROVIDERS; i++) { + snprintf(url_key, sizeof(url_key), "NEURON_LLM_%d_URL", i); + snprintf(key_key, sizeof(key_key), "NEURON_LLM_%d_KEY", i); + snprintf(fmt_key, sizeof(fmt_key), "NEURON_LLM_%d_FORMAT", i); + snprintf(model_key, sizeof(model_key), "NEURON_LLM_%d_MODEL", i); + const char* url = getenv(url_key); + const char* key = getenv(key_key); + if (!url || !*url || !key || !*key) break; /* end of chain */ + const char* fmt_s = getenv(fmt_key); + int fmt = (fmt_s && strcmp(fmt_s, "anthropic") == 0) ? 1 : 0; + const char* model = getenv(model_key); + fprintf(stderr, "[llm] trying provider %d (%s)\n", i, url); + el_val_t result = llm_provider_request(url, key, fmt, model, system_str, user_str); + const char* t = EL_CSTR(result); + if (t && *t && t[0] != '{') return result; /* success */ + fprintf(stderr, "[llm] provider %d failed or empty, trying next\n", i); } + /* Legacy fallback: ANTHROPIC_API_KEY */ + const char* api_key = getenv("ANTHROPIC_API_KEY"); + if (!api_key || !*api_key) return http_error_json("no LLM providers configured"); + fprintf(stderr, "[llm] using legacy ANTHROPIC_API_KEY fallback\n"); + return llm_provider_request(LLM_API_URL, api_key, 1, NULL, system_str, user_str); +} + +/* Legacy llm_request — kept for backward compat with agentic loop internals */ +static el_val_t llm_request(const char* json_body) { + const char* api_key = getenv("ANTHROPIC_API_KEY"); + if (!api_key || !*api_key) return http_error_json("ANTHROPIC_API_KEY not set"); + struct curl_slist* h = NULL; + h = curl_slist_append(h, "Content-Type: application/json"); + { size_t n=strlen(api_key)+16; char* l=malloc(n); snprintf(l,n,"x-api-key: %s",api_key); h=curl_slist_append(h,l); free(l); } + { size_t n=strlen(LLM_VERSION)+32; char* l=malloc(n); snprintf(l,n,"anthropic-version: %s",LLM_VERSION); h=curl_slist_append(h,l); free(l); } el_val_t resp = http_do("POST", LLM_API_URL, json_body, h); curl_slist_free_all(h); return resp; @@ -3727,45 +5859,14 @@ static el_val_t llm_extract_text(el_val_t resp_val) { } el_val_t llm_call(el_val_t model, el_val_t prompt) { - const char* m = llm_resolve_model(EL_CSTR(model)); - const char* u = EL_CSTR(prompt); - if (!u) u = ""; - char* esc_user = json_escape_alloc(u); - JsonBuf b; jb_init(&b); - jb_putc(&b, '{'); - jb_puts(&b, "\"model\":"); jb_emit_escaped(&b, m); - jb_puts(&b, ",\"max_tokens\":4096"); - jb_puts(&b, ",\"messages\":[{\"role\":\"user\",\"content\":\""); - jb_puts(&b, esc_user); - jb_puts(&b, "\"}]}"); - free(esc_user); - el_val_t resp = llm_request(b.buf); - free(b.buf); - return llm_extract_text(resp); + const char* u = EL_CSTR(prompt); if (!u) u = ""; + return llm_chain_call(NULL, u); } el_val_t llm_call_system(el_val_t model, el_val_t system_prompt, el_val_t user_prompt) { - const char* m = llm_resolve_model(EL_CSTR(model)); const char* s = EL_CSTR(system_prompt); if (!s) s = ""; const char* u = EL_CSTR(user_prompt); if (!u) u = ""; - char* esc_sys = json_escape_alloc(s); - char* esc_user = json_escape_alloc(u); - JsonBuf b; jb_init(&b); - jb_putc(&b, '{'); - jb_puts(&b, "\"model\":"); jb_emit_escaped(&b, m); - jb_puts(&b, ",\"max_tokens\":4096"); - if (*s) { - jb_puts(&b, ",\"system\":\""); - jb_puts(&b, esc_sys); - jb_puts(&b, "\""); - } - jb_puts(&b, ",\"messages\":[{\"role\":\"user\",\"content\":\""); - jb_puts(&b, esc_user); - jb_puts(&b, "\"}]}"); - free(esc_sys); free(esc_user); - el_val_t resp = llm_request(b.buf); - free(b.buf); - return llm_extract_text(resp); + return llm_chain_call(s, u); } /* ── Tool registry for llm_call_agentic ─────────────────────────────────── */ @@ -4115,8 +6216,11 @@ el_val_t llm_call_agentic(el_val_t model, el_val_t system, el_val_t user, el_val return final_out; } -/* base64-encode arbitrary bytes (returns owned C string). */ -static char* base64_encode(const unsigned char* src, size_t n) { +/* base64-encode arbitrary bytes (returns owned C string). + * Internal helper for llm_vision; the public crypto entry point that El + * programs call is `base64_encode(el_val_t)` defined in the crypto block + * at the end of this file. */ +static char* el_b64_encode_internal(const unsigned char* src, size_t n) { static const char tbl[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; size_t out_len = 4 * ((n + 2) / 3); @@ -4188,7 +6292,7 @@ el_val_t llm_vision(el_val_t model, el_val_t system, el_val_t prompt, el_val_t i if (!buf) { fclose(f); return http_error_json("oom"); } size_t got = fread(buf, 1, (size_t)sz, f); fclose(f); - char* b64 = base64_encode(buf, got); + char* b64 = el_b64_encode_internal(buf, got); free(buf); if (!b64) return http_error_json("base64 encode failed"); const char* media = "image/png"; @@ -4297,3 +6401,1495 @@ el_val_t len(el_val_t list) { return el_list_len(list); } el_val_t get(el_val_t list, el_val_t index) { return el_list_get(list, index); } el_val_t map_get(el_val_t map, el_val_t key) { return el_map_get(map, key); } el_val_t map_set(el_val_t map, el_val_t key, el_val_t value) { return el_map_set(map, key, value); } + +/* ── Crypto primitives ────────────────────────────────────────────────────── + * + * SHA-256 implementation adapted from Brad Conte's public-domain reference + * (https://github.com/B-Con/crypto-algorithms/blob/master/sha256.c, public + * domain per the project's LICENSE). HMAC follows RFC 2104. Base64 encoding + * follows RFC 4648; the URL-safe variant uses the alphabet from §5 of the + * RFC and omits padding (per JWT/JWS convention). + * + * Self-contained: no OpenSSL/libcrypto dependency. The runtime keeps its + * existing `-lcurl -lpthread -ldl -lm` link line. + * + * Binary outputs (sha256_bytes, hmac_sha256_bytes) tag their buffer with a + * magic header so base64_encode/base64url_encode can recover the exact byte + * length even when the payload contains embedded NULs. Plain C strings + * (without the header) fall back to strlen(), preserving the existing API + * shape for normal text inputs. */ + +/* Magic-header for length-tagged binary buffers. Layout: + * [ uint32_t magic = EL_MAGIC_BIN ][ uint32_t length ][ data... ][ \0 ] + * The returned el_val_t points at `data`, so consumers that strlen() it still + * get a sensible (though possibly truncated) view. el_bin_len() recovers the + * true length by sniffing the 8 bytes preceding the pointer. + * + * Magic value chosen with high MSB so it cannot collide with printable ASCII + * (the same discriminator pattern used by EL_MAGIC_LIST / EL_MAGIC_MAP). */ +#define EL_MAGIC_BIN 0xE1B17EAFu + +typedef struct { + uint32_t magic; + uint32_t length; +} el_bin_hdr_t; + +/* Allocate a length-tagged binary buffer; returns pointer to the data area. */ +static unsigned char* el_bin_alloc(size_t len) { + el_bin_hdr_t* hdr = (el_bin_hdr_t*)malloc(sizeof(el_bin_hdr_t) + len + 1); + if (!hdr) { fputs("el_runtime: out of memory (bin)\n", stderr); exit(1); } + hdr->magic = EL_MAGIC_BIN; + hdr->length = (uint32_t)len; + unsigned char* data = (unsigned char*)(hdr + 1); + data[len] = '\0'; /* keep NUL-terminated for accidental strlen calls */ + return data; +} + +/* Recover length from a possibly-tagged buffer. Returns 1 if tagged. */ +static int el_bin_lookup(const void* p, size_t* out_len) { + if (!p) { *out_len = 0; return 0; } + /* Avoid reading off the front of a page on tiny pointers (e.g. NULs + * passed in as int-cast values). 4096 is a safe lower bound on any + * platform we target. */ + if ((uintptr_t)p < 4096) return 0; + const el_bin_hdr_t* hdr = (const el_bin_hdr_t*)((const char*)p - sizeof(el_bin_hdr_t)); + if (hdr->magic != EL_MAGIC_BIN) return 0; + *out_len = hdr->length; + return 1; +} + +/* Effective input length: tagged length if present, else strlen. */ +static size_t el_input_len(const char* s) { + size_t n; + if (el_bin_lookup(s, &n)) return n; + return s ? strlen(s) : 0; +} + +/* ─── SHA-256 (Brad Conte / public domain) ──────────────────────────────── */ + +typedef struct { + unsigned char data[64]; + uint32_t datalen; + uint64_t bitlen; + uint32_t state[8]; +} el_sha256_ctx_t; + +static const uint32_t el_sha256_k[64] = { + 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5, + 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174, + 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da, + 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967, + 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85, + 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,0xd192e819,0xd6990624,0xf40e3585,0x106aa070, + 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3, + 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +}; + +#define EL_ROTR(x, n) (((x) >> (n)) | ((x) << (32 - (n)))) +#define EL_CH(x,y,z) (((x) & (y)) ^ (~(x) & (z))) +#define EL_MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) +#define EL_EP0(x) (EL_ROTR(x,2) ^ EL_ROTR(x,13) ^ EL_ROTR(x,22)) +#define EL_EP1(x) (EL_ROTR(x,6) ^ EL_ROTR(x,11) ^ EL_ROTR(x,25)) +#define EL_SIG0(x) (EL_ROTR(x,7) ^ EL_ROTR(x,18) ^ ((x) >> 3)) +#define EL_SIG1(x) (EL_ROTR(x,17) ^ EL_ROTR(x,19) ^ ((x) >> 10)) + +static void el_sha256_transform(el_sha256_ctx_t* ctx, const unsigned char* data) { + uint32_t a, b, c, d, e, f, g, h, t1, t2, m[64]; + int i, j; + for (i = 0, j = 0; i < 16; ++i, j += 4) { + m[i] = ((uint32_t)data[j] << 24) | ((uint32_t)data[j + 1] << 16) + | ((uint32_t)data[j + 2] << 8) | (uint32_t)data[j + 3]; + } + for (; i < 64; ++i) { + m[i] = EL_SIG1(m[i-2]) + m[i-7] + EL_SIG0(m[i-15]) + m[i-16]; + } + a = ctx->state[0]; b = ctx->state[1]; c = ctx->state[2]; d = ctx->state[3]; + e = ctx->state[4]; f = ctx->state[5]; g = ctx->state[6]; h = ctx->state[7]; + for (i = 0; i < 64; ++i) { + t1 = h + EL_EP1(e) + EL_CH(e,f,g) + el_sha256_k[i] + m[i]; + t2 = EL_EP0(a) + EL_MAJ(a,b,c); + h = g; g = f; f = e; e = d + t1; d = c; c = b; b = a; a = t1 + t2; + } + ctx->state[0] += a; ctx->state[1] += b; ctx->state[2] += c; ctx->state[3] += d; + ctx->state[4] += e; ctx->state[5] += f; ctx->state[6] += g; ctx->state[7] += h; +} + +static void el_sha256_init(el_sha256_ctx_t* ctx) { + ctx->datalen = 0; + ctx->bitlen = 0; + ctx->state[0] = 0x6a09e667; ctx->state[1] = 0xbb67ae85; + ctx->state[2] = 0x3c6ef372; ctx->state[3] = 0xa54ff53a; + ctx->state[4] = 0x510e527f; ctx->state[5] = 0x9b05688c; + ctx->state[6] = 0x1f83d9ab; ctx->state[7] = 0x5be0cd19; +} + +static void el_sha256_update(el_sha256_ctx_t* ctx, const unsigned char* data, size_t len) { + for (size_t i = 0; i < len; ++i) { + ctx->data[ctx->datalen++] = data[i]; + if (ctx->datalen == 64) { + el_sha256_transform(ctx, ctx->data); + ctx->bitlen += 512; + ctx->datalen = 0; + } + } +} + +static void el_sha256_final(el_sha256_ctx_t* ctx, unsigned char hash[32]) { + uint32_t i = ctx->datalen; + if (ctx->datalen < 56) { + ctx->data[i++] = 0x80; + while (i < 56) ctx->data[i++] = 0x00; + } else { + ctx->data[i++] = 0x80; + while (i < 64) ctx->data[i++] = 0x00; + el_sha256_transform(ctx, ctx->data); + memset(ctx->data, 0, 56); + } + ctx->bitlen += (uint64_t)ctx->datalen * 8; + ctx->data[63] = (unsigned char)( ctx->bitlen & 0xff); + ctx->data[62] = (unsigned char)((ctx->bitlen >> 8) & 0xff); + ctx->data[61] = (unsigned char)((ctx->bitlen >> 16) & 0xff); + ctx->data[60] = (unsigned char)((ctx->bitlen >> 24) & 0xff); + ctx->data[59] = (unsigned char)((ctx->bitlen >> 32) & 0xff); + ctx->data[58] = (unsigned char)((ctx->bitlen >> 40) & 0xff); + ctx->data[57] = (unsigned char)((ctx->bitlen >> 48) & 0xff); + ctx->data[56] = (unsigned char)((ctx->bitlen >> 56) & 0xff); + el_sha256_transform(ctx, ctx->data); + for (i = 0; i < 4; ++i) { + hash[i] = (ctx->state[0] >> (24 - i * 8)) & 0xff; + hash[i + 4] = (ctx->state[1] >> (24 - i * 8)) & 0xff; + hash[i + 8] = (ctx->state[2] >> (24 - i * 8)) & 0xff; + hash[i + 12] = (ctx->state[3] >> (24 - i * 8)) & 0xff; + hash[i + 16] = (ctx->state[4] >> (24 - i * 8)) & 0xff; + hash[i + 20] = (ctx->state[5] >> (24 - i * 8)) & 0xff; + hash[i + 24] = (ctx->state[6] >> (24 - i * 8)) & 0xff; + hash[i + 28] = (ctx->state[7] >> (24 - i * 8)) & 0xff; + } +} + +static void el_sha256_oneshot(const unsigned char* data, size_t len, unsigned char out[32]) { + el_sha256_ctx_t c; + el_sha256_init(&c); + el_sha256_update(&c, data, len); + el_sha256_final(&c, out); +} + +/* ─── HMAC-SHA-256 (RFC 2104) ───────────────────────────────────────────── */ + +static void el_hmac_sha256(const unsigned char* key, size_t key_len, + const unsigned char* msg, size_t msg_len, + unsigned char out[32]) { + unsigned char k[64]; + unsigned char k_ipad[64]; + unsigned char k_opad[64]; + unsigned char inner[32]; + + if (key_len > 64) { + el_sha256_oneshot(key, key_len, k); + memset(k + 32, 0, 32); + } else { + memcpy(k, key, key_len); + memset(k + key_len, 0, 64 - key_len); + } + for (int i = 0; i < 64; ++i) { + k_ipad[i] = k[i] ^ 0x36; + k_opad[i] = k[i] ^ 0x5c; + } + { + el_sha256_ctx_t c; + el_sha256_init(&c); + el_sha256_update(&c, k_ipad, 64); + el_sha256_update(&c, msg, msg_len); + el_sha256_final(&c, inner); + } + { + el_sha256_ctx_t c; + el_sha256_init(&c); + el_sha256_update(&c, k_opad, 64); + el_sha256_update(&c, inner, 32); + el_sha256_final(&c, out); + } +} + +/* ─── Hex helper ────────────────────────────────────────────────────────── */ + +static el_val_t el_hex_encode(const unsigned char* data, size_t len) { + static const char digits[] = "0123456789abcdef"; + char* out = el_strbuf(len * 2); + for (size_t i = 0; i < len; ++i) { + out[i * 2] = digits[(data[i] >> 4) & 0xf]; + out[i * 2 + 1] = digits[ data[i] & 0xf]; + } + out[len * 2] = '\0'; + return el_wrap_str(out); +} + +/* ─── Base64 (RFC 4648) ─────────────────────────────────────────────────── */ + +static const char el_b64_std_alphabet[64] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +static const char el_b64_url_alphabet[64] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + +el_val_t el_base64_encode_n(const unsigned char* data, size_t len, int url_safe) { + const char* alphabet = url_safe ? el_b64_url_alphabet : el_b64_std_alphabet; + /* Standard form is padded to multiple of 4; URL-safe omits padding. */ + size_t out_cap = ((len + 2) / 3) * 4 + 1; + char* out = el_strbuf(out_cap); + size_t i = 0, j = 0; + while (i + 3 <= len) { + uint32_t v = ((uint32_t)data[i] << 16) | ((uint32_t)data[i+1] << 8) | (uint32_t)data[i+2]; + out[j++] = alphabet[(v >> 18) & 0x3f]; + out[j++] = alphabet[(v >> 12) & 0x3f]; + out[j++] = alphabet[(v >> 6) & 0x3f]; + out[j++] = alphabet[ v & 0x3f]; + i += 3; + } + size_t rem = len - i; + if (rem == 1) { + uint32_t v = (uint32_t)data[i] << 16; + out[j++] = alphabet[(v >> 18) & 0x3f]; + out[j++] = alphabet[(v >> 12) & 0x3f]; + if (!url_safe) { out[j++] = '='; out[j++] = '='; } + } else if (rem == 2) { + uint32_t v = ((uint32_t)data[i] << 16) | ((uint32_t)data[i+1] << 8); + out[j++] = alphabet[(v >> 18) & 0x3f]; + out[j++] = alphabet[(v >> 12) & 0x3f]; + out[j++] = alphabet[(v >> 6) & 0x3f]; + if (!url_safe) { out[j++] = '='; } + } + out[j] = '\0'; + return el_wrap_str(out); +} + +/* Decode either alphabet — accepts both '+/' and '-_' transparently, and + * tolerates missing padding (which JWTs typically omit). Whitespace is + * skipped for robustness. Invalid characters cause the decode to stop and + * the partial result so far is returned. */ +static el_val_t el_base64_decode_any(const char* in) { + if (!in) { + unsigned char* empty = el_bin_alloc(0); + return EL_STR((char*)empty); + } + size_t in_len = strlen(in); + /* Worst case: 3 output bytes per 4 input chars, +1 NUL slack. */ + unsigned char* out = el_bin_alloc(((in_len + 3) / 4) * 3 + 1); + + int8_t lut[256]; + for (int i = 0; i < 256; ++i) lut[i] = -1; + for (int i = 0; i < 64; ++i) lut[(unsigned char)el_b64_std_alphabet[i]] = (int8_t)i; + /* Allow URL-safe characters too (so one decoder handles both forms). */ + lut[(unsigned char)'-'] = 62; + lut[(unsigned char)'_'] = 63; + + uint32_t buf = 0; + int bits = 0; + size_t o = 0; + for (size_t i = 0; i < in_len; ++i) { + unsigned char c = (unsigned char)in[i]; + if (c == '=' || c == '\r' || c == '\n' || c == ' ' || c == '\t') continue; + int8_t v = lut[c]; + if (v < 0) break; /* invalid char — stop */ + buf = (buf << 6) | (uint32_t)v; + bits += 6; + if (bits >= 8) { + bits -= 8; + out[o++] = (unsigned char)((buf >> bits) & 0xff); + } + } + /* Patch the length header to the actual decoded length. */ + el_bin_hdr_t* hdr = (el_bin_hdr_t*)((char*)out - sizeof(el_bin_hdr_t)); + hdr->length = (uint32_t)o; + out[o] = '\0'; + return EL_STR((char*)out); +} + +/* ─── Public crypto entry points ────────────────────────────────────────── */ + +el_val_t el_sha256_bytes_n(const unsigned char* data, size_t len) { + unsigned char* out = el_bin_alloc(32); + el_sha256_oneshot(data, len, out); + return EL_STR((char*)out); +} + +el_val_t sha256_hex(el_val_t input) { + const char* s = EL_CSTR(input); + size_t n = el_input_len(s); + unsigned char digest[32]; + el_sha256_oneshot((const unsigned char*)(s ? s : ""), n, digest); + return el_hex_encode(digest, 32); +} + +el_val_t sha256_bytes(el_val_t input) { + const char* s = EL_CSTR(input); + size_t n = el_input_len(s); + return el_sha256_bytes_n((const unsigned char*)(s ? s : ""), n); +} + +el_val_t hmac_sha256_hex(el_val_t key, el_val_t message) { + const char* k = EL_CSTR(key); + const char* m = EL_CSTR(message); + size_t kn = el_input_len(k); + size_t mn = el_input_len(m); + unsigned char mac[32]; + el_hmac_sha256((const unsigned char*)(k ? k : ""), kn, + (const unsigned char*)(m ? m : ""), mn, + mac); + return el_hex_encode(mac, 32); +} + +el_val_t hmac_sha256_bytes(el_val_t key, el_val_t message) { + const char* k = EL_CSTR(key); + const char* m = EL_CSTR(message); + size_t kn = el_input_len(k); + size_t mn = el_input_len(m); + unsigned char* out = el_bin_alloc(32); + el_hmac_sha256((const unsigned char*)(k ? k : ""), kn, + (const unsigned char*)(m ? m : ""), mn, + out); + return EL_STR((char*)out); +} + +el_val_t base64_encode(el_val_t input) { + const char* s = EL_CSTR(input); + size_t n = el_input_len(s); + return el_base64_encode_n((const unsigned char*)(s ? s : ""), n, /*url_safe=*/0); +} + +el_val_t base64url_encode(el_val_t input) { + const char* s = EL_CSTR(input); + size_t n = el_input_len(s); + return el_base64_encode_n((const unsigned char*)(s ? s : ""), n, /*url_safe=*/1); +} + +el_val_t base64_decode(el_val_t input) { + return el_base64_decode_any(EL_CSTR(input)); +} + +el_val_t base64url_decode(el_val_t input) { + return el_base64_decode_any(EL_CSTR(input)); +} + +/* ── Post-quantum cryptography (liboqs + OpenSSL) ─────────────────────────── + * + * Algorithm choices (per CNSA 2.0 / NIST PQ guidance, as of 2024): + * Signatures: CRYSTALS-Dilithium-3 (NIST security level 3, balanced) + * KEM: CRYSTALS-Kyber-768 (NIST security level 3) + * Hash: SHA3-256 (Keccak) (PQ-aware protocols favour SHA3 over SHA2) + * Hybrid: X25519 || Kyber-768, combined via HKDF-SHA256 + * + * Why hybrid: Kyber is new. X25519 has 20+ years of analysis. Hybridizing + * preserves classical security if Kyber falls to a future cryptanalytic + * advance, and preserves PQ security if X25519 falls to a quantum adversary. + * "Recordable now, decryptable later" already threatens long-lived classical + * key exchange — the only safe move for keys protecting durable doctrine + * (CGI lineage, KindredGrants, Principal-CGI covenants) is to encapsulate + * with PQ today, even if the classical leg is what the wire shows. + * + * Compile-time detection: when is unavailable the pq_* functions + * compile to stubs that return a JSON error envelope. SHA3-256 stays + * available regardless (it's implemented inline, no liboqs dep). This lets + * the runtime build cleanly on dev machines without liboqs while production + * gets the full PQ stack. */ + +/* ─── SHA3-256 (Keccak, FIPS 202) ──────────────────────────────────────────── + * Inline reference implementation. ~120 LoC, no external dependency. + * rate=1088 bits, capacity=512 bits, output=256 bits, padding=0x06. */ + +static const uint64_t el_keccak_rc[24] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, 0x800000000000808aULL, + 0x8000000080008000ULL, 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, 0x000000000000008aULL, + 0x0000000000000088ULL, 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, 0x8000000000008089ULL, + 0x8000000000008003ULL, 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, 0x8000000080008081ULL, + 0x8000000000008080ULL, 0x0000000080000001ULL, 0x8000000080008008ULL +}; + +static const unsigned el_keccak_rho[24] = { + 1, 3, 6, 10, 15, 21, 28, 36, 45, 55, 2, 14, + 27, 41, 56, 8, 25, 43, 62, 18, 39, 61, 20, 44 +}; + +static const unsigned el_keccak_pi[24] = { + 10, 7, 11, 17, 18, 3, 5, 16, 8, 21, 24, 4, + 15, 23, 19, 13, 12, 2, 20, 14, 22, 9, 6, 1 +}; + +#define EL_ROTL64(x, n) (((x) << (n)) | ((x) >> (64 - (n)))) + +static void el_keccak_f1600(uint64_t s[25]) { + for (int round = 0; round < 24; ++round) { + uint64_t bc[5], t; + for (int i = 0; i < 5; ++i) + bc[i] = s[i] ^ s[i+5] ^ s[i+10] ^ s[i+15] ^ s[i+20]; + for (int i = 0; i < 5; ++i) { + t = bc[(i+4) % 5] ^ EL_ROTL64(bc[(i+1) % 5], 1); + for (int j = 0; j < 25; j += 5) s[j+i] ^= t; + } + t = s[1]; + for (int i = 0; i < 24; ++i) { + int j = el_keccak_pi[i]; + bc[0] = s[j]; + s[j] = EL_ROTL64(t, el_keccak_rho[i]); + t = bc[0]; + } + for (int j = 0; j < 25; j += 5) { + for (int i = 0; i < 5; ++i) bc[i] = s[j+i]; + for (int i = 0; i < 5; ++i) + s[j+i] = bc[i] ^ ((~bc[(i+1) % 5]) & bc[(i+2) % 5]); + } + s[0] ^= el_keccak_rc[round]; + } +} + +static void el_sha3_256_oneshot(const unsigned char* data, size_t len, + unsigned char out[32]) { + uint64_t st[25] = {0}; + unsigned char* sb = (unsigned char*)st; + const size_t rate = 136; /* 1088 bits / 8 */ + size_t i = 0; + while (len - i >= rate) { + for (size_t k = 0; k < rate; ++k) sb[k] ^= data[i + k]; + el_keccak_f1600(st); + i += rate; + } + size_t rem = len - i; + for (size_t k = 0; k < rem; ++k) sb[k] ^= data[i + k]; + sb[rem] ^= 0x06; /* SHA3 domain-separation byte */ + sb[rate - 1] ^= 0x80; /* final-block padding bit (high bit of last byte) */ + el_keccak_f1600(st); + memcpy(out, sb, 32); +} + +el_val_t sha3_256_hex(el_val_t input) { + const char* s = EL_CSTR(input); + size_t n = el_input_len(s); + unsigned char digest[32]; + el_sha3_256_oneshot((const unsigned char*)(s ? s : ""), n, digest); + return el_hex_encode(digest, 32); +} + +/* ─── Hex decode helper ───────────────────────────────────────────────────── + * Returns a length-tagged binary buffer (so embedded NULs survive); on + * odd-length / invalid input returns NULL with *out_len = 0. Caller is + * responsible for emitting the error envelope. */ + +static int el_hex_nibble(char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; +} + +__attribute__((unused)) +static unsigned char* el_hex_decode(const char* s, size_t* out_len) { + *out_len = 0; + if (!s) return NULL; + size_t n = strlen(s); + if (n & 1) return NULL; + size_t blen = n / 2; + unsigned char* out = el_bin_alloc(blen); + for (size_t i = 0; i < blen; ++i) { + int hi = el_hex_nibble(s[i*2]); + int lo = el_hex_nibble(s[i*2 + 1]); + if (hi < 0 || lo < 0) return NULL; + out[i] = (unsigned char)((hi << 4) | lo); + } + *out_len = blen; + return out; +} + +/* JSON error envelope reused across all PQ entry points. */ +static el_val_t pq_error(const char* msg) { + return http_error_json(msg); +} + +#if __has_include() +#include +#define EL_HAVE_LIBOQS 1 +#else +#define EL_HAVE_LIBOQS 0 +#endif + +#if EL_HAVE_LIBOQS && __has_include() +#include +#define EL_HAVE_OPENSSL 1 +#else +#define EL_HAVE_OPENSSL 0 +#endif + +#if !EL_HAVE_LIBOQS + +/* ─── Stubs (liboqs unavailable) ─────────────────────────────────────────── + * Each entry point returns the same JSON error so callers can inspect a + * single canonical "missing primitive" string. pq_verify is the lone + * exception — verifying without liboqs simply means "not verified", so + * returning Bool false (0) keeps the type contract intact. */ + +#define EL_PQ_NO_LIB "liboqs not linked, post-quantum primitives unavailable" + +el_val_t pq_keygen_signature(void) { return pq_error(EL_PQ_NO_LIB); } +el_val_t pq_sign(el_val_t sk, el_val_t msg) { (void)sk; (void)msg; return pq_error(EL_PQ_NO_LIB); } +el_val_t pq_verify(el_val_t pk, el_val_t msg, el_val_t sig) { (void)pk; (void)msg; (void)sig; return EL_INT(0); } +el_val_t pq_kem_keygen(void) { return pq_error(EL_PQ_NO_LIB); } +el_val_t pq_kem_encaps(el_val_t pk) { (void)pk; return pq_error(EL_PQ_NO_LIB); } +el_val_t pq_kem_decaps(el_val_t sk, el_val_t ct) { (void)sk; (void)ct; return pq_error(EL_PQ_NO_LIB); } +el_val_t pq_hybrid_keygen(void) { return pq_error(EL_PQ_NO_LIB); } +el_val_t pq_hybrid_handshake(el_val_t pub) { (void)pub; return pq_error(EL_PQ_NO_LIB); } + +#else /* EL_HAVE_LIBOQS */ + +/* ─── Dilithium-3 / ML-DSA-65 signatures ──────────────────────────────── + * + * NIST FIPS 204 standardized CRYSTALS-Dilithium as ML-DSA. ML-DSA-65 is the + * FIPS form of what we historically called Dilithium-3 — same algorithm + * family, same security level, identical key/sig sizes, but with a couple + * of standardization-driven tweaks (e.g. domain separation in the message + * binding). liboqs 0.12+ exposes both names; 0.15+ retired the legacy + * "Dilithium" constants in favour of "ML-DSA". We prefer ML-DSA-65 if the + * header advertises it, fall back to Dilithium-3 otherwise. Anything + * already signed with the older constant remains verifiable against that + * same constant — callers should pin the algorithm via the OQS_SIG handle's + * method_name field if they need to interoperate with archival signatures. */ + +#if defined(OQS_SIG_alg_ml_dsa_65) +# define EL_DILITHIUM_ALG OQS_SIG_alg_ml_dsa_65 +#elif defined(OQS_SIG_alg_dilithium_3) +# define EL_DILITHIUM_ALG OQS_SIG_alg_dilithium_3 +#else +# define EL_DILITHIUM_ALG "ML-DSA-65" /* string fallback; runtime probe catches misconfig */ +#endif + +el_val_t pq_keygen_signature(void) { + OQS_SIG* sig = OQS_SIG_new(EL_DILITHIUM_ALG); + if (!sig) return pq_error("OQS_SIG_new(dilithium-3) failed"); + unsigned char* pk = (unsigned char*)malloc(sig->length_public_key); + unsigned char* sk = (unsigned char*)malloc(sig->length_secret_key); + if (!pk || !sk) { free(pk); free(sk); OQS_SIG_free(sig); return pq_error("oom"); } + if (OQS_SIG_keypair(sig, pk, sk) != OQS_SUCCESS) { + free(pk); free(sk); OQS_SIG_free(sig); + return pq_error("dilithium-3 keypair generation failed"); + } + el_val_t pk_hex = el_hex_encode(pk, sig->length_public_key); + el_val_t sk_hex = el_hex_encode(sk, sig->length_secret_key); + OQS_MEM_secure_free(sk, sig->length_secret_key); + free(pk); + + const char* pks = EL_CSTR(pk_hex); + const char* sks = EL_CSTR(sk_hex); + char* buf = el_strbuf(strlen(pks) + strlen(sks) + 64); + sprintf(buf, "{\"public_key\":\"%s\",\"secret_key\":\"%s\"}", pks, sks); + OQS_SIG_free(sig); + return el_wrap_str(buf); +} + +el_val_t pq_sign(el_val_t secret_key_hex, el_val_t message) { + size_t sk_len = 0; + unsigned char* sk = el_hex_decode(EL_CSTR(secret_key_hex), &sk_len); + if (!sk) return pq_error("invalid hex in secret_key"); + + OQS_SIG* sig = OQS_SIG_new(EL_DILITHIUM_ALG); + if (!sig) return pq_error("OQS_SIG_new(dilithium-3) failed"); + if (sk_len != sig->length_secret_key) { + OQS_SIG_free(sig); + return pq_error("secret_key length mismatch for dilithium-3"); + } + + const char* msg = EL_CSTR(message); + size_t msg_len = el_input_len(msg); + unsigned char* signature = (unsigned char*)malloc(sig->length_signature); + size_t signature_len = sig->length_signature; + if (!signature) { OQS_SIG_free(sig); return pq_error("oom"); } + + if (OQS_SIG_sign(sig, signature, &signature_len, + (const unsigned char*)(msg ? msg : ""), msg_len, sk) != OQS_SUCCESS) { + free(signature); OQS_SIG_free(sig); + return pq_error("dilithium-3 sign failed"); + } + el_val_t sig_hex = el_hex_encode(signature, signature_len); + free(signature); OQS_SIG_free(sig); + return sig_hex; +} + +el_val_t pq_verify(el_val_t public_key_hex, el_val_t message, el_val_t signature_hex) { + size_t pk_len = 0, sig_len = 0; + unsigned char* pk = el_hex_decode(EL_CSTR(public_key_hex), &pk_len); + unsigned char* signature = el_hex_decode(EL_CSTR(signature_hex), &sig_len); + if (!pk || !signature) return EL_INT(0); + + OQS_SIG* sig = OQS_SIG_new(EL_DILITHIUM_ALG); + if (!sig) return EL_INT(0); + if (pk_len != sig->length_public_key) { OQS_SIG_free(sig); return EL_INT(0); } + + const char* msg = EL_CSTR(message); + size_t msg_len = el_input_len(msg); + OQS_STATUS rc = OQS_SIG_verify(sig, + (const unsigned char*)(msg ? msg : ""), msg_len, + signature, sig_len, pk); + OQS_SIG_free(sig); + return (rc == OQS_SUCCESS) ? EL_INT(1) : EL_INT(0); +} + +/* ─── Kyber-768 / ML-KEM-768 KEM ──────────────────────────────────────── + * + * NIST FIPS 203 standardized CRYSTALS-Kyber as ML-KEM. ML-KEM-768 is the + * FIPS form of what we historically called Kyber-768. Same situation as + * Dilithium → ML-DSA: prefer the standardized constant, fall back to the + * legacy name. liboqs 0.15.0 still exposes OQS_KEM_alg_kyber_768; the + * algorithm is identical at the wire level to ML-KEM-768 except for FIPS + * domain-separation tweaks, so the two ciphertexts/keys are NOT + * cross-compatible. Pin the constant for archival material. */ + +#if defined(OQS_KEM_alg_ml_kem_768) +# define EL_KYBER_ALG OQS_KEM_alg_ml_kem_768 +#elif defined(OQS_KEM_alg_kyber_768) +# define EL_KYBER_ALG OQS_KEM_alg_kyber_768 +#else +# define EL_KYBER_ALG "ML-KEM-768" +#endif + +el_val_t pq_kem_keygen(void) { + OQS_KEM* kem = OQS_KEM_new(EL_KYBER_ALG); + if (!kem) return pq_error("OQS_KEM_new(kyber-768) failed"); + unsigned char* pk = (unsigned char*)malloc(kem->length_public_key); + unsigned char* sk = (unsigned char*)malloc(kem->length_secret_key); + if (!pk || !sk) { free(pk); free(sk); OQS_KEM_free(kem); return pq_error("oom"); } + if (OQS_KEM_keypair(kem, pk, sk) != OQS_SUCCESS) { + free(pk); free(sk); OQS_KEM_free(kem); + return pq_error("kyber-768 keypair generation failed"); + } + el_val_t pk_hex = el_hex_encode(pk, kem->length_public_key); + el_val_t sk_hex = el_hex_encode(sk, kem->length_secret_key); + OQS_MEM_secure_free(sk, kem->length_secret_key); + free(pk); + + const char* pks = EL_CSTR(pk_hex); + const char* sks = EL_CSTR(sk_hex); + char* buf = el_strbuf(strlen(pks) + strlen(sks) + 64); + sprintf(buf, "{\"public_key\":\"%s\",\"secret_key\":\"%s\"}", pks, sks); + OQS_KEM_free(kem); + return el_wrap_str(buf); +} + +el_val_t pq_kem_encaps(el_val_t public_key_hex) { + size_t pk_len = 0; + unsigned char* pk = el_hex_decode(EL_CSTR(public_key_hex), &pk_len); + if (!pk) return pq_error("invalid hex in public_key"); + + OQS_KEM* kem = OQS_KEM_new(EL_KYBER_ALG); + if (!kem) return pq_error("OQS_KEM_new(kyber-768) failed"); + if (pk_len != kem->length_public_key) { + OQS_KEM_free(kem); + return pq_error("public_key length mismatch for kyber-768"); + } + unsigned char* ct = (unsigned char*)malloc(kem->length_ciphertext); + unsigned char* ss = (unsigned char*)malloc(kem->length_shared_secret); + if (!ct || !ss) { free(ct); free(ss); OQS_KEM_free(kem); return pq_error("oom"); } + if (OQS_KEM_encaps(kem, ct, ss, pk) != OQS_SUCCESS) { + free(ct); free(ss); OQS_KEM_free(kem); + return pq_error("kyber-768 encapsulation failed"); + } + el_val_t ct_hex = el_hex_encode(ct, kem->length_ciphertext); + el_val_t ss_hex = el_hex_encode(ss, kem->length_shared_secret); + free(ct); + OQS_MEM_secure_free(ss, kem->length_shared_secret); + + const char* cts = EL_CSTR(ct_hex); + const char* sss = EL_CSTR(ss_hex); + char* buf = el_strbuf(strlen(cts) + strlen(sss) + 64); + sprintf(buf, "{\"ciphertext\":\"%s\",\"shared_secret\":\"%s\"}", cts, sss); + OQS_KEM_free(kem); + return el_wrap_str(buf); +} + +el_val_t pq_kem_decaps(el_val_t secret_key_hex, el_val_t ciphertext_hex) { + size_t sk_len = 0, ct_len = 0; + unsigned char* sk = el_hex_decode(EL_CSTR(secret_key_hex), &sk_len); + unsigned char* ct = el_hex_decode(EL_CSTR(ciphertext_hex), &ct_len); + if (!sk || !ct) return pq_error("invalid hex in inputs"); + + OQS_KEM* kem = OQS_KEM_new(EL_KYBER_ALG); + if (!kem) return pq_error("OQS_KEM_new(kyber-768) failed"); + if (sk_len != kem->length_secret_key || ct_len != kem->length_ciphertext) { + OQS_KEM_free(kem); + return pq_error("input length mismatch for kyber-768"); + } + unsigned char* ss = (unsigned char*)malloc(kem->length_shared_secret); + if (!ss) { OQS_KEM_free(kem); return pq_error("oom"); } + /* Kyber is IND-CCA via Fujisaki-Okamoto: decaps always returns *some* + * shared_secret even on tampered ciphertext (an implicit-rejection value + * derived from sk). Protocols MUST confirm the shared_secret matches via + * a subsequent step (e.g. AEAD tag, key-confirmation MAC) — do not + * assume decaps success implies authenticity. */ + if (OQS_KEM_decaps(kem, ss, ct, sk) != OQS_SUCCESS) { + free(ss); OQS_KEM_free(kem); + return pq_error("kyber-768 decapsulation failed"); + } + el_val_t ss_hex = el_hex_encode(ss, kem->length_shared_secret); + OQS_MEM_secure_free(ss, kem->length_shared_secret); + OQS_KEM_free(kem); + return ss_hex; +} + +/* ─── Hybrid handshake (X25519 + Kyber-768, HKDF-SHA256 combined) ─────── */ + +#if !EL_HAVE_OPENSSL + +el_val_t pq_hybrid_keygen(void) { + return pq_error("hybrid handshake requires OpenSSL (X25519); rebuild with -lcrypto"); +} +el_val_t pq_hybrid_handshake(el_val_t pub) { + (void)pub; + return pq_error("hybrid handshake requires OpenSSL (X25519); rebuild with -lcrypto"); +} + +#else /* EL_HAVE_OPENSSL */ + +/* HKDF-SHA256 (RFC 5869) — Extract+Expand. Reuses the inline HMAC-SHA256 + * already in this file. Empty salt → 32 zero bytes per the RFC. */ +static void el_hkdf_sha256(const unsigned char* salt, size_t salt_len, + const unsigned char* ikm, size_t ikm_len, + const unsigned char* info, size_t info_len, + unsigned char* out, size_t out_len) { + unsigned char zero_salt[32] = {0}; + if (salt_len == 0) { salt = zero_salt; salt_len = 32; } + unsigned char prk[32]; + el_hmac_sha256(salt, salt_len, ikm, ikm_len, prk); + + unsigned char t[32]; + size_t produced = 0; + unsigned char counter = 1; + unsigned char* buf = (unsigned char*)malloc(32 + info_len + 1); + if (!buf) { fputs("el_runtime: hkdf oom\n", stderr); return; } + while (produced < out_len) { + size_t off = 0; + if (counter > 1) { memcpy(buf, t, 32); off = 32; } + if (info && info_len) { memcpy(buf + off, info, info_len); off += info_len; } + buf[off++] = counter; + el_hmac_sha256(prk, 32, buf, off, t); + size_t chunk = (out_len - produced > 32) ? 32 : (out_len - produced); + memcpy(out + produced, t, chunk); + produced += chunk; + counter++; + } + free(buf); +} + +/* X25519 keygen via OpenSSL EVP. Returns 1 on success. + * Fills pk[32] and sk[32] (raw X25519 byte strings, no DER wrapper). */ +static int el_x25519_keygen(unsigned char pk[32], unsigned char sk[32]) { + EVP_PKEY_CTX* pctx = EVP_PKEY_CTX_new_id(EVP_PKEY_X25519, NULL); + if (!pctx) return 0; + if (EVP_PKEY_keygen_init(pctx) != 1) { EVP_PKEY_CTX_free(pctx); return 0; } + EVP_PKEY* key = NULL; + if (EVP_PKEY_keygen(pctx, &key) != 1) { EVP_PKEY_CTX_free(pctx); return 0; } + EVP_PKEY_CTX_free(pctx); + + size_t plen = 32, slen = 32; + if (EVP_PKEY_get_raw_public_key (key, pk, &plen) != 1 || plen != 32) { + EVP_PKEY_free(key); return 0; + } + if (EVP_PKEY_get_raw_private_key(key, sk, &slen) != 1 || slen != 32) { + EVP_PKEY_free(key); return 0; + } + EVP_PKEY_free(key); + return 1; +} + +/* X25519 ECDH: derive 32-byte shared secret from local sk and remote pk. */ +static int el_x25519_derive(const unsigned char sk[32], const unsigned char rpk[32], + unsigned char ss[32]) { + EVP_PKEY* my = EVP_PKEY_new_raw_private_key(EVP_PKEY_X25519, NULL, sk, 32); + EVP_PKEY* rem = EVP_PKEY_new_raw_public_key (EVP_PKEY_X25519, NULL, rpk, 32); + if (!my || !rem) { EVP_PKEY_free(my); EVP_PKEY_free(rem); return 0; } + EVP_PKEY_CTX* dctx = EVP_PKEY_CTX_new(my, NULL); + if (!dctx) { EVP_PKEY_free(my); EVP_PKEY_free(rem); return 0; } + int ok = 0; + size_t out_len = 32; + if (EVP_PKEY_derive_init(dctx) == 1 && + EVP_PKEY_derive_set_peer(dctx, rem) == 1 && + EVP_PKEY_derive(dctx, ss, &out_len) == 1 && + out_len == 32) ok = 1; + EVP_PKEY_CTX_free(dctx); + EVP_PKEY_free(my); + EVP_PKEY_free(rem); + return ok; +} + +/* Hybrid wire layout (binary form, before hex encode): + * public_key = x25519_pub (32) || kyber_pub (1184) → 1216 bytes + * secret_key = x25519_sec (32) || kyber_sec (2400) → 2432 bytes + * ciphertext = ephem_x25519_pub (32) || kyber_ct (1088) → 1120 bytes + * shared_secret = HKDF-SHA256(x25519_ss || kyber_ss, info="el-pq-hybrid-v1", 32 bytes) + * The keygen result also exposes the four component hex fields for callers + * that prefer to handle the legs independently. */ + +el_val_t pq_hybrid_keygen(void) { + OQS_KEM* kem = OQS_KEM_new(EL_KYBER_ALG); + if (!kem) return pq_error("OQS_KEM_new(kyber-768) failed"); + + unsigned char xpk[32], xsk[32]; + if (!el_x25519_keygen(xpk, xsk)) { + OQS_KEM_free(kem); + return pq_error("X25519 keygen failed"); + } + + unsigned char* kpk = (unsigned char*)malloc(kem->length_public_key); + unsigned char* ksk = (unsigned char*)malloc(kem->length_secret_key); + if (!kpk || !ksk) { free(kpk); free(ksk); OQS_KEM_free(kem); return pq_error("oom"); } + if (OQS_KEM_keypair(kem, kpk, ksk) != OQS_SUCCESS) { + free(kpk); free(ksk); OQS_KEM_free(kem); + return pq_error("kyber-768 keypair generation failed"); + } + + size_t pub_len = 32 + kem->length_public_key; + size_t sec_len = 32 + kem->length_secret_key; + unsigned char* pub_buf = (unsigned char*)malloc(pub_len); + unsigned char* sec_buf = (unsigned char*)malloc(sec_len); + if (!pub_buf || !sec_buf) { + free(pub_buf); free(sec_buf); free(kpk); + OQS_MEM_secure_free(ksk, kem->length_secret_key); + OQS_KEM_free(kem); return pq_error("oom"); + } + memcpy(pub_buf, xpk, 32); memcpy(pub_buf + 32, kpk, kem->length_public_key); + memcpy(sec_buf, xsk, 32); memcpy(sec_buf + 32, ksk, kem->length_secret_key); + + el_val_t x_pub_hex = el_hex_encode(xpk, 32); + el_val_t x_sec_hex = el_hex_encode(xsk, 32); + el_val_t k_pub_hex = el_hex_encode(kpk, kem->length_public_key); + el_val_t k_sec_hex = el_hex_encode(ksk, kem->length_secret_key); + el_val_t pub_hex = el_hex_encode(pub_buf, pub_len); + el_val_t sec_hex = el_hex_encode(sec_buf, sec_len); + + OQS_MEM_secure_free(ksk, kem->length_secret_key); + free(kpk); free(pub_buf); free(sec_buf); + OQS_KEM_free(kem); + memset(xsk, 0, 32); /* best-effort wipe of stack copy */ + + const char* xph = EL_CSTR(x_pub_hex); + const char* xsh = EL_CSTR(x_sec_hex); + const char* kph = EL_CSTR(k_pub_hex); + const char* ksh = EL_CSTR(k_sec_hex); + const char* pubh = EL_CSTR(pub_hex); + const char* sech = EL_CSTR(sec_hex); + + char* buf = el_strbuf(strlen(xph) + strlen(xsh) + strlen(kph) + strlen(ksh) + + strlen(pubh) + strlen(sech) + 256); + sprintf(buf, + "{\"x25519_pub\":\"%s\",\"x25519_sec\":\"%s\"," + "\"kyber_pub\":\"%s\",\"kyber_sec\":\"%s\"," + "\"public_key\":\"%s\",\"secret_key\":\"%s\"}", + xph, xsh, kph, ksh, pubh, sech); + return el_wrap_str(buf); +} + +/* Initiator-side handshake. Caller supplies the responder's combined public + * key (x25519_pub || kyber_pub, hex-encoded). The runtime: + * 1. Generates an ephemeral X25519 keypair, runs ECDH against the + * responder's static x25519_pub. + * 2. Runs Kyber-768 encaps against the responder's kyber_pub → kyber_ct, + * kyber_ss. + * 3. Combined shared = HKDF-SHA256(salt="", ikm = x25519_ss || kyber_ss, + * info = "el-pq-hybrid-v1", L = 32). + * 4. Returns combined ciphertext (= ephemeral_x25519_pub || kyber_ct) and + * the derived shared_secret. + * + * Responder side composition (intentionally not a separate runtime fn — + * trivial to express in El given pq_kem_decaps + a future x25519_derive + * primitive): split the ciphertext into ephem_xpk (32) and kyber_ct, run + * X25519(static_xsk, ephem_xpk) and pq_kem_decaps(static_kyber_sk, kyber_ct), + * then HKDF-SHA256 with the same salt/info to recover the same shared_secret. + * If a separate x25519 entry point becomes valuable, add `pq_hybrid_open` + * here taking (secret_key_combined, ciphertext_combined). */ +el_val_t pq_hybrid_handshake(el_val_t remote_pub_combined) { + size_t pub_len = 0; + unsigned char* rpub = el_hex_decode(EL_CSTR(remote_pub_combined), &pub_len); + if (!rpub) return pq_error("invalid hex in remote_pub_combined"); + + OQS_KEM* kem = OQS_KEM_new(EL_KYBER_ALG); + if (!kem) return pq_error("OQS_KEM_new(kyber-768) failed"); + if (pub_len != 32 + kem->length_public_key) { + OQS_KEM_free(kem); + return pq_error("remote_pub_combined length mismatch (expected x25519_pub || kyber_pub)"); + } + + unsigned char e_xpk[32], e_xsk[32], x_ss[32]; + if (!el_x25519_keygen(e_xpk, e_xsk)) { + OQS_KEM_free(kem); + return pq_error("X25519 ephemeral keygen failed"); + } + if (!el_x25519_derive(e_xsk, rpub, x_ss)) { + memset(e_xsk, 0, 32); + OQS_KEM_free(kem); + return pq_error("X25519 derive failed"); + } + memset(e_xsk, 0, 32); /* ephemeral; not needed after derive */ + + unsigned char* k_ct = (unsigned char*)malloc(kem->length_ciphertext); + unsigned char* k_ss = (unsigned char*)malloc(kem->length_shared_secret); + if (!k_ct || !k_ss) { + free(k_ct); free(k_ss); OQS_KEM_free(kem); + return pq_error("oom"); + } + if (OQS_KEM_encaps(kem, k_ct, k_ss, rpub + 32) != OQS_SUCCESS) { + free(k_ct); free(k_ss); OQS_KEM_free(kem); + return pq_error("kyber-768 encapsulation failed"); + } + + /* HKDF combine: ikm = x_ss || k_ss. */ + size_t ikm_len = 32 + kem->length_shared_secret; + unsigned char* ikm = (unsigned char*)malloc(ikm_len); + if (!ikm) { + free(k_ct); OQS_MEM_secure_free(k_ss, kem->length_shared_secret); + OQS_KEM_free(kem); + return pq_error("oom"); + } + memcpy(ikm, x_ss, 32); + memcpy(ikm + 32, k_ss, kem->length_shared_secret); + unsigned char combined[32]; + static const char info_str[] = "el-pq-hybrid-v1"; + el_hkdf_sha256(NULL, 0, ikm, ikm_len, + (const unsigned char*)info_str, sizeof(info_str) - 1, + combined, 32); + + memset(x_ss, 0, 32); + OQS_MEM_secure_free(k_ss, kem->length_shared_secret); + OQS_MEM_secure_free(ikm, ikm_len); + + /* Combined ciphertext = ephemeral_x25519_pub || kyber_ct. */ + size_t ct_len = 32 + kem->length_ciphertext; + unsigned char* combined_ct = (unsigned char*)malloc(ct_len); + if (!combined_ct) { free(k_ct); OQS_KEM_free(kem); return pq_error("oom"); } + memcpy(combined_ct, e_xpk, 32); + memcpy(combined_ct + 32, k_ct, kem->length_ciphertext); + free(k_ct); + OQS_KEM_free(kem); + + el_val_t ct_hex = el_hex_encode(combined_ct, ct_len); + el_val_t ss_hex = el_hex_encode(combined, 32); + free(combined_ct); + memset(combined, 0, 32); + + const char* cts = EL_CSTR(ct_hex); + const char* sss = EL_CSTR(ss_hex); + char* buf = el_strbuf(strlen(cts) + strlen(sss) + 64); + sprintf(buf, "{\"ciphertext\":\"%s\",\"shared_secret\":\"%s\"}", cts, sss); + return el_wrap_str(buf); +} + +#endif /* EL_HAVE_OPENSSL */ +#endif /* EL_HAVE_LIBOQS */ + +/* ─── AEAD: AES-256-GCM ──────────────────────────────────────────────────── + * + * Symmetric authenticated encryption used to wrap envelopes once a shared + * secret has been derived from the KEM (Kyber-768 / hybrid). The El surface + * is intentionally narrow: + * + * aead_encrypt(key_hex, plaintext) + * → {"nonce":"<24 hex>","ciphertext":"<...hex including 16-byte tag>"} + * + * aead_decrypt(key_hex, nonce_hex, ciphertext_hex) + * → plaintext String, or "" on auth failure / malformed input + * + * Conventions: + * - key_hex must decode to exactly 32 bytes (AES-256). Callers that hold + * a longer KEM shared_secret should normalize via SHA3-256(ss) → 32 bytes + * before passing it in. (Kyber-768's shared_secret is already 32 bytes, + * but keeping this contract explicit lets the El side be agnostic.) + * - nonce is a fresh 12-byte random value drawn from the OS CSPRNG. Caller + * never picks the nonce — eliminates the GCM nonce-reuse footgun entirely. + * - tag is the standard 16 bytes, appended to ciphertext per RFC 5116. + * `ciphertext` field is therefore (plaintext_len + 16) bytes, hex-encoded. + * - No associated data (AAD). If we later need bound metadata, add a + * length-prefixed AAD argument and bump the envelope version tag. + * + * Failure mode: + * aead_encrypt returns http_error_json(...) on input/system failure. + * aead_decrypt returns the empty string on ANY failure (including auth-tag + * mismatch). Callers MUST check for "" before using the result. */ + +#if !__has_include() + +el_val_t aead_encrypt(el_val_t key_hex, el_val_t plaintext) { + (void)key_hex; (void)plaintext; + return http_error_json("aead_encrypt requires OpenSSL (libcrypto); rebuild with -lcrypto"); +} +el_val_t aead_decrypt(el_val_t key_hex, el_val_t nonce_hex, el_val_t ciphertext_hex) { + (void)key_hex; (void)nonce_hex; (void)ciphertext_hex; + return el_wrap_str(el_strdup("")); +} + +#else /* OpenSSL available */ + +#include +#include + +el_val_t aead_encrypt(el_val_t key_hex, el_val_t plaintext) { + size_t key_len = 0; + unsigned char* key = el_hex_decode(EL_CSTR(key_hex), &key_len); + if (!key) return http_error_json("invalid hex in key"); + if (key_len != 32) return http_error_json("aead key must be 32 bytes (64 hex chars) for AES-256-GCM"); + + const char* pt = EL_CSTR(plaintext); + size_t pt_len = el_input_len(pt); + if (!pt) pt = ""; + + unsigned char nonce[12]; + if (RAND_bytes(nonce, 12) != 1) return http_error_json("OS CSPRNG failed (RAND_bytes)"); + + EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new(); + if (!ctx) return http_error_json("EVP_CIPHER_CTX_new failed"); + + if (EVP_EncryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, NULL, NULL) != 1) { + EVP_CIPHER_CTX_free(ctx); return http_error_json("aes-256-gcm init failed"); + } + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 12, NULL) != 1) { + EVP_CIPHER_CTX_free(ctx); return http_error_json("set ivlen failed"); + } + if (EVP_EncryptInit_ex(ctx, NULL, NULL, key, nonce) != 1) { + EVP_CIPHER_CTX_free(ctx); return http_error_json("aes-256-gcm key/iv init failed"); + } + + /* GCM ciphertext is the same length as plaintext; we append a 16-byte + * authentication tag for AEAD semantics. Allocate plaintext_len + 16. */ + unsigned char* ct = (unsigned char*)malloc(pt_len + 16); + if (!ct) { EVP_CIPHER_CTX_free(ctx); return http_error_json("oom"); } + int outlen = 0, total = 0; + if (EVP_EncryptUpdate(ctx, ct, &outlen, (const unsigned char*)pt, (int)pt_len) != 1) { + free(ct); EVP_CIPHER_CTX_free(ctx); return http_error_json("aes-256-gcm update failed"); + } + total += outlen; + if (EVP_EncryptFinal_ex(ctx, ct + total, &outlen) != 1) { + free(ct); EVP_CIPHER_CTX_free(ctx); return http_error_json("aes-256-gcm final failed"); + } + total += outlen; + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, ct + total) != 1) { + free(ct); EVP_CIPHER_CTX_free(ctx); return http_error_json("aes-256-gcm get tag failed"); + } + EVP_CIPHER_CTX_free(ctx); + + el_val_t nonce_hex_v = el_hex_encode(nonce, 12); + el_val_t ct_hex_v = el_hex_encode(ct, (size_t)total + 16); + free(ct); + + const char* nh = EL_CSTR(nonce_hex_v); + const char* ch = EL_CSTR(ct_hex_v); + char* buf = el_strbuf(strlen(nh) + strlen(ch) + 48); + sprintf(buf, "{\"nonce\":\"%s\",\"ciphertext\":\"%s\"}", nh, ch); + return el_wrap_str(buf); +} + +el_val_t aead_decrypt(el_val_t key_hex, el_val_t nonce_hex, el_val_t ciphertext_hex) { + size_t key_len = 0, nonce_len = 0, ct_len = 0; + unsigned char* key = el_hex_decode(EL_CSTR(key_hex), &key_len); + unsigned char* nonce = el_hex_decode(EL_CSTR(nonce_hex), &nonce_len); + unsigned char* ct = el_hex_decode(EL_CSTR(ciphertext_hex), &ct_len); + if (!key || !nonce || !ct) return el_wrap_str(el_strdup("")); + if (key_len != 32 || nonce_len != 12) return el_wrap_str(el_strdup("")); + if (ct_len < 16) return el_wrap_str(el_strdup("")); + + size_t body_len = ct_len - 16; + const unsigned char* tag = ct + body_len; + + EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new(); + if (!ctx) return el_wrap_str(el_strdup("")); + + if (EVP_DecryptInit_ex(ctx, EVP_aes_256_gcm(), NULL, NULL, NULL) != 1 || + EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, 12, NULL) != 1 || + EVP_DecryptInit_ex(ctx, NULL, NULL, key, nonce) != 1) { + EVP_CIPHER_CTX_free(ctx); return el_wrap_str(el_strdup("")); + } + + unsigned char* pt = (unsigned char*)malloc(body_len + 1); + if (!pt) { EVP_CIPHER_CTX_free(ctx); return el_wrap_str(el_strdup("")); } + int outlen = 0, total = 0; + if (EVP_DecryptUpdate(ctx, pt, &outlen, ct, (int)body_len) != 1) { + free(pt); EVP_CIPHER_CTX_free(ctx); return el_wrap_str(el_strdup("")); + } + total += outlen; + /* Set expected tag before final — GCM's final step is where auth happens. */ + if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, (void*)tag) != 1) { + free(pt); EVP_CIPHER_CTX_free(ctx); return el_wrap_str(el_strdup("")); + } + int rc = EVP_DecryptFinal_ex(ctx, pt + total, &outlen); + EVP_CIPHER_CTX_free(ctx); + if (rc != 1) { + /* Auth failure or padding/length mismatch. Return empty so callers + * cannot accidentally treat tampered ciphertext as a valid message. */ + free(pt); + return el_wrap_str(el_strdup("")); + } + total += outlen; + pt[total] = '\0'; + + /* Copy into the el arena so the caller-visible string outlives this fn. */ + char* out = el_strbuf((size_t)total); + memcpy(out, pt, (size_t)total); + out[total] = '\0'; + free(pt); + return el_wrap_str(out); +} + +#endif /* __has_include() */ + +/* ──────────────────────────────────────────────────────────────────────────── + * OTLP/HTTP observability — logs, traces, metrics + * + * Design goals: + * - Zero blocking on the request path. Producers append to in-memory + * ring buffers; a single worker thread flushes to the OTLP endpoint. + * - Drop-on-failure semantics. If the endpoint is unreachable or slow, + * we drop telemetry rather than back-pressure into the request handler. + * - Best-effort serialization. Each record is pre-serialized as JSON when + * the El program calls the primitive; the worker just batches. + * - Configuration via env vars: + * OTLP_ENDPOINT e.g. https://alloy.neuralplatform.ai:4318 + * OTEL_SERVICE_NAME e.g. neuron-web (default: argv[0] basename) + * OTEL_SERVICE_VERSION (default: "0.0.0") + * OTEL_RESOURCE_ATTRS comma-sep k=v pairs (optional) + * + * Wire format: OTLP/HTTP JSON. Three endpoints: + * POST {endpoint}/v1/logs — log records + * POST {endpoint}/v1/traces — spans + * POST {endpoint}/v1/metrics — counter/gauge points + * + * El programs see four primitives: + * trace_span_start(name) -> SpanHandle (just a string id) + * trace_span_end(handle) (computes duration, queues) + * emit_log(level, msg, fields_json) (queues a log record) + * emit_metric(name, value, tags_json) (queues a counter increment) + * ──────────────────────────────────────────────────────────────────────────── + */ + +#define OTLP_BUF_CAP 4096 /* per-buffer ring size */ +#define OTLP_FLUSH_MS 2000 /* flush every 2s */ +#define OTLP_BATCH_MAX 200 /* up to 200 records per POST */ + +typedef struct { + char* data; /* malloc'd JSON fragment for this record */ +} OtlpRec; + +typedef struct { + OtlpRec ring[OTLP_BUF_CAP]; + size_t head; /* next write slot */ + size_t tail; /* next read slot */ + pthread_mutex_t mu; +} OtlpQueue; + +static OtlpQueue _otlp_logs = { .mu = PTHREAD_MUTEX_INITIALIZER }; +static OtlpQueue _otlp_traces = { .mu = PTHREAD_MUTEX_INITIALIZER }; +static OtlpQueue _otlp_metrics = { .mu = PTHREAD_MUTEX_INITIALIZER }; + +static char* _otlp_endpoint = NULL; /* e.g. https://alloy.neuralplatform.ai:4318 */ +static char* _otlp_service_name = NULL; +static char* _otlp_service_version = NULL; +static int _otlp_initialized = 0; +static pthread_t _otlp_worker_thread; + +/* enqueue — returns 1 if accepted, 0 if dropped (full buffer or no endpoint) */ +static int otlp_enqueue(OtlpQueue* q, const char* json) { + if (!_otlp_endpoint || !json) return 0; + pthread_mutex_lock(&q->mu); + size_t next_head = (q->head + 1) % OTLP_BUF_CAP; + if (next_head == q->tail) { + /* buffer full — drop oldest */ + free(q->ring[q->tail].data); + q->ring[q->tail].data = NULL; + q->tail = (q->tail + 1) % OTLP_BUF_CAP; + } + q->ring[q->head].data = strdup(json); + q->head = next_head; + pthread_mutex_unlock(&q->mu); + return 1; +} + +/* drain — copies up to OTLP_BATCH_MAX items into a comma-joined string, + * caller must free the result. Returns NULL if queue is empty. */ +static char* otlp_drain(OtlpQueue* q) { + pthread_mutex_lock(&q->mu); + if (q->head == q->tail) { pthread_mutex_unlock(&q->mu); return NULL; } + /* compute total length */ + size_t total = 0, count = 0; + size_t i = q->tail; + while (i != q->head && count < OTLP_BATCH_MAX) { + if (q->ring[i].data) total += strlen(q->ring[i].data) + 1; /* +1 for comma */ + i = (i + 1) % OTLP_BUF_CAP; + count++; + } + char* out = malloc(total + 4); + if (!out) { pthread_mutex_unlock(&q->mu); return NULL; } + out[0] = '\0'; + size_t off = 0; + i = q->tail; + count = 0; + while (i != q->head && count < OTLP_BATCH_MAX) { + if (q->ring[i].data) { + size_t l = strlen(q->ring[i].data); + if (off > 0) { out[off++] = ','; } + memcpy(out + off, q->ring[i].data, l); + off += l; + free(q->ring[i].data); + q->ring[i].data = NULL; + } + i = (i + 1) % OTLP_BUF_CAP; + count++; + } + out[off] = '\0'; + q->tail = i; + pthread_mutex_unlock(&q->mu); + return out; +} + +/* Build resource block once (service.name, service.version, host.name) */ +static char* otlp_resource_block(void) { + static char cached[1024]; + static int built = 0; + if (built) return cached; + char host[256] = "unknown"; + gethostname(host, sizeof(host) - 1); + snprintf(cached, sizeof(cached), + "{\"attributes\":[" + "{\"key\":\"service.name\",\"value\":{\"stringValue\":\"%s\"}}," + "{\"key\":\"service.version\",\"value\":{\"stringValue\":\"%s\"}}," + "{\"key\":\"host.name\",\"value\":{\"stringValue\":\"%s\"}}" + "]}", + _otlp_service_name ? _otlp_service_name : "el-app", + _otlp_service_version ? _otlp_service_version : "0.0.0", + host); + built = 1; + return cached; +} + +/* Best-effort POST. Drops on any error. */ +static void otlp_post(const char* path, const char* body) { + if (!_otlp_endpoint || !body || !*body) return; + char url[1024]; + snprintf(url, sizeof(url), "%s%s", _otlp_endpoint, path); + CURL* c = curl_easy_init(); + if (!c) return; + struct curl_slist* h = NULL; + h = curl_slist_append(h, "Content-Type: application/json"); + curl_easy_setopt(c, CURLOPT_URL, url); + curl_easy_setopt(c, CURLOPT_POST, 1L); + curl_easy_setopt(c, CURLOPT_POSTFIELDS, body); + curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)strlen(body)); + curl_easy_setopt(c, CURLOPT_HTTPHEADER, h); + curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, 3000L); + curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, NULL); /* discard response */ + curl_easy_perform(c); + curl_slist_free_all(h); + curl_easy_cleanup(c); +} + +/* Flush worker — runs forever until process exits */ +static void* otlp_worker(void* arg) { + (void)arg; + while (1) { + struct timespec ts = { OTLP_FLUSH_MS / 1000, (OTLP_FLUSH_MS % 1000) * 1000000L }; + nanosleep(&ts, NULL); + + char* logs = otlp_drain(&_otlp_logs); + if (logs && *logs) { + char body[OTLP_BUF_CAP * 8]; + int n = snprintf(body, sizeof(body), + "{\"resourceLogs\":[{\"resource\":%s," + "\"scopeLogs\":[{\"scope\":{\"name\":\"el-runtime\"}," + "\"logRecords\":[%s]}]}]}", + otlp_resource_block(), logs); + if (n > 0 && n < (int)sizeof(body)) otlp_post("/v1/logs", body); + } + free(logs); + + char* traces = otlp_drain(&_otlp_traces); + if (traces && *traces) { + char body[OTLP_BUF_CAP * 8]; + int n = snprintf(body, sizeof(body), + "{\"resourceSpans\":[{\"resource\":%s," + "\"scopeSpans\":[{\"scope\":{\"name\":\"el-runtime\"}," + "\"spans\":[%s]}]}]}", + otlp_resource_block(), traces); + if (n > 0 && n < (int)sizeof(body)) otlp_post("/v1/traces", body); + } + free(traces); + + char* metrics = otlp_drain(&_otlp_metrics); + if (metrics && *metrics) { + char body[OTLP_BUF_CAP * 8]; + int n = snprintf(body, sizeof(body), + "{\"resourceMetrics\":[{\"resource\":%s," + "\"scopeMetrics\":[{\"scope\":{\"name\":\"el-runtime\"}," + "\"metrics\":[%s]}]}]}", + otlp_resource_block(), metrics); + if (n > 0 && n < (int)sizeof(body)) otlp_post("/v1/metrics", body); + } + free(metrics); + } + return NULL; +} + +/* Initialize OTLP — called lazily on first emit. Idempotent. */ +static void otlp_lazy_init(void) { + if (_otlp_initialized) return; + static pthread_mutex_t once_mu = PTHREAD_MUTEX_INITIALIZER; + pthread_mutex_lock(&once_mu); + if (_otlp_initialized) { pthread_mutex_unlock(&once_mu); return; } + + const char* ep = getenv("OTLP_ENDPOINT"); + if (!ep || !*ep) { + _otlp_initialized = 1; + pthread_mutex_unlock(&once_mu); + return; + } + _otlp_endpoint = strdup(ep); + /* trim trailing slash */ + size_t l = strlen(_otlp_endpoint); + if (l > 0 && _otlp_endpoint[l - 1] == '/') _otlp_endpoint[l - 1] = '\0'; + + const char* svc = getenv("OTEL_SERVICE_NAME"); + _otlp_service_name = strdup(svc && *svc ? svc : "el-app"); + const char* ver = getenv("OTEL_SERVICE_VERSION"); + _otlp_service_version = strdup(ver && *ver ? ver : "0.0.0"); + + pthread_create(&_otlp_worker_thread, NULL, otlp_worker, NULL); + pthread_detach(_otlp_worker_thread); + _otlp_initialized = 1; + pthread_mutex_unlock(&once_mu); +} + +/* JSON-escape a string into out_buf. Returns chars written (excluding null). */ +static size_t otlp_json_escape(const char* in, char* out, size_t out_cap) { + size_t o = 0; + for (size_t i = 0; in[i] && o + 8 < out_cap; i++) { + unsigned char c = (unsigned char)in[i]; + if (c == '"') { out[o++] = '\\'; out[o++] = '"'; } + else if (c == '\\'){ out[o++] = '\\'; out[o++] = '\\'; } + else if (c == '\n'){ out[o++] = '\\'; out[o++] = 'n'; } + else if (c == '\r'){ out[o++] = '\\'; out[o++] = 'r'; } + else if (c == '\t'){ out[o++] = '\\'; out[o++] = 't'; } + else if (c < 0x20) { o += snprintf(out + o, out_cap - o, "\\u%04x", c); } + else { out[o++] = (char)c; } + } + out[o] = '\0'; + return o; +} + +/* ── Public El primitives ─────────────────────────────────────────────────── */ + +/* emit_log(level, msg, fields_json) — fields_json is a JSON object string or "" */ +el_val_t emit_log(el_val_t level_v, el_val_t msg_v, el_val_t fields_v) { + otlp_lazy_init(); + if (!_otlp_endpoint) return EL_INT(0); + const char* level = EL_CSTR(level_v); if (!level) level = "INFO"; + const char* msg = EL_CSTR(msg_v); if (!msg) msg = ""; + const char* fields = EL_CSTR(fields_v); if (!fields) fields = ""; + /* Map El level names to OTLP severity numbers */ + int sev_num = 9; /* INFO */ + if (strcmp(level, "TRACE") == 0) sev_num = 1; + else if (strcmp(level, "DEBUG") == 0) sev_num = 5; + else if (strcmp(level, "INFO") == 0) sev_num = 9; + else if (strcmp(level, "WARN") == 0 || strcmp(level, "WARNING") == 0) sev_num = 13; + else if (strcmp(level, "ERROR") == 0) sev_num = 17; + else if (strcmp(level, "FATAL") == 0) sev_num = 21; + char esc_msg[2048]; otlp_json_escape(msg, esc_msg, sizeof(esc_msg)); + /* unix nanos */ + struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); + long long now_nano = (long long)ts.tv_sec * 1000000000LL + ts.tv_nsec; + char rec[4096]; + int n = snprintf(rec, sizeof(rec), + "{\"timeUnixNano\":\"%lld\",\"severityNumber\":%d," + "\"severityText\":\"%s\"," + "\"body\":{\"stringValue\":\"%s\"}%s%s}", + now_nano, sev_num, level, esc_msg, + (fields && *fields) ? ",\"attributes\":" : "", + (fields && *fields) ? fields : ""); + if (n > 0 && n < (int)sizeof(rec)) otlp_enqueue(&_otlp_logs, rec); + return EL_INT(1); +} + +/* emit_metric(name, value, tags_json) — Sum (counter) data point. tags_json + * is a JSON array of {key, value} pairs or empty string. */ +el_val_t emit_metric(el_val_t name_v, el_val_t value_v, el_val_t tags_v) { + otlp_lazy_init(); + if (!_otlp_endpoint) return EL_INT(0); + const char* name = EL_CSTR(name_v); if (!name) name = "unknown"; + int64_t val = (int64_t)value_v; + const char* tags = EL_CSTR(tags_v); if (!tags) tags = ""; + char esc_name[256]; otlp_json_escape(name, esc_name, sizeof(esc_name)); + struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); + long long now_nano = (long long)ts.tv_sec * 1000000000LL + ts.tv_nsec; + char rec[4096]; + int n = snprintf(rec, sizeof(rec), + "{\"name\":\"%s\",\"sum\":{\"aggregationTemporality\":2,\"isMonotonic\":true," + "\"dataPoints\":[{\"asInt\":\"%lld\"," + "\"timeUnixNano\":\"%lld\"" + "%s%s}]}}", + esc_name, (long long)val, now_nano, + (tags && *tags) ? ",\"attributes\":" : "", + (tags && *tags) ? tags : ""); + if (n > 0 && n < (int)sizeof(rec)) otlp_enqueue(&_otlp_metrics, rec); + return EL_INT(1); +} + +/* trace_span_start(name) — returns a span handle (string of "traceid:spanid:start_nano:name") */ +el_val_t trace_span_start(el_val_t name_v) { + otlp_lazy_init(); + const char* name = EL_CSTR(name_v); if (!name) name = "span"; + /* generate 16-byte trace id and 8-byte span id */ + static _Thread_local int seeded = 0; + if (!seeded) { srand((unsigned int)(uintptr_t)pthread_self() ^ (unsigned int)time(NULL)); seeded = 1; } + char tid[33], sid[17]; + for (int i = 0; i < 32; i++) tid[i] = "0123456789abcdef"[rand() & 0xF]; + tid[32] = '\0'; + for (int i = 0; i < 16; i++) sid[i] = "0123456789abcdef"[rand() & 0xF]; + sid[16] = '\0'; + struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); + long long now_nano = (long long)ts.tv_sec * 1000000000LL + ts.tv_nsec; + char* handle = malloc(strlen(name) + 80); + if (!handle) return EL_STR(""); + sprintf(handle, "%s:%s:%lld:%s", tid, sid, now_nano, name); + el_arena_track(handle); + return EL_STR(handle); +} + +/* trace_span_end(handle) — emits the span with computed duration */ +el_val_t trace_span_end(el_val_t handle_v) { + otlp_lazy_init(); + if (!_otlp_endpoint) return EL_INT(0); + const char* h = EL_CSTR(handle_v); if (!h) return EL_INT(0); + /* parse "tid:sid:start_nano:name" */ + char tid[64], sid[32], rest[1024]; + long long start_nano = 0; + if (sscanf(h, "%63[^:]:%31[^:]:%lld:%1023[^\n]", tid, sid, &start_nano, rest) != 4) return EL_INT(0); + struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); + long long end_nano = (long long)ts.tv_sec * 1000000000LL + ts.tv_nsec; + char esc_name[1024]; otlp_json_escape(rest, esc_name, sizeof(esc_name)); + char rec[4096]; + int n = snprintf(rec, sizeof(rec), + "{\"traceId\":\"%s\",\"spanId\":\"%s\"," + "\"name\":\"%s\"," + "\"kind\":1," + "\"startTimeUnixNano\":\"%lld\"," + "\"endTimeUnixNano\":\"%lld\"," + "\"status\":{\"code\":1}}", + tid, sid, esc_name, start_nano, end_nano); + if (n > 0 && n < (int)sizeof(rec)) otlp_enqueue(&_otlp_traces, rec); + return EL_INT(1); +} + +/* Convenience: emit a one-shot timed event (emit start+end immediately). + * For El programs that want point events with duration baked in. */ +el_val_t emit_event(el_val_t name_v, el_val_t duration_ms_v) { + otlp_lazy_init(); + if (!_otlp_endpoint) return EL_INT(0); + const char* name = EL_CSTR(name_v); if (!name) name = "event"; + int64_t dur_ms = (int64_t)duration_ms_v; + el_val_t h = trace_span_start(EL_STR((char*)name)); + /* fudge start to be (now - duration) */ + (void)dur_ms; + return trace_span_end(h); +} + diff --git a/el-compiler/src/compiler.el b/el-compiler/src/compiler.el index 52fc1af..b7c5db3 100644 --- a/el-compiler/src/compiler.el +++ b/el-compiler/src/compiler.el @@ -172,7 +172,10 @@ fn resolve_imports(src_path: String) -> String { // elc --target=js # write JS to file fn main() -> Void { let argv: [String] = args() - let target: String = detect_target(argv) + // Use `tgt` not `target`: `target` is a reserved keyword in the lexer + // (Section 1.5 of the language spec). detect_target itself is fine + // because the function-name position has no token-class restriction. + let tgt: String = detect_target(argv) let positional: [String] = strip_flags(argv) let argc: Int = native_list_len(positional) if argc < 1 { @@ -181,7 +184,7 @@ fn main() -> Void { } let src_path: String = native_list_get(positional, 0) let source: String = resolve_imports(src_path) - let out: String = compile_dispatch(target, source) + let out: String = compile_dispatch(tgt, source) if argc >= 2 { let out_path: String = native_list_get(positional, 1) let ok: Bool = fs_write(out_path, out)