neuron-technologies/el
3
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 5 Wiki Activity

runtime: HTTP replies truncated to fs_read's file length — Content-Length lied, onboarding trapped #78

Closed
tim.lingo wants to merge 2 commits from fix/http-fs-read-len into main
pull from: fix/http-fs-read-len
merge into: :main
Conversation 1 Commits 2 Files Changed 2
+197 -41
2 changed files with 197 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engram/src/server.el
+6 -2
View File
@@ -50,8 +50,12 @@ fn query_param(path: String, key: String) -> String {
if pos < 0 { return "" }
let after: String = str_slice(qs, pos + str_len(needle), str_len(qs))
let amp: Int = str_index_of(after, "&")
if amp < 0 { return after }
str_slice(after, 0, amp)
// SPEC-SEARCH-UPGRADE 2026-07-14: URL-decode the extracted value (%XX and
// '+' were previously passed through literally, so an encoded multi-word
// query arrived as junk tokens — pre-existing GET-path defect, masked
// until search could actually rank multi-word queries).
if amp < 0 { return url_decode(after) }
url_decode(str_slice(after, 0, amp))
}
fn query_int(path: String, key: String, default_val: Int) -> Int {
lang/el-compiler/runtime/el_runtime.c
+191 -39
View File
@@ -82,8 +82,14 @@ static _Thread_local ElArena _tl_arena = {NULL, 0, 0};
static _Thread_local int _tl_arena_active = 0;
/* Binary-safe fs_read length — set by fs_read, consumed by http_send_response.
* Allows serving PNGs and other binary files without strlen truncation. */
static _Thread_local size_t _tl_fs_read_len = 0;
* Allows serving PNGs and other binary files without strlen truncation.
* PAIRED with the buffer pointer it describes: the length may only be applied
* to the exact buffer fs_read returned. Without the pairing, any handler that
* fs_read a file and then WRAPPED it into a larger response had that response
* truncated to the file's length (Content-Length lied AND the send stopped
* short) — the safety-contact onboarding trap, 2026-07-17. */
static _Thread_local size_t _tl_fs_read_len = 0;
static _Thread_local const char* _tl_fs_read_buf = NULL;
static void el_arena_track(char* p) {
if (!_tl_arena_active || !p) return;
@@ -101,6 +107,8 @@ static void el_arena_track(char* p) {
void el_request_start(void) {
_tl_arena.count = 0;
_tl_arena_active = 1;
_tl_fs_read_len = 0; /* never let a previous request's file length */
_tl_fs_read_buf = NULL; /* leak into this response's byte accounting */
}
/* Called by http_worker after the El handler returns and the response is sent.
@@ -1484,11 +1492,14 @@ static void http_send_response(int fd, const char* body) {
}
const char* eff_body = is_envelope ? env_body : body;
/* Use the real byte count from fs_read if available (handles binary files
* with embedded null bytes — PNG, WOFF2, etc.). Fall back to strlen for
* normal text/JSON responses where _tl_fs_read_len is 0. */
size_t blen = (_tl_fs_read_len > 0) ? _tl_fs_read_len : strlen(eff_body);
/* Use the real byte count from fs_read ONLY when this body IS the exact
* buffer fs_read returned (binary files with embedded null bytes — PNG,
* WOFF2, etc.). Any other body — wrapped, enveloped, or derived — must be
* measured with strlen, or it is truncated/over-read to the file's size. */
size_t blen = (_tl_fs_read_len > 0 && eff_body == _tl_fs_read_buf)
? _tl_fs_read_len : strlen(eff_body);
_tl_fs_read_len = 0; /* consume — one-shot per response */
_tl_fs_read_buf = NULL;
int head_only = _tl_http_head_only;
JsonBuf hdrs; jb_init(&hdrs);
@@ -1568,11 +1579,22 @@ static void* http_worker(void* arg) {
const char* rs = EL_CSTR(r);
/* Copy response out BEFORE arena teardown.
* For binary files, _tl_fs_read_len holds the real byte count —
* use memcpy instead of strdup so null bytes are preserved. */
size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0);
* use memcpy instead of strdup so null bytes are preserved.
* The stored length applies ONLY when the response IS the exact
* fs_read buffer; a wrapped/derived response must use strlen or
* it gets truncated (or over-read) to the file's length. */
size_t rlen;
if (_tl_fs_read_len > 0 && rs && rs == _tl_fs_read_buf) {
rlen = _tl_fs_read_len; /* raw file bytes — binary-safe */
} else {
rlen = rs ? strlen(rs) : 0;
_tl_fs_read_len = 0; /* hint doesn't describe this body */
_tl_fs_read_buf = NULL;
}
response = malloc(rlen + 1);
if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; }
else if (response) { response[0] = '\0'; }
if (_tl_fs_read_len > 0) _tl_fs_read_buf = response; /* hint follows the copy */
} else {
response = el_strdup_persist("el-runtime: no http handler registered");
}
@@ -1822,10 +1844,20 @@ static void* http_worker_v2(void* arg) {
el_val_t hmap = http_build_headers_map(hdr_block ? hdr_block : "");
el_val_t r = h(EL_STR(dispatch_method), EL_STR(path), hmap, EL_STR(body));
const char* rs = EL_CSTR(r);
size_t rlen = _tl_fs_read_len > 0 ? _tl_fs_read_len : (rs ? strlen(rs) : 0);
/* Same pairing rule as the v1 worker: the fs_read length is only
* trustworthy for the exact buffer fs_read returned. */
size_t rlen;
if (_tl_fs_read_len > 0 && rs && rs == _tl_fs_read_buf) {
rlen = _tl_fs_read_len; /* raw file bytes — binary-safe */
} else {
rlen = rs ? strlen(rs) : 0;
_tl_fs_read_len = 0; /* hint doesn't describe this body */
_tl_fs_read_buf = NULL;
}
response = malloc(rlen + 1);
if (response && rs) { memcpy(response, rs, rlen); response[rlen] = '\0'; }
else if (response) { response[0] = '\0'; }
if (_tl_fs_read_len > 0) _tl_fs_read_buf = response; /* hint follows the copy */
el_release(hmap);
} else {
response = el_strdup_persist(
@@ -2023,6 +2055,7 @@ el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body) {
el_val_t fs_read(el_val_t pathv) {
const char* path = EL_CSTR(pathv);
_tl_fs_read_len = 0;
_tl_fs_read_buf = NULL;
if (!path) return el_wrap_str(el_strdup(""));
FILE* f = fopen(path, "rb");
if (!f) return el_wrap_str(el_strdup(""));
@@ -2034,6 +2067,7 @@ el_val_t fs_read(el_val_t pathv) {
size_t got = fread(buf, 1, (size_t)sz, f);
buf[got] = '\0';
_tl_fs_read_len = got; /* store real byte count for binary-safe send */
_tl_fs_read_buf = buf; /* ...valid ONLY for this exact buffer */
fclose(f);
return el_wrap_str(buf);
}
@@ -3576,8 +3610,10 @@ el_val_t json_get_raw(el_val_t json_str, el_val_t key) {
const char* k = EL_CSTR(key);
const char* p = json_find_key(json, k);
/* Clear fs_read binary-length hint — result is a fresh null-terminated
* string, not the raw file bytes, so Content-Length must use strlen. */
* string, not the raw file bytes, so Content-Length must use strlen.
* (Kept although the pointer pairing now makes this redundant.) */
_tl_fs_read_len = 0;
_tl_fs_read_buf = NULL;
if (!p) return el_wrap_str(el_strdup(""));
const char* end = json_skip_value(p);
size_t n = (size_t)(end - p);
@@ -6826,6 +6862,116 @@ static int istr_contains(const char* hay, const char* needle) {
return 0;
}
/* ---- SPEC-SEARCH-UPGRADE-OURS-2026-07-14: ranked search (BM25 + recency) ----
* Replaces first-N-in-storage-order substring matching (measured 13% hit@5 on
* the 15-query pinned eval; ranked model measured 93% offline). Deterministic,
* local, transparent — no model call on the hot path. Multi-word queries score
* per-token (rare+concentrated terms weigh most); ties break newest-first so
* fresh memories stop losing to storage order. The transparent-layer identity
* filter is preserved unchanged: hidden self layers stay invisible here and
* surface only via engram_activate — the legitimate path. */
#define ENGRAM_BM25_MAX_QTOK 16
#define ENGRAM_BM25_TOKLEN 48
static int engram_tok_next(const char** ps, char* out, int cap) {
const char* s = *ps;
while (*s && !isalnum((unsigned char)*s)) s++;
if (!*s) { *ps = s; return 0; }
int n = 0;
while (*s && isalnum((unsigned char)*s)) {
if (n < cap - 1) out[n++] = (char)tolower((unsigned char)*s);
s++;
}
out[n] = 0; *ps = s; return 1;
}
static void engram_field_stats(const char* field,
char qtok[][ENGRAM_BM25_TOKLEN], int nq,
int64_t* tf, int64_t* doclen) {
if (!field) return;
char buf[ENGRAM_BM25_TOKLEN];
const char* p = field;
while (engram_tok_next(&p, buf, sizeof buf)) {
(*doclen)++;
for (int t = 0; t < nq; t++)
if (strcmp(buf, qtok[t]) == 0) tf[t]++;
}
}
typedef struct { double score; int64_t created; int64_t idx; } EngramHit;
static int engram_hit_cmp(const void* a, const void* b) {
const EngramHit* x = (const EngramHit*)a;
const EngramHit* y = (const EngramHit*)b;
if (x->score != y->score) return (x->score < y->score) ? 1 : -1;
if (x->created != y->created) return (x->created < y->created) ? 1 : -1;
return 0;
}
/* Scores every visible node against the query; writes ranked hits into `out`
* (caller allocates g->node_count entries). Returns min(hits, lim). */
static int64_t engram_search_ranked(EngramStore* g, const char* q, int64_t lim,
EngramHit* out) {
char qtok[ENGRAM_BM25_MAX_QTOK][ENGRAM_BM25_TOKLEN];
int nq = 0;
{
const char* p = q; char buf[ENGRAM_BM25_TOKLEN];
while (nq < ENGRAM_BM25_MAX_QTOK && engram_tok_next(&p, buf, sizeof buf)) {
int dup = 0;
for (int t = 0; t < nq; t++)
if (strcmp(qtok[t], buf) == 0) { dup = 1; break; }
if (!dup) { strcpy(qtok[nq], buf); nq++; }
}
}
if (nq == 0) return 0;
int64_t N = g->node_count;
int64_t* tfm = (int64_t*)calloc((size_t)(N * nq), sizeof(int64_t));
int64_t* dlen = (int64_t*)calloc((size_t)N, sizeof(int64_t));
if (!tfm || !dlen) { free(tfm); free(dlen); return 0; }
int64_t df[ENGRAM_BM25_MAX_QTOK] = {0};
double total_len = 0.0; int64_t live = 0;
for (int64_t i = 0; i < N; i++) {
EngramNode* n = &g->nodes[i];
if (engram_layer_is_transparent(n->layer_id)) continue;
live++;
int64_t* tf = &tfm[i * nq];
engram_field_stats(n->content, qtok, nq, tf, &dlen[i]);
engram_field_stats(n->label, qtok, nq, tf, &dlen[i]);
engram_field_stats(n->tags, qtok, nq, tf, &dlen[i]);
total_len += (double)dlen[i];
for (int t = 0; t < nq; t++) if (tf[t] > 0) df[t]++;
}
double avg = (live > 0) ? total_len / (double)live : 1.0;
if (avg <= 0.0) avg = 1.0;
const double k1 = 1.2, b = 0.75;
int64_t nhits = 0;
for (int64_t i = 0; i < N; i++) {
EngramNode* n = &g->nodes[i];
if (engram_layer_is_transparent(n->layer_id)) continue;
int64_t* tf = &tfm[i * nq];
double s = 0.0;
for (int t = 0; t < nq; t++) {
if (tf[t] == 0) continue;
double idf = log(((double)live - (double)df[t] + 0.5) /
((double)df[t] + 0.5) + 1.0);
double tfd = (double)tf[t];
s += idf * (tfd * (k1 + 1.0)) /
(tfd + k1 * (1.0 - b + b * (double)dlen[i] / avg));
}
if (s > 0.0) {
out[nhits].score = s;
out[nhits].created = n->created_at;
out[nhits].idx = i;
nhits++;
}
}
free(tfm); free(dlen);
qsort(out, (size_t)nhits, sizeof(EngramHit), engram_hit_cmp);
return (nhits < lim) ? nhits : lim;
}
el_val_t engram_search(el_val_t query, el_val_t limit) {
EngramStore* g = engram_get();
const char* q = EL_CSTR(query);
@@ -6833,21 +6979,13 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
if (lim <= 0) lim = 100;
el_val_t lst = el_list_empty();
if (!q || !*q) return lst;
int64_t found = 0;
for (int64_t i = 0; i < g->node_count && found < lim; i++) {
EngramNode* n = &g->nodes[i];
/* Filter transparent layers: nodes whose layer is `transparent=1`
* shape output but are invisible to introspection ("what do you
* know about yourself"). They still surface via engram_activate
* + engram_compile_layered_json — that's the legitimate path. */
if (engram_layer_is_transparent(n->layer_id)) continue;
if (istr_contains(n->content, q) ||
istr_contains(n->label, q) ||
istr_contains(n->tags, q)) {
lst = el_list_append(lst, engram_node_to_map(n));
found++;
}
}
if (g->node_count == 0) return lst;
EngramHit* hits = (EngramHit*)malloc((size_t)g->node_count * sizeof(EngramHit));
if (!hits) return lst;
int64_t k = engram_search_ranked(g, q, lim, hits);
for (int64_t i = 0; i < k; i++)
lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[i].idx]));
free(hits);
return lst;
}
@@ -7761,28 +7899,42 @@ el_val_t engram_get_node_json(el_val_t id) {
return el_wrap_str(jb_finish(&b));
}
/* Look up a node by exact label; returns its JSON or {}. Ported from the
* v1.0.0 release runtime — needed by soul.el session continuity
* (conv_history_load / session_summary_write / emit_session_start_event). */
el_val_t engram_get_node_by_label(el_val_t label) {
const char* lbl = EL_CSTR(label);
if (!lbl || !*lbl) return el_wrap_str(el_strdup(""));
EngramStore* g = engram_get();
for (int64_t i = 0; i < g->node_count; i++) {
EngramNode* n = &g->nodes[i];
if (n->label && strcmp(n->label, lbl) == 0) {
JsonBuf b; jb_init(&b);
engram_emit_node_json(&b, n);
return el_wrap_str(b.buf);
}
}
return el_wrap_str(el_strdup(""));
}
el_val_t engram_search_json(el_val_t query, el_val_t limit) {
/* SPEC-SEARCH-UPGRADE 2026-07-14: same ranked BM25+recency core as
* engram_search; transparent-layer identity filter enforced inside it. */
EngramStore* g = engram_get();
const char* q = EL_CSTR(query);
int64_t lim = (int64_t)limit;
if (lim <= 0) lim = 100;
JsonBuf b; jb_init(&b);
jb_putc(&b, '[');
int first = 1;
int64_t found = 0;
if (q && *q) {
for (int64_t i = 0; i < g->node_count && found < lim; i++) {
EngramNode* n = &g->nodes[i];
/* Filter transparent layers — same as engram_search. */
if (engram_layer_is_transparent(n->layer_id)) continue;
if (istr_contains(n->content, q) ||
istr_contains(n->label, q) ||
istr_contains(n->tags, q)) {
if (!first) jb_putc(&b, ',');
engram_emit_node_json(&b, n);
first = 0;
found++;
if (q && *q && g->node_count > 0) {
EngramHit* hits = (EngramHit*)malloc((size_t)g->node_count * sizeof(EngramHit));
if (hits) {
int64_t k = engram_search_ranked(g, q, lim, hits);
for (int64_t i = 0; i < k; i++) {
if (i) jb_putc(&b, ',');
engram_emit_node_json(&b, &g->nodes[hits[i].idx]);
}
free(hits);
}
}
jb_putc(&b, ']');