From 82e998273ba4a313e26adb510914f81280878d6b Mon Sep 17 00:00:00 2001 From: bigmerge Date: Mon, 17 Aug 2026 08:39:48 -0500 Subject: [PATCH] =?UTF-8?q?self-review=202026-08-17:=20bound=20the=20off-g?= =?UTF-8?q?raph=20ISE=20log=20=E2=80=94=20moving=20telemetry=20off-graph?= =?UTF-8?q?=20moved=20the=20leak,=20it=20did=20not=20close=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-07-16 review fixed telemetry growth in the GRAPH by calling engram_prune_telemetry(48h) on every ISE insert. The 2026-08-xx move to ENGRAM_ISE_OFFGRAPH=1 then routed every state event to a flat append-only log instead — and that path had no retention of any kind. The prune call still exists in server.el, but it now sits in the branch that production never takes, so the fix reads as present while being inert. Measured on the live store: 17.1 MB / 14,305 events over 3.56 days = 4.81 MB/day, unbounded (~1.76 GB/year). engram_ise_log_append now compacts to a byte bound after append. Byte- and not time-bounded on purpose: this is a flat file with no index, so size is the property that has to be bounded, and ftell on the handle already held is O(1) versus an O(file) timestamp scan per append. Default 64 MB retains ~13 days at the measured rate — more history than the 48h the on-graph path kept. Override with ENGRAM_ISE_LOG_MAX_BYTES. Compaction keeps the TAIL, never the head: engram_dreams_json reads the last ~2 MB of this file for dream-recall, so the recent end is the end with a reader, and KEEP (16 MB) stays well clear of that window. Resumes at the first line boundary so the tail never starts mid-record, and only renames over the live log when the tail was written in full — a short write must not destroy history. The honesty rail is unchanged: rotated-out remains "I don't remember", never a synthesized dream. This only makes the forgetting bounded and explicit instead of deferred forever. Verified against a 4,000-event harness at a 200 KB cap: file bounded, newest record retained, oldest dropped, 883 lines with zero malformed records, tail contiguous, no .tmp residue. --- lang/runtime/el_runtime.c | 60 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index 021f3f9..fc5e7a2 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -14379,7 +14379,67 @@ el_val_t engram_ise_log_append(el_val_t content_v){ } } fputs("\"}\n", f); + /* RETENTION (2026-08-17 self-review). The on-graph ISE branch in + * server.el calls engram_prune_telemetry(48h) on every insert, but that + * branch is DEAD in production: ENGRAM_ISE_OFFGRAPH=1 is the live + * setting, so every state event lands here instead — and this path had + * no retention of any kind. Measured: 17.1 MB / 14,305 events over 3.56 + * days = 4.81 MB/day, growing without bound (~1.76 GB/year). The graph + * got its telemetry-growth fix on 2026-07-16; moving telemetry off-graph + * moved the leak rather than closing it. + * + * Byte-bounded rather than time-bounded on purpose: this is a flat + * append-only file with no index, so size is the property that actually + * has to be bounded, and a byte check is O(1) against the handle we + * already hold (ftell) instead of an O(file) timestamp scan per append. + * At the measured rate the 64 MB default retains ~13 days — comfortably + * more history than the 48h the on-graph path kept. + * + * Compaction keeps the TAIL, never the head: engram_dreams_json reads + * the last ~2 MB of this file for dream-recall, so the recent end is the + * end that has a reader. KEEP is held well above that 2 MB window so + * recall is never truncated by a rotation. The honesty rail is + * preserved exactly as before — rotated-out remains "I don't remember", + * never a synthesized dream; this only makes the forgetting bounded and + * explicit instead of deferred forever. */ + long pos = ftell(f); fclose(f); + { + long maxb = 64L*1024L*1024L; + long keepb = 16L*1024L*1024L; + const char* mv = getenv("ENGRAM_ISE_LOG_MAX_BYTES"); + if (mv && *mv) { long v = atol(mv); if (v > 0) maxb = v; } + if (keepb > maxb/2) keepb = maxb/2; + if (pos > 0 && pos > maxb) { + FILE* rf = fopen(path, "rb"); + if (rf) { + if (fseek(rf, pos - keepb, SEEK_SET) == 0) { + char* buf = (char*)malloc((size_t)keepb + 1); + if (buf) { + size_t rd = fread(buf, 1, (size_t)keepb, rf); + buf[rd] = 0; + /* Resume at the first LINE boundary so the tail never + * begins with a half-written JSON record. */ + char* start = memchr(buf, '\n', rd); + start = start ? start + 1 : buf; + size_t keep_n = rd - (size_t)(start - buf); + char tmp[4096]; + snprintf(tmp, sizeof tmp, "%s/state-events.jsonl.tmp", dir); + FILE* wf = fopen(tmp, "wb"); + if (wf) { + int ok = (fwrite(start, 1, keep_n, wf) == keep_n); + fclose(wf); + /* Only replace the live log if the tail was written + * in full — a short write must not destroy history. */ + if (ok) rename(tmp, path); else remove(tmp); + } + free(buf); + } + } + fclose(rf); + } + } + } return EL_INT(1); }