neuron-technologies/el
3
0
Fork 0
Code Issues 4 Pull Requests 3 Actions Packages Projects Releases 5 Wiki Activity

self-review 2026-07-31: strip emb from consumer API JSON; cumulative eviction/breakthrough counters

Browse Source 
Every node object on consumer read routes (/api/nodes, /api/search,
activation results, neighbors, compiled context) carried the full ~5.7KB
emb vector — responses 10-50x oversized, blowing MCP token limits.
engram_emit_node_json now takes include_emb; only engram_save passes 1,
so persistence and the /api/sync//api/edges replication paths (which
serve engram_save output) keep embeddings intact.

_eg_act_wm_evicted/_eg_act_breakthroughs were reset at the top of every
engram_activate, so act_stats reported only the last call and the 60s
heartbeat missed nearly all events (curiosity runs 2 activates per 30s).
Both are now monotonic process-lifetime totals; consumers diff readings.
This commit is contained in:
Will Anderson
2026-07-31 08:41:33 -05:00
parent 7f66529510
commit 599073cb92
2 changed files with 41 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engram/dist/engram
Vendored
BIN
View File
Binary file not shown.
lang/releases/v1.0.0-20260501/el_runtime.c
+41 -24
View File
@@ -6022,10 +6022,17 @@ static int64_t _eg_embed_breaker_until = 0;
* The executive-filter pathologies this system has repeatedly debugged by
* inference (WM flooded with breakthrough-floor nodes, silent cap evictions,
* embedder wedged behind the breaker) were all invisible: no ISE, no stats
* field. These per-call counters record what the LAST engram_activate did;
* engram_act_stats_json() exposes them so the soul heartbeat can emit them. */
static int64_t _eg_act_breakthroughs = 0; /* forced promotions at the floor */
static int64_t _eg_act_wm_evicted = 0; /* Pass 4 over-cap evictions */
* field. engram_act_stats_json() exposes them so the soul heartbeat can emit
* them.
*
* CUMULATIVE CONTRACT (2026-07-31 self-review): these are monotonic totals
* for the process lifetime, like pulse/sync_added_total — NOT per-call. The
* original per-call reset meant engram_act_stats_json only reported the LAST
* activate call, and the 60s heartbeat (2 curiosity activates per 30s in
* between) missed nearly every eviction/breakthrough event. Consumers wanting
* rates keep the previous reading and diff. Restart legitimately resets to 0. */
static int64_t _eg_act_breakthroughs = 0; /* forced promotions at the floor, cumulative */
static int64_t _eg_act_wm_evicted = 0; /* Pass 4 over-cap evictions, cumulative */
static int64_t engram_now_ms(void); /* defined in the store section below */
@@ -6724,7 +6731,7 @@ static el_val_t engram_node_to_map(const EngramNode* n) {
/* (Node JSON serialization is provided by `engram_emit_node_json` further
* down in the persistence section — reused by the *_json builtins below.) */
static void engram_emit_node_json(JsonBuf* b, const EngramNode* n);
static void engram_emit_node_json(JsonBuf* b, const EngramNode* n, int include_emb);
static void engram_emit_edge_json(JsonBuf* b, const EngramEdge* e);
/* Salience may arrive either as a float bit-pattern or as a small integer
@@ -7659,9 +7666,11 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
int64_t now_ms = engram_now_ms();
/* Reset per-call observability counters (2026-07-27). */
_eg_act_breakthroughs = 0;
_eg_act_wm_evicted = 0;
/* Observability counters: _eg_act_breakthroughs/_eg_act_wm_evicted are
* CUMULATIVE for the process lifetime and intentionally NOT reset here
* (2026-07-31 self-review — the old per-call reset made the 60s heartbeat
* miss nearly all events between beats; see the definition site).
* ctx_cos stays per-call: it is a gauge of THIS query vs the centroid. */
_eg_act_ctx_cos = -2.0;
/* ── Embedding backfill + query embedding (2026-07-24, bl-b2d1c944) ──
@@ -8411,7 +8420,13 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
/* ── Engram persistence (JSON snapshot) ─────────────────────────────────── */
static void engram_emit_node_json(JsonBuf* b, const EngramNode* n) {
/* include_emb (2026-07-31 self-review): the ~5.7KB "emb" vector belongs ONLY
* in persistence/replication output (engram_save → snapshot.json, which also
* backs /api/sync and /api/edges via scratch exports). Consumer read routes
* (/api/nodes, /api/search, activation results, neighbors, compiled context)
* were shipping it on every node — responses 10-50x oversized, blowing MCP
* token limits. Pass include_emb=1 only from engram_save. */
static void engram_emit_node_json(JsonBuf* b, const EngramNode* n, int include_emb) {
jb_putc(b, '{');
jb_puts(b, "\"id\":"); jb_emit_escaped(b, n->id ? n->id : "");
jb_puts(b, ",\"content\":"); jb_emit_escaped(b, n->content ? n->content : "");
@@ -8454,7 +8469,7 @@ static void engram_emit_node_json(JsonBuf* b, const EngramNode* n) {
* insensitive to 4-sig-fig rounding, and this keeps snapshot bloat to
* ~5KB per embedded node without a base64 codec. Absent field = not
* embedded; the lazy backfill re-embeds eventually if dropped. */
if (n->emb && n->emb_dim > 0) {
if (include_emb && n->emb && n->emb_dim > 0) {
jb_puts(b, ",\"emb\":\"");
for (int32_t j = 0; j < n->emb_dim; j++) {
snprintf(tmp, sizeof(tmp), "%s%.4g", j ? "," : "",
@@ -8492,7 +8507,7 @@ el_val_t engram_save(el_val_t path) {
jb_puts(&b, "{\"nodes\":[");
for (int64_t i = 0; i < g->node_count; i++) {
if (i > 0) jb_putc(&b, ',');
engram_emit_node_json(&b, &g->nodes[i]);
engram_emit_node_json(&b, &g->nodes[i], 1);
}
jb_puts(&b, "],\"edges\":[");
for (int64_t i = 0; i < g->edge_count; i++) {
@@ -9025,7 +9040,7 @@ el_val_t engram_get_node_json(el_val_t id) {
EngramNode* n = engram_find_node(sid);
if (!n) return el_wrap_str(el_strdup("{}"));
JsonBuf b; jb_init(&b);
engram_emit_node_json(&b, n);
engram_emit_node_json(&b, n, 0);
return el_wrap_str(b.buf);
}
@@ -9050,7 +9065,7 @@ el_val_t engram_get_node_by_label(el_val_t label) {
EngramNode* n = &g->nodes[i];
if (n->label && strcmp(n->label, lbl) == 0) {
JsonBuf b; jb_init(&b);
engram_emit_node_json(&b, n);
engram_emit_node_json(&b, n, 0);
return el_wrap_str(b.buf);
}
}
@@ -9091,7 +9106,7 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
int64_t end = nhits < lim ? nhits : lim;
for (int64_t k = 0; k < end; k++) {
if (!first) jb_putc(&b, ',');
engram_emit_node_json(&b, &g->nodes[hits[k].idx]);
engram_emit_node_json(&b, &g->nodes[hits[k].idx], 0);
first = 0;
}
free(hits);
@@ -9123,7 +9138,7 @@ el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset) {
int first = 1;
for (int64_t i = off; i < end; i++) {
if (!first) jb_putc(&b, ',');
engram_emit_node_json(&b, &g->nodes[idx[i]]);
engram_emit_node_json(&b, &g->nodes[idx[i]], 0);
first = 0;
}
free(idx);
@@ -9160,7 +9175,7 @@ el_val_t engram_scan_nodes_by_type_json(el_val_t type_v, el_val_t limit, el_val_
int first = 1;
for (int64_t i = off; i < end; i++) {
if (!first) jb_putc(&b, ',');
engram_emit_node_json(&b, &g->nodes[idx[i]]);
engram_emit_node_json(&b, &g->nodes[idx[i]], 0);
first = 0;
}
free(idx);
@@ -9224,7 +9239,7 @@ el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t di
if (!n) continue;
if (!first) jb_putc(&b, ',');
jb_puts(&b, "{\"node\":");
engram_emit_node_json(&b, n);
engram_emit_node_json(&b, n, 0);
jb_puts(&b, ",\"edge\":");
engram_emit_edge_json(&b, e);
char tmp[64]; snprintf(tmp, sizeof(tmp), ",\"hops\":%lld}", (long long)(h + 1));
@@ -9267,7 +9282,7 @@ el_val_t engram_activate_json(el_val_t query, el_val_t depth) {
if (i > 0) jb_putc(&b, ',');
jb_puts(&b, "{\"node\":");
if (n) {
engram_emit_node_json(&b, n);
engram_emit_node_json(&b, n, 0);
} else {
jb_puts(&b, "{}");
}
@@ -9404,9 +9419,11 @@ el_val_t engram_stats_json(void) {
return el_wrap_str(el_strdup(buf));
}
/* engram_act_stats_json — per-call activation observability + embedder
* breaker state. (2026-07-27 self-review.) wm_evicted/breakthroughs describe
* the LAST engram_activate call on this store; embed_breaker_open=1 means
/* engram_act_stats_json — activation observability + embedder breaker state.
* (2026-07-27 self-review; counters made cumulative 2026-07-31.)
* wm_evicted/breakthroughs are monotonic process-lifetime totals across ALL
* engram_activate calls on this store (diff successive readings for rates;
* they reset to 0 only on restart); embed_breaker_open=1 means
* eg_embed_fetch is currently refusing calls (semantic activation silently
* degraded to lexical until the cooldown expires). The soul heartbeat folds
* this into its ISE so the pathologies are diagnosable from telemetry
@@ -9593,7 +9610,7 @@ el_val_t engram_compile_layered_json(el_val_t intent, el_val_t depth) {
jb_puts(&b, "[LAYER 0 — STRUCTURAL]\n");
wrote_layer0 = 1;
}
engram_emit_node_json(&b, n);
engram_emit_node_json(&b, n, 0);
jb_putc(&b, '\n');
}
@@ -9606,7 +9623,7 @@ el_val_t engram_compile_layered_json(el_val_t intent, el_val_t depth) {
jb_puts(&b, "[ENGRAM CONTEXT]\n");
wrote_normal = 1;
}
engram_emit_node_json(&b, n);
engram_emit_node_json(&b, n, 0);
jb_putc(&b, '\n');
}
@@ -9656,7 +9673,7 @@ el_val_t engram_query_range(el_val_t start_ms_v, el_val_t end_ms_v) {
jb_putc(&b, '[');
for (int64_t i = 0; i < mc; i++) {
if (i > 0) jb_putc(&b, ',');
engram_emit_node_json(&b, &g->nodes[idx[i]]);
engram_emit_node_json(&b, &g->nodes[idx[i]], 0);
}
jb_putc(&b, ']');
free(idx);