runtime: engram_edges_json — read edges without a whole-graph file round trip
El SDK CI - dev / build-and-test (pull_request) Failing after 13m4s

/api/graph/edges answered a read query by calling engram_save() to serialize
the ENTIRE graph to disk (128 MB) and then fs_read-ing it back. Two defects in
one line, and both bit production on 2026-08-15:

  1. The path it wrote was ~/.neuron/engram/snapshot.json — the engram
     server's CANONICAL store. A READ route overwriting the persistence
     owner's canonical file. This defect had been fixed once (export moved to
     a scratch path); it came back when the hand-written dispatch block was
     replaced by @route dispatch and the unfixed copy is the one that
     survived the merge.
  2. Cost: a full snapshot write, a 128 MB read, and a parse of the whole
     graph, per request, to return a bounded slice.

Calling it tonight overwrote the canonical snapshot and immediately preceded
an engram crash loop.

engram_edges_json(limit, offset) is the builtin that route's own TODO asked
for ("Future: add an engram_edges_json() builtin and drop the file round trip
entirely"). It walks g->edges directly and emits every persisted field.

limit <= 0 defaults to 1000, not unbounded: this is the endpoint that fell
over, and an unbounded default would preserve the failure mode under a new
name. Callers page explicitly.

Registered in codegen.el's builtin_arity (both plain and __ spellings) and
wrapped in el_seed.c per the project's C-builtin recipe.
This commit is contained in:
bigmerge
2026-08-15 20:10:48 -05:00
parent 7351fb0a8d
commit 4e24d7d3f1
4 changed files with 55 additions and 0 deletions
+2
View File
@@ -2764,6 +2764,7 @@ fn builtin_arity(name: String) -> Int {
if str_eq(name, "__engram_node_full_in") { return 9 }
if str_eq(name, "__engram_connect_in") { return 5 }
if str_eq(name, "__engram_scan_nodes_json") { return 2 }
if str_eq(name, "__engram_edges_json") { return 2 }
if str_eq(name, "__generate") { return 1 }
// Filesystem
if str_eq(name, "fs_read") { return 1 }
@@ -2862,6 +2863,7 @@ fn builtin_arity(name: String) -> Int {
if str_eq(name, "engram_get_node_by_label") { return 1 }
if str_eq(name, "engram_search_json") { return 2 }
if str_eq(name, "engram_scan_nodes_json") { return 2 }
if str_eq(name, "engram_edges_json") { return 2 }
if str_eq(name, "engram_neighbors_json") { return 3 }
if str_eq(name, "engram_activate_json") { return 2 }
if str_eq(name, "engram_stats_json") { return 0 }
+44
View File
@@ -18329,3 +18329,47 @@ el_val_t engram_recall_json(el_val_t query, el_val_t limit) {
el_val_t cgi_principal(void) { return EL_STR(_el_cgi_principal ? _el_cgi_principal : ""); }
el_val_t cgi_network(void) { return EL_STR(_el_cgi_network ? _el_cgi_network : ""); }
el_val_t cgi_engram(void) { return EL_STR(_el_cgi_engram ? _el_cgi_engram : ""); }
/* engram_edges_json(limit, offset) — emit edges straight from the store.
*
* Replaces a serialize-and-reread round trip that took production down on
* 2026-08-15: /api/graph/edges called engram_save() to write the ENTIRE graph
* to disk (128 MB) and then fs_read it back, just to answer a read query for
* edges. One debug request cost a full snapshot write, a 128 MB read, and the
* peak memory to hold it on top of being O(whole graph) for a bounded slice.
* The route's own comment had already named the fix: "Future: add an
* engram_edges_json() builtin and drop the file round trip entirely."
*
* limit <= 0 defaults to 1000 rather than unbounded: this is the endpoint that
* fell over, and an unbounded default would preserve the failure mode under a
* different name. Pass an explicit limit to page.
*/
el_val_t engram_edges_json(el_val_t limit, el_val_t offset) {
EngramStore* g = engram_get();
int64_t lim = (int64_t)limit; if (lim <= 0) lim = 1000;
int64_t off = (int64_t)offset; if (off < 0) off = 0;
JsonBuf b; jb_init(&b);
jb_putc(&b, '[');
int64_t emitted = 0;
char t[192];
for (int64_t i = off; i < g->edge_count && emitted < lim; i++) {
EngramEdge* e = &g->edges[i];
if (emitted > 0) jb_putc(&b, ',');
jb_puts(&b, "{\"id\":"); jb_emit_escaped(&b, e->id ? e->id : "");
jb_puts(&b, ",\"from_id\":"); jb_emit_escaped(&b, e->from_id ? e->from_id : "");
jb_puts(&b, ",\"to_id\":"); jb_emit_escaped(&b, e->to_id ? e->to_id : "");
jb_puts(&b, ",\"relation\":"); jb_emit_escaped(&b, e->relation ? e->relation : "");
snprintf(t, sizeof t,
",\"weight\":%.6g,\"hebb\":%.6g,\"confidence\":%.6g,"
"\"created_at\":%lld,\"updated_at\":%lld,\"last_fired\":%lld,"
"\"inhibitory\":%d,\"layer_id\":%u}",
e->weight, e->hebb, e->confidence,
(long long)e->created_at, (long long)e->updated_at,
(long long)e->last_fired, e->inhibitory, (unsigned)e->layer_id);
jb_puts(&b, t);
emitted++;
}
jb_putc(&b, ']');
return el_wrap_str(b.buf);
}
+4
View File
@@ -1021,6 +1021,10 @@ el_val_t el_mem_check(void);
* which is lexical by design — see the note at the definition. */
el_val_t engram_recall_json(el_val_t query, el_val_t limit);
/* Edges straight from the store — replaces the engram_save()+fs_read()
* whole-graph round trip that /api/graph/edges used to do. */
el_val_t engram_edges_json(el_val_t limit, el_val_t offset);
/* CGI identity accessors (read-only). */
el_val_t cgi_principal(void);
el_val_t cgi_network(void);
+5
View File
@@ -1371,6 +1371,11 @@ el_val_t __engram_scan_nodes_json(el_val_t limit, el_val_t offset) {
return engram_scan_nodes_json(limit, offset);
}
el_val_t engram_edges_json(el_val_t limit, el_val_t offset);
el_val_t __engram_edges_json(el_val_t limit, el_val_t offset) {
return engram_edges_json(limit, offset);
}
el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset) {
return engram_scan_nodes_by_type_json(node_type, limit, offset);
}