M-INTEROCEPTION P0: add read-only engram_scan_nodes_emb_json builtin
Read routes (GET /api/embeddings, /api/graph/dump) need node embedding vectors, but every consumer emit path deliberately drops the ~5.7KB emb vector (include_emb=0) to stay under MCP token limits. Rather than perturb that shared path, add a dedicated additive builtin that pages nodes WITH their dense vector, emitting id/node_type/label/created_at/emb_dim plus emb as a JSON array whose length equals emb_dim (so a consumer can verify the vector round-trips). Un-embedded nodes emit emb_dim:0 / emb:[]. Same salience-sorted, transparent-layer-skipped, bounded pagination as engram_scan_nodes_json; default page 256. Purely additive: engram_emit_node_json and the default include_emb=0 are untouched, so every existing endpoint is byte-identical to trunk (verified: scan_nodes_json still carries no emb). The HTTP route wiring in server.el is DEFERRED to cutover per the elc-drift blocker (regenerating engram/dist diverges ~285 lines with no source change); the builtin is exercised directly by the pure-C gate instead. Measured on a copy: len(emb)==emb_dim for all nodes, pagination disjoint, existing path unchanged; full 256-node x 768-dim page = 16.7 ms / 1.18 MB. ASan+UBSan clean.
This commit is contained in:
@@ -11936,6 +11936,58 @@ el_val_t engram_scan_nodes_by_type_json(el_val_t type_v, el_val_t limit, el_val_
|
||||
return el_wrap_str(b.buf);
|
||||
}
|
||||
|
||||
/* engram_scan_nodes_emb_json — READ-ONLY, ADDITIVE (M-INTEROCEPTION P0).
|
||||
* Paginated dump of nodes WITH their dense embedding vector, for the
|
||||
* GET /api/embeddings + /api/graph/dump read routes. Purely additive: it does
|
||||
* NOT touch engram_emit_node_json or the default include_emb=0 anywhere. It
|
||||
* emits a compact, self-describing record per node — id, node_type, label,
|
||||
* created_at, emb_dim, and emb as a JSON array of %.4g floats (length ==
|
||||
* emb_dim, so a consumer can verify the vector round-trips). Nodes without an
|
||||
* embedding are emitted with emb_dim:0 and emb:[]. Same salience-sorted,
|
||||
* transparent-layer-skipped pagination as engram_scan_nodes_json.
|
||||
* Default limit is bounded (256) to keep one page's payload sane. */
|
||||
el_val_t engram_scan_nodes_emb_json(el_val_t limit, el_val_t offset) {
|
||||
EngramStore* g = engram_get();
|
||||
int64_t lim = (int64_t)limit; if (lim <= 0) lim = 256;
|
||||
int64_t off = (int64_t)offset; if (off < 0) off = 0;
|
||||
JsonBuf b; jb_init(&b);
|
||||
jb_putc(&b, '[');
|
||||
if (g->node_count == 0) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
|
||||
int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t));
|
||||
if (!idx) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
|
||||
int64_t live = 0;
|
||||
for (int64_t i = 0; i < g->node_count; i++) {
|
||||
if (engram_layer_is_transparent(g->nodes[i].layer_id)) continue;
|
||||
idx[live++] = i;
|
||||
}
|
||||
engram_sort_indices_by_salience(idx, live, g->nodes);
|
||||
int64_t end = off + lim;
|
||||
if (end > live) end = live;
|
||||
int first = 1;
|
||||
char tmp[80];
|
||||
for (int64_t i = off; i < end; i++) {
|
||||
const EngramNode* n = &g->nodes[idx[i]];
|
||||
if (!first) jb_putc(&b, ',');
|
||||
first = 0;
|
||||
jb_putc(&b, '{');
|
||||
jb_puts(&b, "\"id\":"); jb_emit_escaped(&b, n->id ? n->id : "");
|
||||
jb_puts(&b, ",\"node_type\":"); jb_emit_escaped(&b, n->node_type ? n->node_type : "");
|
||||
jb_puts(&b, ",\"label\":"); jb_emit_escaped(&b, n->label ? n->label : "");
|
||||
snprintf(tmp, sizeof(tmp), ",\"created_at\":%lld", (long long)n->created_at); jb_puts(&b, tmp);
|
||||
int32_t dim = (n->emb && n->emb_dim > 0) ? n->emb_dim : 0;
|
||||
snprintf(tmp, sizeof(tmp), ",\"emb_dim\":%d", dim); jb_puts(&b, tmp);
|
||||
jb_puts(&b, ",\"emb\":[");
|
||||
for (int32_t j = 0; j < dim; j++) {
|
||||
snprintf(tmp, sizeof(tmp), "%s%.4g", j ? "," : "", (double)n->emb[j]);
|
||||
jb_puts(&b, tmp);
|
||||
}
|
||||
jb_puts(&b, "]}");
|
||||
}
|
||||
free(idx);
|
||||
jb_putc(&b, ']');
|
||||
return el_wrap_str(b.buf);
|
||||
}
|
||||
|
||||
el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction) {
|
||||
/* Re-implement here directly so we serialize without going through
|
||||
* the ElList path. Walks BFS to max_depth, emits {node, edge, hops}
|
||||
|
||||
@@ -621,6 +621,7 @@ el_val_t engram_get_node_by_label(el_val_t label);
|
||||
el_val_t engram_search_json(el_val_t query, el_val_t limit);
|
||||
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_scan_nodes_emb_json(el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction);
|
||||
el_val_t engram_activate_json(el_val_t query, el_val_t depth);
|
||||
el_val_t engram_stats_json(void);
|
||||
|
||||
@@ -1086,6 +1086,10 @@ el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el
|
||||
return engram_scan_nodes_by_type_json(node_type, limit, offset);
|
||||
}
|
||||
|
||||
el_val_t __engram_scan_nodes_emb_json(el_val_t limit, el_val_t offset) {
|
||||
return engram_scan_nodes_emb_json(limit, offset);
|
||||
}
|
||||
|
||||
el_val_t __engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction) {
|
||||
return engram_neighbors_json(node_id, max_depth, direction);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user