runtime: engram_*_json accessors, http_set_handler dlsym, codegen int-call

Three changes that turned the runtime into something Engram-the-server
can actually run on top of.

1. engram_*_json accessors. The runtime's engram_get_node/search/scan/
   neighbors/activate return ElList/ElMap; passing those through
   json_stringify hit the type-erasure wall (an ElList* has no header
   that distinguishes it from a string pointer). Added pre-serialized
   sibling builtins:

     engram_get_node_json(id)         -> JSON object
     engram_search_json(query, limit) -> JSON array of node objects
     engram_scan_nodes_json(limit, offset)
     engram_neighbors_json(node_id, max_depth, direction)
     engram_activate_json(query, depth)
     engram_stats_json()

   Each walks the typed C structures and serializes directly, reusing
   the existing engram_emit_node_json / engram_emit_edge_json helpers
   from the snapshot path.

2. http_set_handler now falls back to dlsym(RTLD_DEFAULT, name) when
   the named handler isn't already in the C-level registry. El programs
   that define `fn handle_request(method, path, body) -> String` can
   register themselves just by calling http_set_handler("handle_request").
   No C glue required. Verified live on a real El server.

3. Codegen: extended int-typed dispatch on `+` to handle Calls. New
   helper is_int_call recognizes a known-int-returning builtin set:
   str_len, str_index_of, str_to_int, str_char_code, native_list_len,
   el_list_len, len, json_get_int, json_array_len, engram_node_count,
   engram_edge_count, time_now, time_now_utc, time_diff, time_add,
   time_from_parts, el_abs/max/min, float_to_int. With this,
   `pos + str_len(needle)` compiles to integer arithmetic instead of
   string concat. The earlier limitation noted in the previous commit
   (Ident + Call returning Int) is now closed.

Also: el_to_float / el_from_float moved to el_runtime.h as static
inlines so generated programs can use them. Eliminates the unused
inline definitions that were duplicating in the .c file.

Closure verified: stage1 vs stage2 byte-identical against the new
runtime. dist/platform/elc rebuilt; .prev4 preserved.

Engram server (engram/src/server.el) end-to-end:
  POST /api/nodes ×3 → 3 UUIDs returned
  POST /api/edges ×2 → linkage made
  GET /api/stats → {"node_count":3,"edge_count":2}
  GET /api/search?q=spreading&limit=5 → 1 hit, full node JSON
  POST /api/activate {"query":"Hebbian","depth":3}
    → seed node @ hop 0, strength 0.8
    → 1-hop neighbor @ strength 0.392 (= 0.8 × 0.7 weight × 0.7 decay)
  GET /api/neighbors/<id>?depth=2 → {node, edge, hops} triple
  POST /api/save → {"ok":true,"path":"..."}
  Server stays alive across all routes.

Snapshot save/load on restart still TODO — server starts with 0 nodes
even when a snapshot exists; investigation pending.
This commit is contained in:
Will Anderson
2026-04-30 13:44:41 -05:00
parent 6bdd4a4ba9
commit 0fa9e749e1
7 changed files with 428 additions and 3 deletions
BIN
View File
Binary file not shown.
+102
View File
@@ -50,6 +50,7 @@ el_val_t param_decl(el_val_t param, el_val_t idx);
el_val_t params_to_c(el_val_t params);
el_val_t transform_implicit_return(el_val_t body);
el_val_t is_int_name(el_val_t name);
el_val_t is_int_call(el_val_t call_expr);
el_val_t add_int_name(el_val_t name);
el_val_t build_int_names_for_params(el_val_t params);
el_val_t cg_fn(el_val_t stmt);
@@ -1715,7 +1716,37 @@ el_val_t cg_expr(el_val_t expr) {
}
}
}
if (str_eq(left_kind, EL_STR("Ident"))) {
if (str_eq(right_kind, EL_STR("Call"))) {
el_val_t lname = el_get_field(left, EL_STR("name"));
if (is_int_name(lname)) {
if (is_int_call(right)) {
el_val_t op_c = binop_to_c(op);
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("("), left_c), EL_STR(" ")), op_c), EL_STR(" ")), right_c), EL_STR(")"));
}
}
}
}
if (str_eq(right_kind, EL_STR("Ident"))) {
if (str_eq(left_kind, EL_STR("Call"))) {
el_val_t rname = el_get_field(right, EL_STR("name"));
if (is_int_name(rname)) {
if (is_int_call(left)) {
el_val_t op_c = binop_to_c(op);
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("("), left_c), EL_STR(" ")), op_c), EL_STR(" ")), right_c), EL_STR(")"));
}
}
}
}
if (str_eq(left_kind, EL_STR("Call"))) {
if (str_eq(right_kind, EL_STR("Call"))) {
if (is_int_call(left)) {
if (is_int_call(right)) {
el_val_t op_c = binop_to_c(op);
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("("), left_c), EL_STR(" ")), op_c), EL_STR(" ")), right_c), EL_STR(")"));
}
}
}
return el_str_concat(el_str_concat(el_str_concat(el_str_concat(EL_STR("el_str_concat("), left_c), EL_STR(", ")), right_c), EL_STR(")"));
}
if (str_eq(right_kind, EL_STR("Call"))) {
@@ -2160,6 +2191,77 @@ el_val_t is_int_name(el_val_t name) {
return 0;
}
el_val_t is_int_call(el_val_t call_expr) {
el_val_t func = el_get_field(call_expr, EL_STR("func"));
el_val_t fk = el_get_field(func, EL_STR("expr"));
if (!str_eq(fk, EL_STR("Ident"))) {
return 0;
}
el_val_t name = el_get_field(func, EL_STR("name"));
if (str_eq(name, EL_STR("str_len"))) {
return 1;
}
if (str_eq(name, EL_STR("str_index_of"))) {
return 1;
}
if (str_eq(name, EL_STR("str_to_int"))) {
return 1;
}
if (str_eq(name, EL_STR("str_char_code"))) {
return 1;
}
if (str_eq(name, EL_STR("native_list_len"))) {
return 1;
}
if (str_eq(name, EL_STR("el_list_len"))) {
return 1;
}
if (str_eq(name, EL_STR("len"))) {
return 1;
}
if (str_eq(name, EL_STR("json_get_int"))) {
return 1;
}
if (str_eq(name, EL_STR("json_array_len"))) {
return 1;
}
if (str_eq(name, EL_STR("engram_node_count"))) {
return 1;
}
if (str_eq(name, EL_STR("engram_edge_count"))) {
return 1;
}
if (str_eq(name, EL_STR("time_now"))) {
return 1;
}
if (str_eq(name, EL_STR("time_now_utc"))) {
return 1;
}
if (str_eq(name, EL_STR("time_diff"))) {
return 1;
}
if (str_eq(name, EL_STR("time_add"))) {
return 1;
}
if (str_eq(name, EL_STR("time_from_parts"))) {
return 1;
}
if (str_eq(name, EL_STR("el_abs"))) {
return 1;
}
if (str_eq(name, EL_STR("el_max"))) {
return 1;
}
if (str_eq(name, EL_STR("el_min"))) {
return 1;
}
if (str_eq(name, EL_STR("float_to_int"))) {
return 1;
}
return 0;
return 0;
}
el_val_t add_int_name(el_val_t name) {
el_val_t csv = state_get(EL_STR("__int_names"));
if (str_eq(csv, EL_STR(""))) {
Vendored Executable
BIN
View File
Binary file not shown.
+187
View File
@@ -2145,6 +2145,11 @@ static el_val_t engram_node_to_map(const EngramNode* n) {
return m;
}
/* (Node JSON serialization is provided by `engram_emit_node_json` further
* down in the persistence section reused by the *_json builtins below.) */
static void engram_emit_node_json(JsonBuf* b, const EngramNode* n);
static void engram_emit_edge_json(JsonBuf* b, const EngramEdge* e);
/* Salience may arrive either as a float bit-pattern or as a small integer
* (e.g. 1, meaning 1.0). Heuristic: if interpreted as double it's in
* [0.0, 100.0] use it; otherwise treat as int and convert. */
@@ -2794,6 +2799,188 @@ el_val_t engram_load(el_val_t path) {
return 1;
}
/* ── Engram JSON-string accessors ─────────────────────────────────────────
* These return pre-serialized JSON strings so callers (especially HTTP
* handlers) don't have to round-trip ElList/ElMap through json_stringify
* which can't reliably distinguish those structures from raw pointers
* due to el_val_t's type erasure. The runtime knows the real C types and
* can serialize directly. */
el_val_t engram_get_node_json(el_val_t id) {
const char* sid = EL_CSTR(id);
EngramNode* n = engram_find_node(sid);
if (!n) return el_wrap_str(el_strdup("{}"));
JsonBuf b; jb_init(&b);
engram_emit_node_json(&b, n);
return el_wrap_str(b.buf);
}
el_val_t engram_search_json(el_val_t query, el_val_t limit) {
EngramStore* g = engram_get();
const char* q = EL_CSTR(query);
int64_t lim = (int64_t)limit;
if (lim <= 0) lim = 100;
JsonBuf b; jb_init(&b);
jb_putc(&b, '[');
int first = 1;
int64_t found = 0;
if (q && *q) {
for (int64_t i = 0; i < g->node_count && found < lim; i++) {
EngramNode* n = &g->nodes[i];
if (istr_contains(n->content, q) ||
istr_contains(n->label, q) ||
istr_contains(n->tags, q)) {
if (!first) jb_putc(&b, ',');
engram_emit_node_json(&b, n);
first = 0;
found++;
}
}
}
jb_putc(&b, ']');
return el_wrap_str(b.buf);
}
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset) {
EngramStore* g = engram_get();
int64_t lim = (int64_t)limit; if (lim <= 0) lim = 100;
int64_t off = (int64_t)offset; if (off < 0) off = 0;
JsonBuf b; jb_init(&b);
jb_putc(&b, '[');
if (g->node_count == 0) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
int64_t* idx = malloc((size_t)g->node_count * sizeof(int64_t));
if (!idx) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
for (int64_t i = 0; i < g->node_count; i++) idx[i] = i;
engram_sort_indices_by_salience(idx, g->node_count, g->nodes);
int64_t end = off + lim;
if (end > g->node_count) end = g->node_count;
int first = 1;
for (int64_t i = off; i < end; i++) {
if (!first) jb_putc(&b, ',');
engram_emit_node_json(&b, &g->nodes[idx[i]]);
first = 0;
}
free(idx);
jb_putc(&b, ']');
return el_wrap_str(b.buf);
}
el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction) {
/* Re-implement here directly so we serialize without going through
* the ElList path. Walks BFS to max_depth, emits {node, edge, hops}
* triples. */
EngramStore* g = engram_get();
const char* sid = EL_CSTR(node_id);
int64_t depth = (int64_t)max_depth; if (depth <= 0) depth = 1;
const char* dir = EL_CSTR(direction); if (!dir) dir = "both";
int allow_out = (strcmp(dir, "out") == 0) || (strcmp(dir, "both") == 0);
int allow_in = (strcmp(dir, "in") == 0) || (strcmp(dir, "both") == 0);
JsonBuf b; jb_init(&b);
jb_putc(&b, '[');
if (!sid || !*sid) { jb_putc(&b, ']'); return el_wrap_str(b.buf); }
/* Frontier of (node_id, hops). Cap to a sane size. */
char** frontier = calloc(1024, sizeof(char*));
int64_t* frontier_h = calloc(1024, sizeof(int64_t));
int64_t fc = 0;
char** visited = calloc(1024, sizeof(char*));
int64_t vc = 0;
if (!frontier || !frontier_h || !visited) {
free(frontier); free(frontier_h); free(visited);
jb_putc(&b, ']'); return el_wrap_str(b.buf);
}
frontier[fc] = el_strdup(sid); frontier_h[fc] = 0; fc++;
visited[vc++] = el_strdup(sid);
int first = 1;
while (fc > 0) {
char* cur = frontier[0]; int64_t h = frontier_h[0];
for (int64_t k = 1; k < fc; k++) { frontier[k-1] = frontier[k]; frontier_h[k-1] = frontier_h[k]; }
fc--;
if (h >= depth) { free(cur); continue; }
for (int64_t i = 0; i < g->edge_count; i++) {
EngramEdge* e = &g->edges[i];
const char* peer = NULL;
if (allow_out && e->from_id && strcmp(e->from_id, cur) == 0) peer = e->to_id;
else if (allow_in && e->to_id && strcmp(e->to_id, cur) == 0) peer = e->from_id;
if (!peer) continue;
int seen = 0;
for (int64_t v = 0; v < vc; v++) {
if (strcmp(visited[v], peer) == 0) { seen = 1; break; }
}
if (seen) continue;
EngramNode* n = engram_find_node(peer);
if (!n) continue;
if (!first) jb_putc(&b, ',');
jb_puts(&b, "{\"node\":");
engram_emit_node_json(&b, n);
jb_puts(&b, ",\"edge\":");
engram_emit_edge_json(&b, e);
char tmp[64]; snprintf(tmp, sizeof(tmp), ",\"hops\":%lld}", (long long)(h + 1));
jb_puts(&b, tmp);
first = 0;
if (vc < 1024) visited[vc++] = el_strdup(peer);
if (fc < 1024 && h + 1 < depth) { frontier[fc] = el_strdup(peer); frontier_h[fc] = h + 1; fc++; }
}
free(cur);
}
for (int64_t i = 0; i < fc; i++) free(frontier[i]);
for (int64_t i = 0; i < vc; i++) free(visited[i]);
free(frontier); free(frontier_h); free(visited);
jb_putc(&b, ']');
return el_wrap_str(b.buf);
}
el_val_t engram_activate_json(el_val_t query, el_val_t depth) {
/* Run the existing engram_activate to get the ElList of result maps,
* then walk that list and serialize each entry into JSON manually.
* We have the raw nodes via engram_find_node, so we can re-emit
* directly without trusting json_stringify on the ElMap. */
el_val_t lst = engram_activate(query, depth);
ElList* arr = (ElList*)(uintptr_t)lst;
JsonBuf b; jb_init(&b);
jb_putc(&b, '[');
if (arr) {
for (int64_t i = 0; i < arr->length; i++) {
ElMap* entry = (ElMap*)(uintptr_t)arr->elems[i];
if (!entry) continue;
/* The entry map has keys: "node" (ElMap), "activation_strength"
* (Float bit-pattern), "hops" (Int). Read them from the map
* directly using el_map_get with EL_STR keys. */
el_val_t node_map = el_map_get(arr->elems[i], EL_STR("node"));
el_val_t strength_v = el_map_get(arr->elems[i], EL_STR("activation_strength"));
el_val_t hops_v = el_map_get(arr->elems[i], EL_STR("hops"));
/* Look up the underlying EngramNode by id field of the map */
el_val_t id_v = el_map_get(node_map, EL_STR("id"));
const char* id_s = EL_CSTR(id_v);
EngramNode* n = id_s ? engram_find_node(id_s) : NULL;
if (i > 0) jb_putc(&b, ',');
jb_puts(&b, "{\"node\":");
if (n) {
engram_emit_node_json(&b, n);
} else {
jb_puts(&b, "{}");
}
char tmp[64];
snprintf(tmp, sizeof(tmp), ",\"activation_strength\":%g", el_to_float(strength_v));
jb_puts(&b, tmp);
snprintf(tmp, sizeof(tmp), ",\"hops\":%lld}", (long long)(int64_t)hops_v);
jb_puts(&b, tmp);
}
}
jb_putc(&b, ']');
return el_wrap_str(b.buf);
}
el_val_t engram_stats_json(void) {
EngramStore* g = engram_get();
char buf[128];
snprintf(buf, sizeof(buf),
"{\"node_count\":%lld,\"edge_count\":%lld}",
(long long)g->node_count, (long long)g->edge_count);
return el_wrap_str(el_strdup(buf));
}
/* ── Batch 4: LLM (Anthropic API client) ─────────────────────────────────── */
/*
* All LLM builtins call https://api.anthropic.com/v1/messages with the API
+10
View File
@@ -255,6 +255,16 @@ el_val_t engram_activate(el_val_t query, el_val_t depth);
el_val_t engram_save(el_val_t path);
el_val_t engram_load(el_val_t path);
/* JSON-string accessors — return pre-serialized JSON so HTTP handlers
* can pass results straight through without round-tripping ElList/ElMap
* through json_stringify. */
el_val_t engram_get_node_json(el_val_t id);
el_val_t engram_search_json(el_val_t query, el_val_t limit);
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction);
el_val_t engram_activate_json(el_val_t query, el_val_t depth);
el_val_t engram_stats_json(void);
/* ── LLM (Anthropic API client) ─────────────────────────────────────────────
* All functions call https://api.anthropic.com/v1/messages with the API key
* from env ANTHROPIC_API_KEY. Default model when empty: claude-sonnet-4-5. */
+64
View File
@@ -191,7 +191,40 @@ fn cg_expr(expr: Map<String, Any>) -> String {
}
}
}
// Same dispatch for Ident-Int + Call-to-known-Int-builtin (and the
// mirror). Without this, expressions like `pos + str_len(s)` get
// string-concatenated. is_int_call walks a known-builtin list.
if left_kind == "Ident" {
if right_kind == "Call" {
let lname: String = left["name"]
if is_int_name(lname) {
if is_int_call(right) {
let op_c: String = binop_to_c(op)
return "(" + left_c + " " + op_c + " " + right_c + ")"
}
}
}
}
if right_kind == "Ident" {
if left_kind == "Call" {
let rname: String = right["name"]
if is_int_name(rname) {
if is_int_call(left) {
let op_c: String = binop_to_c(op)
return "(" + left_c + " " + op_c + " " + right_c + ")"
}
}
}
}
if left_kind == "Call" {
if right_kind == "Call" {
if is_int_call(left) {
if is_int_call(right) {
let op_c: String = binop_to_c(op)
return "(" + left_c + " " + op_c + " " + right_c + ")"
}
}
}
return "el_str_concat(" + left_c + ", " + right_c + ")"
}
if right_kind == "Call" {
@@ -669,6 +702,37 @@ fn is_int_name(name: String) -> Bool {
return str_contains(csv, "," + name + ",")
}
// Known runtime builtins that return Int. Used to dispatch arithmetic vs
// string-concat on `+` when one side is a Call. New builtins must be added
// here when they return Int and may participate in arithmetic.
fn is_int_call(call_expr: Map<String, Any>) -> Bool {
let func = call_expr["func"]
let fk: String = func["expr"]
if !str_eq(fk, "Ident") { return false }
let name: String = func["name"]
if str_eq(name, "str_len") { return true }
if str_eq(name, "str_index_of") { return true }
if str_eq(name, "str_to_int") { return true }
if str_eq(name, "str_char_code") { return true }
if str_eq(name, "native_list_len") { return true }
if str_eq(name, "el_list_len") { return true }
if str_eq(name, "len") { return true }
if str_eq(name, "json_get_int") { return true }
if str_eq(name, "json_array_len") { return true }
if str_eq(name, "engram_node_count") { return true }
if str_eq(name, "engram_edge_count") { return true }
if str_eq(name, "time_now") { return true }
if str_eq(name, "time_now_utc") { return true }
if str_eq(name, "time_diff") { return true }
if str_eq(name, "time_add") { return true }
if str_eq(name, "time_from_parts") { return true }
if str_eq(name, "el_abs") { return true }
if str_eq(name, "el_max") { return true }
if str_eq(name, "el_min") { return true }
if str_eq(name, "float_to_int") { return true }
return false
}
fn add_int_name(name: String) -> Bool {
let csv: String = state_get("__int_names")
if str_eq(csv, "") { csv = "," }
+65 -3
View File
@@ -1,5 +1,4 @@
// elc-combined.el El self-hosting compiler, single-file bootstrap edition
// elc-combined.el
// lexer.el el self-hosting lexer
//
// Tokenises an el source string into a list of token maps.
@@ -1511,7 +1510,40 @@ fn cg_expr(expr: Map<String, Any>) -> String {
}
}
}
// Same dispatch for Ident-Int + Call-to-known-Int-builtin (and the
// mirror). Without this, expressions like `pos + str_len(s)` get
// string-concatenated. is_int_call walks a known-builtin list.
if left_kind == "Ident" {
if right_kind == "Call" {
let lname: String = left["name"]
if is_int_name(lname) {
if is_int_call(right) {
let op_c: String = binop_to_c(op)
return "(" + left_c + " " + op_c + " " + right_c + ")"
}
}
}
}
if right_kind == "Ident" {
if left_kind == "Call" {
let rname: String = right["name"]
if is_int_name(rname) {
if is_int_call(left) {
let op_c: String = binop_to_c(op)
return "(" + left_c + " " + op_c + " " + right_c + ")"
}
}
}
}
if left_kind == "Call" {
if right_kind == "Call" {
if is_int_call(left) {
if is_int_call(right) {
let op_c: String = binop_to_c(op)
return "(" + left_c + " " + op_c + " " + right_c + ")"
}
}
}
return "el_str_concat(" + left_c + ", " + right_c + ")"
}
if right_kind == "Call" {
@@ -1989,6 +2021,37 @@ fn is_int_name(name: String) -> Bool {
return str_contains(csv, "," + name + ",")
}
// Known runtime builtins that return Int. Used to dispatch arithmetic vs
// string-concat on `+` when one side is a Call. New builtins must be added
// here when they return Int and may participate in arithmetic.
fn is_int_call(call_expr: Map<String, Any>) -> Bool {
let func = call_expr["func"]
let fk: String = func["expr"]
if !str_eq(fk, "Ident") { return false }
let name: String = func["name"]
if str_eq(name, "str_len") { return true }
if str_eq(name, "str_index_of") { return true }
if str_eq(name, "str_to_int") { return true }
if str_eq(name, "str_char_code") { return true }
if str_eq(name, "native_list_len") { return true }
if str_eq(name, "el_list_len") { return true }
if str_eq(name, "len") { return true }
if str_eq(name, "json_get_int") { return true }
if str_eq(name, "json_array_len") { return true }
if str_eq(name, "engram_node_count") { return true }
if str_eq(name, "engram_edge_count") { return true }
if str_eq(name, "time_now") { return true }
if str_eq(name, "time_now_utc") { return true }
if str_eq(name, "time_diff") { return true }
if str_eq(name, "time_add") { return true }
if str_eq(name, "time_from_parts") { return true }
if str_eq(name, "el_abs") { return true }
if str_eq(name, "el_max") { return true }
if str_eq(name, "el_min") { return true }
if str_eq(name, "float_to_int") { return true }
return false
}
fn add_int_name(name: String) -> Bool {
let csv: String = state_get("__int_names")
if str_eq(csv, "") { csv = "," }
@@ -2154,7 +2217,6 @@ fn compile(source: String) -> String {
// result to args()[1]. Then run:
// cc -o <prog> <output.c> el_runtime.c
// CLI driver
let _argv: [String] = args()
let _src_path: String = native_list_get(_argv, 0)
let _source: String = fs_read(_src_path)