diff --git a/engram/src/server.el b/engram/src/server.el index 40f7bac..571ad3c 100644 --- a/engram/src/server.el +++ b/engram/src/server.el @@ -288,6 +288,27 @@ fn route_create_node(method: String, path: String, body: String) -> String { salience, importance, confidence, tier, tags ) + // GEOMETRY INGEST (2026-08-16 self-review): this route accepted an "emb" + // field, returned 200 with a fresh id, and stored NOTHING — engram_node_full + // has no vector parameter, so the caller's geometry was silently discarded + // and the node came back emb_dim=None / embedded:false. Measured live while + // trying to admit a voice signal. The consequence was structural, not + // cosmetic: text was the only entry medium, so any non-text modality had to + // be DESCRIBED in prose and what we then reasoned over was the geometry of + // the description, not of the signal. + // + // "emb" is little-endian float32 hex (dim*8 chars) — the encoding the + // perception vessel's /voice/embed already emits, so a realizer's output + // moves in with no float-array round trip. "dim" defaults to the vector's + // implied width. Off-dimension vectors are stored but not inserted into the + // resident index (its build loop filters on emb_dim), so a modality vector + // is durable and addressable without perturbing the canonical index. + let emb_hex: String = json_get_string(body, "emb") + let emb_set: Int = if str_eq(emb_hex, "") { 0 } else { + let dim_raw: String = json_get_raw(body, "dim") + let dim: Int = if str_eq(dim_raw, "") { str_len(emb_hex) / 8 } else { json_get_int(body, "dim") } + engram_node_set_emb(id, emb_hex, dim) + } let saved: Int = persist_node(id) // ORPHAN PREVENTION (ENGRAM_AUTOCONNECT): connect the fresh node to its // nearest embedded neighbors so it never enters the graph edgeless. @@ -298,7 +319,11 @@ fn route_create_node(method: String, path: String, body: String) -> String { if added > 0 { let sv2: Int = persist_edges_since(ec0) } added } else { 0 } - "{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\",\"connected\":" + int_to_str(connected) + "}" + // Report whether the supplied geometry actually landed. The old response + // was success-shaped regardless — 200 with an id while the vector was + // discarded — which is how the drop went unnoticed. A caller can now + // assert on emb_set instead of trusting the status code. + "{\"id\":\"" + id + "\",\"content\":\"" + content + "\",\"node_type\":\"" + node_type + "\",\"connected\":" + int_to_str(connected) + ",\"emb_set\":" + int_to_str(emb_set) + "}" } fn route_get_node(method: String, path: String, body: String) -> String { diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index 8abc646..199fa90 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -8507,6 +8507,80 @@ el_val_t engram_node_count(void) { return (el_val_t)engram_get()->node_count; } +/* engram_node_set_emb — attach GEOMETRY to an existing node. + * + * WHY THIS EXISTS (2026-08-16). Until now no ingest path could carry a + * vector. engram_node / engram_node_full / engram_node_layered take text + * only, and the sole way a node acquired an embedding was + * engram_embed_backfill DERIVING one from n->content. That made text the + * mandatory entry medium: any non-text modality (audio, image, sensor) + * had to be described in prose first, and the geometry we then reasoned + * over was the geometry OF THE DESCRIPTION, not of the signal. Measured + * consequence: POST /api/nodes accepted an "emb" field, returned 200 with + * a fresh id, and stored emb_dim=None / embedded:false — the vector was + * silently discarded because no parameter existed to receive it. + * + * `hex` is little-endian float32, the encoding the perception vessel's + * /voice/embed already emits, so a realizer's output moves in without a + * JSON float-array round trip. Length must be exactly dim*8 hex chars. + * + * DIMENSION POLICY: dim need NOT equal the canonical text-embedding dim. + * A modality vector of a different width is stored and is simply not + * inserted into the resident HNSW index, whose build loop already filters + * on `n->emb_dim == dim`. So off-dimension geometry is durable and + * addressable without perturbing the canonical index. + * + * Setting emb also makes the node ineligible for embed_backfill (which + * only fills nodes with no emb), so a realizer's vector is never + * overwritten by a text-derived one. + * + * Returns 1 on success, 0 on unknown id / malformed hex / bad dim. */ +el_val_t engram_node_set_emb(el_val_t id, el_val_t hex, el_val_t dim) { + const char* sid = EL_CSTR(id); + const char* sh = EL_CSTR(hex); + int32_t d = (int32_t)(int64_t)dim; + /* Bound the allocation. No max-dim constant existed because no caller + * could supply a dim before this function; 8192 is generous for any + * realizer (canonical text embeddings are 768, MFCC voice stats 64) + * while keeping a malformed `dim` from requesting an unbounded malloc. */ + if (!sid || !*sid || !sh || d <= 0 || d > 8192) return (el_val_t)0; + + size_t need = (size_t)d * 8u; /* 4 bytes → 8 hex chars per float */ + if (strlen(sh) != need) return (el_val_t)0; + + EngramNode* n = engram_find_node(sid); + if (!n) return (el_val_t)0; + + float* v = (float*)malloc(sizeof(float) * (size_t)d); + if (!v) return (el_val_t)0; + + for (int32_t i = 0; i < d; i++) { + uint32_t w = 0; + for (int k = 0; k < 8; k++) { + char c = sh[(size_t)i * 8u + (size_t)k]; + uint32_t nib; + if (c >= '0' && c <= '9') nib = (uint32_t)(c - '0'); + else if (c >= 'a' && c <= 'f') nib = (uint32_t)(c - 'a' + 10); + else if (c >= 'A' && c <= 'F') nib = (uint32_t)(c - 'A' + 10); + else { free(v); return (el_val_t)0; } + w = (w << 4) | nib; + } + /* Hex is emitted little-endian byte order; rebuild the word. */ + uint32_t le = ((w & 0x000000FFu) << 24) | ((w & 0x0000FF00u) << 8) | + ((w & 0x00FF0000u) >> 8) | ((w & 0xFF000000u) >> 24); + float f; + memcpy(&f, &le, sizeof(f)); + v[i] = f; + } + + free(n->emb); + n->emb = v; + n->emb_dim = d; + n->updated_at = engram_now_ms(); + if (engram_store_enabled()) eg_store_put_node(n); + return (el_val_t)1; +} + /* ── Telemetry retention ──────────────────────────────────────────────────── * (2026-07-16 self-review) InternalStateEvent nodes are append-only telemetry * (heartbeat, curiosity_scan, engram_sync) written ~3/min by the awareness @@ -11302,6 +11376,15 @@ static void engram_emit_node_json(JsonBuf* b, const EngramNode* n, int include_e snprintf(tmp, sizeof(tmp), ",\"wm_anchor\":%g", n->wm_anchor); jb_puts(b, tmp); snprintf(tmp, sizeof(tmp), ",\"base_level\":%g", engram_bll_base_level(n, engram_now_ms())); jb_puts(b, tmp); + /* GEOMETRY VISIBILITY (2026-08-16 self-review): the node document never + * said whether the node carried a vector, so a read-back could not tell + * "has geometry" from "text only". Not cosmetic — it is exactly how a + * real ingest drop and a mere reporting gap became indistinguishable, + * and I misdiagnosed one as the other for an hour. Always emit the width + * and the boolean; the vector itself stays behind include_emb since it + * is large and most callers do not want it inline. */ + snprintf(tmp, sizeof(tmp), ",\"emb_dim\":%d,\"embedded\":%s", + (int)n->emb_dim, (n->emb && n->emb_dim > 0) ? "true" : "false"); jb_puts(b, tmp); /* Base-level access history: chronological (oldest→newest) compact * string. Loaders replay it through engram_bll_record_access; absent * field = empty ring (optimized-form fallback). (2026-07-22) */ diff --git a/lang/runtime/el_runtime.h b/lang/runtime/el_runtime.h index 25ea292..10e337b 100644 --- a/lang/runtime/el_runtime.h +++ b/lang/runtime/el_runtime.h @@ -613,6 +613,11 @@ void engram_strengthen(el_val_t node_id); void engram_forget(el_val_t node_id); el_val_t engram_prune_telemetry(el_val_t older_than_ms); el_val_t engram_node_count(void); +/* Attach geometry to an existing node. `hex` is little-endian float32, + * exactly dim*8 hex chars — the encoding realizers already emit. Lets a + * non-text modality enter as geometry instead of being described in prose + * and embedded as its description. Returns 1 on success, 0 otherwise. */ +el_val_t engram_node_set_emb(el_val_t id, el_val_t hex, el_val_t dim); el_val_t engram_search(el_val_t query, el_val_t limit); el_val_t engram_scan_nodes(el_val_t limit, el_val_t offset); void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t relation);