From c2d8a07c7b083ba4dd81fbe529add6df5feb2dc5 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Sat, 15 Aug 2026 14:44:18 -0500 Subject: [PATCH] transduce: name the invisible mechanism, fix a silent-failure bug, drop a CRUD verb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ingest and transduce are complements, not synonyms: ingest is the conscious, deliberate act of pointing at a source (ingest_file/dir/url/llm/stream stay named exactly that); transduce is the automatic, invisible mechanism inside it that converts extracted surface content into geometry (renamed build_prose/build_structured -> transduce_prose/transduce_structured, the functions that actually turn raw text into a node+edge manifold). Real bug found and fixed along the way: the final /api/load-merge response was never checked for an error. A total failure (bad auth, network down, anything) silently reported nodes_added:0/edges_added:0 — indistinguishable from a benign 'everything was already known' outcome. Verified live: with a wrong key, the tool now honestly returns {"error":"load-merge failed: unauthorized",...} instead of a misleading zero. Also dropped a CRUD-verb smell: the per-decision println said CREATE (a database-log verb for something that hasn't actually been written to the server yet — it's a local, tentative decision pending the batch merge). Renamed to FORM. The dead-code eg_create_node (defined, never called) renamed to eg_crystallize_node and annotated honestly as unused, since if it's ever wired up it represents the real server-confirmed write, unlike the local FORM guess. Not yet re-verified end-to-end against a real successful write: the ingest-test sandbox (nsbx up ingest-test) is itself currently broken — it prints a green "ready" banner after its own readiness check fails, and nothing is actually listening. Filed separately; not in scope here. --- ingest/src/ingest.el | 48 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/ingest/src/ingest.el b/ingest/src/ingest.el index 562be86..42dfcbb 100644 --- a/ingest/src/ingest.el +++ b/ingest/src/ingest.el @@ -2,9 +2,13 @@ // // The source-polymorphic ingest(source) primitive: point it at a directory, // file, url, llm-query, structured-primitive set, or stream; it EXTRACTS the -// real content faithfully (no invention), TRANSFORMS it into a DISCRETE +// real content faithfully (no invention), TRANSDUCES it into a DISCRETE // MANIFOLD (multiple nodes + internal edges — meaning-structure, never a -// single blob), and MERGES that manifold into the engram geometry: shared +// single blob; the conversion from extracted surface content into geometry +// is automatic and invisible to the caller, the way digestion is invisible +// to the one who chose to eat — ingest is the conscious act, transduce is +// the mechanism underneath it, and it is no less real for being unseen), +// and MERGES that manifold into the engram geometry: shared // meanings DEDUP onto existing nodes (search + exact/cosine match), genuinely // new meanings add nodes, relations add edges. Every node enters with // PROVENANCE + grounding-level + stewardship class from the moment of entry. @@ -128,8 +132,11 @@ fn eg_get(path: String) -> String { return http_get(eg_base() + path) } -// create a node with full provenance-bearing metadata; returns the new node id -fn eg_create_node(content: String, ntype: String, tier: String, +// crystallize a node with full provenance-bearing metadata (server-confirmed +// write — unlike the local FORM decision in merge_manifold, this is real); +// returns the new node id. Not currently called by any live path (dead code, +// kept for a future single-node ad-hoc write use case) — 2026-08-15. +fn eg_crystallize_node(content: String, ntype: String, tier: String, sal: String, imp: String, conf: String, tags: String) -> String { let inner: String = "\"content\":" + j_q(content) + @@ -308,7 +315,7 @@ fn merge_manifold(nodes: [String], edges: [String]) -> String { snap_nodes = snap_nodes + sep + njson sn_count = sn_count + 1 created = created + 1 - println(" CREATE " + real + " :: " + head80(content)) + println(" FORM " + real + " :: " + head80(content)) } } lids = el_list_append(lids, lid) @@ -351,6 +358,25 @@ fn merge_manifold(nodes: [String], edges: [String]) -> String { let path: String = ingest_snap_path() fs_write(path, snap) let resp: String = eg_post("/api/load-merge", "\"path\":" + j_q(path)) + + // HONESTY GATE: the local FORM/DEDUP/EDGE decisions above are real (they + // describe what this manifold contains), but they are NOT confirmation of + // a server write — only this response is. If the server returned an error + // (bad auth, network failure, anything), nodes_added/edges_added silently + // default to 0 via json_get_int, which reads identically to "everything + // was already known" — a real failure and a benign no-op must never look + // the same. Surface the distinction explicitly rather than let a caller + // (or a human) infer success from a quiet zero. + let srv_err: String = json_get_string(resp, "error") + if !str_eq(srv_err, "") { + return "{\"error\":" + j_q("load-merge failed: " + srv_err) + + ",\"nodes_formed_locally\":" + int_to_str(created) + + ",\"nodes_deduped_locally\":" + int_to_str(deduped) + + ",\"manifold_nodes\":" + int_to_str(nn) + + ",\"manifold_edges\":" + int_to_str(ne) + + ",\"note\":" + j_q("nothing below this manifold was confirmed persisted by the server") + "}" + } + let nadd: Int = json_get_int(resp, "nodes_added") let eadd: Int = json_get_int(resp, "edges_added") @@ -381,7 +407,7 @@ fn head80(s: String) -> String { // Edges: doc-root -contains-> chunk; chunk -precedes-> next chunk; // most-recent-heading -section_of-> chunk. Content is verbatim (substring of // the source) — pure extraction of ground truth. -fn build_prose(nodes: [String], edges: [String], text: String, +fn transduce_prose(nodes: [String], edges: [String], text: String, prov: String, ground: String, steward: String, root_lid: String, root_title: String) -> [String] { // returns [nodes_json_list_encoded, edges_json_list_encoded] is awkward in @@ -450,7 +476,7 @@ fn build_prose(nodes: [String], edges: [String], text: String, // connective geometry, meaning saturates); numeric attributes fold into the // primitive's content (unique values, no dedup benefit). This is knowledge // represented as geometry, not prose — the path speech/music/image ingest on. -fn build_structured(nodes: [String], edges: [String], js: String, +fn transduce_structured(nodes: [String], edges: [String], js: String, prov: String, ground: String, steward: String, root_lid: String) -> [String] { // grounding integrity: the SOURCE may declare its own epistemic grounding @@ -600,11 +626,11 @@ fn ingest_file(path: String) -> String { } let prov: String = "file:" + path if ends_with_ci(path, ".json") { - let packed: [String] = build_structured(el_list_empty(), el_list_empty(), + let packed: [String] = transduce_structured(el_list_empty(), el_list_empty(), text, prov, default_ground(), default_steward(), "ds:" + basename(path)) return merge_packed(packed) } - let packed: [String] = build_prose(el_list_empty(), el_list_empty(), + let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(), text, prov, default_ground(), default_steward(), "doc:" + basename(path), basename(path)) return merge_packed(packed) @@ -645,7 +671,7 @@ fn ingest_dir(path: String) -> String { fn ingest_url(url: String) -> String { let body: String = http_get(url) if str_eq(body, "") { return "{\"error\":\"empty fetch\",\"url\":" + j_q(url) + "}" } - let packed: [String] = build_prose(el_list_empty(), el_list_empty(), + let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(), body, "url:" + url, "extracted", "public-web", "url:" + url, url) return merge_packed(packed) @@ -660,7 +686,7 @@ fn ingest_llm(query: String) -> String { let resp: String = http_post_json("http://127.0.0.1:11434/api/generate", body) let answer: String = json_get_string(resp, "response") if str_eq(answer, "") { return "{\"error\":\"no model response\"}" } - let packed: [String] = build_prose(el_list_empty(), el_list_empty(), + let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(), answer, "llm:" + model + ":" + query, "candidate-provisional", "guide-provisional", "llm:" + query, "guide answer: " + query) return merge_packed(packed)