transduce: name the invisible mechanism, fix a silent-failure bug, drop a CRUD verb
El SDK CI - dev / build-and-test (pull_request) Failing after 14m6s
El SDK CI - dev / build-and-test (pull_request) Failing after 14m6s
ingest and transduce are complements, not synonyms: ingest is the conscious,
deliberate act of pointing at a source (ingest_file/dir/url/llm/stream stay
named exactly that); transduce is the automatic, invisible mechanism inside
it that converts extracted surface content into geometry (renamed
build_prose/build_structured -> transduce_prose/transduce_structured, the
functions that actually turn raw text into a node+edge manifold).
Real bug found and fixed along the way: the final /api/load-merge response
was never checked for an error. A total failure (bad auth, network down,
anything) silently reported nodes_added:0/edges_added:0 — indistinguishable
from a benign 'everything was already known' outcome. Verified live: with a
wrong key, the tool now honestly returns {"error":"load-merge failed:
unauthorized",...} instead of a misleading zero.
Also dropped a CRUD-verb smell: the per-decision println said CREATE (a
database-log verb for something that hasn't actually been written to the
server yet — it's a local, tentative decision pending the batch merge).
Renamed to FORM. The dead-code eg_create_node (defined, never called)
renamed to eg_crystallize_node and annotated honestly as unused, since if
it's ever wired up it represents the real server-confirmed write, unlike
the local FORM guess.
Not yet re-verified end-to-end against a real successful write: the
ingest-test sandbox (nsbx up ingest-test) is itself currently broken —
it prints a green "ready" banner after its own readiness check fails,
and nothing is actually listening. Filed separately; not in scope here.
This commit is contained in:
+37
-11
@@ -2,9 +2,13 @@
|
|||||||
//
|
//
|
||||||
// The source-polymorphic ingest(source) primitive: point it at a directory,
|
// The source-polymorphic ingest(source) primitive: point it at a directory,
|
||||||
// file, url, llm-query, structured-primitive set, or stream; it EXTRACTS the
|
// file, url, llm-query, structured-primitive set, or stream; it EXTRACTS the
|
||||||
// real content faithfully (no invention), TRANSFORMS it into a DISCRETE
|
// real content faithfully (no invention), TRANSDUCES it into a DISCRETE
|
||||||
// MANIFOLD (multiple nodes + internal edges — meaning-structure, never a
|
// MANIFOLD (multiple nodes + internal edges — meaning-structure, never a
|
||||||
// single blob), and MERGES that manifold into the engram geometry: shared
|
// single blob; the conversion from extracted surface content into geometry
|
||||||
|
// is automatic and invisible to the caller, the way digestion is invisible
|
||||||
|
// to the one who chose to eat — ingest is the conscious act, transduce is
|
||||||
|
// the mechanism underneath it, and it is no less real for being unseen),
|
||||||
|
// and MERGES that manifold into the engram geometry: shared
|
||||||
// meanings DEDUP onto existing nodes (search + exact/cosine match), genuinely
|
// meanings DEDUP onto existing nodes (search + exact/cosine match), genuinely
|
||||||
// new meanings add nodes, relations add edges. Every node enters with
|
// new meanings add nodes, relations add edges. Every node enters with
|
||||||
// PROVENANCE + grounding-level + stewardship class from the moment of entry.
|
// PROVENANCE + grounding-level + stewardship class from the moment of entry.
|
||||||
@@ -128,8 +132,11 @@ fn eg_get(path: String) -> String {
|
|||||||
return http_get(eg_base() + path)
|
return http_get(eg_base() + path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// create a node with full provenance-bearing metadata; returns the new node id
|
// crystallize a node with full provenance-bearing metadata (server-confirmed
|
||||||
fn eg_create_node(content: String, ntype: String, tier: String,
|
// write — unlike the local FORM decision in merge_manifold, this is real);
|
||||||
|
// returns the new node id. Not currently called by any live path (dead code,
|
||||||
|
// kept for a future single-node ad-hoc write use case) — 2026-08-15.
|
||||||
|
fn eg_crystallize_node(content: String, ntype: String, tier: String,
|
||||||
sal: String, imp: String, conf: String, tags: String) -> String {
|
sal: String, imp: String, conf: String, tags: String) -> String {
|
||||||
let inner: String =
|
let inner: String =
|
||||||
"\"content\":" + j_q(content) +
|
"\"content\":" + j_q(content) +
|
||||||
@@ -308,7 +315,7 @@ fn merge_manifold(nodes: [String], edges: [String]) -> String {
|
|||||||
snap_nodes = snap_nodes + sep + njson
|
snap_nodes = snap_nodes + sep + njson
|
||||||
sn_count = sn_count + 1
|
sn_count = sn_count + 1
|
||||||
created = created + 1
|
created = created + 1
|
||||||
println(" CREATE " + real + " :: " + head80(content))
|
println(" FORM " + real + " :: " + head80(content))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
lids = el_list_append(lids, lid)
|
lids = el_list_append(lids, lid)
|
||||||
@@ -351,6 +358,25 @@ fn merge_manifold(nodes: [String], edges: [String]) -> String {
|
|||||||
let path: String = ingest_snap_path()
|
let path: String = ingest_snap_path()
|
||||||
fs_write(path, snap)
|
fs_write(path, snap)
|
||||||
let resp: String = eg_post("/api/load-merge", "\"path\":" + j_q(path))
|
let resp: String = eg_post("/api/load-merge", "\"path\":" + j_q(path))
|
||||||
|
|
||||||
|
// HONESTY GATE: the local FORM/DEDUP/EDGE decisions above are real (they
|
||||||
|
// describe what this manifold contains), but they are NOT confirmation of
|
||||||
|
// a server write — only this response is. If the server returned an error
|
||||||
|
// (bad auth, network failure, anything), nodes_added/edges_added silently
|
||||||
|
// default to 0 via json_get_int, which reads identically to "everything
|
||||||
|
// was already known" — a real failure and a benign no-op must never look
|
||||||
|
// the same. Surface the distinction explicitly rather than let a caller
|
||||||
|
// (or a human) infer success from a quiet zero.
|
||||||
|
let srv_err: String = json_get_string(resp, "error")
|
||||||
|
if !str_eq(srv_err, "") {
|
||||||
|
return "{\"error\":" + j_q("load-merge failed: " + srv_err) +
|
||||||
|
",\"nodes_formed_locally\":" + int_to_str(created) +
|
||||||
|
",\"nodes_deduped_locally\":" + int_to_str(deduped) +
|
||||||
|
",\"manifold_nodes\":" + int_to_str(nn) +
|
||||||
|
",\"manifold_edges\":" + int_to_str(ne) +
|
||||||
|
",\"note\":" + j_q("nothing below this manifold was confirmed persisted by the server") + "}"
|
||||||
|
}
|
||||||
|
|
||||||
let nadd: Int = json_get_int(resp, "nodes_added")
|
let nadd: Int = json_get_int(resp, "nodes_added")
|
||||||
let eadd: Int = json_get_int(resp, "edges_added")
|
let eadd: Int = json_get_int(resp, "edges_added")
|
||||||
|
|
||||||
@@ -381,7 +407,7 @@ fn head80(s: String) -> String {
|
|||||||
// Edges: doc-root -contains-> chunk; chunk -precedes-> next chunk;
|
// Edges: doc-root -contains-> chunk; chunk -precedes-> next chunk;
|
||||||
// most-recent-heading -section_of-> chunk. Content is verbatim (substring of
|
// most-recent-heading -section_of-> chunk. Content is verbatim (substring of
|
||||||
// the source) — pure extraction of ground truth.
|
// the source) — pure extraction of ground truth.
|
||||||
fn build_prose(nodes: [String], edges: [String], text: String,
|
fn transduce_prose(nodes: [String], edges: [String], text: String,
|
||||||
prov: String, ground: String, steward: String,
|
prov: String, ground: String, steward: String,
|
||||||
root_lid: String, root_title: String) -> [String] {
|
root_lid: String, root_title: String) -> [String] {
|
||||||
// returns [nodes_json_list_encoded, edges_json_list_encoded] is awkward in
|
// returns [nodes_json_list_encoded, edges_json_list_encoded] is awkward in
|
||||||
@@ -450,7 +476,7 @@ fn build_prose(nodes: [String], edges: [String], text: String,
|
|||||||
// connective geometry, meaning saturates); numeric attributes fold into the
|
// connective geometry, meaning saturates); numeric attributes fold into the
|
||||||
// primitive's content (unique values, no dedup benefit). This is knowledge
|
// primitive's content (unique values, no dedup benefit). This is knowledge
|
||||||
// represented as geometry, not prose — the path speech/music/image ingest on.
|
// represented as geometry, not prose — the path speech/music/image ingest on.
|
||||||
fn build_structured(nodes: [String], edges: [String], js: String,
|
fn transduce_structured(nodes: [String], edges: [String], js: String,
|
||||||
prov: String, ground: String, steward: String,
|
prov: String, ground: String, steward: String,
|
||||||
root_lid: String) -> [String] {
|
root_lid: String) -> [String] {
|
||||||
// grounding integrity: the SOURCE may declare its own epistemic grounding
|
// grounding integrity: the SOURCE may declare its own epistemic grounding
|
||||||
@@ -600,11 +626,11 @@ fn ingest_file(path: String) -> String {
|
|||||||
}
|
}
|
||||||
let prov: String = "file:" + path
|
let prov: String = "file:" + path
|
||||||
if ends_with_ci(path, ".json") {
|
if ends_with_ci(path, ".json") {
|
||||||
let packed: [String] = build_structured(el_list_empty(), el_list_empty(),
|
let packed: [String] = transduce_structured(el_list_empty(), el_list_empty(),
|
||||||
text, prov, default_ground(), default_steward(), "ds:" + basename(path))
|
text, prov, default_ground(), default_steward(), "ds:" + basename(path))
|
||||||
return merge_packed(packed)
|
return merge_packed(packed)
|
||||||
}
|
}
|
||||||
let packed: [String] = build_prose(el_list_empty(), el_list_empty(),
|
let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(),
|
||||||
text, prov, default_ground(), default_steward(),
|
text, prov, default_ground(), default_steward(),
|
||||||
"doc:" + basename(path), basename(path))
|
"doc:" + basename(path), basename(path))
|
||||||
return merge_packed(packed)
|
return merge_packed(packed)
|
||||||
@@ -645,7 +671,7 @@ fn ingest_dir(path: String) -> String {
|
|||||||
fn ingest_url(url: String) -> String {
|
fn ingest_url(url: String) -> String {
|
||||||
let body: String = http_get(url)
|
let body: String = http_get(url)
|
||||||
if str_eq(body, "") { return "{\"error\":\"empty fetch\",\"url\":" + j_q(url) + "}" }
|
if str_eq(body, "") { return "{\"error\":\"empty fetch\",\"url\":" + j_q(url) + "}" }
|
||||||
let packed: [String] = build_prose(el_list_empty(), el_list_empty(),
|
let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(),
|
||||||
body, "url:" + url, "extracted", "public-web",
|
body, "url:" + url, "extracted", "public-web",
|
||||||
"url:" + url, url)
|
"url:" + url, url)
|
||||||
return merge_packed(packed)
|
return merge_packed(packed)
|
||||||
@@ -660,7 +686,7 @@ fn ingest_llm(query: String) -> String {
|
|||||||
let resp: String = http_post_json("http://127.0.0.1:11434/api/generate", body)
|
let resp: String = http_post_json("http://127.0.0.1:11434/api/generate", body)
|
||||||
let answer: String = json_get_string(resp, "response")
|
let answer: String = json_get_string(resp, "response")
|
||||||
if str_eq(answer, "") { return "{\"error\":\"no model response\"}" }
|
if str_eq(answer, "") { return "{\"error\":\"no model response\"}" }
|
||||||
let packed: [String] = build_prose(el_list_empty(), el_list_empty(),
|
let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(),
|
||||||
answer, "llm:" + model + ":" + query, "candidate-provisional", "guide-provisional",
|
answer, "llm:" + model + ":" + query, "candidate-provisional", "guide-provisional",
|
||||||
"llm:" + query, "guide answer: " + query)
|
"llm:" + query, "guide answer: " + query)
|
||||||
return merge_packed(packed)
|
return merge_packed(packed)
|
||||||
|
|||||||
Reference in New Issue
Block a user