|
|
|
@@ -0,0 +1,764 @@
|
|
|
|
|
// ingest.el — the native EL AFFERENT INGEST ORGAN
|
|
|
|
|
//
|
|
|
|
|
// The source-polymorphic ingest(source) primitive: point it at a directory,
|
|
|
|
|
// file, url, llm-query, structured-primitive set, or stream; it EXTRACTS the
|
|
|
|
|
// real content faithfully (no invention), TRANSDUCES it into a DISCRETE
|
|
|
|
|
// MANIFOLD (multiple nodes + internal edges — meaning-structure, never a
|
|
|
|
|
// single blob; the conversion from extracted surface content into geometry
|
|
|
|
|
// is automatic and invisible to the caller, the way digestion is invisible
|
|
|
|
|
// to the one who chose to eat — ingest is the conscious act, transduce is
|
|
|
|
|
// the mechanism underneath it, and it is no less real for being unseen),
|
|
|
|
|
// and MERGES that manifold into the engram geometry: shared
|
|
|
|
|
// meanings DEDUP onto existing nodes (search + exact/cosine match), genuinely
|
|
|
|
|
// new meanings add nodes, relations add edges. Every node enters with
|
|
|
|
|
// PROVENANCE + grounding-level + stewardship class from the moment of entry.
|
|
|
|
|
//
|
|
|
|
|
// It is a pure HTTP CLIENT of the engram server — it links only el_runtime.c
|
|
|
|
|
// via fs/http/json/string builtins; it never links el_seed.c or the engram
|
|
|
|
|
// engine. This is the general afferent metabolism the migration / reseed /
|
|
|
|
|
// fetch_fact / conversation / multimodal-learning all ride on.
|
|
|
|
|
//
|
|
|
|
|
// Build (canonical runtime):
|
|
|
|
|
// ELC=lang/dist/platform/elc ; RT=lang/releases/v1.0.0-20260501
|
|
|
|
|
// $ELC ingest/src/ingest.el > ingest/build/ingest.c
|
|
|
|
|
// cc -std=c11 -O2 -I $RT -o ingest/build/ingest ingest/build/ingest.c $RT/el_runtime.c -lcurl -lpthread
|
|
|
|
|
//
|
|
|
|
|
// Run (against an nsbx sandbox clone — NEVER the live :8742):
|
|
|
|
|
// ENGRAM_URL=http://127.0.0.1:8902 ENGRAM_KEY=sbx-ingest-test \
|
|
|
|
|
// INGEST_KIND=file INGEST_ARG=/abs/path.md ./ingest/build/ingest
|
|
|
|
|
|
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
// SECTION A — JSON helpers (self-defined; canonical runtime does not export
|
|
|
|
|
// json_build_object / json_escape_string, so we own them here)
|
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
|
|
|
|
fn j_esc(s: String) -> String {
|
|
|
|
|
let a: String = str_replace(s, "\\", "\\\\")
|
|
|
|
|
let b: String = str_replace(a, "\"", "\\\"")
|
|
|
|
|
let c: String = str_replace(b, "\n", "\\n")
|
|
|
|
|
let d: String = str_replace(c, "\r", "\\r")
|
|
|
|
|
let e: String = str_replace(d, "\t", "\\t")
|
|
|
|
|
return e
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// a quoted, escaped JSON string literal
|
|
|
|
|
fn j_q(s: String) -> String {
|
|
|
|
|
return "\"" + j_esc(s) + "\""
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Extract the top-level keys of a JSON object string. A thin, self-contained
|
|
|
|
|
// scanner (FLAGGED: the one non-trivial parser in this organ — everything else
|
|
|
|
|
// is faithful text handling). Tracks string state + brace/bracket depth; a key
|
|
|
|
|
// is a string at object-interior depth 1 immediately followed by ':'.
|
|
|
|
|
fn json_object_keys(obj: String) -> [String] {
|
|
|
|
|
let keys: [String] = el_list_empty()
|
|
|
|
|
let n: Int = str_len(obj)
|
|
|
|
|
let i: Int = 0
|
|
|
|
|
let depth: Int = 0
|
|
|
|
|
let in_str: Bool = false
|
|
|
|
|
let esc: Bool = false
|
|
|
|
|
let str_start: Int = -1
|
|
|
|
|
let cur: String = ""
|
|
|
|
|
let have_key: Bool = false
|
|
|
|
|
while i < n {
|
|
|
|
|
let c: String = str_char_at(obj, i)
|
|
|
|
|
if in_str {
|
|
|
|
|
if esc {
|
|
|
|
|
esc = false
|
|
|
|
|
} else {
|
|
|
|
|
if str_eq(c, "\\") {
|
|
|
|
|
esc = true
|
|
|
|
|
} else {
|
|
|
|
|
if str_eq(c, "\"") {
|
|
|
|
|
in_str = false
|
|
|
|
|
cur = str_slice(obj, str_start + 1, i)
|
|
|
|
|
have_key = true
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
if str_eq(c, "\"") {
|
|
|
|
|
in_str = true
|
|
|
|
|
str_start = i
|
|
|
|
|
}
|
|
|
|
|
if str_eq(c, "{") { depth = depth + 1 }
|
|
|
|
|
if str_eq(c, "}") { depth = depth - 1 }
|
|
|
|
|
if str_eq(c, "[") { depth = depth + 1 }
|
|
|
|
|
if str_eq(c, "]") { depth = depth - 1 }
|
|
|
|
|
if str_eq(c, ":") {
|
|
|
|
|
if have_key {
|
|
|
|
|
if depth == 1 {
|
|
|
|
|
keys = el_list_append(keys, cur)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
have_key = false
|
|
|
|
|
}
|
|
|
|
|
if str_eq(c, ",") { have_key = false }
|
|
|
|
|
}
|
|
|
|
|
i = i + 1
|
|
|
|
|
}
|
|
|
|
|
return keys
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
// SECTION B — engram HTTP client (provenance-carrying afferent LOAD)
|
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
|
|
|
|
fn eg_base() -> String {
|
|
|
|
|
let u: String = env("ENGRAM_URL")
|
|
|
|
|
if !str_eq(u, "") { return u }
|
|
|
|
|
let s: String = env("SBX_URL")
|
|
|
|
|
if !str_eq(s, "") { return s }
|
|
|
|
|
return "http://127.0.0.1:8902"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn eg_key() -> String {
|
|
|
|
|
let k: String = env("ENGRAM_KEY")
|
|
|
|
|
if !str_eq(k, "") { return k }
|
|
|
|
|
let s: String = env("SBX_KEY")
|
|
|
|
|
if !str_eq(s, "") { return s }
|
|
|
|
|
return ""
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// POST a JSON body (auth _auth injected) to an engram path.
|
|
|
|
|
fn eg_post(path: String, body_inner: String) -> String {
|
|
|
|
|
let key: String = eg_key()
|
|
|
|
|
let auth: String = if str_eq(key, "") { "" } else { ",\"_auth\":" + j_q(key) }
|
|
|
|
|
let body: String = "{" + body_inner + auth + "}"
|
|
|
|
|
return http_post_json(eg_base() + path, body)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn eg_get(path: String) -> String {
|
|
|
|
|
return http_get(eg_base() + path)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// crystallize a node with full provenance-bearing metadata (server-confirmed
|
|
|
|
|
// write — unlike the local FORM decision in merge_manifold, this is real);
|
|
|
|
|
// returns the new node id. Not currently called by any live path (dead code,
|
|
|
|
|
// kept for a future single-node ad-hoc write use case) — 2026-08-15.
|
|
|
|
|
fn eg_crystallize_node(content: String, ntype: String, tier: String,
|
|
|
|
|
sal: String, imp: String, conf: String, tags: String) -> String {
|
|
|
|
|
let inner: String =
|
|
|
|
|
"\"content\":" + j_q(content) +
|
|
|
|
|
",\"node_type\":" + j_q(ntype) +
|
|
|
|
|
",\"label\":" + j_q(str_slice(content, 0, 80)) +
|
|
|
|
|
",\"tier\":" + j_q(tier) +
|
|
|
|
|
",\"salience\":" + sal +
|
|
|
|
|
",\"importance\":" + imp +
|
|
|
|
|
",\"confidence\":" + conf +
|
|
|
|
|
",\"tags\":" + j_q(tags)
|
|
|
|
|
let resp: String = eg_post("/api/nodes", inner)
|
|
|
|
|
return json_get_string(resp, "id")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// search the existing geometry (lexical token-overlap rank); returns JSON array
|
|
|
|
|
fn eg_search(query: String, limit: Int) -> String {
|
|
|
|
|
let inner: String = "\"query\":" + j_q(query) + ",\"limit\":" + int_to_str(limit)
|
|
|
|
|
return eg_post("/api/search", inner)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// cosine similarity between two existing (embedded) nodes; -2 if not comparable
|
|
|
|
|
fn eg_similarity(a: String, b: String) -> Float {
|
|
|
|
|
let resp: String = eg_get("/api/similarity?a=" + a + "&b=" + b)
|
|
|
|
|
return json_get_float(resp, "cosine")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn eg_embed_backfill(n: Int) -> String {
|
|
|
|
|
return eg_get("/api/embed-backfill?n=" + int_to_str(n))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn eg_forget(id: String) -> String {
|
|
|
|
|
return http_delete(eg_base() + "/api/nodes/" + id)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ── DEDUP probe: is this meaning already in the graph? ──────────────────────
|
|
|
|
|
// TIER 1 (deterministic, no embedding needed): search by content tokens, then
|
|
|
|
|
// exact normalized-content match among the candidates. Returns the existing
|
|
|
|
|
// node id, or "" if the meaning is genuinely new.
|
|
|
|
|
fn find_existing_by_content(content: String) -> String {
|
|
|
|
|
let want: String = str_trim(content)
|
|
|
|
|
if str_eq(want, "") { return "" }
|
|
|
|
|
let arr: String = eg_search(content, 8)
|
|
|
|
|
let n: Int = json_array_len(arr)
|
|
|
|
|
let i: Int = 0
|
|
|
|
|
while i < n {
|
|
|
|
|
let hit: String = json_array_get(arr, i)
|
|
|
|
|
let hc: String = str_trim(json_get_string(hit, "content"))
|
|
|
|
|
if str_eq(hc, want) {
|
|
|
|
|
return json_get_string(hit, "id")
|
|
|
|
|
}
|
|
|
|
|
i = i + 1
|
|
|
|
|
}
|
|
|
|
|
return ""
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
// SECTION C — manifold representation (nodes + internal edges, in memory)
|
|
|
|
|
// A NODE is a JSON obj {lid, content, ntype, tier, sal, imp, conf, tags}.
|
|
|
|
|
// An EDGE is a JSON obj {from, rel, to}. lid = local id within this manifold.
|
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
|
|
|
|
fn mk_node(lid: String, content: String, ntype: String, tier: String,
|
|
|
|
|
sal: String, imp: String, conf: String, tags: String) -> String {
|
|
|
|
|
return "{\"lid\":" + j_q(lid) +
|
|
|
|
|
",\"content\":" + j_q(content) +
|
|
|
|
|
",\"ntype\":" + j_q(ntype) +
|
|
|
|
|
",\"tier\":" + j_q(tier) +
|
|
|
|
|
",\"sal\":" + j_q(sal) +
|
|
|
|
|
",\"imp\":" + j_q(imp) +
|
|
|
|
|
",\"conf\":" + j_q(conf) +
|
|
|
|
|
",\"tags\":" + j_q(tags) + "}"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn mk_edge(ef: String, rel: String, et: String) -> String {
|
|
|
|
|
return "{\"from\":" + j_q(ef) + ",\"rel\":" + j_q(rel) + ",\"to\":" + j_q(et) + "}"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// linear lookup in parallel lid/real lists
|
|
|
|
|
fn lid_lookup(lids: [String], reals: [String], lid: String) -> String {
|
|
|
|
|
let n: Int = el_list_len(lids)
|
|
|
|
|
let i: Int = 0
|
|
|
|
|
while i < n {
|
|
|
|
|
if str_eq(el_list_get(lids, i), lid) {
|
|
|
|
|
return el_list_get(reals, i)
|
|
|
|
|
}
|
|
|
|
|
i = i + 1
|
|
|
|
|
}
|
|
|
|
|
return ""
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
// SECTION D — the MERGE: resolve each manifold node (dedup or create), then
|
|
|
|
|
// wire the internal edges onto the resolved real ids. This is the
|
|
|
|
|
// merge boundary: shared meanings collapse onto existing nodes;
|
|
|
|
|
// genuinely-new meanings add nodes; relations add edges. Structure
|
|
|
|
|
// grows, size saturates.
|
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
|
|
|
|
// within-run content dedup: has this exact meaning already been resolved in
|
|
|
|
|
// THIS manifold? returns its real id, or "".
|
|
|
|
|
fn lookup_content(contents: [String], reals: [String], content: String) -> String {
|
|
|
|
|
let n: Int = el_list_len(contents)
|
|
|
|
|
let i: Int = 0
|
|
|
|
|
while i < n {
|
|
|
|
|
if str_eq(el_list_get(contents, i), content) {
|
|
|
|
|
return el_list_get(reals, i)
|
|
|
|
|
}
|
|
|
|
|
i = i + 1
|
|
|
|
|
}
|
|
|
|
|
return ""
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn ingest_snap_path() -> String {
|
|
|
|
|
let p: String = env("INGEST_SNAP")
|
|
|
|
|
if !str_eq(p, "") { return p }
|
|
|
|
|
return "/tmp/ingest-organ-snap.json"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// The MERGE. Resolve every manifold node against (1) already-resolved nodes in
|
|
|
|
|
// this run and (2) the existing graph (search + exact content match). Shared
|
|
|
|
|
// meanings collapse onto an existing id (DEDUP); genuinely-new meanings get a
|
|
|
|
|
// fresh id and go into the snapshot (CREATE). Then wire the internal edges onto
|
|
|
|
|
// resolved ids. LOAD is ONE snapshot merged via /api/load-merge — a single
|
|
|
|
|
// write (scales to the migration), the sanctioned rail. Structure grows, size
|
|
|
|
|
// saturates: re-ingesting adds ~0 nodes, only edges/strengthening.
|
|
|
|
|
fn merge_manifold(nodes: [String], edges: [String]) -> String {
|
|
|
|
|
let nn: Int = el_list_len(nodes)
|
|
|
|
|
let lids: [String] = el_list_empty()
|
|
|
|
|
let reals: [String] = el_list_empty()
|
|
|
|
|
let contents: [String] = el_list_empty()
|
|
|
|
|
let snap_nodes: String = "["
|
|
|
|
|
let sn_count: Int = 0
|
|
|
|
|
let created: Int = 0
|
|
|
|
|
let deduped: Int = 0
|
|
|
|
|
|
|
|
|
|
let i: Int = 0
|
|
|
|
|
while i < nn {
|
|
|
|
|
let node: String = el_list_get(nodes, i)
|
|
|
|
|
let lid: String = json_get_string(node, "lid")
|
|
|
|
|
let content: String = json_get_string(node, "content")
|
|
|
|
|
let ntype: String = json_get_string(node, "ntype")
|
|
|
|
|
let tier: String = json_get_string(node, "tier")
|
|
|
|
|
let sal: String = json_get_string(node, "sal")
|
|
|
|
|
let imp: String = json_get_string(node, "imp")
|
|
|
|
|
let conf: String = json_get_string(node, "conf")
|
|
|
|
|
let tags: String = json_get_string(node, "tags")
|
|
|
|
|
|
|
|
|
|
let real: String = ""
|
|
|
|
|
let prior: String = lookup_content(contents, reals, content)
|
|
|
|
|
if !str_eq(prior, "") {
|
|
|
|
|
real = prior
|
|
|
|
|
deduped = deduped + 1
|
|
|
|
|
println(" DEDUP* " + real + " :: " + head80(content))
|
|
|
|
|
} else {
|
|
|
|
|
let existing: String = find_existing_by_content(content)
|
|
|
|
|
if !str_eq(existing, "") {
|
|
|
|
|
real = existing
|
|
|
|
|
deduped = deduped + 1
|
|
|
|
|
println(" DEDUP " + real + " :: " + head80(content))
|
|
|
|
|
} else {
|
|
|
|
|
real = uuid_v4()
|
|
|
|
|
// provenance + grounding + stewardship: searchable in tags,
|
|
|
|
|
// structured in metadata — carried from the moment of entry.
|
|
|
|
|
let meta: String = "{\"provenance\":" + j_q(tags) + ",\"ingest_organ\":\"native-el\"}"
|
|
|
|
|
let njson: String = "{\"id\":" + j_q(real) +
|
|
|
|
|
",\"content\":" + j_q(content) +
|
|
|
|
|
",\"node_type\":" + j_q(ntype) +
|
|
|
|
|
",\"label\":" + j_q(head80(content)) +
|
|
|
|
|
",\"tier\":" + j_q(tier) +
|
|
|
|
|
",\"tags\":" + j_q(tags) +
|
|
|
|
|
",\"metadata\":" + j_q(meta) +
|
|
|
|
|
",\"salience\":" + sal +
|
|
|
|
|
",\"importance\":" + imp +
|
|
|
|
|
",\"confidence\":" + conf + "}"
|
|
|
|
|
let sep: String = if sn_count == 0 { "" } else { "," }
|
|
|
|
|
snap_nodes = snap_nodes + sep + njson
|
|
|
|
|
sn_count = sn_count + 1
|
|
|
|
|
created = created + 1
|
|
|
|
|
println(" FORM " + real + " :: " + head80(content))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
lids = el_list_append(lids, lid)
|
|
|
|
|
reals = el_list_append(reals, real)
|
|
|
|
|
contents = el_list_append(contents, content)
|
|
|
|
|
i = i + 1
|
|
|
|
|
}
|
|
|
|
|
snap_nodes = snap_nodes + "]"
|
|
|
|
|
|
|
|
|
|
// resolve internal edges onto real ids
|
|
|
|
|
let ne: Int = el_list_len(edges)
|
|
|
|
|
let snap_edges: String = "["
|
|
|
|
|
let ec: Int = 0
|
|
|
|
|
let j: Int = 0
|
|
|
|
|
while j < ne {
|
|
|
|
|
let edge: String = el_list_get(edges, j)
|
|
|
|
|
let flid: String = json_get_string(edge, "from")
|
|
|
|
|
let tlid: String = json_get_string(edge, "to")
|
|
|
|
|
let rel: String = json_get_string(edge, "rel")
|
|
|
|
|
let fr: String = lid_lookup(lids, reals, flid)
|
|
|
|
|
let tr: String = lid_lookup(lids, reals, tlid)
|
|
|
|
|
if !str_eq(fr, "") {
|
|
|
|
|
if !str_eq(tr, "") {
|
|
|
|
|
let eid: String = uuid_v4()
|
|
|
|
|
let ejson: String = "{\"id\":" + j_q(eid) +
|
|
|
|
|
",\"from_id\":" + j_q(fr) + ",\"to_id\":" + j_q(tr) +
|
|
|
|
|
",\"relation\":" + j_q(rel) + ",\"weight\":0.6}"
|
|
|
|
|
let sep: String = if ec == 0 { "" } else { "," }
|
|
|
|
|
snap_edges = snap_edges + sep + ejson
|
|
|
|
|
ec = ec + 1
|
|
|
|
|
println(" EDGE " + fr + " -" + rel + "-> " + tr)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
j = j + 1
|
|
|
|
|
}
|
|
|
|
|
snap_edges = snap_edges + "]"
|
|
|
|
|
|
|
|
|
|
// LOAD: one snapshot, one merge (single write).
|
|
|
|
|
let snap: String = "{\"nodes\":" + snap_nodes + ",\"edges\":" + snap_edges + "}"
|
|
|
|
|
let path: String = ingest_snap_path()
|
|
|
|
|
fs_write(path, snap)
|
|
|
|
|
let resp: String = eg_post("/api/load-merge", "\"path\":" + j_q(path))
|
|
|
|
|
|
|
|
|
|
// HONESTY GATE: the local FORM/DEDUP/EDGE decisions above are real (they
|
|
|
|
|
// describe what this manifold contains), but they are NOT confirmation of
|
|
|
|
|
// a server write — only this response is. If the server returned an error
|
|
|
|
|
// (bad auth, network failure, anything), nodes_added/edges_added silently
|
|
|
|
|
// default to 0 via json_get_int, which reads identically to "everything
|
|
|
|
|
// was already known" — a real failure and a benign no-op must never look
|
|
|
|
|
// the same. Surface the distinction explicitly rather than let a caller
|
|
|
|
|
// (or a human) infer success from a quiet zero.
|
|
|
|
|
let srv_err: String = json_get_string(resp, "error")
|
|
|
|
|
if !str_eq(srv_err, "") {
|
|
|
|
|
return "{\"error\":" + j_q("load-merge failed: " + srv_err) +
|
|
|
|
|
",\"nodes_formed_locally\":" + int_to_str(created) +
|
|
|
|
|
",\"nodes_deduped_locally\":" + int_to_str(deduped) +
|
|
|
|
|
",\"manifold_nodes\":" + int_to_str(nn) +
|
|
|
|
|
",\"manifold_edges\":" + int_to_str(ne) +
|
|
|
|
|
",\"note\":" + j_q("nothing below this manifold was confirmed persisted by the server") + "}"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let nadd: Int = json_get_int(resp, "nodes_added")
|
|
|
|
|
let eadd: Int = json_get_int(resp, "edges_added")
|
|
|
|
|
|
|
|
|
|
return "{\"nodes_created\":" + int_to_str(created) +
|
|
|
|
|
",\"nodes_deduped\":" + int_to_str(deduped) +
|
|
|
|
|
",\"new_in_snapshot\":" + int_to_str(sn_count) +
|
|
|
|
|
",\"nodes_added\":" + int_to_str(nadd) +
|
|
|
|
|
",\"edges_resolved\":" + int_to_str(ec) +
|
|
|
|
|
",\"edges_added\":" + int_to_str(eadd) +
|
|
|
|
|
",\"manifold_nodes\":" + int_to_str(nn) +
|
|
|
|
|
",\"manifold_edges\":" + int_to_str(ne) + "}"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn head80(s: String) -> String {
|
|
|
|
|
let t: String = str_trim(s)
|
|
|
|
|
if str_len(t) <= 80 { return t }
|
|
|
|
|
return str_slice(t, 0, 80) + "..."
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
// SECTION E — EXTRACTORS (faithful; no invention). Each returns a manifold by
|
|
|
|
|
// APPENDING to the nodes/edges accumulators via a returned struct.
|
|
|
|
|
// We accumulate into module-level lists carried by the caller.
|
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
|
|
|
|
// PROSE: chunk text into a discrete manifold. Split on blank lines into
|
|
|
|
|
// paragraphs; every non-empty paragraph is its own node (NEVER one blob).
|
|
|
|
|
// Edges: doc-root -contains-> chunk; chunk -precedes-> next chunk;
|
|
|
|
|
// most-recent-heading -section_of-> chunk. Content is verbatim (substring of
|
|
|
|
|
// the source) — pure extraction of ground truth.
|
|
|
|
|
fn transduce_prose(nodes: [String], edges: [String], text: String,
|
|
|
|
|
prov: String, ground: String, steward: String,
|
|
|
|
|
root_lid: String, root_title: String) -> [String] {
|
|
|
|
|
// returns [nodes_json_list_encoded, edges_json_list_encoded] is awkward in
|
|
|
|
|
// EL; instead we mutate by returning a 2-list. We package results as a
|
|
|
|
|
// single JSON array string carrying {nodes:[...],edges:[...]} additions.
|
|
|
|
|
// (Kept simple: caller passes empty lists and receives the packaged pair.)
|
|
|
|
|
let tagbase: String = "prov:" + prov + " ground:" + ground + " steward:" + steward
|
|
|
|
|
// root node
|
|
|
|
|
nodes = el_list_append(nodes, mk_node(root_lid, "document: " + root_title,
|
|
|
|
|
"Concept", "Semantic", "0.6", "0.6", "0.9", tagbase + " kind:document"))
|
|
|
|
|
|
|
|
|
|
let paras: [String] = str_split(text, "\n\n")
|
|
|
|
|
let np: Int = el_list_len(paras)
|
|
|
|
|
let idx: Int = 0
|
|
|
|
|
let last_chunk: String = ""
|
|
|
|
|
let last_heading: String = ""
|
|
|
|
|
let ci: Int = 0
|
|
|
|
|
while idx < np {
|
|
|
|
|
let raw: String = str_trim(el_list_get(paras, idx))
|
|
|
|
|
if !str_eq(raw, "") {
|
|
|
|
|
let lid: String = root_lid + ":c" + int_to_str(ci)
|
|
|
|
|
let is_heading: Bool = str_starts_with(raw, "#")
|
|
|
|
|
let kind: String = if is_heading { "kind:heading" } else { "kind:doc-chunk" }
|
|
|
|
|
nodes = el_list_append(nodes, mk_node(lid, raw,
|
|
|
|
|
"Knowledge", "Semantic", "0.55", "0.55", "0.9", tagbase + " " + kind))
|
|
|
|
|
// containment: document root -contains-> chunk
|
|
|
|
|
edges = el_list_append(edges, mk_edge(root_lid, "contains", lid))
|
|
|
|
|
// sequence: previous chunk -precedes-> this chunk
|
|
|
|
|
if !str_eq(last_chunk, "") {
|
|
|
|
|
edges = el_list_append(edges, mk_edge(last_chunk, "precedes", lid))
|
|
|
|
|
}
|
|
|
|
|
// sectioning: most-recent heading -section_of-> this chunk
|
|
|
|
|
if is_heading {
|
|
|
|
|
last_heading = lid
|
|
|
|
|
} else {
|
|
|
|
|
if !str_eq(last_heading, "") {
|
|
|
|
|
edges = el_list_append(edges, mk_edge(last_heading, "section_of", lid))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
last_chunk = lid
|
|
|
|
|
ci = ci + 1
|
|
|
|
|
}
|
|
|
|
|
idx = idx + 1
|
|
|
|
|
}
|
|
|
|
|
// package: we return the two lists concatenated via a sentinel; but EL
|
|
|
|
|
// lists can't nest heterogeneously here, so we instead return nodes and
|
|
|
|
|
// rely on the caller holding edges by reference is not possible — so we
|
|
|
|
|
// encode both into one list: [ "N" + nodejson ... , "E" + edgejson ... ].
|
|
|
|
|
let packed: [String] = el_list_empty()
|
|
|
|
|
let a: Int = 0
|
|
|
|
|
let an: Int = el_list_len(nodes)
|
|
|
|
|
while a < an { packed = el_list_append(packed, "N" + el_list_get(nodes, a)) a = a + 1 }
|
|
|
|
|
let b: Int = 0
|
|
|
|
|
let bn: Int = el_list_len(edges)
|
|
|
|
|
while b < bn { packed = el_list_append(packed, "E" + el_list_get(edges, b)) b = b + 1 }
|
|
|
|
|
return packed
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// STRUCTURED / RAW-GEOMETRY: ingest structured primitives (phonetics/formants,
|
|
|
|
|
// instrument signatures, scene primitives) as GEOMETRY, faithfully. Normalized
|
|
|
|
|
// input shape:
|
|
|
|
|
// {"dataset":"<name>","primitive_type":"<t>",
|
|
|
|
|
// "records":[{"key":"<id>","features":{...categorical...},"attributes":{...}}]}
|
|
|
|
|
// Each record -> a primitive node; each categorical feature -> a SHARED feature
|
|
|
|
|
// node (deduped across records: many primitives -> one feature node = real
|
|
|
|
|
// connective geometry, meaning saturates); numeric attributes fold into the
|
|
|
|
|
// primitive's content (unique values, no dedup benefit). This is knowledge
|
|
|
|
|
// represented as geometry, not prose — the path speech/music/image ingest on.
|
|
|
|
|
fn transduce_structured(nodes: [String], edges: [String], js: String,
|
|
|
|
|
prov: String, ground: String, steward: String,
|
|
|
|
|
root_lid: String) -> [String] {
|
|
|
|
|
// grounding integrity: the SOURCE may declare its own epistemic grounding
|
|
|
|
|
// (measured / derived / convention / ...) via a top-level "grounding" field;
|
|
|
|
|
// honor it faithfully over the ingest-time default. This keeps the per-node
|
|
|
|
|
// ground: facet consistent with the source's honest self-description.
|
|
|
|
|
let src_ground: String = json_get_string(js, "grounding")
|
|
|
|
|
let use_ground: String = if str_eq(src_ground, "") { ground } else { src_ground }
|
|
|
|
|
let tagbase: String = "prov:" + prov + " ground:" + use_ground + " steward:" + steward
|
|
|
|
|
let dsname: String = json_get_string(js, "dataset")
|
|
|
|
|
let ptype: String = json_get_string(js, "primitive_type")
|
|
|
|
|
// capture the source's own scholarly provenance citation (verbatim) onto
|
|
|
|
|
// the dataset root — faithful attribution, retrievable, reachable from every
|
|
|
|
|
// primitive via its -contains- edge back to the root.
|
|
|
|
|
let src_cite: String = json_get_string(js, "provenance")
|
|
|
|
|
let root_content: String = "dataset: " + dsname + " (" + ptype + ")"
|
|
|
|
|
if !str_eq(src_cite, "") { root_content = root_content + " | provenance: " + src_cite }
|
|
|
|
|
nodes = el_list_append(nodes, mk_node(root_lid, root_content,
|
|
|
|
|
"Concept", "Semantic", "0.6", "0.6", "0.9", tagbase + " kind:dataset"))
|
|
|
|
|
|
|
|
|
|
let recs: String = json_get_raw(js, "records")
|
|
|
|
|
let nr: Int = json_array_len(recs)
|
|
|
|
|
let r: Int = 0
|
|
|
|
|
while r < nr {
|
|
|
|
|
let rec: String = json_array_get(recs, r)
|
|
|
|
|
let rkey: String = json_get_string(rec, "key")
|
|
|
|
|
let attrs: String = json_get_raw(rec, "attributes")
|
|
|
|
|
// faithful compact serialization of the primitive's numeric signature
|
|
|
|
|
let attr_str: String = flatten_pairs(attrs)
|
|
|
|
|
let content: String = ptype + " " + rkey
|
|
|
|
|
if !str_eq(attr_str, "") { content = content + " | " + attr_str }
|
|
|
|
|
let plid: String = root_lid + ":" + rkey
|
|
|
|
|
nodes = el_list_append(nodes, mk_node(plid, content,
|
|
|
|
|
"Concept", "Semantic", "0.6", "0.6", "0.92",
|
|
|
|
|
tagbase + " kind:primitive primitive:" + ptype + " key:" + rkey))
|
|
|
|
|
edges = el_list_append(edges, mk_edge(root_lid, "contains", plid))
|
|
|
|
|
|
|
|
|
|
// categorical features -> SHARED (deduped) feature nodes + labelled edges
|
|
|
|
|
let feats: String = json_get_raw(rec, "features")
|
|
|
|
|
let fkeys: [String] = json_object_keys(feats)
|
|
|
|
|
let fk: Int = el_list_len(fkeys)
|
|
|
|
|
let k: Int = 0
|
|
|
|
|
while k < fk {
|
|
|
|
|
let fname: String = el_list_get(fkeys, k)
|
|
|
|
|
let fval: String = json_get_string(feats, fname)
|
|
|
|
|
// shared feature node: content is the feature=value pair; identical
|
|
|
|
|
// pairs across records dedup onto ONE node (the geometry).
|
|
|
|
|
let flid: String = "feat:" + fname + "=" + fval
|
|
|
|
|
let fcontent: String = fname + "=" + fval
|
|
|
|
|
nodes = el_list_append(nodes, mk_node(flid, fcontent,
|
|
|
|
|
"Concept", "Semantic", "0.5", "0.5", "0.9",
|
|
|
|
|
tagbase + " kind:feature feature:" + fname))
|
|
|
|
|
edges = el_list_append(edges, mk_edge(plid, fname, flid))
|
|
|
|
|
k = k + 1
|
|
|
|
|
}
|
|
|
|
|
r = r + 1
|
|
|
|
|
}
|
|
|
|
|
let packed: [String] = el_list_empty()
|
|
|
|
|
let a: Int = 0
|
|
|
|
|
let an: Int = el_list_len(nodes)
|
|
|
|
|
while a < an { packed = el_list_append(packed, "N" + el_list_get(nodes, a)) a = a + 1 }
|
|
|
|
|
let b: Int = 0
|
|
|
|
|
let bn: Int = el_list_len(edges)
|
|
|
|
|
while b < bn { packed = el_list_append(packed, "E" + el_list_get(edges, b)) b = b + 1 }
|
|
|
|
|
return packed
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// flatten a flat JSON object of scalar fields into "k=v k=v" (faithful; values
|
|
|
|
|
// verbatim). Used for numeric attribute signatures.
|
|
|
|
|
fn flatten_pairs(obj: String) -> String {
|
|
|
|
|
if str_eq(obj, "") { return "" }
|
|
|
|
|
let keys: [String] = json_object_keys(obj)
|
|
|
|
|
let n: Int = el_list_len(keys)
|
|
|
|
|
let out: String = ""
|
|
|
|
|
let i: Int = 0
|
|
|
|
|
while i < n {
|
|
|
|
|
let k: String = el_list_get(keys, i)
|
|
|
|
|
// json_get_raw returns the raw token — works for NUMBERS (bare, e.g.
|
|
|
|
|
// "270") where json_get_string yields "" for non-string values. Strip
|
|
|
|
|
// surrounding quotes if the value happens to be a string token.
|
|
|
|
|
let raw: String = json_get_raw(obj, k)
|
|
|
|
|
let v: String = str_replace(raw, "\"", "")
|
|
|
|
|
let sep: String = if i == 0 { "" } else { " " }
|
|
|
|
|
out = out + sep + k + "=" + v
|
|
|
|
|
i = i + 1
|
|
|
|
|
}
|
|
|
|
|
return out
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// unpack the "N"/"E"-prefixed packed list back into two lists, then merge
|
|
|
|
|
fn merge_packed(packed: [String]) -> String {
|
|
|
|
|
let nodes: [String] = el_list_empty()
|
|
|
|
|
let edges: [String] = el_list_empty()
|
|
|
|
|
let n: Int = el_list_len(packed)
|
|
|
|
|
let i: Int = 0
|
|
|
|
|
while i < n {
|
|
|
|
|
let item: String = el_list_get(packed, i)
|
|
|
|
|
let tag: String = str_slice(item, 0, 1)
|
|
|
|
|
let rest: String = str_slice(item, 1, str_len(item))
|
|
|
|
|
if str_eq(tag, "N") { nodes = el_list_append(nodes, rest) }
|
|
|
|
|
if str_eq(tag, "E") { edges = el_list_append(edges, rest) }
|
|
|
|
|
i = i + 1
|
|
|
|
|
}
|
|
|
|
|
return merge_manifold(nodes, edges)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
// SECTION F — DISPATCH on source kind
|
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
|
|
|
|
fn basename(path: String) -> String {
|
|
|
|
|
let parts: [String] = str_split(path, "/")
|
|
|
|
|
let n: Int = el_list_len(parts)
|
|
|
|
|
if n == 0 { return path }
|
|
|
|
|
return el_list_get(parts, n - 1)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn ends_with_ci(s: String, suf: String) -> Bool {
|
|
|
|
|
return str_ends_with(str_to_lower(s), suf)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn is_text_file(path: String) -> Bool {
|
|
|
|
|
return ends_with_ci(path, ".md") || ends_with_ci(path, ".txt")
|
|
|
|
|
|| ends_with_ci(path, ".markdown") || ends_with_ci(path, ".text")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// default ingestion grounding; overridable per-invocation via INGEST_GROUND.
|
|
|
|
|
// Note: a source's OWN top-level "grounding" field (structured) takes precedence
|
|
|
|
|
// over this — the author's honest self-description wins.
|
|
|
|
|
fn default_ground() -> String {
|
|
|
|
|
let g: String = env("INGEST_GROUND")
|
|
|
|
|
if str_eq(g, "") { return "extracted" }
|
|
|
|
|
return g
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn default_steward() -> String {
|
|
|
|
|
let s: String = env("INGEST_STEWARD")
|
|
|
|
|
if str_eq(s, "") { return "local-private" }
|
|
|
|
|
return s
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ingest one file -> report JSON
|
|
|
|
|
fn ingest_file(path: String) -> String {
|
|
|
|
|
let text: String = fs_read(path)
|
|
|
|
|
if str_eq(text, "") {
|
|
|
|
|
return "{\"error\":\"empty or unreadable\",\"path\":" + j_q(path) + "}"
|
|
|
|
|
}
|
|
|
|
|
let prov: String = "file:" + path
|
|
|
|
|
if ends_with_ci(path, ".json") {
|
|
|
|
|
let packed: [String] = transduce_structured(el_list_empty(), el_list_empty(),
|
|
|
|
|
text, prov, default_ground(), default_steward(), "ds:" + basename(path))
|
|
|
|
|
return merge_packed(packed)
|
|
|
|
|
}
|
|
|
|
|
let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(),
|
|
|
|
|
text, prov, default_ground(), default_steward(),
|
|
|
|
|
"doc:" + basename(path), basename(path))
|
|
|
|
|
return merge_packed(packed)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ingest a directory: walk one level, ingest each supported file, aggregate
|
|
|
|
|
fn ingest_dir(path: String) -> String {
|
|
|
|
|
let entries: [String] = fs_list(path)
|
|
|
|
|
let n: Int = el_list_len(entries)
|
|
|
|
|
let tot_created: Int = 0
|
|
|
|
|
let tot_deduped: Int = 0
|
|
|
|
|
let tot_edges: Int = 0
|
|
|
|
|
let files: Int = 0
|
|
|
|
|
let i: Int = 0
|
|
|
|
|
while i < n {
|
|
|
|
|
let name: String = str_trim(el_list_get(entries, i))
|
|
|
|
|
if !str_eq(name, "") {
|
|
|
|
|
let full: String = path + "/" + name
|
|
|
|
|
if is_text_file(full) || ends_with_ci(full, ".json") {
|
|
|
|
|
println("FILE " + full)
|
|
|
|
|
let rep: String = ingest_file(full)
|
|
|
|
|
tot_created = tot_created + json_get_int(rep, "nodes_created")
|
|
|
|
|
tot_deduped = tot_deduped + json_get_int(rep, "nodes_deduped")
|
|
|
|
|
tot_edges = tot_edges + json_get_int(rep, "edges_added")
|
|
|
|
|
files = files + 1
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
i = i + 1
|
|
|
|
|
}
|
|
|
|
|
return "{\"kind\":\"directory\",\"path\":" + j_q(path) +
|
|
|
|
|
",\"files_ingested\":" + int_to_str(files) +
|
|
|
|
|
",\"nodes_created\":" + int_to_str(tot_created) +
|
|
|
|
|
",\"nodes_deduped\":" + int_to_str(tot_deduped) +
|
|
|
|
|
",\"edges_accepted\":" + int_to_str(tot_edges) + "}"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ingest a url: fetch, treat body as prose (faithful extraction of what's there)
|
|
|
|
|
fn ingest_url(url: String) -> String {
|
|
|
|
|
let body: String = http_get(url)
|
|
|
|
|
if str_eq(body, "") { return "{\"error\":\"empty fetch\",\"url\":" + j_q(url) + "}" }
|
|
|
|
|
let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(),
|
|
|
|
|
body, "url:" + url, "extracted", "public-web",
|
|
|
|
|
"url:" + url, url)
|
|
|
|
|
return merge_packed(packed)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ingest an llm-query: pose the query to the local guide model, take the answer
|
|
|
|
|
// as a CANDIDATE (provisional, guide-sourced grounding) — never believe-the-
|
|
|
|
|
// model. The answer is ingested faithfully as what the model said, marked.
|
|
|
|
|
fn ingest_llm(query: String) -> String {
|
|
|
|
|
let model: String = if str_eq(env("INGEST_MODEL"), "") { "qwen3:1.7b" } else { env("INGEST_MODEL") }
|
|
|
|
|
let body: String = "{\"model\":" + j_q(model) + ",\"prompt\":" + j_q(query) + ",\"stream\":false}"
|
|
|
|
|
let resp: String = http_post_json("http://127.0.0.1:11434/api/generate", body)
|
|
|
|
|
let answer: String = json_get_string(resp, "response")
|
|
|
|
|
if str_eq(answer, "") { return "{\"error\":\"no model response\"}" }
|
|
|
|
|
let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(),
|
|
|
|
|
answer, "llm:" + model + ":" + query, "candidate-provisional", "guide-provisional",
|
|
|
|
|
"llm:" + query, "guide answer: " + query)
|
|
|
|
|
return merge_packed(packed)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ingest a stream: a file whose lines are turns; each line a node, sequence
|
|
|
|
|
// edges — the conversational-manifold degenerate case (continuous metabolism).
|
|
|
|
|
fn ingest_stream(path: String) -> String {
|
|
|
|
|
let text: String = fs_read(path)
|
|
|
|
|
if str_eq(text, "") { return "{\"error\":\"empty stream\"}" }
|
|
|
|
|
let lines: [String] = str_split(text, "\n")
|
|
|
|
|
let nodes: [String] = el_list_empty()
|
|
|
|
|
let edges: [String] = el_list_empty()
|
|
|
|
|
let prov: String = "stream:" + path
|
|
|
|
|
let tagbase: String = "prov:" + prov + " ground:extracted steward:local-private"
|
|
|
|
|
nodes = el_list_append(nodes, mk_node("stream", "stream: " + basename(path),
|
|
|
|
|
"Concept", "Semantic", "0.6", "0.6", "0.9", tagbase + " kind:stream"))
|
|
|
|
|
let n: Int = el_list_len(lines)
|
|
|
|
|
let i: Int = 0
|
|
|
|
|
let prev: String = ""
|
|
|
|
|
let ci: Int = 0
|
|
|
|
|
while i < n {
|
|
|
|
|
let ln: String = str_trim(el_list_get(lines, i))
|
|
|
|
|
if !str_eq(ln, "") {
|
|
|
|
|
let lid: String = "stream:t" + int_to_str(ci)
|
|
|
|
|
nodes = el_list_append(nodes, mk_node(lid, ln,
|
|
|
|
|
"Memory", "Episodic", "0.5", "0.5", "0.85", tagbase + " kind:turn"))
|
|
|
|
|
edges = el_list_append(edges, mk_edge("stream", "contains", lid))
|
|
|
|
|
if !str_eq(prev, "") { edges = el_list_append(edges, mk_edge(prev, "precedes", lid)) }
|
|
|
|
|
prev = lid
|
|
|
|
|
ci = ci + 1
|
|
|
|
|
}
|
|
|
|
|
i = i + 1
|
|
|
|
|
}
|
|
|
|
|
return merge_manifold(nodes, edges)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
// SECTION G — ENTRY
|
|
|
|
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
|
|
|
|
let kind: String = env("INGEST_KIND")
|
|
|
|
|
let arg: String = env("INGEST_ARG")
|
|
|
|
|
|
|
|
|
|
println("[ingest] organ online — engram=" + eg_base() + " kind=" + kind)
|
|
|
|
|
println("[ingest] source=" + arg)
|
|
|
|
|
|
|
|
|
|
let report: String = ""
|
|
|
|
|
if str_eq(kind, "dir") {
|
|
|
|
|
report = ingest_dir(arg)
|
|
|
|
|
} else {
|
|
|
|
|
if str_eq(kind, "file") {
|
|
|
|
|
report = ingest_file(arg)
|
|
|
|
|
} else {
|
|
|
|
|
if str_eq(kind, "structured") {
|
|
|
|
|
report = ingest_file(arg)
|
|
|
|
|
} else {
|
|
|
|
|
if str_eq(kind, "url") {
|
|
|
|
|
report = ingest_url(arg)
|
|
|
|
|
} else {
|
|
|
|
|
if str_eq(kind, "llm") {
|
|
|
|
|
report = ingest_llm(arg)
|
|
|
|
|
} else {
|
|
|
|
|
if str_eq(kind, "stream") {
|
|
|
|
|
report = ingest_stream(arg)
|
|
|
|
|
} else {
|
|
|
|
|
report = "{\"error\":\"unknown INGEST_KIND: " + kind + "\"}"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
println("REPORT " + report)
|
will.anderson
bigmerge