diff --git a/ingest/.gitignore b/ingest/.gitignore new file mode 100644 index 0000000..567609b --- /dev/null +++ b/ingest/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/ingest/src/ingest.el b/ingest/src/ingest.el index 42dfcbb..1d47432 100644 --- a/ingest/src/ingest.el +++ b/ingest/src/ingest.el @@ -1,17 +1,28 @@ // ingest.el — the native EL AFFERENT INGEST ORGAN // // The source-polymorphic ingest(source) primitive: point it at a directory, -// file, url, llm-query, structured-primitive set, or stream; it EXTRACTS the -// real content faithfully (no invention), TRANSDUCES it into a DISCRETE -// MANIFOLD (multiple nodes + internal edges — meaning-structure, never a -// single blob; the conversion from extracted surface content into geometry -// is automatic and invisible to the caller, the way digestion is invisible -// to the one who chose to eat — ingest is the conscious act, transduce is -// the mechanism underneath it, and it is no less real for being unseen), -// and MERGES that manifold into the engram geometry: shared -// meanings DEDUP onto existing nodes (search + exact/cosine match), genuinely -// new meanings add nodes, relations add edges. Every node enters with -// PROVENANCE + grounding-level + stewardship class from the moment of entry. +// file, url, llm-query, or stream; it EXTRACTS the real content faithfully +// (no invention), TRANSDUCES it into a DISCRETE MANIFOLD (multiple nodes + +// internal edges — meaning-structure, never a single blob; the conversion +// from extracted surface content into geometry is automatic and invisible +// to the caller, the way digestion is invisible to the one who chose to +// eat — ingest is the conscious act, transduce is the mechanism underneath +// it, and it is no less real for being unseen), and MERGES that manifold +// into the engram geometry: shared meanings DEDUP onto existing nodes +// (search + exact/cosine match), genuinely new meanings add nodes, +// relations add edges. Every node enters with PROVENANCE + grounding-level +// + stewardship class from the moment of entry. +// +// transduce() is THE single mechanism — one function, polymorphic, with no +// content-type branch inside it. It does not ask whether a payload is +// prose, structured data, or raw/opaque bytes (audio, or anything else); +// it runs one boundary-scan-with-fixed-window-fallback chunking algorithm +// and one dedup mechanism on whatever bytes it's handed, unconditionally. +// Any deeper structure a payload might have (shared fields, relationships, +// what a chunk of audio "means") is NOT interpreted here — that's left +// entirely to the engram's own mechanisms (embedding, spreading activation, +// dedup) acting on this real geometry over time. This organ claims zero +// semantic understanding of any payload it transduces. // // It is a pure HTTP CLIENT of the engram server — it links only el_runtime.c // via fs/http/json/string builtins; it never links el_seed.c or the engram @@ -46,60 +57,6 @@ fn j_q(s: String) -> String { return "\"" + j_esc(s) + "\"" } -// Extract the top-level keys of a JSON object string. A thin, self-contained -// scanner (FLAGGED: the one non-trivial parser in this organ — everything else -// is faithful text handling). Tracks string state + brace/bracket depth; a key -// is a string at object-interior depth 1 immediately followed by ':'. -fn json_object_keys(obj: String) -> [String] { - let keys: [String] = el_list_empty() - let n: Int = str_len(obj) - let i: Int = 0 - let depth: Int = 0 - let in_str: Bool = false - let esc: Bool = false - let str_start: Int = -1 - let cur: String = "" - let have_key: Bool = false - while i < n { - let c: String = str_char_at(obj, i) - if in_str { - if esc { - esc = false - } else { - if str_eq(c, "\\") { - esc = true - } else { - if str_eq(c, "\"") { - in_str = false - cur = str_slice(obj, str_start + 1, i) - have_key = true - } - } - } - } else { - if str_eq(c, "\"") { - in_str = true - str_start = i - } - if str_eq(c, "{") { depth = depth + 1 } - if str_eq(c, "}") { depth = depth - 1 } - if str_eq(c, "[") { depth = depth + 1 } - if str_eq(c, "]") { depth = depth - 1 } - if str_eq(c, ":") { - if have_key { - if depth == 1 { - keys = el_list_append(keys, cur) - } - } - have_key = false - } - if str_eq(c, ",") { have_key = false } - } - i = i + 1 - } - return keys -} - // ═══════════════════════════════════════════════════════════════════════════ // SECTION B — engram HTTP client (provenance-carrying afferent LOAD) // ═══════════════════════════════════════════════════════════════════════════ @@ -402,168 +359,125 @@ fn head80(s: String) -> String { // We accumulate into module-level lists carried by the caller. // ═══════════════════════════════════════════════════════════════════════════ -// PROSE: chunk text into a discrete manifold. Split on blank lines into -// paragraphs; every non-empty paragraph is its own node (NEVER one blob). -// Edges: doc-root -contains-> chunk; chunk -precedes-> next chunk; -// most-recent-heading -section_of-> chunk. Content is verbatim (substring of -// the source) — pure extraction of ground truth. -fn transduce_prose(nodes: [String], edges: [String], text: String, - prov: String, ground: String, steward: String, - root_lid: String, root_title: String) -> [String] { - // returns [nodes_json_list_encoded, edges_json_list_encoded] is awkward in - // EL; instead we mutate by returning a 2-list. We package results as a - // single JSON array string carrying {nodes:[...],edges:[...]} additions. - // (Kept simple: caller passes empty lists and receives the packaged pair.) +// TRANSDUCE — the single mechanism. Takes ANY payload (prose, structured +// data, raw/opaque bytes — audio, whatever) as one opaque string and turns +// it into a discrete manifold: nodes + internal edges. There is no +// content-type branch anywhere in this function. It never asks "is this +// text," "is this JSON," "is this audio" — it runs ONE algorithm on the +// bytes it is given, unconditionally: +// +// 1. BOUNDARY SCAN — split on "\n\n". This is a property of the bytes +// (does a blank-line-style marker occur in them, yes or no), not a +// classification of what the content IS. Prose paragraphs split on it +// because that's how prose is typically written; that's a fact about +// the bytes, not a rule this function knows about prose. Anything else +// that happens to contain the same marker splits on it too, and +// anything that doesn't, doesn't — same code path either way. +// 2. FIXED-WINDOW FALLBACK — if step 1 found no boundary (0 or 1 non-empty +// piece), the payload is cut into fixed-size windows instead. Same +// chunk-per-node, edge-per-adjacency structure as step 1 produces; only +// the source of the cut point differs. +// +// Every resulting chunk becomes its own node (never one blob), wired with +// the same edges regardless of what's inside a chunk: root -contains-> +// chunk, chunk -precedes-> next chunk, and — if a chunk happens to start +// with "#" — most-recent-heading -section_of-> chunk. That "#" check is a +// structural marker (a fact about a chunk's first byte), not a decision +// about whether this run is "the text case": chunks from any payload that +// never happen to start with "#" simply never trigger it. +// +// Dedup is the existing, fully generic mechanism (find_existing_by_content, +// via merge_manifold downstream of merge_packed) applied uniformly to every +// chunk from every payload — there is no separate "structured" dedup path. +// Any deeper structure that might exist inside a payload (shared fields, +// repeated records, relationships) is NOT pre-computed here; that's left to +// the engram's own mechanisms (embedding, spreading activation, dedup) +// acting on this geometry over time, which is the whole point of handing it +// raw bytes instead of a hand-coded interpretation of them. +// +// Byte-safety note: `source` must already be a string this function can +// safely str_split/str_slice. Protecting it from silent truncation (El +// strings are NUL-unsafe under strlen-based ops; fs_read()'s result +// truncates at the first embedded NUL, which is routine in real binary +// bytes) is a MECHANICAL fidelity concern that belongs to whatever produced +// `source` (see ingest_file's file_source_string below) — not a +// content-type judgment made in here. transduce() never learns whether a +// chunk is plain text or a base64-encoded raw-byte window; every chunk is +// handled identically either way. +fn transduce(nodes: [String], edges: [String], source: String, + prov: String, ground: String, steward: String, + root_lid: String, root_title: String) -> [String] { let tagbase: String = "prov:" + prov + " ground:" + ground + " steward:" + steward - // root node - nodes = el_list_append(nodes, mk_node(root_lid, "document: " + root_title, - "Concept", "Semantic", "0.6", "0.6", "0.9", tagbase + " kind:document")) + nodes = el_list_append(nodes, mk_node(root_lid, "source: " + root_title, + "Concept", "Semantic", "0.6", "0.6", "0.9", tagbase + " kind:source")) - let paras: [String] = str_split(text, "\n\n") - let np: Int = el_list_len(paras) - let idx: Int = 0 + // step 1: universal boundary scan + let boundary_parts: [String] = str_split(source, "\n\n") + let chunks: [String] = el_list_empty() + let bp_n: Int = el_list_len(boundary_parts) + let bp_i: Int = 0 + while bp_i < bp_n { + let piece: String = str_trim(el_list_get(boundary_parts, bp_i)) + if !str_eq(piece, "") { chunks = el_list_append(chunks, piece) } + bp_i = bp_i + 1 + } + + // step 2: no boundary found -> fixed-size windows over the whole + // payload. 4096 chars/node: low kilobytes — big enough to keep node + // count sane on a large unbroken payload, small enough that each node + // stays a legible, individually embeddable/dedupable unit rather than + // one giant blob. + if el_list_len(chunks) <= 1 { + chunks = el_list_empty() + let total: Int = str_len(source) + let win: Int = 4096 + let off: Int = 0 + while off < total { + let endp: Int = if off + win < total { off + win } else { total } + let piece: String = str_slice(source, off, endp) + if !str_eq(piece, "") { chunks = el_list_append(chunks, piece) } + off = off + win + } + } + + let nc: Int = el_list_len(chunks) + let ci: Int = 0 let last_chunk: String = "" let last_heading: String = "" - let ci: Int = 0 - while idx < np { - let raw: String = str_trim(el_list_get(paras, idx)) - if !str_eq(raw, "") { - let lid: String = root_lid + ":c" + int_to_str(ci) - let is_heading: Bool = str_starts_with(raw, "#") - let kind: String = if is_heading { "kind:heading" } else { "kind:doc-chunk" } - nodes = el_list_append(nodes, mk_node(lid, raw, - "Knowledge", "Semantic", "0.55", "0.55", "0.9", tagbase + " " + kind)) - // containment: document root -contains-> chunk - edges = el_list_append(edges, mk_edge(root_lid, "contains", lid)) - // sequence: previous chunk -precedes-> this chunk - if !str_eq(last_chunk, "") { - edges = el_list_append(edges, mk_edge(last_chunk, "precedes", lid)) - } - // sectioning: most-recent heading -section_of-> this chunk - if is_heading { - last_heading = lid - } else { - if !str_eq(last_heading, "") { - edges = el_list_append(edges, mk_edge(last_heading, "section_of", lid)) - } - } - last_chunk = lid - ci = ci + 1 + while ci < nc { + let raw: String = el_list_get(chunks, ci) + let lid: String = root_lid + ":c" + int_to_str(ci) + let is_heading: Bool = str_starts_with(raw, "#") + let kind: String = if is_heading { "kind:heading" } else { "kind:chunk" } + nodes = el_list_append(nodes, mk_node(lid, raw, + "Knowledge", "Semantic", "0.55", "0.55", "0.9", tagbase + " " + kind)) + // containment: root -contains-> chunk + edges = el_list_append(edges, mk_edge(root_lid, "contains", lid)) + // sequence: previous chunk -precedes-> this chunk + if !str_eq(last_chunk, "") { + edges = el_list_append(edges, mk_edge(last_chunk, "precedes", lid)) } - idx = idx + 1 - } - // package: we return the two lists concatenated via a sentinel; but EL - // lists can't nest heterogeneously here, so we instead return nodes and - // rely on the caller holding edges by reference is not possible — so we - // encode both into one list: [ "N" + nodejson ... , "E" + edgejson ... ]. - let packed: [String] = el_list_empty() - let a: Int = 0 - let an: Int = el_list_len(nodes) - while a < an { packed = el_list_append(packed, "N" + el_list_get(nodes, a)) a = a + 1 } - let b: Int = 0 - let bn: Int = el_list_len(edges) - while b < bn { packed = el_list_append(packed, "E" + el_list_get(edges, b)) b = b + 1 } - return packed -} - -// STRUCTURED / RAW-GEOMETRY: ingest structured primitives (phonetics/formants, -// instrument signatures, scene primitives) as GEOMETRY, faithfully. Normalized -// input shape: -// {"dataset":"","primitive_type":"", -// "records":[{"key":"","features":{...categorical...},"attributes":{...}}]} -// Each record -> a primitive node; each categorical feature -> a SHARED feature -// node (deduped across records: many primitives -> one feature node = real -// connective geometry, meaning saturates); numeric attributes fold into the -// primitive's content (unique values, no dedup benefit). This is knowledge -// represented as geometry, not prose — the path speech/music/image ingest on. -fn transduce_structured(nodes: [String], edges: [String], js: String, - prov: String, ground: String, steward: String, - root_lid: String) -> [String] { - // grounding integrity: the SOURCE may declare its own epistemic grounding - // (measured / derived / convention / ...) via a top-level "grounding" field; - // honor it faithfully over the ingest-time default. This keeps the per-node - // ground: facet consistent with the source's honest self-description. - let src_ground: String = json_get_string(js, "grounding") - let use_ground: String = if str_eq(src_ground, "") { ground } else { src_ground } - let tagbase: String = "prov:" + prov + " ground:" + use_ground + " steward:" + steward - let dsname: String = json_get_string(js, "dataset") - let ptype: String = json_get_string(js, "primitive_type") - // capture the source's own scholarly provenance citation (verbatim) onto - // the dataset root — faithful attribution, retrievable, reachable from every - // primitive via its -contains- edge back to the root. - let src_cite: String = json_get_string(js, "provenance") - let root_content: String = "dataset: " + dsname + " (" + ptype + ")" - if !str_eq(src_cite, "") { root_content = root_content + " | provenance: " + src_cite } - nodes = el_list_append(nodes, mk_node(root_lid, root_content, - "Concept", "Semantic", "0.6", "0.6", "0.9", tagbase + " kind:dataset")) - - let recs: String = json_get_raw(js, "records") - let nr: Int = json_array_len(recs) - let r: Int = 0 - while r < nr { - let rec: String = json_array_get(recs, r) - let rkey: String = json_get_string(rec, "key") - let attrs: String = json_get_raw(rec, "attributes") - // faithful compact serialization of the primitive's numeric signature - let attr_str: String = flatten_pairs(attrs) - let content: String = ptype + " " + rkey - if !str_eq(attr_str, "") { content = content + " | " + attr_str } - let plid: String = root_lid + ":" + rkey - nodes = el_list_append(nodes, mk_node(plid, content, - "Concept", "Semantic", "0.6", "0.6", "0.92", - tagbase + " kind:primitive primitive:" + ptype + " key:" + rkey)) - edges = el_list_append(edges, mk_edge(root_lid, "contains", plid)) - - // categorical features -> SHARED (deduped) feature nodes + labelled edges - let feats: String = json_get_raw(rec, "features") - let fkeys: [String] = json_object_keys(feats) - let fk: Int = el_list_len(fkeys) - let k: Int = 0 - while k < fk { - let fname: String = el_list_get(fkeys, k) - let fval: String = json_get_string(feats, fname) - // shared feature node: content is the feature=value pair; identical - // pairs across records dedup onto ONE node (the geometry). - let flid: String = "feat:" + fname + "=" + fval - let fcontent: String = fname + "=" + fval - nodes = el_list_append(nodes, mk_node(flid, fcontent, - "Concept", "Semantic", "0.5", "0.5", "0.9", - tagbase + " kind:feature feature:" + fname)) - edges = el_list_append(edges, mk_edge(plid, fname, flid)) - k = k + 1 + // sectioning: most-recent heading -section_of-> this chunk + if is_heading { + last_heading = lid + } else { + if !str_eq(last_heading, "") { + edges = el_list_append(edges, mk_edge(last_heading, "section_of", lid)) + } } - r = r + 1 + last_chunk = lid + ci = ci + 1 } - let packed: [String] = el_list_empty() - let a: Int = 0 - let an: Int = el_list_len(nodes) - while a < an { packed = el_list_append(packed, "N" + el_list_get(nodes, a)) a = a + 1 } - let b: Int = 0 - let bn: Int = el_list_len(edges) - while b < bn { packed = el_list_append(packed, "E" + el_list_get(edges, b)) b = b + 1 } - return packed -} -// flatten a flat JSON object of scalar fields into "k=v k=v" (faithful; values -// verbatim). Used for numeric attribute signatures. -fn flatten_pairs(obj: String) -> String { - if str_eq(obj, "") { return "" } - let keys: [String] = json_object_keys(obj) - let n: Int = el_list_len(keys) - let out: String = "" - let i: Int = 0 - while i < n { - let k: String = el_list_get(keys, i) - // json_get_raw returns the raw token — works for NUMBERS (bare, e.g. - // "270") where json_get_string yields "" for non-string values. Strip - // surrounding quotes if the value happens to be a string token. - let raw: String = json_get_raw(obj, k) - let v: String = str_replace(raw, "\"", "") - let sep: String = if i == 0 { "" } else { " " } - out = out + sep + k + "=" + v - i = i + 1 - } - return out + // package both lists into one, "N"/"E"-prefixed (see merge_packed). + let packed: [String] = el_list_empty() + let pn_i: Int = 0 + let pn_n: Int = el_list_len(nodes) + while pn_i < pn_n { packed = el_list_append(packed, "N" + el_list_get(nodes, pn_i)) pn_i = pn_i + 1 } + let pe_i: Int = 0 + let pe_n: Int = el_list_len(edges) + while pe_i < pe_n { packed = el_list_append(packed, "E" + el_list_get(edges, pe_i)) pe_i = pe_i + 1 } + return packed } // unpack the "N"/"E"-prefixed packed list back into two lists, then merge @@ -594,18 +508,7 @@ fn basename(path: String) -> String { return el_list_get(parts, n - 1) } -fn ends_with_ci(s: String, suf: String) -> Bool { - return str_ends_with(str_to_lower(s), suf) -} - -fn is_text_file(path: String) -> Bool { - return ends_with_ci(path, ".md") || ends_with_ci(path, ".txt") - || ends_with_ci(path, ".markdown") || ends_with_ci(path, ".text") -} - // default ingestion grounding; overridable per-invocation via INGEST_GROUND. -// Note: a source's OWN top-level "grounding" field (structured) takes precedence -// over this — the author's honest self-description wins. fn default_ground() -> String { let g: String = env("INGEST_GROUND") if str_eq(g, "") { return "extracted" } @@ -618,25 +521,67 @@ fn default_steward() -> String { return s } -// ingest one file -> report JSON +// Mechanical fidelity guard — NOT a content-type test. fs_read()'s el_val_t +// result truncates at the first embedded NUL byte under El's strlen-based +// string ops (see fs_size's doc comment in runtime/el_runtime.h); comparing +// its length against fs_size() (a real stat()-based byte count) is a +// technical fact about whether the string channel captured the file intact +// — computed the same way for a poem, a JSON file, or a WAV, and saying +// nothing about what the file IS. When the counts agree, `text` is +// trustworthy verbatim. When they don't (silent truncation happened), +// rebuild the payload as base64-encoded fixed-size windows read directly +// off disk (fs_read_b64_chunk — binary-safe in C), joined with the same +// "\n\n" boundary marker transduce()'s generic scan already looks for, so +// transduce() sees one ordinary boundary-delimited payload and runs its one +// algorithm on it exactly as it would on prose — it never learns that a +// fidelity problem occurred upstream, let alone why. +fn file_source_string(path: String, text: String, real_size: Int) -> String { + if real_size <= 0 { return text } + if str_len(text) == real_size { return text } + // 3072 raw bytes -> 4096 base64 chars (3 divides evenly into base64's + // 3-byte/4-char ratio); keeps each resulting node's content a clean, + // bounded, low-kilobytes unit, same order of magnitude as the fixed + // fallback window in transduce() itself. + let win: Int = 3072 + let out: String = "" + let off: Int = 0 + let first: Bool = true + while off < real_size { + let chunk_b64: String = fs_read_b64_chunk(path, off, win) + if str_eq(chunk_b64, "") { + off = real_size + } else { + let sep: String = if first { "" } else { "\n\n" } + out = out + sep + chunk_b64 + first = false + off = off + win + } + } + return out +} + +// ingest one file -> report JSON. Uniform for every file regardless of +// extension or content — transduce() decides nothing about content-type, so +// neither does this function; it only decides whether the raw bytes made it +// through the read intact (file_source_string), which is a fidelity +// question, not a format one. fn ingest_file(path: String) -> String { + let real_size: Int = fs_size(path) let text: String = fs_read(path) - if str_eq(text, "") { + let source: String = file_source_string(path, text, real_size) + if str_eq(source, "") { return "{\"error\":\"empty or unreadable\",\"path\":" + j_q(path) + "}" } let prov: String = "file:" + path - if ends_with_ci(path, ".json") { - let packed: [String] = transduce_structured(el_list_empty(), el_list_empty(), - text, prov, default_ground(), default_steward(), "ds:" + basename(path)) - return merge_packed(packed) - } - let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(), - text, prov, default_ground(), default_steward(), + let packed: [String] = transduce(el_list_empty(), el_list_empty(), + source, prov, default_ground(), default_steward(), "doc:" + basename(path), basename(path)) return merge_packed(packed) } -// ingest a directory: walk one level, ingest each supported file, aggregate +// ingest a directory: walk one level, ingest every file found, aggregate. +// No extension filter — transduce() handles any payload uniformly now, so +// there is no content-type gate at the directory boundary either. fn ingest_dir(path: String) -> String { let entries: [String] = fs_list(path) let n: Int = el_list_len(entries) @@ -649,14 +594,12 @@ fn ingest_dir(path: String) -> String { let name: String = str_trim(el_list_get(entries, i)) if !str_eq(name, "") { let full: String = path + "/" + name - if is_text_file(full) || ends_with_ci(full, ".json") { - println("FILE " + full) - let rep: String = ingest_file(full) - tot_created = tot_created + json_get_int(rep, "nodes_created") - tot_deduped = tot_deduped + json_get_int(rep, "nodes_deduped") - tot_edges = tot_edges + json_get_int(rep, "edges_added") - files = files + 1 - } + println("FILE " + full) + let rep: String = ingest_file(full) + tot_created = tot_created + json_get_int(rep, "nodes_created") + tot_deduped = tot_deduped + json_get_int(rep, "nodes_deduped") + tot_edges = tot_edges + json_get_int(rep, "edges_added") + files = files + 1 } i = i + 1 } @@ -667,11 +610,12 @@ fn ingest_dir(path: String) -> String { ",\"edges_accepted\":" + int_to_str(tot_edges) + "}" } -// ingest a url: fetch, treat body as prose (faithful extraction of what's there) +// ingest a url: fetch, hand the body straight to transduce (faithful +// extraction of what's there — no interpretation of what it is) fn ingest_url(url: String) -> String { let body: String = http_get(url) if str_eq(body, "") { return "{\"error\":\"empty fetch\",\"url\":" + j_q(url) + "}" } - let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(), + let packed: [String] = transduce(el_list_empty(), el_list_empty(), body, "url:" + url, "extracted", "public-web", "url:" + url, url) return merge_packed(packed) @@ -686,7 +630,7 @@ fn ingest_llm(query: String) -> String { let resp: String = http_post_json("http://127.0.0.1:11434/api/generate", body) let answer: String = json_get_string(resp, "response") if str_eq(answer, "") { return "{\"error\":\"no model response\"}" } - let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(), + let packed: [String] = transduce(el_list_empty(), el_list_empty(), answer, "llm:" + model + ":" + query, "candidate-provisional", "guide-provisional", "llm:" + query, "guide answer: " + query) return merge_packed(packed) @@ -728,6 +672,19 @@ fn ingest_stream(path: String) -> String { // SECTION G — ENTRY // ═══════════════════════════════════════════════════════════════════════════ +// INGEST_KIND selects an ACQUISITION mechanism only — dir/file/url/llm/ +// stream — i.e. which RPC shape to use to go get the bytes (walk a +// directory, open a file, fetch a URL, query an LLM, read a turn-stream). +// That is a genuinely unavoidable choice at the process-entry boundary +// (nothing about the string "/tmp/x" tells you whether it's a file to read +// or a stream to read line-by-line, or distinguishes an LLM query from a +// path), so it cannot be dropped the way content-type dispatch was. +// It is NOT a content-type flag: it says nothing about what's inside the +// bytes once fetched, and none of the five ingest_* functions it selects +// among interpret their payload differently by content shape anymore — +// they all hand off to the single, format-agnostic transduce(). The old +// "structured" value (a caller-declared alias for "file", used only to hint +// the now-removed JSON-vs-prose branch) is gone along with that branch. let kind: String = env("INGEST_KIND") let arg: String = env("INGEST_ARG") @@ -741,20 +698,16 @@ if str_eq(kind, "dir") { if str_eq(kind, "file") { report = ingest_file(arg) } else { - if str_eq(kind, "structured") { - report = ingest_file(arg) + if str_eq(kind, "url") { + report = ingest_url(arg) } else { - if str_eq(kind, "url") { - report = ingest_url(arg) + if str_eq(kind, "llm") { + report = ingest_llm(arg) } else { - if str_eq(kind, "llm") { - report = ingest_llm(arg) + if str_eq(kind, "stream") { + report = ingest_stream(arg) } else { - if str_eq(kind, "stream") { - report = ingest_stream(arg) - } else { - report = "{\"error\":\"unknown INGEST_KIND: " + kind + "\"}" - } + report = "{\"error\":\"unknown INGEST_KIND: " + kind + "\"}" } } } diff --git a/lang/el-compiler/src/codegen.el b/lang/el-compiler/src/codegen.el index 9b86040..d13410c 100644 --- a/lang/el-compiler/src/codegen.el +++ b/lang/el-compiler/src/codegen.el @@ -2766,6 +2766,8 @@ fn builtin_arity(name: String) -> Int { if str_eq(name, "fs_read") { return 1 } if str_eq(name, "fs_write") { return 2 } if str_eq(name, "fs_list") { return 1 } + if str_eq(name, "fs_size") { return 1 } + if str_eq(name, "fs_read_b64_chunk") { return 3 } // JSON if str_eq(name, "json_get") { return 2 } if str_eq(name, "json_parse") { return 1 } diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index dfb9fa0..a896549 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -2226,6 +2226,22 @@ el_val_t fs_exists(el_val_t pathv) { return (el_val_t)(stat(path, &st) == 0 ? 1 : 0); } +/* fs_size — real on-disk byte count of a file, via stat() (not strlen). + * Needed alongside fs_read_b64_chunk() below: fs_read()'s el_val_t string + * result is NUL-terminated and length-unsafe for arbitrary binary content + * (str_len/str_slice fall back to strlen when the buffer isn't a tagged + * binary value — see el_input_len), so any caller that needs to walk a + * binary file in fixed-size windows (e.g. transduce()'s raw/opaque chunking + * of audio bytes) must learn the true length here instead of from the + * decoded string. Returns -1 if the path doesn't exist or isn't stat-able. */ +el_val_t fs_size(el_val_t pathv) { + const char* path = EL_CSTR(pathv); + if (!path || !*path) return -1; + struct stat st; + if (stat(path, &st) != 0) return -1; + return (el_val_t)st.st_size; +} + /* fs_mkdir — create directory at path with mode 0755, mkdir -p semantics. * Returns 1 if path exists or was created (incl. all parents); 0 on failure. * Walks the path component-by-component so missing intermediate dirs are @@ -16469,6 +16485,41 @@ el_val_t el_base64_encode_n(const unsigned char* data, size_t len, int url_safe) return el_wrap_str(out); } +/* fs_read_b64_chunk — binary-safe windowed file read: read up to `length` + * raw bytes starting at byte `offset` from `path` and return them base64- + * encoded (RFC 4648, standard alphabet, padded). The raw bytes are read into + * a local C buffer and base64-encoded directly here — they never pass + * through an el_val_t string as raw bytes, so embedded NUL bytes (common in + * real PCM audio) never hit a strlen()-based code path. This mirrors the + * existing llm_vision() image-attachment path (read file -> base64 in C -> + * hand back a plain-ASCII string) and the http_*_to_file() rationale above: + * bypass the string wrapper entirely for the part that must stay binary. + * + * Returns "" if the path can't be opened, offset is negative or past EOF, + * or length <= 0. A short final chunk (less than `length` bytes remaining) + * is returned truncated to what's actually on disk — never padded/invented. */ +el_val_t fs_read_b64_chunk(el_val_t pathv, el_val_t offsetv, el_val_t lengthv) { + const char* path = EL_CSTR(pathv); + int64_t offset = (int64_t)offsetv; + int64_t length = (int64_t)lengthv; + if (!path || !*path || offset < 0 || length <= 0) return el_wrap_str(el_strdup("")); + FILE* f = fopen(path, "rb"); + if (!f) return el_wrap_str(el_strdup("")); + fseek(f, 0, SEEK_END); + long sz = ftell(f); + if (sz < 0 || offset >= sz) { fclose(f); return el_wrap_str(el_strdup("")); } + fseek(f, (long)offset, SEEK_SET); + long remain = sz - (long)offset; + size_t want = (size_t)(((int64_t)remain < length) ? remain : length); + unsigned char* buf = malloc(want > 0 ? want : 1); + if (!buf) { fclose(f); return el_wrap_str(el_strdup("")); } + size_t got = fread(buf, 1, want, f); + fclose(f); + el_val_t out = el_base64_encode_n(buf, got, /*url_safe=*/0); + free(buf); + return out; +} + /* Decode either alphabet — accepts both '+/' and '-_' transparently, and * tolerates missing padding (which JWTs typically omit). Whitespace is * skipped for robustness. Invalid characters cause the decode to stop and diff --git a/lang/runtime/el_runtime.h b/lang/runtime/el_runtime.h index 6824b81..d126993 100644 --- a/lang/runtime/el_runtime.h +++ b/lang/runtime/el_runtime.h @@ -235,6 +235,20 @@ el_val_t fs_list(el_val_t path); el_val_t fs_exists(el_val_t path); el_val_t fs_mkdir(el_val_t path); /* mkdir -p, mode 0755 */ +/* Real on-disk byte count via stat() — not strlen(). Use this (not + * str_len(fs_read(path))) when a file may contain binary content, since + * fs_read()'s result truncates at the first embedded NUL under strlen-based + * string ops. Returns -1 if the path doesn't exist. */ +el_val_t fs_size(el_val_t path); + +/* Binary-safe windowed read: read up to `length` bytes starting at byte + * `offset` from `path` and return them base64-encoded. Bytes are read and + * encoded in C without ever passing through an el_val_t string as raw + * bytes, so embedded NULs (routine in PCM audio) can't truncate the result. + * Returns "" on any failure or when offset is past EOF; a final short + * window returns only the bytes that actually exist on disk. */ +el_val_t fs_read_b64_chunk(el_val_t path, el_val_t offset, el_val_t length); + /* Length-explicit binary write. `length` is an Int (el_val_t holding the * byte count). The caller knows the length from context — typically because * `bytes` came from base64_decode (which produces a magic-tagged binary