Merge pull request 'ingest: unify transduce_prose/transduce_structured into one transduce()' (#117) from feat/transduce-unify into dev

lang/el-compiler/src/codegen.el

@@ -2766,6 +2766,8 @@ fn builtin_arity(name: String) -> Int {
     if str_eq(name, "fs_read") { return 1 }
     if str_eq(name, "fs_write") { return 2 }
     if str_eq(name, "fs_list") { return 1 }
+    if str_eq(name, "fs_size") { return 1 }
+    if str_eq(name, "fs_read_b64_chunk") { return 3 }
     // JSON
     if str_eq(name, "json_get") { return 2 }
     if str_eq(name, "json_parse") { return 1 }

lang/runtime/el_runtime.c

@@ -2226,6 +2226,22 @@ el_val_t fs_exists(el_val_t pathv) {
     return (el_val_t)(stat(path, &st) == 0 ? 1 : 0);
 }
 
+/* fs_size — real on-disk byte count of a file, via stat() (not strlen).
+ * Needed alongside fs_read_b64_chunk() below: fs_read()'s el_val_t string
+ * result is NUL-terminated and length-unsafe for arbitrary binary content
+ * (str_len/str_slice fall back to strlen when the buffer isn't a tagged
+ * binary value — see el_input_len), so any caller that needs to walk a
+ * binary file in fixed-size windows (e.g. transduce()'s raw/opaque chunking
+ * of audio bytes) must learn the true length here instead of from the
+ * decoded string. Returns -1 if the path doesn't exist or isn't stat-able. */
+el_val_t fs_size(el_val_t pathv) {
+    const char* path = EL_CSTR(pathv);
+    if (!path || !*path) return -1;
+    struct stat st;
+    if (stat(path, &st) != 0) return -1;
+    return (el_val_t)st.st_size;
+}
+
 /* fs_mkdir — create directory at path with mode 0755, mkdir -p semantics.
  * Returns 1 if path exists or was created (incl. all parents); 0 on failure.
  * Walks the path component-by-component so missing intermediate dirs are
@@ -16469,6 +16485,41 @@ el_val_t el_base64_encode_n(const unsigned char* data, size_t len, int url_safe)
     return el_wrap_str(out);
 }
 
+/* fs_read_b64_chunk — binary-safe windowed file read: read up to `length`
+ * raw bytes starting at byte `offset` from `path` and return them base64-
+ * encoded (RFC 4648, standard alphabet, padded). The raw bytes are read into
+ * a local C buffer and base64-encoded directly here — they never pass
+ * through an el_val_t string as raw bytes, so embedded NUL bytes (common in
+ * real PCM audio) never hit a strlen()-based code path. This mirrors the
+ * existing llm_vision() image-attachment path (read file -> base64 in C ->
+ * hand back a plain-ASCII string) and the http_*_to_file() rationale above:
+ * bypass the string wrapper entirely for the part that must stay binary.
+ *
+ * Returns "" if the path can't be opened, offset is negative or past EOF,
+ * or length <= 0. A short final chunk (less than `length` bytes remaining)
+ * is returned truncated to what's actually on disk — never padded/invented. */
+el_val_t fs_read_b64_chunk(el_val_t pathv, el_val_t offsetv, el_val_t lengthv) {
+    const char* path = EL_CSTR(pathv);
+    int64_t offset = (int64_t)offsetv;
+    int64_t length = (int64_t)lengthv;
+    if (!path || !*path || offset < 0 || length <= 0) return el_wrap_str(el_strdup(""));
+    FILE* f = fopen(path, "rb");
+    if (!f) return el_wrap_str(el_strdup(""));
+    fseek(f, 0, SEEK_END);
+    long sz = ftell(f);
+    if (sz < 0 || offset >= sz) { fclose(f); return el_wrap_str(el_strdup("")); }
+    fseek(f, (long)offset, SEEK_SET);
+    long remain = sz - (long)offset;
+    size_t want = (size_t)(((int64_t)remain < length) ? remain : length);
+    unsigned char* buf = malloc(want > 0 ? want : 1);
+    if (!buf) { fclose(f); return el_wrap_str(el_strdup("")); }
+    size_t got = fread(buf, 1, want, f);
+    fclose(f);
+    el_val_t out = el_base64_encode_n(buf, got, /*url_safe=*/0);
+    free(buf);
+    return out;
+}
+
 /* Decode either alphabet — accepts both '+/' and '-_' transparently, and
  * tolerates missing padding (which JWTs typically omit). Whitespace is
  * skipped for robustness. Invalid characters cause the decode to stop and

lang/runtime/el_runtime.h

@@ -235,6 +235,20 @@ el_val_t  fs_list(el_val_t path);
 el_val_t  fs_exists(el_val_t path);
 el_val_t  fs_mkdir(el_val_t path);   /* mkdir -p, mode 0755                */
 
+/* Real on-disk byte count via stat() — not strlen(). Use this (not
+ * str_len(fs_read(path))) when a file may contain binary content, since
+ * fs_read()'s result truncates at the first embedded NUL under strlen-based
+ * string ops. Returns -1 if the path doesn't exist. */
+el_val_t  fs_size(el_val_t path);
+
+/* Binary-safe windowed read: read up to `length` bytes starting at byte
+ * `offset` from `path` and return them base64-encoded. Bytes are read and
+ * encoded in C without ever passing through an el_val_t string as raw
+ * bytes, so embedded NULs (routine in PCM audio) can't truncate the result.
+ * Returns "" on any failure or when offset is past EOF; a final short
+ * window returns only the bytes that actually exist on disk. */
+el_val_t  fs_read_b64_chunk(el_val_t path, el_val_t offset, el_val_t length);
+
 /* Length-explicit binary write. `length` is an Int (el_val_t holding the
  * byte count). The caller knows the length from context — typically because
  * `bytes` came from base64_decode (which produces a magic-tagged binary