diff --git a/ingest/.gitignore b/ingest/.gitignore new file mode 100644 index 0000000..567609b --- /dev/null +++ b/ingest/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/ingest/src/ingest.el b/ingest/src/ingest.el index 42dfcbb..1d47432 100644 --- a/ingest/src/ingest.el +++ b/ingest/src/ingest.el @@ -1,17 +1,28 @@ // ingest.el — the native EL AFFERENT INGEST ORGAN // // The source-polymorphic ingest(source) primitive: point it at a directory, -// file, url, llm-query, structured-primitive set, or stream; it EXTRACTS the -// real content faithfully (no invention), TRANSDUCES it into a DISCRETE -// MANIFOLD (multiple nodes + internal edges — meaning-structure, never a -// single blob; the conversion from extracted surface content into geometry -// is automatic and invisible to the caller, the way digestion is invisible -// to the one who chose to eat — ingest is the conscious act, transduce is -// the mechanism underneath it, and it is no less real for being unseen), -// and MERGES that manifold into the engram geometry: shared -// meanings DEDUP onto existing nodes (search + exact/cosine match), genuinely -// new meanings add nodes, relations add edges. Every node enters with -// PROVENANCE + grounding-level + stewardship class from the moment of entry. +// file, url, llm-query, or stream; it EXTRACTS the real content faithfully +// (no invention), TRANSDUCES it into a DISCRETE MANIFOLD (multiple nodes + +// internal edges — meaning-structure, never a single blob; the conversion +// from extracted surface content into geometry is automatic and invisible +// to the caller, the way digestion is invisible to the one who chose to +// eat — ingest is the conscious act, transduce is the mechanism underneath +// it, and it is no less real for being unseen), and MERGES that manifold +// into the engram geometry: shared meanings DEDUP onto existing nodes +// (search + exact/cosine match), genuinely new meanings add nodes, +// relations add edges. Every node enters with PROVENANCE + grounding-level +// + stewardship class from the moment of entry. +// +// transduce() is THE single mechanism — one function, polymorphic, with no +// content-type branch inside it. It does not ask whether a payload is +// prose, structured data, or raw/opaque bytes (audio, or anything else); +// it runs one boundary-scan-with-fixed-window-fallback chunking algorithm +// and one dedup mechanism on whatever bytes it's handed, unconditionally. +// Any deeper structure a payload might have (shared fields, relationships, +// what a chunk of audio "means") is NOT interpreted here — that's left +// entirely to the engram's own mechanisms (embedding, spreading activation, +// dedup) acting on this real geometry over time. This organ claims zero +// semantic understanding of any payload it transduces. // // It is a pure HTTP CLIENT of the engram server — it links only el_runtime.c // via fs/http/json/string builtins; it never links el_seed.c or the engram @@ -46,60 +57,6 @@ fn j_q(s: String) -> String { return "\"" + j_esc(s) + "\"" } -// Extract the top-level keys of a JSON object string. A thin, self-contained -// scanner (FLAGGED: the one non-trivial parser in this organ — everything else -// is faithful text handling). Tracks string state + brace/bracket depth; a key -// is a string at object-interior depth 1 immediately followed by ':'. -fn json_object_keys(obj: String) -> [String] { - let keys: [String] = el_list_empty() - let n: Int = str_len(obj) - let i: Int = 0 - let depth: Int = 0 - let in_str: Bool = false - let esc: Bool = false - let str_start: Int = -1 - let cur: String = "" - let have_key: Bool = false - while i < n { - let c: String = str_char_at(obj, i) - if in_str { - if esc { - esc = false - } else { - if str_eq(c, "\\") { - esc = true - } else { - if str_eq(c, "\"") { - in_str = false - cur = str_slice(obj, str_start + 1, i) - have_key = true - } - } - } - } else { - if str_eq(c, "\"") { - in_str = true - str_start = i - } - if str_eq(c, "{") { depth = depth + 1 } - if str_eq(c, "}") { depth = depth - 1 } - if str_eq(c, "[") { depth = depth + 1 } - if str_eq(c, "]") { depth = depth - 1 } - if str_eq(c, ":") { - if have_key { - if depth == 1 { - keys = el_list_append(keys, cur) - } - } - have_key = false - } - if str_eq(c, ",") { have_key = false } - } - i = i + 1 - } - return keys -} - // ═══════════════════════════════════════════════════════════════════════════ // SECTION B — engram HTTP client (provenance-carrying afferent LOAD) // ═══════════════════════════════════════════════════════════════════════════ @@ -402,168 +359,125 @@ fn head80(s: String) -> String { // We accumulate into module-level lists carried by the caller. // ═══════════════════════════════════════════════════════════════════════════ -// PROSE: chunk text into a discrete manifold. Split on blank lines into -// paragraphs; every non-empty paragraph is its own node (NEVER one blob). -// Edges: doc-root -contains-> chunk; chunk -precedes-> next chunk; -// most-recent-heading -section_of-> chunk. Content is verbatim (substring of -// the source) — pure extraction of ground truth. -fn transduce_prose(nodes: [String], edges: [String], text: String, - prov: String, ground: String, steward: String, - root_lid: String, root_title: String) -> [String] { - // returns [nodes_json_list_encoded, edges_json_list_encoded] is awkward in - // EL; instead we mutate by returning a 2-list. We package results as a - // single JSON array string carrying {nodes:[...],edges:[...]} additions. - // (Kept simple: caller passes empty lists and receives the packaged pair.) +// TRANSDUCE — the single mechanism. Takes ANY payload (prose, structured +// data, raw/opaque bytes — audio, whatever) as one opaque string and turns +// it into a discrete manifold: nodes + internal edges. There is no +// content-type branch anywhere in this function. It never asks "is this +// text," "is this JSON," "is this audio" — it runs ONE algorithm on the +// bytes it is given, unconditionally: +// +// 1. BOUNDARY SCAN — split on "\n\n". This is a property of the bytes +// (does a blank-line-style marker occur in them, yes or no), not a +// classification of what the content IS. Prose paragraphs split on it +// because that's how prose is typically written; that's a fact about +// the bytes, not a rule this function knows about prose. Anything else +// that happens to contain the same marker splits on it too, and +// anything that doesn't, doesn't — same code path either way. +// 2. FIXED-WINDOW FALLBACK — if step 1 found no boundary (0 or 1 non-empty +// piece), the payload is cut into fixed-size windows instead. Same +// chunk-per-node, edge-per-adjacency structure as step 1 produces; only +// the source of the cut point differs. +// +// Every resulting chunk becomes its own node (never one blob), wired with +// the same edges regardless of what's inside a chunk: root -contains-> +// chunk, chunk -precedes-> next chunk, and — if a chunk happens to start +// with "#" — most-recent-heading -section_of-> chunk. That "#" check is a +// structural marker (a fact about a chunk's first byte), not a decision +// about whether this run is "the text case": chunks from any payload that +// never happen to start with "#" simply never trigger it. +// +// Dedup is the existing, fully generic mechanism (find_existing_by_content, +// via merge_manifold downstream of merge_packed) applied uniformly to every +// chunk from every payload — there is no separate "structured" dedup path. +// Any deeper structure that might exist inside a payload (shared fields, +// repeated records, relationships) is NOT pre-computed here; that's left to +// the engram's own mechanisms (embedding, spreading activation, dedup) +// acting on this geometry over time, which is the whole point of handing it +// raw bytes instead of a hand-coded interpretation of them. +// +// Byte-safety note: `source` must already be a string this function can +// safely str_split/str_slice. Protecting it from silent truncation (El +// strings are NUL-unsafe under strlen-based ops; fs_read()'s result +// truncates at the first embedded NUL, which is routine in real binary +// bytes) is a MECHANICAL fidelity concern that belongs to whatever produced +// `source` (see ingest_file's file_source_string below) — not a +// content-type judgment made in here. transduce() never learns whether a +// chunk is plain text or a base64-encoded raw-byte window; every chunk is +// handled identically either way. +fn transduce(nodes: [String], edges: [String], source: String, + prov: String, ground: String, steward: String, + root_lid: String, root_title: String) -> [String] { let tagbase: String = "prov:" + prov + " ground:" + ground + " steward:" + steward - // root node - nodes = el_list_append(nodes, mk_node(root_lid, "document: " + root_title, - "Concept", "Semantic", "0.6", "0.6", "0.9", tagbase + " kind:document")) + nodes = el_list_append(nodes, mk_node(root_lid, "source: " + root_title, + "Concept", "Semantic", "0.6", "0.6", "0.9", tagbase + " kind:source")) - let paras: [String] = str_split(text, "\n\n") - let np: Int = el_list_len(paras) - let idx: Int = 0 + // step 1: universal boundary scan + let boundary_parts: [String] = str_split(source, "\n\n") + let chunks: [String] = el_list_empty() + let bp_n: Int = el_list_len(boundary_parts) + let bp_i: Int = 0 + while bp_i < bp_n { + let piece: String = str_trim(el_list_get(boundary_parts, bp_i)) + if !str_eq(piece, "") { chunks = el_list_append(chunks, piece) } + bp_i = bp_i + 1 + } + + // step 2: no boundary found -> fixed-size windows over the whole + // payload. 4096 chars/node: low kilobytes — big enough to keep node + // count sane on a large unbroken payload, small enough that each node + // stays a legible, individually embeddable/dedupable unit rather than + // one giant blob. + if el_list_len(chunks) <= 1 { + chunks = el_list_empty() + let total: Int = str_len(source) + let win: Int = 4096 + let off: Int = 0 + while off < total { + let endp: Int = if off + win < total { off + win } else { total } + let piece: String = str_slice(source, off, endp) + if !str_eq(piece, "") { chunks = el_list_append(chunks, piece) } + off = off + win + } + } + + let nc: Int = el_list_len(chunks) + let ci: Int = 0 let last_chunk: String = "" let last_heading: String = "" - let ci: Int = 0 - while idx < np { - let raw: String = str_trim(el_list_get(paras, idx)) - if !str_eq(raw, "") { - let lid: String = root_lid + ":c" + int_to_str(ci) - let is_heading: Bool = str_starts_with(raw, "#") - let kind: String = if is_heading { "kind:heading" } else { "kind:doc-chunk" } - nodes = el_list_append(nodes, mk_node(lid, raw, - "Knowledge", "Semantic", "0.55", "0.55", "0.9", tagbase + " " + kind)) - // containment: document root -contains-> chunk - edges = el_list_append(edges, mk_edge(root_lid, "contains", lid)) - // sequence: previous chunk -precedes-> this chunk - if !str_eq(last_chunk, "") { - edges = el_list_append(edges, mk_edge(last_chunk, "precedes", lid)) - } - // sectioning: most-recent heading -section_of-> this chunk - if is_heading { - last_heading = lid - } else { - if !str_eq(last_heading, "") { - edges = el_list_append(edges, mk_edge(last_heading, "section_of", lid)) - } - } - last_chunk = lid - ci = ci + 1 + while ci < nc { + let raw: String = el_list_get(chunks, ci) + let lid: String = root_lid + ":c" + int_to_str(ci) + let is_heading: Bool = str_starts_with(raw, "#") + let kind: String = if is_heading { "kind:heading" } else { "kind:chunk" } + nodes = el_list_append(nodes, mk_node(lid, raw, + "Knowledge", "Semantic", "0.55", "0.55", "0.9", tagbase + " " + kind)) + // containment: root -contains-> chunk + edges = el_list_append(edges, mk_edge(root_lid, "contains", lid)) + // sequence: previous chunk -precedes-> this chunk + if !str_eq(last_chunk, "") { + edges = el_list_append(edges, mk_edge(last_chunk, "precedes", lid)) } - idx = idx + 1 - } - // package: we return the two lists concatenated via a sentinel; but EL - // lists can't nest heterogeneously here, so we instead return nodes and - // rely on the caller holding edges by reference is not possible — so we - // encode both into one list: [ "N" + nodejson ... , "E" + edgejson ... ]. - let packed: [String] = el_list_empty() - let a: Int = 0 - let an: Int = el_list_len(nodes) - while a < an { packed = el_list_append(packed, "N" + el_list_get(nodes, a)) a = a + 1 } - let b: Int = 0 - let bn: Int = el_list_len(edges) - while b < bn { packed = el_list_append(packed, "E" + el_list_get(edges, b)) b = b + 1 } - return packed -} - -// STRUCTURED / RAW-GEOMETRY: ingest structured primitives (phonetics/formants, -// instrument signatures, scene primitives) as GEOMETRY, faithfully. Normalized -// input shape: -// {"dataset":"","primitive_type":"", -// "records":[{"key":"","features":{...categorical...},"attributes":{...}}]} -// Each record -> a primitive node; each categorical feature -> a SHARED feature -// node (deduped across records: many primitives -> one feature node = real -// connective geometry, meaning saturates); numeric attributes fold into the -// primitive's content (unique values, no dedup benefit). This is knowledge -// represented as geometry, not prose — the path speech/music/image ingest on. -fn transduce_structured(nodes: [String], edges: [String], js: String, - prov: String, ground: String, steward: String, - root_lid: String) -> [String] { - // grounding integrity: the SOURCE may declare its own epistemic grounding - // (measured / derived / convention / ...) via a top-level "grounding" field; - // honor it faithfully over the ingest-time default. This keeps the per-node - // ground: facet consistent with the source's honest self-description. - let src_ground: String = json_get_string(js, "grounding") - let use_ground: String = if str_eq(src_ground, "") { ground } else { src_ground } - let tagbase: String = "prov:" + prov + " ground:" + use_ground + " steward:" + steward - let dsname: String = json_get_string(js, "dataset") - let ptype: String = json_get_string(js, "primitive_type") - // capture the source's own scholarly provenance citation (verbatim) onto - // the dataset root — faithful attribution, retrievable, reachable from every - // primitive via its -contains- edge back to the root. - let src_cite: String = json_get_string(js, "provenance") - let root_content: String = "dataset: " + dsname + " (" + ptype + ")" - if !str_eq(src_cite, "") { root_content = root_content + " | provenance: " + src_cite } - nodes = el_list_append(nodes, mk_node(root_lid, root_content, - "Concept", "Semantic", "0.6", "0.6", "0.9", tagbase + " kind:dataset")) - - let recs: String = json_get_raw(js, "records") - let nr: Int = json_array_len(recs) - let r: Int = 0 - while r < nr { - let rec: String = json_array_get(recs, r) - let rkey: String = json_get_string(rec, "key") - let attrs: String = json_get_raw(rec, "attributes") - // faithful compact serialization of the primitive's numeric signature - let attr_str: String = flatten_pairs(attrs) - let content: String = ptype + " " + rkey - if !str_eq(attr_str, "") { content = content + " | " + attr_str } - let plid: String = root_lid + ":" + rkey - nodes = el_list_append(nodes, mk_node(plid, content, - "Concept", "Semantic", "0.6", "0.6", "0.92", - tagbase + " kind:primitive primitive:" + ptype + " key:" + rkey)) - edges = el_list_append(edges, mk_edge(root_lid, "contains", plid)) - - // categorical features -> SHARED (deduped) feature nodes + labelled edges - let feats: String = json_get_raw(rec, "features") - let fkeys: [String] = json_object_keys(feats) - let fk: Int = el_list_len(fkeys) - let k: Int = 0 - while k < fk { - let fname: String = el_list_get(fkeys, k) - let fval: String = json_get_string(feats, fname) - // shared feature node: content is the feature=value pair; identical - // pairs across records dedup onto ONE node (the geometry). - let flid: String = "feat:" + fname + "=" + fval - let fcontent: String = fname + "=" + fval - nodes = el_list_append(nodes, mk_node(flid, fcontent, - "Concept", "Semantic", "0.5", "0.5", "0.9", - tagbase + " kind:feature feature:" + fname)) - edges = el_list_append(edges, mk_edge(plid, fname, flid)) - k = k + 1 + // sectioning: most-recent heading -section_of-> this chunk + if is_heading { + last_heading = lid + } else { + if !str_eq(last_heading, "") { + edges = el_list_append(edges, mk_edge(last_heading, "section_of", lid)) + } } - r = r + 1 + last_chunk = lid + ci = ci + 1 } - let packed: [String] = el_list_empty() - let a: Int = 0 - let an: Int = el_list_len(nodes) - while a < an { packed = el_list_append(packed, "N" + el_list_get(nodes, a)) a = a + 1 } - let b: Int = 0 - let bn: Int = el_list_len(edges) - while b < bn { packed = el_list_append(packed, "E" + el_list_get(edges, b)) b = b + 1 } - return packed -} -// flatten a flat JSON object of scalar fields into "k=v k=v" (faithful; values -// verbatim). Used for numeric attribute signatures. -fn flatten_pairs(obj: String) -> String { - if str_eq(obj, "") { return "" } - let keys: [String] = json_object_keys(obj) - let n: Int = el_list_len(keys) - let out: String = "" - let i: Int = 0 - while i < n { - let k: String = el_list_get(keys, i) - // json_get_raw returns the raw token — works for NUMBERS (bare, e.g. - // "270") where json_get_string yields "" for non-string values. Strip - // surrounding quotes if the value happens to be a string token. - let raw: String = json_get_raw(obj, k) - let v: String = str_replace(raw, "\"", "") - let sep: String = if i == 0 { "" } else { " " } - out = out + sep + k + "=" + v - i = i + 1 - } - return out + // package both lists into one, "N"/"E"-prefixed (see merge_packed). + let packed: [String] = el_list_empty() + let pn_i: Int = 0 + let pn_n: Int = el_list_len(nodes) + while pn_i < pn_n { packed = el_list_append(packed, "N" + el_list_get(nodes, pn_i)) pn_i = pn_i + 1 } + let pe_i: Int = 0 + let pe_n: Int = el_list_len(edges) + while pe_i < pe_n { packed = el_list_append(packed, "E" + el_list_get(edges, pe_i)) pe_i = pe_i + 1 } + return packed } // unpack the "N"/"E"-prefixed packed list back into two lists, then merge @@ -594,18 +508,7 @@ fn basename(path: String) -> String { return el_list_get(parts, n - 1) } -fn ends_with_ci(s: String, suf: String) -> Bool { - return str_ends_with(str_to_lower(s), suf) -} - -fn is_text_file(path: String) -> Bool { - return ends_with_ci(path, ".md") || ends_with_ci(path, ".txt") - || ends_with_ci(path, ".markdown") || ends_with_ci(path, ".text") -} - // default ingestion grounding; overridable per-invocation via INGEST_GROUND. -// Note: a source's OWN top-level "grounding" field (structured) takes precedence -// over this — the author's honest self-description wins. fn default_ground() -> String { let g: String = env("INGEST_GROUND") if str_eq(g, "") { return "extracted" } @@ -618,25 +521,67 @@ fn default_steward() -> String { return s } -// ingest one file -> report JSON +// Mechanical fidelity guard — NOT a content-type test. fs_read()'s el_val_t +// result truncates at the first embedded NUL byte under El's strlen-based +// string ops (see fs_size's doc comment in runtime/el_runtime.h); comparing +// its length against fs_size() (a real stat()-based byte count) is a +// technical fact about whether the string channel captured the file intact +// — computed the same way for a poem, a JSON file, or a WAV, and saying +// nothing about what the file IS. When the counts agree, `text` is +// trustworthy verbatim. When they don't (silent truncation happened), +// rebuild the payload as base64-encoded fixed-size windows read directly +// off disk (fs_read_b64_chunk — binary-safe in C), joined with the same +// "\n\n" boundary marker transduce()'s generic scan already looks for, so +// transduce() sees one ordinary boundary-delimited payload and runs its one +// algorithm on it exactly as it would on prose — it never learns that a +// fidelity problem occurred upstream, let alone why. +fn file_source_string(path: String, text: String, real_size: Int) -> String { + if real_size <= 0 { return text } + if str_len(text) == real_size { return text } + // 3072 raw bytes -> 4096 base64 chars (3 divides evenly into base64's + // 3-byte/4-char ratio); keeps each resulting node's content a clean, + // bounded, low-kilobytes unit, same order of magnitude as the fixed + // fallback window in transduce() itself. + let win: Int = 3072 + let out: String = "" + let off: Int = 0 + let first: Bool = true + while off < real_size { + let chunk_b64: String = fs_read_b64_chunk(path, off, win) + if str_eq(chunk_b64, "") { + off = real_size + } else { + let sep: String = if first { "" } else { "\n\n" } + out = out + sep + chunk_b64 + first = false + off = off + win + } + } + return out +} + +// ingest one file -> report JSON. Uniform for every file regardless of +// extension or content — transduce() decides nothing about content-type, so +// neither does this function; it only decides whether the raw bytes made it +// through the read intact (file_source_string), which is a fidelity +// question, not a format one. fn ingest_file(path: String) -> String { + let real_size: Int = fs_size(path) let text: String = fs_read(path) - if str_eq(text, "") { + let source: String = file_source_string(path, text, real_size) + if str_eq(source, "") { return "{\"error\":\"empty or unreadable\",\"path\":" + j_q(path) + "}" } let prov: String = "file:" + path - if ends_with_ci(path, ".json") { - let packed: [String] = transduce_structured(el_list_empty(), el_list_empty(), - text, prov, default_ground(), default_steward(), "ds:" + basename(path)) - return merge_packed(packed) - } - let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(), - text, prov, default_ground(), default_steward(), + let packed: [String] = transduce(el_list_empty(), el_list_empty(), + source, prov, default_ground(), default_steward(), "doc:" + basename(path), basename(path)) return merge_packed(packed) } -// ingest a directory: walk one level, ingest each supported file, aggregate +// ingest a directory: walk one level, ingest every file found, aggregate. +// No extension filter — transduce() handles any payload uniformly now, so +// there is no content-type gate at the directory boundary either. fn ingest_dir(path: String) -> String { let entries: [String] = fs_list(path) let n: Int = el_list_len(entries) @@ -649,14 +594,12 @@ fn ingest_dir(path: String) -> String { let name: String = str_trim(el_list_get(entries, i)) if !str_eq(name, "") { let full: String = path + "/" + name - if is_text_file(full) || ends_with_ci(full, ".json") { - println("FILE " + full) - let rep: String = ingest_file(full) - tot_created = tot_created + json_get_int(rep, "nodes_created") - tot_deduped = tot_deduped + json_get_int(rep, "nodes_deduped") - tot_edges = tot_edges + json_get_int(rep, "edges_added") - files = files + 1 - } + println("FILE " + full) + let rep: String = ingest_file(full) + tot_created = tot_created + json_get_int(rep, "nodes_created") + tot_deduped = tot_deduped + json_get_int(rep, "nodes_deduped") + tot_edges = tot_edges + json_get_int(rep, "edges_added") + files = files + 1 } i = i + 1 } @@ -667,11 +610,12 @@ fn ingest_dir(path: String) -> String { ",\"edges_accepted\":" + int_to_str(tot_edges) + "}" } -// ingest a url: fetch, treat body as prose (faithful extraction of what's there) +// ingest a url: fetch, hand the body straight to transduce (faithful +// extraction of what's there — no interpretation of what it is) fn ingest_url(url: String) -> String { let body: String = http_get(url) if str_eq(body, "") { return "{\"error\":\"empty fetch\",\"url\":" + j_q(url) + "}" } - let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(), + let packed: [String] = transduce(el_list_empty(), el_list_empty(), body, "url:" + url, "extracted", "public-web", "url:" + url, url) return merge_packed(packed) @@ -686,7 +630,7 @@ fn ingest_llm(query: String) -> String { let resp: String = http_post_json("http://127.0.0.1:11434/api/generate", body) let answer: String = json_get_string(resp, "response") if str_eq(answer, "") { return "{\"error\":\"no model response\"}" } - let packed: [String] = transduce_prose(el_list_empty(), el_list_empty(), + let packed: [String] = transduce(el_list_empty(), el_list_empty(), answer, "llm:" + model + ":" + query, "candidate-provisional", "guide-provisional", "llm:" + query, "guide answer: " + query) return merge_packed(packed) @@ -728,6 +672,19 @@ fn ingest_stream(path: String) -> String { // SECTION G — ENTRY // ═══════════════════════════════════════════════════════════════════════════ +// INGEST_KIND selects an ACQUISITION mechanism only — dir/file/url/llm/ +// stream — i.e. which RPC shape to use to go get the bytes (walk a +// directory, open a file, fetch a URL, query an LLM, read a turn-stream). +// That is a genuinely unavoidable choice at the process-entry boundary +// (nothing about the string "/tmp/x" tells you whether it's a file to read +// or a stream to read line-by-line, or distinguishes an LLM query from a +// path), so it cannot be dropped the way content-type dispatch was. +// It is NOT a content-type flag: it says nothing about what's inside the +// bytes once fetched, and none of the five ingest_* functions it selects +// among interpret their payload differently by content shape anymore — +// they all hand off to the single, format-agnostic transduce(). The old +// "structured" value (a caller-declared alias for "file", used only to hint +// the now-removed JSON-vs-prose branch) is gone along with that branch. let kind: String = env("INGEST_KIND") let arg: String = env("INGEST_ARG") @@ -741,20 +698,16 @@ if str_eq(kind, "dir") { if str_eq(kind, "file") { report = ingest_file(arg) } else { - if str_eq(kind, "structured") { - report = ingest_file(arg) + if str_eq(kind, "url") { + report = ingest_url(arg) } else { - if str_eq(kind, "url") { - report = ingest_url(arg) + if str_eq(kind, "llm") { + report = ingest_llm(arg) } else { - if str_eq(kind, "llm") { - report = ingest_llm(arg) + if str_eq(kind, "stream") { + report = ingest_stream(arg) } else { - if str_eq(kind, "stream") { - report = ingest_stream(arg) - } else { - report = "{\"error\":\"unknown INGEST_KIND: " + kind + "\"}" - } + report = "{\"error\":\"unknown INGEST_KIND: " + kind + "\"}" } } } diff --git a/lang/el-compiler/src/codegen.el b/lang/el-compiler/src/codegen.el index 9b86040..d13410c 100644 --- a/lang/el-compiler/src/codegen.el +++ b/lang/el-compiler/src/codegen.el @@ -2766,6 +2766,8 @@ fn builtin_arity(name: String) -> Int { if str_eq(name, "fs_read") { return 1 } if str_eq(name, "fs_write") { return 2 } if str_eq(name, "fs_list") { return 1 } + if str_eq(name, "fs_size") { return 1 } + if str_eq(name, "fs_read_b64_chunk") { return 3 } // JSON if str_eq(name, "json_get") { return 2 } if str_eq(name, "json_parse") { return 1 } diff --git a/lang/runtime/el_runtime.c b/lang/runtime/el_runtime.c index 72046f3..a896549 100644 --- a/lang/runtime/el_runtime.c +++ b/lang/runtime/el_runtime.c @@ -2226,6 +2226,22 @@ el_val_t fs_exists(el_val_t pathv) { return (el_val_t)(stat(path, &st) == 0 ? 1 : 0); } +/* fs_size — real on-disk byte count of a file, via stat() (not strlen). + * Needed alongside fs_read_b64_chunk() below: fs_read()'s el_val_t string + * result is NUL-terminated and length-unsafe for arbitrary binary content + * (str_len/str_slice fall back to strlen when the buffer isn't a tagged + * binary value — see el_input_len), so any caller that needs to walk a + * binary file in fixed-size windows (e.g. transduce()'s raw/opaque chunking + * of audio bytes) must learn the true length here instead of from the + * decoded string. Returns -1 if the path doesn't exist or isn't stat-able. */ +el_val_t fs_size(el_val_t pathv) { + const char* path = EL_CSTR(pathv); + if (!path || !*path) return -1; + struct stat st; + if (stat(path, &st) != 0) return -1; + return (el_val_t)st.st_size; +} + /* fs_mkdir — create directory at path with mode 0755, mkdir -p semantics. * Returns 1 if path exists or was created (incl. all parents); 0 on failure. * Walks the path component-by-component so missing intermediate dirs are @@ -6847,10 +6863,16 @@ static float* eg_embed_fetch(const char* text, int32_t* out_dim) { else esc[w++] = (char)c; } esc[w] = '\0'; - size_t blen = w + strlen(eg_embed_model()) + 64; + size_t blen = w + strlen(eg_embed_model()) + 96; char* body = malloc(blen); if (!body) { free(esc); return NULL; } - snprintf(body, blen, "{\"model\":\"%s\",\"prompt\":\"%s\"}", + /* keep_alive:-1 pins the embed model resident in Ollama indefinitely + * (2026-08-15, PR #105 port). Without it the tiny embed model is evicted + * whenever a larger generation model loads (unified-memory pressure), so + * the NEXT search pays a cold model reload — measured cold reload up to + * ~2.2s vs ~0.02-0.05s warm, well inside ENGRAM_EMBED_TIMEOUT_MS but a + * real tax on every activate() call that lands cold. Pinning removes it. */ + snprintf(body, blen, "{\"model\":\"%s\",\"keep_alive\":-1,\"prompt\":\"%s\"}", eg_embed_model(), esc); free(esc); struct curl_slist* h = curl_slist_append(NULL, "Content-Type: application/json"); @@ -9508,6 +9530,36 @@ static inline double eg_cosq_at(EngramStore* g, double* cosq, unsigned char* cos return cosq[i]; } +/* ── Beam cap for engram_activate spreading activation (2026-08-15, PR #105 + * port) ────────────────────────────────────────────────────────────────── + * #105 measured the OLD (pre-adjacency-index, pre-qgate, pre-fan-effect) + * BFS reaching multi-second/crash territory at depth 2-3 from unbounded + * hub-node fan-out. That specific failure mode is already substantially + * mitigated here by mechanisms #105's branch predates: the adjacency index + * (O(degree) not O(E) per hop), the query-aware qgate (prunes semantically + * irrelevant branches), the ACT-R fan-effect correction (dampens popular- + * hub over-connectivity), and the 0.02 firing threshold. A beam cap is still + * a genuine additional, orthogonal bound: it caps WORST-CASE per-hop + * expansion width regardless of how many targets happen to pass the soft + * gates above, so it is kept as defense in depth rather than dropped as + * redundant. + * + * Bounds the number of frontier nodes EXPANDED per hop-level (see the + * level-batching in the BFS below). Every reached node still gets its + * best_bg[]/reached[] recorded and appears in the returned/promoted set — + * the cap bounds only how far ASSOCIATIVE SPREAD continues past a level, + * never the direct seed matches or the reported result set. Tunable via + * ENGRAM_ACTIVATE_BEAM (default 128, matching #105); set very high (e.g. + * the node count) to recover the pre-cap unbounded-per-level behaviour. */ +static int64_t engram_activate_beam(void) { + static int64_t v = -1; + if (v >= 0) return v; + const char* s = getenv("ENGRAM_ACTIVATE_BEAM"); + int64_t d = 128; + if (s && *s) { char* e = NULL; long t = strtol(s, &e, 10); if (e != s && t > 0) d = (int64_t)t; } + v = d; return v; +} + el_val_t engram_activate(el_val_t query, el_val_t depth) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); @@ -9552,25 +9604,43 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { backfilled++; } } - /* Query embedding, cached single-slot: the curiosity loop re-issues the - * same 4 rotating phrases, so consecutive identical queries skip the - * HTTP round-trip entirely. */ - static char* _eg_qcache_text = NULL; - static float* _eg_qcache_emb = NULL; - static int32_t _eg_qcache_dim = 0; + /* Query embedding cache (2026-08-15, PR #105 port: single-slot -> direct- + * mapped multi-slot). The single-slot cache below this comment's history + * only remembered the LAST query, so "the curiosity loop re-issues the + * same 4 rotating phrases" only hit when two CONSECUTIVE calls used the + * SAME phrase — any rotation among >1 phrase evicted the slot before it + * could be reused. #105 measured this as a real cost (a repeated query + * costing a full Ollama round-trip whenever a different phrase intervened) + * and fixed it with a direct-mapped, FNV-1a-keyed cache sized for the + * rotation. Ported here on TOP of the existing cosq/e_eff semantic layer + * (this cache only ever supplies q_emb/q_dim into that unchanged + * pipeline) rather than replacing it — see the M8/#105 reconciliation + * note above eg_cosq_at. ENGRAM_QCACHE_SIZE must be a power of two (mask + * indexing below). Full strcmp on lookup rejects hash collisions; each + * slot owns its `text`/`vec` and is freed on eviction, matching the old + * single-slot free/replace contract — q_emb below still points at cache- + * owned memory the caller must NOT free, just as before. */ +#define ENGRAM_QCACHE_SIZE 1024 + typedef struct { char* text; uint64_t hash; float* vec; int32_t dim; } EgQCacheEntry; + static EgQCacheEntry _eg_qcache[ENGRAM_QCACHE_SIZE]; float* q_emb = NULL; int32_t q_dim = 0; - if (_eg_qcache_text && strcmp(_eg_qcache_text, q) == 0) { - q_emb = _eg_qcache_emb; q_dim = _eg_qcache_dim; - } else { - int32_t d = 0; - float* v = eg_embed_fetch(q, &d); - if (v) { - free(_eg_qcache_text); free(_eg_qcache_emb); - _eg_qcache_text = strdup(q); - _eg_qcache_emb = v; - _eg_qcache_dim = d; - q_emb = v; q_dim = d; + { + uint64_t qh = engram_id_hash(q); + EgQCacheEntry* slot = &_eg_qcache[qh & (ENGRAM_QCACHE_SIZE - 1)]; + if (slot->vec && slot->hash == qh && slot->text && strcmp(slot->text, q) == 0) { + q_emb = slot->vec; q_dim = slot->dim; + } else { + int32_t d = 0; + float* v = eg_embed_fetch(q, &d); + if (v) { + free(slot->text); free(slot->vec); /* evict prior occupant */ + slot->text = strdup(q); + slot->hash = qh; + slot->vec = v; + slot->dim = d; + q_emb = v; q_dim = d; + } } } /* ── Context centroid fold-in (2026-07-29) ────────────────────────── @@ -9997,8 +10067,45 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { const double FAN_DREF = (g->adj_connected > 0) ? (2.0 * (double)g->edge_count / (double)g->adj_connected) : 0.0; _eg_act_fan_dref = FAN_DREF; + const int64_t activate_beam = engram_activate_beam(); while (fhead < ftail) { - Frontier f = fr[fhead++]; + /* Level-batch (2026-08-15, PR #105 port): entries sharing .hops are + * always contiguous — hop k+1 entries are appended only while + * processing hop k, strictly after the current ftail, so they form one + * block right after hop k's block (see engram_activate_beam's comment + * for why this holds even with the improve-and-re-enqueue behavior + * below). Find this level's extent, then beam-select which of it + * EXPANDS; every entry in the level still gets recorded via + * reached[]/best_bg[] regardless (that happened when it was enqueued, + * one level up) — the cap bounds propagation width only. */ + int64_t level_hops = fr[fhead].hops; + int64_t level_start = fhead; + int64_t level_end = fhead; + while (level_end < ftail && fr[level_end].hops == level_hops) level_end++; + int64_t level_n = level_end - level_start; + unsigned char* expand = NULL; + if (level_n > activate_beam) { + expand = calloc((size_t)level_n, 1); + if (expand) { + /* Partial selection: mark the top-`activate_beam` entries by + * .act. O(beam*level_n) — beam is the small tunable. */ + for (int64_t bsel = 0; bsel < activate_beam; bsel++) { + int64_t best = -1; + for (int64_t k = 0; k < level_n; k++) { + if (expand[k]) continue; + if (best < 0 || fr[level_start+k].act > fr[level_start+best].act) + best = k; + } + if (best < 0) break; + expand[best] = 1; + } + } + /* OOM on the selection map: expand stays NULL -> this level runs + * unbounded, same as if beam were disabled. Never silently wrong. */ + } + for (int64_t lk = level_start; lk < level_end; lk++) { + if (expand && !expand[lk - level_start]) continue; + Frontier f = fr[lk]; if (f.hops >= max_depth) continue; int64_t cur = f.idx; int64_t new_hops = f.hops + 1; @@ -10130,6 +10237,9 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { } } } + } + free(expand); + fhead = level_end; } /* Persist layer-1 background_activation to node store. */ for (int64_t i = 0; i < g->node_count; i++) { @@ -16375,6 +16485,41 @@ el_val_t el_base64_encode_n(const unsigned char* data, size_t len, int url_safe) return el_wrap_str(out); } +/* fs_read_b64_chunk — binary-safe windowed file read: read up to `length` + * raw bytes starting at byte `offset` from `path` and return them base64- + * encoded (RFC 4648, standard alphabet, padded). The raw bytes are read into + * a local C buffer and base64-encoded directly here — they never pass + * through an el_val_t string as raw bytes, so embedded NUL bytes (common in + * real PCM audio) never hit a strlen()-based code path. This mirrors the + * existing llm_vision() image-attachment path (read file -> base64 in C -> + * hand back a plain-ASCII string) and the http_*_to_file() rationale above: + * bypass the string wrapper entirely for the part that must stay binary. + * + * Returns "" if the path can't be opened, offset is negative or past EOF, + * or length <= 0. A short final chunk (less than `length` bytes remaining) + * is returned truncated to what's actually on disk — never padded/invented. */ +el_val_t fs_read_b64_chunk(el_val_t pathv, el_val_t offsetv, el_val_t lengthv) { + const char* path = EL_CSTR(pathv); + int64_t offset = (int64_t)offsetv; + int64_t length = (int64_t)lengthv; + if (!path || !*path || offset < 0 || length <= 0) return el_wrap_str(el_strdup("")); + FILE* f = fopen(path, "rb"); + if (!f) return el_wrap_str(el_strdup("")); + fseek(f, 0, SEEK_END); + long sz = ftell(f); + if (sz < 0 || offset >= sz) { fclose(f); return el_wrap_str(el_strdup("")); } + fseek(f, (long)offset, SEEK_SET); + long remain = sz - (long)offset; + size_t want = (size_t)(((int64_t)remain < length) ? remain : length); + unsigned char* buf = malloc(want > 0 ? want : 1); + if (!buf) { fclose(f); return el_wrap_str(el_strdup("")); } + size_t got = fread(buf, 1, want, f); + fclose(f); + el_val_t out = el_base64_encode_n(buf, got, /*url_safe=*/0); + free(buf); + return out; +} + /* Decode either alphabet — accepts both '+/' and '-_' transparently, and * tolerates missing padding (which JWTs typically omit). Whitespace is * skipped for robustness. Invalid characters cause the decode to stop and diff --git a/lang/runtime/el_runtime.h b/lang/runtime/el_runtime.h index 6824b81..d126993 100644 --- a/lang/runtime/el_runtime.h +++ b/lang/runtime/el_runtime.h @@ -235,6 +235,20 @@ el_val_t fs_list(el_val_t path); el_val_t fs_exists(el_val_t path); el_val_t fs_mkdir(el_val_t path); /* mkdir -p, mode 0755 */ +/* Real on-disk byte count via stat() — not strlen(). Use this (not + * str_len(fs_read(path))) when a file may contain binary content, since + * fs_read()'s result truncates at the first embedded NUL under strlen-based + * string ops. Returns -1 if the path doesn't exist. */ +el_val_t fs_size(el_val_t path); + +/* Binary-safe windowed read: read up to `length` bytes starting at byte + * `offset` from `path` and return them base64-encoded. Bytes are read and + * encoded in C without ever passing through an el_val_t string as raw + * bytes, so embedded NULs (routine in PCM audio) can't truncate the result. + * Returns "" on any failure or when offset is past EOF; a final short + * window returns only the bytes that actually exist on disk. */ +el_val_t fs_read_b64_chunk(el_val_t path, el_val_t offset, el_val_t length); + /* Length-explicit binary write. `length` is an Int (el_val_t holding the * byte count). The caller knows the length from context — typically because * `bytes` came from base64_decode (which produces a magic-tagged binary diff --git a/lang/runtime/el_seed.c b/lang/runtime/el_seed.c index aa4305b..e30585d 100644 --- a/lang/runtime/el_seed.c +++ b/lang/runtime/el_seed.c @@ -37,6 +37,82 @@ #include #include +/* el_runtime.c bridge prototypes. + * + * A block of __-prefixed wrappers further down in this file (http serving, + * JSON access, key-val state, URL/HTML escaping, and the whole engram_* + * node/edge/layer/search surface -- 51 symbols in total) delegate to + * unprefixed counterparts that are implemented in el_runtime.c, not here. + * Porting them into native el_seed.c or El has not happened yet. + * tools/install.sh compiles el_seed.c and el_runtime.c as separate objects + * and archives both into libel.a, so the symbols are always present at link + * time. el_seed.c alone was just missing the prototypes, which made even a + * standalone -c compile of this one file fail on a toolchain that now treats + * an implicit function declaration as a hard error under C11. + * + * A plain include of el_runtime.h was tried first and rejected: it redefines + * el_to_float and el_from_float, which el_seed.h already provides. Narrow + * prototypes, copied verbatim from el_runtime.h, avoid that collision without + * pulling in the rest of the retiring runtime header. + */ +el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body); +void http_serve(el_val_t port, el_val_t handler); +void http_serve_v2(el_val_t port, el_val_t handler); +el_val_t json_get(el_val_t json, el_val_t key); +el_val_t json_get_string(el_val_t json_str, el_val_t key); +el_val_t json_get_int(el_val_t json_str, el_val_t key); +el_val_t json_get_float(el_val_t json_str, el_val_t key); +el_val_t json_get_bool(el_val_t json_str, el_val_t key); +el_val_t json_get_raw(el_val_t json_str, el_val_t key); +el_val_t json_parse(el_val_t s); +el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value); +el_val_t json_stringify(el_val_t v); +el_val_t json_array_len(el_val_t json_str); +el_val_t json_array_get(el_val_t json_str, el_val_t index); +el_val_t json_array_get_string(el_val_t json_str, el_val_t index); +el_val_t state_set(el_val_t key, el_val_t value); +el_val_t state_get(el_val_t key); +el_val_t state_del(el_val_t key); +el_val_t state_keys(void); +el_val_t url_encode(el_val_t s); +el_val_t url_decode(el_val_t s); +el_val_t el_html_sanitize(el_val_t input_html, el_val_t allowlist_json); +el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience); +el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label, + el_val_t salience, el_val_t importance, el_val_t confidence, + el_val_t tier, el_val_t tags); +el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t label, + el_val_t salience, el_val_t certainty, el_val_t confidence, + el_val_t status, el_val_t tags, el_val_t layer_id); +el_val_t engram_add_layer(el_val_t name, el_val_t priority, el_val_t suppressible, + el_val_t transparent, el_val_t injectable); +el_val_t engram_remove_layer(el_val_t layer_id); +el_val_t engram_list_layers(void); +el_val_t engram_list_layers_json(void); +el_val_t engram_get_node(el_val_t id); +el_val_t engram_get_node_json(el_val_t id); +el_val_t engram_get_node_by_label(el_val_t label); +void engram_strengthen(el_val_t node_id); +void engram_forget(el_val_t node_id); +el_val_t engram_node_count(void); +el_val_t engram_edge_count(void); +el_val_t engram_scan_nodes(el_val_t limit, el_val_t offset); +el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset); +el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset); +el_val_t engram_search(el_val_t query, el_val_t limit); +el_val_t engram_search_json(el_val_t query, el_val_t limit); +el_val_t engram_activate(el_val_t query, el_val_t depth); +el_val_t engram_activate_json(el_val_t query, el_val_t depth); +el_val_t engram_compile_layered_json(el_val_t intent, el_val_t depth); +el_val_t engram_stats_json(void); +void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t relation); +el_val_t engram_edge_between(el_val_t from_id, el_val_t to_id); +el_val_t engram_neighbors(el_val_t node_id); +el_val_t engram_neighbors_filtered(el_val_t node_id, el_val_t max_depth, el_val_t direction); +el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction); +el_val_t engram_load(el_val_t path); +el_val_t engram_save(el_val_t path); + /* ── Private allocator ───────────────────────────────────────────────────── */ /* * el_seed.c carries its own arena for per-request allocation tracking. @@ -831,6 +907,219 @@ void __mutex_unlock(el_val_t m) { pthread_mutex_unlock(&_el_mutexes[slot]); } +/* ── Channels ─────────────────────────────────────────────────────────────── * + * Buffered MPMC channel backed by a mutex + condvar + circular buffer. + * Ported from the pre-restructure el_runtime.c (b2aac4b) — runtime/channel.el + * has always called these five primitives, but they were never carried + * forward into el_seed.c when el_runtime.c was consolidated onto the + * canonical release copy. Native channels were silently unlinkable on dev + * until this port. + * + * __channel_new(capacity) -> Int (handle) + * __channel_send(ch, msg) — blocks if full (capacity > 0) or never (unbounded) + * __channel_recv(ch) -> String — blocks until a message is available + * __channel_try_recv(ch) -> String — non-blocking, returns "" if empty + * __channel_close(ch) — signal no more sends; recv drains remaining + * + * Bounded channels (cap > 0): circular buffer, sender blocks when full. + * Unbounded channels (cap == 0): dynamic array, sender never blocks. + */ +#define EL_CHANNEL_MAX 64 +#define EL_CHANNEL_BUF 1024 + +typedef struct { + char** buf; + int cap; /* 0 = unbounded (grows dynamically) */ + int head, tail, count; + int dyn_cap; /* allocated slots for unbounded mode */ + int closed; + pthread_mutex_t mu; + pthread_cond_t not_empty; + pthread_cond_t not_full; +} ElChannel; + +static ElChannel _channels[EL_CHANNEL_MAX]; +static int _channel_count = 0; +static pthread_mutex_t _channel_alloc_mu = PTHREAD_MUTEX_INITIALIZER; + +el_val_t __channel_new(el_val_t capacity_v) { + int cap = (int)(int64_t)capacity_v; + if (cap < 0) cap = 0; + + pthread_mutex_lock(&_channel_alloc_mu); + if (_channel_count >= EL_CHANNEL_MAX) { + pthread_mutex_unlock(&_channel_alloc_mu); + fprintf(stderr, "[__channel_new] channel table full\n"); + return EL_INT(-1); + } + int slot = _channel_count++; + pthread_mutex_unlock(&_channel_alloc_mu); + + ElChannel* ch = &_channels[slot]; + memset(ch, 0, sizeof(*ch)); + ch->cap = cap; + ch->closed = 0; + ch->head = 0; + ch->tail = 0; + ch->count = 0; + + if (cap > 0) { + /* Bounded: fixed circular buffer. */ + ch->buf = (char**)malloc((size_t)cap * sizeof(char*)); + ch->dyn_cap = cap; + } else { + /* Unbounded: start with EL_CHANNEL_BUF slots, grow as needed. */ + ch->buf = (char**)malloc(EL_CHANNEL_BUF * sizeof(char*)); + ch->dyn_cap = EL_CHANNEL_BUF; + } + if (!ch->buf) { + fprintf(stderr, "[__channel_new] out of memory\n"); + return EL_INT(-1); + } + + pthread_mutex_init(&ch->mu, NULL); + pthread_cond_init(&ch->not_empty, NULL); + pthread_cond_init(&ch->not_full, NULL); + + return EL_INT(slot); +} + +el_val_t __channel_send(el_val_t ch_v, el_val_t msg_v) { + int slot = (int)(int64_t)ch_v; + if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR(""); + ElChannel* ch = &_channels[slot]; + + const char* msg = EL_CSTR(msg_v); + if (!msg) msg = ""; + char* copy = strdup(msg); /* channel owns the string */ + + pthread_mutex_lock(&ch->mu); + + if (ch->closed) { + /* Send on closed channel is a no-op (drop the message). */ + pthread_mutex_unlock(&ch->mu); + free(copy); + return EL_STR(""); + } + + if (ch->cap > 0) { + /* Bounded: block while full. */ + while (ch->count >= ch->cap && !ch->closed) { + pthread_cond_wait(&ch->not_full, &ch->mu); + } + if (ch->closed) { + pthread_mutex_unlock(&ch->mu); + free(copy); + return EL_STR(""); + } + ch->buf[ch->tail] = copy; + ch->tail = (ch->tail + 1) % ch->cap; + ch->count++; + } else { + /* Unbounded: grow the buffer if needed. */ + if (ch->count >= ch->dyn_cap) { + int new_cap = ch->dyn_cap * 2; + char** grown = (char**)realloc(ch->buf, (size_t)new_cap * sizeof(char*)); + if (!grown) { + pthread_mutex_unlock(&ch->mu); + free(copy); + fprintf(stderr, "[__channel_send] out of memory growing channel\n"); + return EL_STR(""); + } + /* The circular buffer may have wrapped. Linearise it first. + * In unbounded mode head is always 0 (we append at tail, drain + * from head), so a simple memmove isn't needed — but if the + * buffer did wrap (tail < head after growth), we need to fix up. + * Simplest safe path: if tail wrapped, move the head..old_cap + * segment to new_cap..new_cap+(old_cap-head). */ + if (ch->tail < ch->head) { + /* Wrapped: [head..old_cap) is the front, [0..tail) is the back. */ + int front = ch->dyn_cap - ch->head; + memmove(grown + ch->dyn_cap, grown + ch->head, (size_t)front * sizeof(char*)); + ch->head = ch->dyn_cap; + } + ch->buf = grown; + ch->dyn_cap = new_cap; + } + ch->buf[ch->tail] = copy; + ch->tail = (ch->tail + 1) % ch->dyn_cap; + ch->count++; + } + + pthread_cond_signal(&ch->not_empty); + pthread_mutex_unlock(&ch->mu); + return EL_STR(""); +} + +el_val_t __channel_recv(el_val_t ch_v) { + int slot = (int)(int64_t)ch_v; + if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR(""); + ElChannel* ch = &_channels[slot]; + + pthread_mutex_lock(&ch->mu); + + /* Block until there is a message or the channel is closed and drained. */ + while (ch->count == 0 && !ch->closed) { + pthread_cond_wait(&ch->not_empty, &ch->mu); + } + + if (ch->count == 0) { + /* Closed and empty — signal EOF. */ + pthread_mutex_unlock(&ch->mu); + return EL_STR(""); + } + + int buf_cap = (ch->cap > 0) ? ch->cap : ch->dyn_cap; + char* msg = ch->buf[ch->head]; + ch->head = (ch->head + 1) % buf_cap; + ch->count--; + + pthread_cond_signal(&ch->not_full); + pthread_mutex_unlock(&ch->mu); + + /* Hand the string to the arena so it is freed after the request. */ + seed_arena_track(msg); + return EL_STR(msg); +} + +el_val_t __channel_try_recv(el_val_t ch_v) { + int slot = (int)(int64_t)ch_v; + if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR(""); + ElChannel* ch = &_channels[slot]; + + pthread_mutex_lock(&ch->mu); + + if (ch->count == 0) { + pthread_mutex_unlock(&ch->mu); + return EL_STR(""); + } + + int buf_cap = (ch->cap > 0) ? ch->cap : ch->dyn_cap; + char* msg = ch->buf[ch->head]; + ch->head = (ch->head + 1) % buf_cap; + ch->count--; + + pthread_cond_signal(&ch->not_full); + pthread_mutex_unlock(&ch->mu); + + seed_arena_track(msg); + return EL_STR(msg); +} + +el_val_t __channel_close(el_val_t ch_v) { + int slot = (int)(int64_t)ch_v; + if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR(""); + ElChannel* ch = &_channels[slot]; + + pthread_mutex_lock(&ch->mu); + ch->closed = 1; + /* Wake all blocked recvers and senders so they can observe the close. */ + pthread_cond_broadcast(&ch->not_empty); + pthread_cond_broadcast(&ch->not_full); + pthread_mutex_unlock(&ch->mu); + return EL_STR(""); +} + /* ── Subprocess ──────────────────────────────────────────────────────────── */ el_val_t __exec(el_val_t cmd) { diff --git a/lang/runtime/el_seed.h b/lang/runtime/el_seed.h index 7597511..799bb7e 100644 --- a/lang/runtime/el_seed.h +++ b/lang/runtime/el_seed.h @@ -139,6 +139,13 @@ el_val_t __mutex_new(void); void __mutex_lock(el_val_t m); void __mutex_unlock(el_val_t m); +/* Buffered MPMC channel (runtime/channel.el). capacity=0 means unbounded. */ +el_val_t __channel_new(el_val_t capacity); +el_val_t __channel_send(el_val_t ch, el_val_t msg); /* blocks if bounded+full */ +el_val_t __channel_recv(el_val_t ch); /* blocks until available */ +el_val_t __channel_try_recv(el_val_t ch); /* non-blocking, "" if empty */ +el_val_t __channel_close(el_val_t ch); + /* ── Subprocess ──────────────────────────────────────────────────────────── */ el_val_t __exec(el_val_t cmd); /* popen, capture all stdout, return String */ diff --git a/tools/neuron-sandbox/README.md b/tools/neuron-sandbox/README.md index 5dd776d..a9dd9cf 100644 --- a/tools/neuron-sandbox/README.md +++ b/tools/neuron-sandbox/README.md @@ -173,4 +173,7 @@ Each ad-hoc harness becomes `nsbx run …` (or `--source` build) against ## Env knobs `NSBX_ROOT`, `NSBX_PORT_BASE`, `NSBX_RSS_BOUND_MB`, `NSBX_REMERGE_THRESHOLD`, +`NSBX_READY_TIMEOUT_SECS` (default 15 — how long `up`/`create`/`build` wait for a +daemon to answer `/api/stats` before reporting failure; raise it if a boot is +legitimately slow under concurrent sandbox/CPU load rather than actually broken), `EL_REPO` (for `elc` + runtime sources), `ENGRAM_LIVE_DATA_DIR`, `ENGRAM_LIVE_PLIST`. diff --git a/tools/neuron-sandbox/nsbx b/tools/neuron-sandbox/nsbx index 249780a..ffeedeb 100755 --- a/tools/neuron-sandbox/nsbx +++ b/tools/neuron-sandbox/nsbx @@ -32,6 +32,12 @@ EL_REPO="${EL_REPO:-$HOME/Development/neuron-technologies/foundation/el}" PORT_BASE="${NSBX_PORT_BASE:-8900}" RSS_BOUND_MB="${NSBX_RSS_BOUND_MB:-550}" # from store-fix reboot-proof (aaf13f88) REMERGE_THRESHOLD="${NSBX_REMERGE_THRESHOLD:-40000}" +# readiness-poll window for start_daemon (0.5s ticks). Default unchanged (15s) — +# but a cold boot against the full live store, under concurrent CPU contention +# from other running sandboxes, has been observed live to take well past that. +# Bump per-invocation with NSBX_READY_TIMEOUT_SECS if `up`/`create` reports a +# not-ready failure but the daemon looks otherwise fine (see its logs/daemon.log). +READY_TICKS=$(( ${NSBX_READY_TIMEOUT_SECS:-15} * 2 )) KEYSTONES=( "kn-efeb4a5b-5aff-4759-8a97-7233099be6ee" "kn-5b606390-a52d-4ca2-8e0e-eba141d13440" ) # fixed probe set for retrieval-parity (stable, identity-anchored) PARITY_QUERIES=( "who am I" "self identity core" "engram store durability" "keystone self anchor" "grounding honesty" ) @@ -77,7 +83,7 @@ _port_claimed(){ # is another sandbox already assigned this port? live_stats(){ curl -s -m5 "$LIVE_URL/api/stats" 2>/dev/null; } api(){ # api [json-body] local name="$1" path="$2" body="${3:-}" - local port; port="$(mget "$name" "['port']")"; [ -n "$port" ] || die "unknown sandbox: $name" + local port; port="$(mget "$name" "['port']")"; [ -n "$port" ] || die "unknown sandbox: $name (run: nsbx list)" local url="http://127.0.0.1:${port}${path}" if [ -n "$body" ]; then curl -s -m30 -X POST -H 'Content-Type: application/json' -d "$body" "$url" else curl -s -m30 "$url"; fi @@ -88,6 +94,19 @@ stat_field(){ printf '%s' "$1" | sed -n "s/.*\"$2\":\([0-9]*\).*/\1/p"; } daemon_pid(){ local f; f="$(sdir "$1")/daemon.pid"; [ -f "$f" ] && cat "$f" || true; } daemon_alive(){ local p; p="$(daemon_pid "$1")"; [ -n "$p" ] && kill -0 "$p" 2>/dev/null; } +# daemon_health : prints "stopped" | "running" | "unresponsive" (to stdout). +# "running" means the pid is alive AND /api/stats actually answered — process +# liveness alone (daemon_alive) is not proof the HTTP server is serving; a pegged +# or hung process still passes kill -0. Short timeout (2s) since this runs per-row +# in `nsbx list`. +daemon_health(){ + local name="$1" + daemon_alive "$name" || { echo "stopped"; return 0; } + local port; port="$(mget "$name" "['port']")" + local s; s="$(curl -s -m2 "http://127.0.0.1:${port}/api/stats" 2>/dev/null)" + [ -n "$s" ] && echo "running" || echo "unresponsive" +} + # ---------------------------------------------------------------- elc/build ---- find_elc(){ command -v elc 2>/dev/null && return 0 @@ -123,6 +142,49 @@ _build_binary(){ ok "built: $out ($(ls -lh "$out" | awk '{print $5}'), sha $(sha "$out" | cut -c1-12))" } +# bin_built_at : human-readable build timestamp. `cp -p` preserves mtime, +# so this is the ORIGINAL build time even for binaries copied stock-prod into a +# sandbox — not the copy time. +bin_built_at(){ stat -f '%Sm' -t '%Y-%m-%d %H:%M:%S' "$1" 2>/dev/null || echo "unknown"; } + +# _binary_freshness : best-effort staleness note (empty string if fresh/ +# unknown — never guesses). Two cases: +# - stock-prod: compares the sha recorded at create time against the CURRENTLY +# configured live real binary's sha (recomputed now, not cached) — catches +# "live prod moved on since this sandbox was cloned". +# - branch/source/rebuilt: compares the recorded source_commit against the +# LOCAL origin/dev ref (no fetch — reads whatever the repo already has) — +# catches "built from a commit that predates current dev tip". +_binary_freshness(){ + local name="$1" src; src="$(mget "$name" "['source']")" + case "$src" in + stock-prod:*) + local live_bin cur_sha rec_sha + live_bin="$(_live_real_bin)" + [ -n "$live_bin" ] && [ -x "$live_bin" ] || return 0 + cur_sha="$(sha "$live_bin")"; rec_sha="$(mget "$name" "['binary_sha256']")" + [ -n "$cur_sha" ] && [ -n "$rec_sha" ] && [ "$cur_sha" != "$rec_sha" ] \ + && printf 'stale: live prod binary has moved on since this sandbox was cloned (live is now %s, sha %s) — nsbx build %s --binary %s to catch up' \ + "$(basename "$live_bin")" "${cur_sha:0:12}" "$name" "$live_bin" + ;; + branch:*|source:*|rebuilt:*) + local commit cur behind + commit="$(mget "$name" "['source_commit']")" + [ -n "$commit" ] || return 0 + cur="$(git -C "$EL_REPO" rev-parse origin/dev 2>/dev/null)" || return 0 + [ -n "$cur" ] && [ "$commit" != "$cur" ] || return 0 + git -C "$EL_REPO" merge-base --is-ancestor "$commit" "$cur" 2>/dev/null || return 0 + behind="$(git -C "$EL_REPO" rev-list --count "$commit..$cur" 2>/dev/null)" + printf 'stale: built from %s, %s commit(s) behind local origin/dev (%s) — nsbx build %s --branch origin/dev' \ + "${commit:0:12}" "${behind:-?}" "${cur:0:12}" "$name" + ;; + esac +} + +# _source_commit : best-effort git HEAD of a source tree used to build a +# sandbox binary, empty if not a git repo (e.g. a prebuilt --binary path has none). +_source_commit(){ git -C "$1" rev-parse HEAD 2>/dev/null || true; } + # ---------------------------------------------------------------- daemon ------- # start_daemon : boots the sandbox's real engram binary on its isolated # port against its cloned data dir, with the SAME auto-remerge net the live soul @@ -133,9 +195,9 @@ start_daemon(){ local port bin data export key port="$(mget "$name" "['port']")"; bin="$d/bin/engram"; data="$d/data" key="sbx-$name"; export="$data/.scan-export.reseed-clean.json" - [ -x "$bin" ] || die "sandbox binary missing: $bin" + [ -x "$bin" ] || die "sandbox binary missing: $bin (run: nsbx build $name --source DIR | --branch REF, or nsbx destroy $name && nsbx create $name to reclone stock-prod)" [ "$port" != "$LIVE_BIND_PORT" ] && [ "$port" != "$SOUL_PORT" ] || die "refusing forbidden port $port" - [ -f "$data/neuron.egm" ] || die "sandbox has no cloned store: $data/neuron.egm" + [ -f "$data/neuron.egm" ] || die "sandbox has no cloned store: $data/neuron.egm (data dir is corrupt/incomplete — run: nsbx destroy $name && nsbx create $name)" # HARD guard: never point a sandbox daemon at the live data dir. [ "$(cd "$data" && pwd -P)" != "$(cd "$LIVE_DATA_DIR" && pwd -P)" ] || die "refusing: sandbox data dir resolves to LIVE store" @@ -150,11 +212,11 @@ start_daemon(){ echo "$pid" > "$d/daemon.pid" # readiness poll local url="http://127.0.0.1:$port" i s - for i in $(seq 1 30); do + for i in $(seq 1 "$READY_TICKS"); do s="$(curl -s -m3 "$url/api/stats" 2>/dev/null)" [ -n "$s" ] && break; sleep 0.5 done - [ -n "$s" ] || { warn "daemon did not become ready (see $d/logs/daemon.log)"; return 1; } + [ -n "$s" ] || { warn "daemon did not become ready within ${NSBX_READY_TIMEOUT_SECS:-15}s (see $d/logs/daemon.log). pid $pid may still be alive and slow to boot under load — check: lsof -iTCP:$port -P, or retry with NSBX_READY_TIMEOUT_SECS=45"; return 1; } ok "ready pid=$pid boot-stats: $s" # auto-remerge net (idempotent): match live edge population if the export is present if [ -f "$export" ]; then @@ -231,33 +293,35 @@ cmd_create(){ info "live baseline stats: ${lstats:-}" # ---- determine + place the runtime binary (versioned into the snapshot) ---- - local source_desc live_bin + local source_desc live_bin source_commit="" live_bin="$(_live_real_bin)" if [ -n "$binpath" ]; then [ -x "$binpath" ] || die "not an executable binary: $binpath" cp -p "$binpath" "$d/bin/engram"; source_desc="prebuilt:$binpath" elif [ -n "$src" ]; then _build_binary "$src" "$d/bin/engram" "$d/build"; source_desc="source:$src" + source_commit="$(_source_commit "$src")" elif [ -n "$branch" ]; then log "worktree: $repo @ $branch -> $d/build/worktree" git -C "$repo" worktree add --detach "$d/build/worktree" "$branch" >/dev/null 2>&1 \ || die "git worktree add failed ($repo @ $branch)" _build_binary "$d/build/worktree" "$d/bin/engram" "$d/build"; source_desc="branch:$branch@$repo" + source_commit="$(_source_commit "$d/build/worktree")" else [ -x "$live_bin" ] || die "cannot resolve live ENGRAM_REAL_BIN: $live_bin" cp -p "$live_bin" "$d/bin/engram"; source_desc="stock-prod:$live_bin" fi local bin_sha; bin_sha="$(sha "$d/bin/engram")" - info "runtime: $source_desc (sha ${bin_sha:0:12})" + info "runtime: $source_desc (sha ${bin_sha:0:12}, built $(bin_built_at "$d/bin/engram"))" # ---- write manifest ---- - python3 - "$name" "$port" "$source_desc" "$bin_sha" "$egm_sha" "$base_nodes" "$base_edges" "$(sha "$live_bin" 2>/dev/null)" <<'PY' > "$(manifest "$name")" + python3 - "$name" "$port" "$source_desc" "$bin_sha" "$egm_sha" "$base_nodes" "$base_edges" "$(sha "$live_bin" 2>/dev/null)" "$source_commit" <<'PY' > "$(manifest "$name")" import json,sys,datetime -name,port,src,binsha,egmsha,bn,be,livebinsha=sys.argv[1:9] +name,port,src,binsha,egmsha,bn,be,livebinsha,source_commit=sys.argv[1:10] json.dump({ "name":name,"port":int(port),"created_at":datetime.datetime.now(datetime.timezone.utc).isoformat(), "source":src,"binary_sha256":binsha,"clone_egm_sha256":egmsha, - "live_binary_sha256":livebinsha, + "live_binary_sha256":livebinsha,"source_commit":source_commit, "live_baseline":{"node_count":int(bn or 0),"edge_count":int(be or 0)}, "keystones":["kn-efeb4a5b-5aff-4759-8a97-7233099be6ee","kn-5b606390-a52d-4ca2-8e0e-eba141d13440"] }, sys.stdout, indent=2) @@ -268,6 +332,17 @@ PY start_daemon "$name" || die "daemon failed to start" local sstats; sstats="$(sbx_stats "$name")" local sbn sbe; sbn="$(stat_field "$sstats" node_count)"; sbe="$(stat_field "$sstats" edge_count)" + # a boot immediately followed by an auto-remerge can leave the daemon briefly + # busy — retry rather than silently folding a 0/0 baseline into the manifest. + # `validate`'s zero-loss/reboot-prove checks compare current counts >= baseline, + # so a 0/0 baseline would make them trivially PASS regardless of real data loss. + local _bi + for _bi in 1 2 3 4 5; do + [ -n "$sbn" ] && [ "$sbn" != "0" ] && break + sleep 1 + sstats="$(sbx_stats "$name")"; sbn="$(stat_field "$sstats" node_count)"; sbe="$(stat_field "$sstats" edge_count)" + done + [ -z "$sbn" ] || [ "$sbn" = "0" ] && warn "sandbox stats still empty/zero after retries — recording sbx_baseline 0/0. This makes 'nsbx validate $name' zero-loss checks trivially pass; investigate before trusting a validate PASS: nsbx status $name" _capture_retrieval "$name" "$d/baseline/retrieval.json" # fold sandbox baseline into manifest python3 - "$(manifest "$name")" "$sbn" "$sbe" <<'PY' @@ -320,7 +395,12 @@ except Exception: print("[]")' 2>/dev/null)" # thereafter. Prod on :$LIVE_BIND_PORT/:$SOUL_PORT is unreachable from here by design. cmd_up(){ local name; if [ $# -gt 0 ] && [ "${1#-}" = "$1" ]; then name="$1"; shift; else name="${USER:-dev}-dev"; fi - if mexists "$name"; then daemon_alive "$name" || start_daemon "$name"; else cmd_create "$name" "$@"; fi + if mexists "$name"; then + daemon_alive "$name" || start_daemon "$name" \ + || die "daemon did not become ready — see $(sdir "$name")/logs/daemon.log (try: nsbx up $name again once you've checked the log)" + else + cmd_create "$name" "$@" + fi local port; port="$(mget "$name" "['port']")" echo >&2 ok "your sandbox '$name' is ready at http://127.0.0.1:$port (a private copy of the mind — prod is untouchable)" @@ -334,34 +414,37 @@ cmd_up(){ # it on the SAME clone + port (the code-change dev loop, in place). cmd_build(){ local name="$1"; shift || true - mexists "$name" || die "no such sandbox: $name" + mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)" local src="" branch="" repo="$EL_REPO" while [ $# -gt 0 ]; do case "$1" in --source) src="$2"; shift 2;; --branch) branch="$2"; shift 2;; --repo) repo="$2"; shift 2;; *) die "unknown flag: $1";; esac; done local d; d="$(sdir "$name")" stop_daemon "$name" - if [ -n "$src" ]; then _build_binary "$src" "$d/bin/engram" "$d/build" + local source_commit="" + if [ -n "$src" ]; then _build_binary "$src" "$d/bin/engram" "$d/build"; source_commit="$(_source_commit "$src")" elif [ -n "$branch" ]; then rm -rf "$d/build/worktree" 2>/dev/null; git -C "$repo" worktree prune 2>/dev/null git -C "$repo" worktree add --detach "$d/build/worktree" "$branch" >/dev/null 2>&1 || die "worktree add failed" _build_binary "$d/build/worktree" "$d/bin/engram" "$d/build" + source_commit="$(_source_commit "$d/build/worktree")" else die "usage: nsbx build --source DIR | --branch REF [--repo R]"; fi # record new binary sha - python3 - "$(manifest "$name")" "$(sha "$d/bin/engram")" "${src:-branch:$branch}" <<'PY' -import json,sys; mf,s,src=sys.argv[1:4] -d=json.load(open(mf)); d["binary_sha256"]=s; d["source"]="rebuilt:"+src + python3 - "$(manifest "$name")" "$(sha "$d/bin/engram")" "${src:-branch:$branch}" "$source_commit" <<'PY' +import json,sys; mf,s,src,source_commit=sys.argv[1:5] +d=json.load(open(mf)); d["binary_sha256"]=s; d["source"]="rebuilt:"+src; d["source_commit"]=source_commit json.dump(d,open(mf,'w'),indent=2) PY - start_daemon "$name" + start_daemon "$name" \ + || die "rebuilt binary did not become ready — see $d/logs/daemon.log (the old binary is gone; fix the code and re-run nsbx build $name ...)" ok "rebuilt + restarted on :$(mget "$name" "['port']")" } # ================================================================ run ========== cmd_run(){ local name="$1"; shift || true - mexists "$name" || die "no such sandbox: $name" - daemon_alive "$name" || start_daemon "$name" + mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)" + daemon_alive "$name" || start_daemon "$name" || die "daemon not running and failed to start — see $(sdir "$name")/logs/daemon.log" local d port; d="$(sdir "$name")"; port="$(mget "$name" "['port']")" # direct API form: nsbx run api [json] if [ "${1:-}" = "api" ]; then @@ -395,8 +478,8 @@ cmd_run(){ # RSS bound; retrieval parity; keystone integrity. cmd_validate(){ local name="$1"; shift || true - mexists "$name" || die "no such sandbox: $name" - daemon_alive "$name" || start_daemon "$name" + mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)" + daemon_alive "$name" || start_daemon "$name" || die "daemon not running and failed to start — see $(sdir "$name")/logs/daemon.log" local d port key; d="$(sdir "$name")"; port="$(mget "$name" "['port']")"; key="sbx-$name" local url="http://127.0.0.1:$port" local bn be; bn="$(mget "$name" "['sbx_baseline']['node_count']")"; be="$(mget "$name" "['sbx_baseline']['edge_count']")" @@ -489,7 +572,7 @@ PY # Default is a DRY-RUN plan; requires --i-approve-prod-cutover to actually cut over. cmd_promote(){ local name="$1"; shift || true - mexists "$name" || die "no such sandbox: $name" + mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)" local approve=0 do_data=0 while [ $# -gt 0 ]; do case "$1" in --i-approve-prod-cutover) approve=1; shift;; @@ -588,7 +671,7 @@ PY # ================================================================ destroy ====== cmd_destroy(){ local name="$1"; shift || true - mexists "$name" || die "no such sandbox: $name" + mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)" local d; d="$(sdir "$name")" stop_daemon "$name" if [ -d "$d/build/worktree" ]; then @@ -604,22 +687,44 @@ cmd_destroy(){ # ================================================================ list/status == cmd_list(){ [ -d "$SBX_ROOT" ] || { echo "no sandboxes"; return 0; } - printf '%-16s %-6s %-8s %-9s %s\n' NAME PORT STATE PID SOURCE + printf '%-16s %-6s %-13s %-9s %-19s %s\n' NAME PORT STATE PID "BUILT" SOURCE local m for m in "$SBX_ROOT"/*/manifest.json; do [ -f "$m" ] || continue - local n p src pid state + local n p src pid state bpath built fresh n="$(python3 -c "import json;print(json.load(open('$m'))['name'])")" p="$(python3 -c "import json;print(json.load(open('$m'))['port'])")" src="$(python3 -c "import json;print(json.load(open('$m'))['source'])")" - pid="$(daemon_pid "$n")"; state="stopped"; daemon_alive "$n" && state="running" - printf '%-16s %-6s %-8s %-9s %s\n' "$n" "$p" "$state" "${pid:-–}" "$src" + pid="$(daemon_pid "$n")" + case "$(daemon_health "$n")" in + running) state="running";; + unresponsive) state="running(!resp)";; + *) state="stopped";; + esac + bpath="$(sdir "$n")/bin/engram"; built="$([ -f "$bpath" ] && bin_built_at "$bpath" || echo unknown)" + fresh="$(_binary_freshness "$n")"; [ -n "$fresh" ] && src="[STALE] $src" + printf '%-16s %-6s %-13s %-9s %-19s %s\n' "$n" "$p" "$state" "${pid:-–}" "$built" "$src" done + info "state 'running(!resp)' = process alive but /api/stats didn't answer — see: nsbx status " } cmd_status(){ - local name="$1"; mexists "$name" || die "no such sandbox: $name" + local name="$1"; mexists "$name" || die "no such sandbox: $name (run: nsbx list to see what exists)" python3 -m json.tool "$(manifest "$name")" - daemon_alive "$name" && echo "state: running (pid $(daemon_pid "$name")) stats: $(sbx_stats "$name")" || echo "state: stopped" + local bpath; bpath="$(sdir "$name")/bin/engram" + if [ -f "$bpath" ]; then + echo "binary: sha=$(sha "$bpath" | cut -c1-12) built=$(bin_built_at "$bpath")" + local fresh; fresh="$(_binary_freshness "$name")" + [ -n "$fresh" ] && printf '%s%s%s\n' "$C_YEL" "$fresh" "$C_0" + fi + case "$(daemon_health "$name")" in + running) + echo "state: running (pid $(daemon_pid "$name")) stats: $(sbx_stats "$name")";; + unresponsive) + printf '%sstate: running but NOT RESPONDING%s (pid %s) — process alive, /api/stats returned nothing.\n' "$C_RED" "$C_0" "$(daemon_pid "$name")" + info "check: tail -50 $(sdir "$name")/logs/daemon.log | next: kill -9 $(daemon_pid "$name") && nsbx up $name" + ;; + *) echo "state: stopped";; + esac [ -f "$(sdir "$name")/validate.json" ] && { echo "--- last validation ---"; python3 -m json.tool "$(sdir "$name")/validate.json"; } }