Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 will.anderson
|
c7a78ab1eb |
+5
-17
@@ -41,29 +41,17 @@ fn strip_query(path: String) -> String {
|
||||
str_slice(path, 0, q)
|
||||
}
|
||||
|
||||
// query_param — extract one query-string value, URL-DECODED.
|
||||
//
|
||||
// The decode step was missing (found 2026-08-15): a claim sent as
|
||||
// "test%20claim" arrived at engram_assert_json still percent-encoded and was
|
||||
// stored/compared that way, so any value containing a space, &, =, or non-ASCII
|
||||
// character silently became a different string than the caller sent. Affects
|
||||
// every GET route that reads params this way, not just /api/assert.
|
||||
fn query_param(path: String, key: String) -> String {
|
||||
let q: Int = str_index_of(path, "?")
|
||||
if q < 0 { return "" }
|
||||
let qs: String = str_slice(path, q + 1, str_len(path))
|
||||
// Anchor the match to a real key boundary: prefixing "&" and searching for
|
||||
// "&key=" means "q" can never match inside "faq=". (Found 2026-08-15:
|
||||
// "?faq=X&q=Y" returned X for key "q" — a silently wrong value, not an
|
||||
// error.) The leading "&" makes the first parameter match the same way.
|
||||
let hay: String = "&" + qs
|
||||
let needle: String = "&" + key + "="
|
||||
let pos: Int = str_index_of(hay, needle)
|
||||
let needle: String = key + "="
|
||||
let pos: Int = str_index_of(qs, needle)
|
||||
if pos < 0 { return "" }
|
||||
let after: String = str_slice(hay, pos + str_len(needle), str_len(hay))
|
||||
let after: String = str_slice(qs, pos + str_len(needle), str_len(qs))
|
||||
let amp: Int = str_index_of(after, "&")
|
||||
let raw: String = if amp < 0 { after } else { str_slice(after, 0, amp) }
|
||||
return __url_decode(raw)
|
||||
if amp < 0 { return after }
|
||||
str_slice(after, 0, amp)
|
||||
}
|
||||
|
||||
fn query_int(path: String, key: String, default_val: Int) -> Int {
|
||||
|
||||
+12
-23
@@ -62,45 +62,34 @@ This is where almost all work belongs. El programs are source files that get com
|
||||
|
||||
This is the self-contained C OS-boundary layer. It provides the `__`-prefixed primitives that compiled El programs call: libcurl HTTP, pthreads, filesystem I/O, arena allocation, etc. It is **not generated** — it is maintained by hand.
|
||||
|
||||
The runtime is native El (`runtime/*.el`) over a C OS-boundary. **Status (verified 2026-08-15):** the migration to a seed-only boundary is *in progress, not done*. Two files exist:
|
||||
- `runtime/el_runtime.c` (~860 KB) — **LIVE**. Holds the engram store (`EngramStore engram_global`) plus the `http_*`/`json_*`/`state_*`/`engram_*` impls. It is the authoritative single-file link target for the compiler, and `tools/install.sh` compiles it into `libel.a`. This is where a new C builtin's *implementation* must currently live to be linkable.
|
||||
- `runtime/el_seed.c` — the intended hand-maintained `__`-prefixed seed (thin wrappers over the above). It is compiled alongside `el_runtime.c` by `tools/install.sh`, but does **not** compile standalone yet (see the build-path caveat under "Rebuilding the Compiler").
|
||||
The old `el_runtime.c` has been archived to `runtime/legacy/`. The runtime is now native El (`runtime/*.el`). `el_seed.c` replaces `el_runtime.c` as the sole C compilation dependency.
|
||||
|
||||
**Only edit these when you genuinely need OS-level access** (raw sockets, GPU calls, new libcurl features, a new engram store op). For everything else, write El.
|
||||
**Only edit `el_seed.c` when you genuinely need OS-level access** (raw sockets, GPU calls, new libcurl features). For everything else, write El.
|
||||
|
||||
When you add a C builtin (verbatim-emit recipe — the El name is emitted as the exact C symbol; `builtin_arity` is an arity guard only, not a dispatch table):
|
||||
1. Implement the C function in `el_runtime.c` (and declare it in `el_runtime.h`).
|
||||
2. Add a `__`-prefixed thin wrapper in `el_seed.c` and declare it in `el_seed.h`.
|
||||
3. Add the name to `builtin_arity` in `el-compiler/src/codegen.el` — add **both** the plain and `__`-prefixed spellings.
|
||||
4. Rebuild the elc binary (see below) and confirm the self-host fixpoint is byte-identical.
|
||||
|
||||
Worked example: the `engram_assert_json` (op_assert seam) and `engram_node_full_in`/`engram_connect_in` (purview write-side) primitives added 2026-08-15 follow exactly this recipe.
|
||||
When you do add a C builtin:
|
||||
1. Add the C function to `el_seed.c`
|
||||
2. Declare it in `el_seed.h`
|
||||
3. Add it to the `builtin_arity` table in `el-compiler/src/codegen.el` (so the compiler knows the arg count)
|
||||
4. Rebuild the elc binary (see below)
|
||||
|
||||
---
|
||||
|
||||
## Rebuilding the Compiler
|
||||
|
||||
After changing any `.el` source in `el-compiler/src/` (run from the `lang/` dir):
|
||||
After changing any `.el` source in `el-compiler/src/`:
|
||||
|
||||
```bash
|
||||
# 1. Stage2: current elc compiles the (modified) compiler to C
|
||||
cd /Users/will/Development/neuron-technologies/foundation/el
|
||||
./dist/platform/elc elc-cli.el > elc-new.c
|
||||
# 2. Build the new compiler. The C link target is el_runtime.c — it holds the
|
||||
# engram store + http/json/state impls the compiler output calls. el_runtime.c
|
||||
# self-hosts elc on its own; el_seed.c is the (aspirational) seed layer and does
|
||||
# NOT compile standalone under clang (missing prototypes for the el_runtime.c
|
||||
# symbols it wraps — see caveat below), so link el_runtime.c here.
|
||||
cc -std=c11 -I runtime -lcurl -lpthread \
|
||||
-o dist/platform/elc-new \
|
||||
elc-new.c runtime/el_runtime.c
|
||||
# 3. Verify self-hosting FIXPOINT (stage3 == stage2 output, byte-identical):
|
||||
elc-new.c runtime/el_seed.c
|
||||
# Verify self-hosting:
|
||||
./dist/platform/elc-new elc-cli.el > elc-verify.c
|
||||
diff elc-new.c elc-verify.c # must be identical
|
||||
diff elc-new.c elc-verify.c # should be identical
|
||||
mv dist/platform/elc-new dist/platform/elc
|
||||
```
|
||||
|
||||
> **Build-path caveat (verified 2026-08-15).** `el_seed.c` is the intended hand-maintained OS-boundary seed, but it does **not** compile standalone under modern clang: it wraps ~16 unprefixed `el_runtime.c` symbols (`http_serve`, `json_*`, `state_*`, `http_response`) without prototypes, and clang treats implicit declarations as errors (C99+). The productionised install (`tools/install.sh`) builds `libel.a` from **both** `el_seed.o` + `el_runtime.o` together, which is why linking succeeds there. To make `el_seed.c` build on its own, add prototypes for those symbols (or `#include "el_runtime.h"`, reconciling the `__http_serve` return-type mismatch first). Until then, `el_runtime.c` is the authoritative single-file link target for the compiler.
|
||||
|
||||
After changing `el_seed.c` only (no El source changes), rebuild downstream programs but do NOT need to rebuild the compiler binary itself — the seed is linked at the application level, not the compiler level.
|
||||
|
||||
---
|
||||
|
||||
@@ -2760,9 +2760,6 @@ fn builtin_arity(name: String) -> Int {
|
||||
if str_eq(name, "__engram_neighbors_filtered") { return 3 }
|
||||
if str_eq(name, "__engram_activate") { return 2 }
|
||||
if str_eq(name, "__engram_activate_json") { return 2 }
|
||||
if str_eq(name, "__engram_op_assert_json") { return 2 }
|
||||
if str_eq(name, "__engram_node_full_in") { return 9 }
|
||||
if str_eq(name, "__engram_connect_in") { return 5 }
|
||||
if str_eq(name, "__engram_scan_nodes_json") { return 2 }
|
||||
if str_eq(name, "__generate") { return 1 }
|
||||
// Filesystem
|
||||
@@ -2865,9 +2862,6 @@ fn builtin_arity(name: String) -> Int {
|
||||
if str_eq(name, "engram_neighbors_json") { return 3 }
|
||||
if str_eq(name, "engram_activate_json") { return 2 }
|
||||
if str_eq(name, "engram_stats_json") { return 0 }
|
||||
if str_eq(name, "engram_op_assert_json") { return 2 }
|
||||
if str_eq(name, "engram_node_full_in") { return 9 }
|
||||
if str_eq(name, "engram_connect_in") { return 5 }
|
||||
// LLM
|
||||
if str_eq(name, "llm_call") { return 2 }
|
||||
if str_eq(name, "llm_call_system") { return 3 }
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
#!/bin/bash
|
||||
# build_vindex_bench.sh — build the vindex_bench oracle/proof harness, with
|
||||
# the real ggml + hand-rolled-Metal batch-cosine strategies on Darwin and a
|
||||
# zero-dependency CPU-only stub everywhere else. Mirrors the two-step recipe
|
||||
# documented in vindex_bench.c's own header comment; this script exists so
|
||||
# that recipe is one command, not a copy-pasted paragraph.
|
||||
#
|
||||
# Darwin build links FOUR strategy translation units:
|
||||
# eg_cosine_batch.c — the Factory (always)
|
||||
# eg_cosine_batch_strategy_cpu.c — universal fallback (always)
|
||||
# eg_cosine_batch_strategy_ggml.c — ggml + dynamic Metal backend plugin
|
||||
# eg_cosine_batch_strategy_metal_hand.m — PR #114's original hand-rolled
|
||||
# Metal shader, preserved as one
|
||||
# selectable strategy
|
||||
# plus -DEG_HAVE_STRATEGY_GGML -DEG_HAVE_STRATEGY_METAL_HAND so the Factory
|
||||
# (and vindex_bench.c's own direct strategy comparison) knows both exist.
|
||||
#
|
||||
# ggml is resolved via `brew --prefix ggml` when available (portable across
|
||||
# Intel /usr/local and Apple Silicon /opt/homebrew installs), falling back to
|
||||
# /opt/homebrew if brew isn't on PATH. Override with GGML_PREFIX=... env var.
|
||||
#
|
||||
# Usage: ./build_vindex_bench.sh [output_path]
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
OUT="${1:-./vindex_bench}"
|
||||
CC="${CC:-cc}"
|
||||
|
||||
if [ "$(uname -s)" = "Darwin" ]; then
|
||||
GGML_PREFIX="${GGML_PREFIX:-$(brew --prefix ggml 2>/dev/null || echo /opt/homebrew)}"
|
||||
echo "== Darwin: building with the ggml + hand-rolled-Metal strategies (ggml prefix: $GGML_PREFIX) =="
|
||||
|
||||
"$CC" -O2 -std=c11 -x objective-c \
|
||||
-c eg_cosine_batch_strategy_metal_hand.m -o /tmp/eg_cosine_batch_strategy_metal_hand.o \
|
||||
-framework Metal -framework Foundation
|
||||
|
||||
"$CC" -O2 -std=c11 -DEG_HAVE_STRATEGY_GGML -DEG_HAVE_STRATEGY_METAL_HAND -w \
|
||||
-I"$GGML_PREFIX/include" \
|
||||
vindex_bench.c engram_vindex.c \
|
||||
eg_cosine_batch.c eg_cosine_batch_strategy_cpu.c eg_cosine_batch_strategy_ggml.c \
|
||||
/tmp/eg_cosine_batch_strategy_metal_hand.o \
|
||||
-L"$GGML_PREFIX/lib" -lggml -lggml-base \
|
||||
-Wl,-rpath,"$GGML_PREFIX/lib" \
|
||||
-lm -framework Metal -framework Foundation -o "$OUT"
|
||||
else
|
||||
echo "== non-Darwin: building with the CPU-only fallback strategy (no ggml, no Metal) =="
|
||||
"$CC" -O2 -std=c11 -w vindex_bench.c engram_vindex.c \
|
||||
eg_cosine_batch.c eg_cosine_batch_strategy_cpu.c \
|
||||
-lm -o "$OUT"
|
||||
fi
|
||||
|
||||
echo "built: $OUT"
|
||||
@@ -1,121 +0,0 @@
|
||||
/* eg_cosine_batch.c — the Factory. Implements the stable public interface
|
||||
* declared in eg_cosine_batch.h by selecting ONE concrete
|
||||
* EgCosineBatchStrategy (eg_cosine_batch_strategy.h) and dispatching every
|
||||
* call to it. This is the ONLY file that branches on EG_HAVE_STRATEGY_*
|
||||
* (build-time: which strategy .c/.m files were actually compiled in for
|
||||
* this platform) — call sites never see those macros.
|
||||
*
|
||||
* Selection is lazy (first call) and cached — mirrors the lazy-init caching
|
||||
* every individual strategy already does internally, so there is no added
|
||||
* per-call cost after the first.
|
||||
*
|
||||
* Selection mechanism (env var + build-time + runtime capability probe, all
|
||||
* three, exactly as directed):
|
||||
* - BUILD-TIME decides which strategies exist to choose from at all: a
|
||||
* Darwin build compiles+links the ggml strategy and the hand-rolled
|
||||
* Metal strategy (EG_HAVE_STRATEGY_GGML / EG_HAVE_STRATEGY_METAL_HAND
|
||||
* both defined); a non-Darwin build compiles neither, matching PR #114's
|
||||
* original Linux behavior exactly (CPU-fallback only, no Objective-C
|
||||
* compiler or Metal frameworks required).
|
||||
* - RUNTIME CAPABILITY PROBE: each candidate strategy's own available()
|
||||
* does the real, cheap-after-first-call check (device present, backend
|
||||
* plugin loaded, pipeline compiled) — never assumed from build-time
|
||||
* alone. A build that HAS the ggml strategy compiled in but is running
|
||||
* on hardware/software where it can't actually initialize (backend
|
||||
* plugin missing, no GPU) correctly falls through to the next candidate.
|
||||
* - ENV VAR gives explicit, debuggable override for either axis:
|
||||
* EL_COSINE_BATCH_STRATEGY = "ggml" | "metal" | "cpu" | unset/"auto"
|
||||
* forces a specific strategy (falling back to cpu if the forced one
|
||||
* isn't actually available), or leaves the default auto-preference
|
||||
* order in place.
|
||||
* EL_METAL_COSINE = 0/n/N/f/F (back-compat with PR #114's vindex_bench
|
||||
* gate) disables ALL GPU-backed strategies outright, same as before.
|
||||
*
|
||||
* DEFAULT preference order when nothing is forced: ggml, then hand-rolled
|
||||
* Metal, then CPU fallback — first candidate whose available() reports true
|
||||
* wins. This is what makes "stop hand-rolling GPU kernels, use ggml" real
|
||||
* rather than nominal: ggml is what actually runs by default on this
|
||||
* machine today (see the PR body for the measured numbers backing that).
|
||||
*/
|
||||
#include "eg_cosine_batch.h"
|
||||
#include "eg_cosine_batch_strategy.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
static bool g_selected = false;
|
||||
static const EgCosineBatchStrategy* g_active = NULL;
|
||||
|
||||
static bool eg_env_truthy_off(const char* v) {
|
||||
return v && (v[0]=='0' || v[0]=='n' || v[0]=='N' || v[0]=='f' || v[0]=='F');
|
||||
}
|
||||
|
||||
static const EgCosineBatchStrategy* eg_select_strategy(void) {
|
||||
if (g_selected) return g_active;
|
||||
g_selected = true;
|
||||
|
||||
const EgCosineBatchStrategy* cpu = eg_cosine_batch_strategy_cpu();
|
||||
const char* force = getenv("EL_COSINE_BATCH_STRATEGY");
|
||||
const char* legacy_off = getenv("EL_METAL_COSINE");
|
||||
|
||||
if (eg_env_truthy_off(legacy_off)) { g_active = cpu; return g_active; }
|
||||
|
||||
if (force && strcmp(force, "cpu") == 0) { g_active = cpu; return g_active; }
|
||||
|
||||
if (force && strcmp(force, "ggml") == 0) {
|
||||
#ifdef EG_HAVE_STRATEGY_GGML
|
||||
const EgCosineBatchStrategy* s = eg_cosine_batch_strategy_ggml();
|
||||
if (s->available()) { g_active = s; return g_active; }
|
||||
#endif
|
||||
g_active = cpu; return g_active;
|
||||
}
|
||||
|
||||
if (force && strcmp(force, "metal") == 0) {
|
||||
#ifdef EG_HAVE_STRATEGY_METAL_HAND
|
||||
const EgCosineBatchStrategy* s = eg_cosine_batch_strategy_metal_hand();
|
||||
if (s->available()) { g_active = s; return g_active; }
|
||||
#endif
|
||||
g_active = cpu; return g_active;
|
||||
}
|
||||
|
||||
/* auto (unset, or any other value): ggml -> metal-hand -> cpu, first
|
||||
* available wins. */
|
||||
#ifdef EG_HAVE_STRATEGY_GGML
|
||||
{
|
||||
const EgCosineBatchStrategy* s = eg_cosine_batch_strategy_ggml();
|
||||
if (s->available()) { g_active = s; return g_active; }
|
||||
}
|
||||
#endif
|
||||
#ifdef EG_HAVE_STRATEGY_METAL_HAND
|
||||
{
|
||||
const EgCosineBatchStrategy* s = eg_cosine_batch_strategy_metal_hand();
|
||||
if (s->available()) { g_active = s; return g_active; }
|
||||
}
|
||||
#endif
|
||||
g_active = cpu;
|
||||
return g_active;
|
||||
}
|
||||
|
||||
bool eg_cosine_batch_available(void) {
|
||||
return eg_select_strategy()->available();
|
||||
}
|
||||
|
||||
const char* eg_cosine_batch_strategy_name(void) {
|
||||
return eg_select_strategy()->name;
|
||||
}
|
||||
|
||||
bool eg_cosine_batch(const float* query, int32_t qdim,
|
||||
const float* const* node_ptrs,
|
||||
const int32_t* node_dims,
|
||||
int32_t n,
|
||||
double* out_scores) {
|
||||
return eg_select_strategy()->batch(query, qdim, node_ptrs, node_dims, n, out_scores);
|
||||
}
|
||||
|
||||
bool eg_cosine_batch_multi(const float* queries, int32_t qdim, int32_t nq,
|
||||
const float* const* node_ptrs,
|
||||
const int32_t* node_dims,
|
||||
int32_t n,
|
||||
double* out_scores) {
|
||||
return eg_select_strategy()->batch_multi(queries, qdim, nq, node_ptrs, node_dims, n, out_scores);
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
/* eg_cosine_batch.h — stable Adapter interface over batch-cosine-similarity
|
||||
* BACKEND STRATEGIES. This header is the ONE thing call sites (el_runtime.c,
|
||||
* vindex_bench.c, ...) talk to. Plain C11, safe to #include on every
|
||||
* platform — the symbols declared here always exist and always link,
|
||||
* regardless of what backend actually runs underneath. Zero #ifdef at call
|
||||
* sites: which concrete strategy executes (ggml/Metal, hand-rolled Metal, or
|
||||
* the always-false CPU fallback) is resolved once, lazily, inside
|
||||
* eg_cosine_batch.c's factory — see eg_cosine_batch_strategy.h for that.
|
||||
*
|
||||
* This supersedes eg_metal_cosine.h (PR #114's single hand-rolled-Metal-only
|
||||
* bridge). The contract is UNCHANGED — same shapes, same sentinel, same
|
||||
* never-partial guarantee, same "caller must always be prepared to fall back
|
||||
* to its own scalar per-node loop" rule — only the name changed, because the
|
||||
* thing behind it is no longer "the Metal bridge," it is "whichever batch-
|
||||
* cosine strategy the factory picked." eg_metal_cosine.h's original doc
|
||||
* comments (byte-for-byte, this file is the direct descendant) are preserved
|
||||
* below since they remain the precise spec any strategy must honor.
|
||||
*
|
||||
* On ANY failure at ANY step — no compute device, compile/init error, alloc
|
||||
* failure, bad args — every function here returns false and writes nothing.
|
||||
* Out-params are either fully populated or left completely untouched, never
|
||||
* partial. Callers MUST always be prepared to fall back to their own scalar
|
||||
* per-node CPU loop unconditionally. These functions must never crash, throw,
|
||||
* or hang the calling process — several call sites run inside a long-lived
|
||||
* daemon's request-handling hot path.
|
||||
*/
|
||||
#ifndef EG_COSINE_BATCH_H
|
||||
#define EG_COSINE_BATCH_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* Batched cosine similarity: one query vector against `n` node vectors.
|
||||
*
|
||||
* query — qdim floats, the query embedding. Raw/unnormalized.
|
||||
* qdim — query dimensionality (e.g. 768 for nomic-embed-text).
|
||||
* node_ptrs — array of n pointers, node_ptrs[i] pointing at a (possibly
|
||||
* differently-owned, possibly NULL) float vector for node i.
|
||||
* NOT required to be contiguous — every strategy performs the
|
||||
* gather into a packed row-major matrix internally, exactly
|
||||
* mirroring how EngramNode.emb is one malloc per node.
|
||||
* node_dims — array of n ints, node_dims[i] = that node's real emb_dim
|
||||
* (0 or mismatched vs qdim => that node scores -2.0, matching
|
||||
* eg_cosine's null/dim-mismatch/zero-norm sentinel exactly).
|
||||
* n — number of nodes.
|
||||
* out_scores — caller-owned array of n doubles; out_scores[i] is filled
|
||||
* with the cosine similarity of node i against query, or
|
||||
* -2.0 for a null/dim-mismatched/zero-norm node — bit-for-bit
|
||||
* the same contract as eg_cosine(node_ptrs[i], query, qdim).
|
||||
*
|
||||
* Returns true iff a real backend strategy ran and out_scores was fully
|
||||
* populated. Returns false (out_scores left untouched) on ANY failure or
|
||||
* unavailability — no compute device, compile/init failure, allocation
|
||||
* failure, n<=0, qdim<=0, null query/node_ptrs/node_dims/out_scores.
|
||||
*/
|
||||
bool eg_cosine_batch(const float* query, int32_t qdim,
|
||||
const float* const* node_ptrs,
|
||||
const int32_t* node_dims,
|
||||
int32_t n,
|
||||
double* out_scores);
|
||||
|
||||
/* True iff a real (non-CPU-fallback) strategy is available right now (cheap
|
||||
* after the first call — cached). Purely informational (e.g. a startup log
|
||||
* line or /api/stats field); callers should still treat a false return from
|
||||
* eg_cosine_batch()/eg_cosine_batch_multi() itself as the authoritative
|
||||
* fallback signal, not this function. */
|
||||
bool eg_cosine_batch_available(void);
|
||||
|
||||
/* Which concrete strategy is currently selected — "ggml", "metal-hand",
|
||||
* or "cpu-fallback". Purely informational/diagnostic, same spirit as
|
||||
* eg_cosine_batch_available(). Never NULL. */
|
||||
const char* eg_cosine_batch_strategy_name(void);
|
||||
|
||||
/* Multi-query batched cosine: nq query vectors against the SAME n node
|
||||
* vectors, in one call. A real strategy uploads/prepares the node population
|
||||
* ONCE and reuses it for every query, instead of nq separate
|
||||
* eg_cosine_batch() calls each paying the full gather+upload cost — PR #114
|
||||
* measured this necessary: at N~=13.7k/dim=768, repeating the single-query
|
||||
* call per query was slower than the CPU baseline; batching queries together
|
||||
* is what makes a GPU-backed path a real win at this shape. Use this
|
||||
* whenever multiple queries will run against an unchanged (or
|
||||
* rarely-changing) node population; use eg_cosine_batch() for a genuinely
|
||||
* one-off comparison.
|
||||
*
|
||||
* queries — nq*qdim floats, row-major (query i at queries+i*qdim).
|
||||
* out_scores — caller-owned nq*n doubles, row-major
|
||||
* (out_scores[i*n+j] = cosine(queries[i], node j)), same
|
||||
* -2.0 sentinel semantics as eg_cosine_batch().
|
||||
*
|
||||
* Returns true iff a real strategy ran and out_scores was fully populated
|
||||
* (all nq*n entries); false (untouched) on any failure/unavailability. */
|
||||
bool eg_cosine_batch_multi(const float* queries, int32_t qdim, int32_t nq,
|
||||
const float* const* node_ptrs,
|
||||
const int32_t* node_dims,
|
||||
int32_t n,
|
||||
double* out_scores);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* EG_COSINE_BATCH_H */
|
||||
@@ -1,156 +0,0 @@
|
||||
/* eg_cosine_batch.metal — batched cosine similarity, one query vs N node vectors.
|
||||
*
|
||||
* GPU-shaped counterpart to eg_cosine() in el_runtime.c: same math, same
|
||||
* dim-mismatch/zero-norm sentinel (-2.0), applied to N independent rows in
|
||||
* parallel instead of one pair at a time in a CPU loop.
|
||||
*
|
||||
* Semantics MUST match eg_cosine() exactly:
|
||||
* - inputs are raw, UNNORMALIZED vectors (nomic-embed-text magnitudes are
|
||||
* not 1.0) — this kernel computes the full dot/(|a|*|b|) cosine, not a
|
||||
* plain dot product.
|
||||
* - a node whose declared dim differs from the query dim, or whose norm is
|
||||
* zero, scores exactly -2.0 (below any valid cosine in [-1,1]), so a
|
||||
* caller doing `if (score > threshold)` behaves identically whether the
|
||||
* scalar or the batched path filled the array.
|
||||
*
|
||||
* Precision: Apple GPUs do not support double in Metal Shading Language —
|
||||
* everything here is float32. eg_cosine accumulates in CPU double, but its
|
||||
* *inputs* are float32 embeddings, so the achievable precision ceiling is
|
||||
* bounded by the input data regardless of accumulator width. To keep the
|
||||
* float32 reduction from drifting relative to the double-accumulated CPU
|
||||
* result across dim=768 terms, each thread accumulates with 4 independent
|
||||
* partial sums (unrolled) rather than one running scalar — the same
|
||||
* error-reduction trick already used by the CPU brute-force loop in
|
||||
* vindex_bench.c. The measured float-vs-double delta is reported in the PR
|
||||
* description; this is not assumed to be "close enough" without measurement.
|
||||
*/
|
||||
#include <metal_stdlib>
|
||||
using namespace metal;
|
||||
|
||||
/* Per-dispatch invariants. `dim` is the query's dimensionality — the
|
||||
* dimensionality every comparable node vector must match. */
|
||||
struct EgCosineParams {
|
||||
uint n; /* number of node rows */
|
||||
uint dim; /* vector width (both query and node rows are `dim` wide in
|
||||
* the packed buffer; node_dims[] carries each node's REAL
|
||||
* embedded dim for the mismatch check) */
|
||||
};
|
||||
|
||||
/* One thread per node row. node_matrix is n*dim floats, row-major, packed at
|
||||
* `dim` stride regardless of a row's real dim (the CPU side zero-pads or
|
||||
* skips packing rows that don't match — see eg_cosine_batch_metal in
|
||||
* eg_metal_cosine.m for the exact packing contract). node_dims[i] is the
|
||||
* node's true emb_dim, used only for the mismatch sentinel — never used to
|
||||
* index, since every row is packed at uniform `dim` stride. */
|
||||
kernel void eg_cosine_batch_kernel(
|
||||
device const float* query [[buffer(0)]],
|
||||
device const float* node_matrix [[buffer(1)]],
|
||||
device const int* node_dims [[buffer(2)]],
|
||||
constant EgCosineParams& p [[buffer(3)]],
|
||||
device float* out_scores [[buffer(4)]],
|
||||
uint gid [[thread_position_in_grid]])
|
||||
{
|
||||
if (gid >= p.n) return;
|
||||
|
||||
if (node_dims[gid] != int(p.dim)) {
|
||||
out_scores[gid] = -2.0f;
|
||||
return;
|
||||
}
|
||||
|
||||
device const float* row = node_matrix + (uint64_t)gid * (uint64_t)p.dim;
|
||||
|
||||
/* 4-way partial accumulation — same shape as vindex_bench.c's brute_topk
|
||||
* unroll, done here for float32 accuracy rather than raw throughput. */
|
||||
float dot0 = 0.0f, dot1 = 0.0f, dot2 = 0.0f, dot3 = 0.0f;
|
||||
float na0 = 0.0f, na1 = 0.0f, na2 = 0.0f, na3 = 0.0f;
|
||||
float nb0 = 0.0f, nb1 = 0.0f, nb2 = 0.0f, nb3 = 0.0f;
|
||||
|
||||
uint d = 0;
|
||||
uint dim4 = p.dim & ~3u;
|
||||
for (; d < dim4; d += 4) {
|
||||
float a0 = row[d], b0 = query[d];
|
||||
float a1 = row[d+1], b1 = query[d+1];
|
||||
float a2 = row[d+2], b2 = query[d+2];
|
||||
float a3 = row[d+3], b3 = query[d+3];
|
||||
dot0 += a0*b0; dot1 += a1*b1; dot2 += a2*b2; dot3 += a3*b3;
|
||||
na0 += a0*a0; na1 += a1*a1; na2 += a2*a2; na3 += a3*a3;
|
||||
nb0 += b0*b0; nb1 += b1*b1; nb2 += b2*b2; nb3 += b3*b3;
|
||||
}
|
||||
float dot = (dot0 + dot1) + (dot2 + dot3);
|
||||
float na = (na0 + na1) + (na2 + na3);
|
||||
float nb = (nb0 + nb1) + (nb2 + nb3);
|
||||
for (; d < p.dim; d++) {
|
||||
float a = row[d], b = query[d];
|
||||
dot += a*b; na += a*a; nb += b*b;
|
||||
}
|
||||
|
||||
if (na <= 0.0f || nb <= 0.0f) {
|
||||
out_scores[gid] = -2.0f;
|
||||
return;
|
||||
}
|
||||
out_scores[gid] = dot / sqrt(na * nb);
|
||||
}
|
||||
|
||||
/* ── multi-query variant ──────────────────────────────────────────────────
|
||||
* Same per-pair math as eg_cosine_batch_kernel, but amortizes ONE upload of
|
||||
* node_matrix (the expensive part at real store size — 13k*768 floats is
|
||||
* ~42MB) across `nq` queries instead of re-uploading it once per query.
|
||||
* Measured need: a naive one-query-at-a-time loop calling the single-query
|
||||
* kernel nq times was SLOWER than the CPU oracle at N≈13.7k (re-gather +
|
||||
* re-upload dominated the actual compute) — this is the fix, not a
|
||||
* hypothetical optimization.
|
||||
*
|
||||
* 2D grid: x = node index [0,n), y = query index [0,nq). out_scores is
|
||||
* nq*n, row-major by query (out_scores[qid*n + nid]). */
|
||||
struct EgCosineMultiParams { uint n; uint dim; uint nq; };
|
||||
|
||||
kernel void eg_cosine_batch_multi_kernel(
|
||||
device const float* queries [[buffer(0)]], /* nq*dim */
|
||||
device const float* node_matrix [[buffer(1)]], /* n*dim */
|
||||
device const int* node_dims [[buffer(2)]], /* n */
|
||||
constant EgCosineMultiParams& p [[buffer(3)]],
|
||||
device float* out_scores [[buffer(4)]], /* nq*n */
|
||||
uint2 gid [[thread_position_in_grid]])
|
||||
{
|
||||
uint nid = gid.x, qid = gid.y;
|
||||
if (nid >= p.n || qid >= p.nq) return;
|
||||
|
||||
uint64_t out_idx = (uint64_t)qid * (uint64_t)p.n + (uint64_t)nid;
|
||||
|
||||
if (node_dims[nid] != int(p.dim)) {
|
||||
out_scores[out_idx] = -2.0f;
|
||||
return;
|
||||
}
|
||||
|
||||
device const float* row = node_matrix + (uint64_t)nid * (uint64_t)p.dim;
|
||||
device const float* query = queries + (uint64_t)qid * (uint64_t)p.dim;
|
||||
|
||||
float dot0 = 0.0f, dot1 = 0.0f, dot2 = 0.0f, dot3 = 0.0f;
|
||||
float na0 = 0.0f, na1 = 0.0f, na2 = 0.0f, na3 = 0.0f;
|
||||
float nb0 = 0.0f, nb1 = 0.0f, nb2 = 0.0f, nb3 = 0.0f;
|
||||
|
||||
uint d = 0;
|
||||
uint dim4 = p.dim & ~3u;
|
||||
for (; d < dim4; d += 4) {
|
||||
float a0 = row[d], b0 = query[d];
|
||||
float a1 = row[d+1], b1 = query[d+1];
|
||||
float a2 = row[d+2], b2 = query[d+2];
|
||||
float a3 = row[d+3], b3 = query[d+3];
|
||||
dot0 += a0*b0; dot1 += a1*b1; dot2 += a2*b2; dot3 += a3*b3;
|
||||
na0 += a0*a0; na1 += a1*a1; na2 += a2*a2; na3 += a3*a3;
|
||||
nb0 += b0*b0; nb1 += b1*b1; nb2 += b2*b2; nb3 += b3*b3;
|
||||
}
|
||||
float dot = (dot0 + dot1) + (dot2 + dot3);
|
||||
float na = (na0 + na1) + (na2 + na3);
|
||||
float nb = (nb0 + nb1) + (nb2 + nb3);
|
||||
for (; d < p.dim; d++) {
|
||||
float a = row[d], b = query[d];
|
||||
dot += a*b; na += a*a; nb += b*b;
|
||||
}
|
||||
|
||||
if (na <= 0.0f || nb <= 0.0f) {
|
||||
out_scores[out_idx] = -2.0f;
|
||||
return;
|
||||
}
|
||||
out_scores[out_idx] = dot / sqrt(na * nb);
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
/* eg_cosine_batch_strategy.h — internal Strategy interface, NOT for call
|
||||
* sites (they use eg_cosine_batch.h). Only eg_cosine_batch.c's factory and
|
||||
* the concrete strategy implementation files include this.
|
||||
*
|
||||
* Each concrete strategy exposes exactly one getter returning a pointer to a
|
||||
* static, immutable EgCosineBatchStrategy vtable. Which getters actually
|
||||
* exist as linkable symbols is a BUILD-TIME concern (decided by
|
||||
* build_vindex_bench.sh / the engram daemon's own build, via which .c/.m
|
||||
* files get compiled per platform) gated by the EG_HAVE_STRATEGY_* macros
|
||||
* below — the factory in eg_cosine_batch.c is the ONLY place that branches
|
||||
* on those macros. Call sites never see them; that's the whole point of the
|
||||
* Adapter in eg_cosine_batch.h.
|
||||
*
|
||||
* Three concrete strategies exist:
|
||||
* eg_cosine_batch_strategy_ggml() — ggml + dynamically-loaded Metal
|
||||
* backend plugin. Darwin only in
|
||||
* this build; the default
|
||||
* preferred strategy wherever
|
||||
* available. EG_HAVE_STRATEGY_GGML.
|
||||
* eg_cosine_batch_strategy_metal_hand() — the original hand-rolled Metal
|
||||
* compute shader from PR #114
|
||||
* (eg_cosine_batch.metal),
|
||||
* preserved verbatim as a
|
||||
* selectable fallback strategy,
|
||||
* not deleted. Darwin only.
|
||||
* EG_HAVE_STRATEGY_METAL_HAND.
|
||||
* eg_cosine_batch_strategy_cpu() — universal always-false
|
||||
* fallback. Always compiled, on
|
||||
* every platform; this is what a
|
||||
* non-Darwin build links
|
||||
* exclusively (matching PR #114's
|
||||
* eg_metal_cosine_stub.c), and
|
||||
* what any platform falls back
|
||||
* to when no real strategy is
|
||||
* available at runtime.
|
||||
*/
|
||||
#ifndef EG_COSINE_BATCH_STRATEGY_H
|
||||
#define EG_COSINE_BATCH_STRATEGY_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct EgCosineBatchStrategy {
|
||||
/* Stable, short, lowercase-hyphenated identifier — what
|
||||
* eg_cosine_batch_strategy_name() surfaces. Never NULL. */
|
||||
const char* name;
|
||||
|
||||
/* Cheap after the first call (lazy init, cached internally). Must never
|
||||
* throw/crash/hang — mirrors eg_cosine_batch_available()'s contract. */
|
||||
bool (*available)(void);
|
||||
|
||||
/* Same shape/contract as eg_cosine_batch() in eg_cosine_batch.h. */
|
||||
bool (*batch)(const float* query, int32_t qdim,
|
||||
const float* const* node_ptrs, const int32_t* node_dims,
|
||||
int32_t n, double* out_scores);
|
||||
|
||||
/* Same shape/contract as eg_cosine_batch_multi() in eg_cosine_batch.h. */
|
||||
bool (*batch_multi)(const float* queries, int32_t qdim, int32_t nq,
|
||||
const float* const* node_ptrs, const int32_t* node_dims,
|
||||
int32_t n, double* out_scores);
|
||||
} EgCosineBatchStrategy;
|
||||
|
||||
#ifdef EG_HAVE_STRATEGY_GGML
|
||||
const EgCosineBatchStrategy* eg_cosine_batch_strategy_ggml(void);
|
||||
#endif
|
||||
|
||||
#ifdef EG_HAVE_STRATEGY_METAL_HAND
|
||||
const EgCosineBatchStrategy* eg_cosine_batch_strategy_metal_hand(void);
|
||||
#endif
|
||||
|
||||
/* Always declared/linked, on every platform/build. */
|
||||
const EgCosineBatchStrategy* eg_cosine_batch_strategy_cpu(void);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* EG_COSINE_BATCH_STRATEGY_H */
|
||||
@@ -1,45 +0,0 @@
|
||||
/* eg_cosine_batch_strategy_cpu.c — plain-C, zero-dependency universal
|
||||
* fallback strategy. Always returns false / unavailable. Direct descendant
|
||||
* of PR #114's eg_metal_cosine_stub.c, generalized from "the Metal stub" to
|
||||
* "the strategy vtable's universal fallback entry" now that multiple real
|
||||
* strategies can exist.
|
||||
*
|
||||
* Always compiled, on every platform. On Darwin builds it is the last-resort
|
||||
* strategy the factory falls back to when neither ggml nor the hand-rolled
|
||||
* Metal strategy is available at runtime (no device, compile failure, ...).
|
||||
* On non-Darwin builds it is the ONLY strategy compiled in at all — no
|
||||
* Objective-C, no Metal frameworks, no ggml/Metal backend plugin — so
|
||||
* eg_cosine_batch()/eg_cosine_batch_multi() always return false there and
|
||||
* every call site's existing CPU fallback runs unconditionally, exactly as
|
||||
* before this PR.
|
||||
*/
|
||||
#include "eg_cosine_batch_strategy.h"
|
||||
|
||||
static bool cpu_available(void) {
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool cpu_batch(const float* query, int32_t qdim,
|
||||
const float* const* node_ptrs, const int32_t* node_dims,
|
||||
int32_t n, double* out_scores) {
|
||||
(void)query; (void)qdim; (void)node_ptrs; (void)node_dims; (void)n; (void)out_scores;
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool cpu_batch_multi(const float* queries, int32_t qdim, int32_t nq,
|
||||
const float* const* node_ptrs, const int32_t* node_dims,
|
||||
int32_t n, double* out_scores) {
|
||||
(void)queries; (void)qdim; (void)nq; (void)node_ptrs; (void)node_dims; (void)n; (void)out_scores;
|
||||
return false;
|
||||
}
|
||||
|
||||
static const EgCosineBatchStrategy g_cpu_strategy = {
|
||||
.name = "cpu-fallback",
|
||||
.available = cpu_available,
|
||||
.batch = cpu_batch,
|
||||
.batch_multi = cpu_batch_multi,
|
||||
};
|
||||
|
||||
const EgCosineBatchStrategy* eg_cosine_batch_strategy_cpu(void) {
|
||||
return &g_cpu_strategy;
|
||||
}
|
||||
@@ -1,515 +0,0 @@
|
||||
/* eg_cosine_batch_strategy_ggml.c — the GGML Strategy, and the preferred
|
||||
* default whenever it is available (see the factory's selection order in
|
||||
* eg_cosine_batch.c).
|
||||
*
|
||||
* WHY: directive from Will Anderson — stop hand-rolling GPU kernels, use a
|
||||
* real, proven, permissively-licensed library instead. ggml (the compute
|
||||
* library underneath llama.cpp, MIT licensed) is already installed on this
|
||||
* machine as a standalone Homebrew package (`brew info ggml`), independent
|
||||
* of llama.cpp itself. This file is a genuinely bounded COMPUTE UTILITY —
|
||||
* batch cosine-similarity math — analogous to a VBD Accessor calling out to
|
||||
* infrastructure. It is explicitly NOT the engram's reasoning/persistence
|
||||
* core; using ggml here does not cross the "own the core" line, because
|
||||
* batch cosine math is infrastructure, not the graph traversal / activation
|
||||
* spreading / "thinking" that IS the core and stays 100% own-code.
|
||||
*
|
||||
* ── The real API shape (verified against the installed headers + a
|
||||
* standalone probe program, not assumed from memory of other tensor
|
||||
* libraries) ──────────────────────────────────────────────────────────
|
||||
*
|
||||
* ggml ships its CPU and Metal implementations as DYNAMICALLY LOADED PLUGIN
|
||||
* .so files (confirmed by nm: `ggml_backend_metal_init` is NOT an exported
|
||||
* symbol of libggml.dylib/libggml-base.dylib — it exists ONLY inside
|
||||
* libggml-metal.so under $(brew --prefix ggml)/libexec/). You cannot link
|
||||
* `-lggml-metal`; you must go through ggml's backend REGISTRY:
|
||||
*
|
||||
* 1. ggml_backend_load_all_from_path(dir) — dlopen()s every backend plugin
|
||||
* .so found in `dir` and registers its device(s). We point this at
|
||||
* $(brew --prefix ggml)/libexec (resolved once, at build+init time; see
|
||||
* eg_ggml_backend_dir() below) rather than relying on
|
||||
* ggml_backend_load_all()'s own default search heuristics, which are
|
||||
* tuned for an installed llama.cpp-style app bundle layout, not an
|
||||
* arbitrary `cc`-built binary invoked from an arbitrary cwd — the exact
|
||||
* same "must not silently fall back to CPU for reasons that have
|
||||
* nothing to do with GPU availability" concern PR #114's hand-rolled
|
||||
* bridge already documented for its own embedded-shader-source choice.
|
||||
* 2. ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU) — find the
|
||||
* registered Metal device.
|
||||
* 3. ggml_backend_dev_init(dev, NULL) — get a live ggml_backend_t.
|
||||
* 4. Build a tiny ggml_context (no_alloc=true; it holds only tensor
|
||||
* metadata, not data), declare 2D F32 tensors, ggml_mul_mat(nodes,
|
||||
* query) — ggml's documented convention: A is [k cols, n rows], B is
|
||||
* [k cols, m rows] (transposed internally), result is [n cols, m rows]
|
||||
* — i.e. mul_mat(node_matrix[dim,n], query_matrix[dim,nq]) yields
|
||||
* out[n,nq] where out[j*n+i] = dot(node_i, query_j). A row-major
|
||||
* (dim,n) node matrix and a row-major (dim,nq) query matrix is EXACTLY
|
||||
* the packed layout the hand-rolled Metal kernel already used — one
|
||||
* matmul replaces the whole per-row dot-product loop.
|
||||
* 5. ggml_backend_alloc_ctx_tensors(ctx, backend) to actually allocate
|
||||
* device buffers for those tensors, ggml_backend_tensor_set() to upload,
|
||||
* ggml_backend_graph_compute() to run, ggml_backend_tensor_get() to
|
||||
* read back.
|
||||
*
|
||||
* This exact sequence was verified end-to-end in a standalone probe (build
|
||||
* it yourself: see the PR description) against a plain-C CPU dot product —
|
||||
* bit-for-bit correct within float rounding. Real numbers against the
|
||||
* el_runtime.c CPU oracle are reported in the PR body via vindex_bench.
|
||||
*
|
||||
* ggml_mul_mat only computes the raw dot products — it has no notion of
|
||||
* "cosine" or of this codebase's -2.0 dim-mismatch/null/zero-norm sentinel.
|
||||
* Per the adapter's directive: gather only VALID, uniform-dim rows into the
|
||||
* packed matrix sent to the GPU (skipping null/mismatched rows entirely,
|
||||
* rather than the hand-rolled kernel's zero-pad-and-sentinel-in-shader
|
||||
* approach), then scatter -2.0 back for every row that was excluded — same
|
||||
* gather/scatter contract eg_cosine_batch.h documents. Norms (||node||,
|
||||
* ||query||) are computed on the CPU host in the same pass that already
|
||||
* touches every element to gather/convert — essentially free — using the
|
||||
* same 4-way-partial-sum accumulation the hand-rolled kernel and the CPU
|
||||
* oracle both use, so the float32 error profile stays comparable across all
|
||||
* three strategies. Only the O(n*dim*nq) dot-product matmul — the actual
|
||||
* expensive part — is offloaded to the GPU.
|
||||
*
|
||||
* ── Precision: the ne11<=8 chunking, and why it is not optional ──────────
|
||||
*
|
||||
* The claim in the first version of this file — "ggml_mul_mat on F32 x F32
|
||||
* inputs computes in F32 on the Metal backend" — is WRONG, and the 0.9933
|
||||
* id-recall it shipped with (vs the hand-rolled kernel's 0.9997) was the
|
||||
* symptom. ggml-metal has two F32xF32 matmul kernels and picks between them
|
||||
* purely on ne11 (the number of B rows == our query count):
|
||||
*
|
||||
* ne11 <= 8 -> kernel_mul_mv_ext_f32_f32_* / kernel_mul_mv_f32_f32_*
|
||||
* templated <float, float> — genuine F32 accumulation.
|
||||
* ne11 > 8 -> kernel_mul_mm_f32_f32, which is templated
|
||||
* <half, half4x4, simdgroup_half8x8, half, half2x4,
|
||||
* simdgroup_half8x8, ...> — i.e. BOTH operands are narrowed
|
||||
* to F16 and accumulated in simdgroup_half8x8 tiles, even
|
||||
* though the tensors are GGML_TYPE_F32 on both sides.
|
||||
*
|
||||
* (Read it yourself, no guessing — the kernel templates are literal strings
|
||||
* in the shipped plugin:
|
||||
* strings $(brew --prefix ggml)/libexec/libggml-metal.so \
|
||||
* | grep -E 'host_name\("kernel_mul_m[mv]_f32_f32'
|
||||
* and the runtime pick is visible with GGML_METAL_DEBUG-style logging as
|
||||
* "compiling pipeline: base = 'kernel_mul_mm_f32_f32'".)
|
||||
*
|
||||
* The previous code issued ONE ggml_mul_mat with ne11 = nq (300 in the
|
||||
* benchmark), landing squarely on the F16 mul_mm path. Measured on this
|
||||
* machine (M4 Pro), n=13415 x dim=768 x nq=300, against a CPU double-
|
||||
* accumulated oracle:
|
||||
*
|
||||
* ne11=300 (one mul_mat, the old code) : mean |Δdot| = 1.038e-05
|
||||
* ne11=8 (chunked, this code) : mean |Δdot| = 3.863e-09
|
||||
*
|
||||
* — a ~2700x reduction in dot-product error, which is exactly the gap that
|
||||
* showed up as 0.9933-vs-0.9997 recall.
|
||||
*
|
||||
* ggml_mul_mat_set_prec(t, GGML_PREC_F32) does NOT fix this. It was tried:
|
||||
* the error was bit-identical with and without it (1.038e-05 either way),
|
||||
* because ggml-metal only consults the prec flag on paths that have an F32
|
||||
* variant to switch to, and there is no F32-accumulating mul_mm kernel in
|
||||
* this build to select. The ONLY lever from outside ggml is ne11.
|
||||
*
|
||||
* So: instead of one mul_mat with ne11=nq, we emit ceil(nq/8) mul_mats, each
|
||||
* over an ne11<=8 ggml_view_2d slice of the same query tensor, all into ONE
|
||||
* graph and ONE ggml_backend_graph_compute. The node matrix is still uploaded
|
||||
* exactly once and still read by the GPU as one shared operand — the whole
|
||||
* point of batch_multi is preserved.
|
||||
*
|
||||
* The cost is real, and stated rather than buried. Timing the whole
|
||||
* batch_multi() call (gather + norms + upload + GPU + scatter) on the real
|
||||
* shape, median of 15 reps after a discarded warm-up, three separate runs:
|
||||
*
|
||||
* unchunked (old, F16 mm) : 13.19 / 13.35 / 14.42 ms -> ~0.044 ms/query
|
||||
* chunked (this code) : 19.92 / 20.08 / 20.23 ms -> ~0.067 ms/query
|
||||
* hand-rolled Metal : 17.74 / 17.88 / 17.99 ms -> ~0.060 ms/query
|
||||
*
|
||||
* So correctness here costs about +6.7ms per 300-query batch (~1.5x on this
|
||||
* call), and leaves us ~12% behind the hand-rolled kernel instead of ~35%
|
||||
* ahead of it. That is not free and should not be sold as free. The reason it
|
||||
* cannot be recovered inside ggml: an fp32 matmul on Metal has to re-stream
|
||||
* the whole node matrix once per <=8 queries (38 dispatches x ~41MB here),
|
||||
* where the F16 mul_mm kernel tiles it in threadgroup memory and reads it far
|
||||
* fewer times. ggml's Metal backend ships no fp32 TILED matmul, so on this
|
||||
* backend "fast" and "fp32" are genuinely exclusive — the hand-rolled kernel
|
||||
* escapes the choice only because it is an fp32 kernel written for this one
|
||||
* shape. Trading precision back for speed is a one-line env change; trading
|
||||
* the other way was not available before this commit at all.
|
||||
*
|
||||
* 8 is not a magic number we invented — it is ggml-metal's own mul_mm
|
||||
* threshold, measured by sweeping ne11 and watching both the error and which
|
||||
* pipeline ggml compiles (9 flips to mul_mm and the error jumps back to
|
||||
* 1.0e-05 in the same step). EL_GGML_MULMAT_CHUNK overrides it: raise it to
|
||||
* trade this precision back for throughput, or set it >= nq to reproduce the
|
||||
* old single-mul_mat behaviour exactly. If a future ggml moves the threshold,
|
||||
* the worst case is that we silently land back on mul_mm — the same accuracy
|
||||
* we shipped before, never a correctness break.
|
||||
*
|
||||
* ── Cold start: what is and is not ours to fix ───────────────────────────
|
||||
*
|
||||
* The ~7.8s first-call cost reported for the first version of this file is
|
||||
* NOT this file re-initialising per call (init is, and always was, cached
|
||||
* behind g_init_attempted below). It is Apple's Metal shader cache missing
|
||||
* on ggml's embedded metallib — ggml-metal ships ~650 kernels in one
|
||||
* __ggml_metallib section, and the first newLibraryWithData of it on a given
|
||||
* machine costs seconds ("ggml_metal_library_init: loaded in 7.670 sec")
|
||||
* while the driver populates ~/…/C/com.apple.metal/. That cache is keyed on
|
||||
* the library, not on our binary, and is shared across processes: the very
|
||||
* next run of a DIFFERENT binary linking the same ggml reports
|
||||
* "loaded in 0.009 sec". So it is a once-per-machine, per-ggml-version cost,
|
||||
* not a per-process one, and nothing this file does can avoid it — the
|
||||
* hand-rolled strategy escapes it only because its shader is two small
|
||||
* kernels instead of six hundred.
|
||||
*
|
||||
* The residual warm init IS ours to look at, and the answer there is "there
|
||||
* was nothing much to win": ggml_backend_load_all_from_path() dlopens every
|
||||
* plugin in the directory (three CPU micro-arch variants + BLAS + Metal) when
|
||||
* we only ever use Metal, so we now load the single Metal plugin instead —
|
||||
* but measured warm that is 44.7-52.4ms against 46.9-58.9ms, i.e. the same
|
||||
* number inside noise, because libggml-metal.so's own init dominates. Warm
|
||||
* ggml init lands at 44-53ms, against 36-117ms for the hand-rolled strategy's
|
||||
* device+pipeline setup. Cold start was never the real defect here; precision
|
||||
* was.
|
||||
*/
|
||||
#include "eg_cosine_batch_strategy.h"
|
||||
|
||||
#include <ggml.h>
|
||||
#include <ggml-backend.h>
|
||||
#include <ggml-alloc.h>
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <math.h>
|
||||
|
||||
/* ── lazy, one-time backend init, cached ─────────────────────────────────── */
|
||||
static bool g_init_attempted = false;
|
||||
static bool g_init_ok = false;
|
||||
static ggml_backend_t g_backend = NULL;
|
||||
|
||||
/* Where to look for the dynamically-loaded backend plugin .so files.
|
||||
* EL_GGML_BACKEND_PATH overrides for non-standard installs; otherwise we try
|
||||
* the Homebrew opt-prefix symlink (stable across ggml point-version bumps —
|
||||
* $(brew --prefix ggml)/libexec — confirmed to exist and contain
|
||||
* libggml-metal.so / libggml-cpu-*.so / libggml-blas.so on this machine),
|
||||
* falling back to ggml's own default search (ggml_backend_load_all()) in
|
||||
* case a different install layout (e.g. a from-source build with a
|
||||
* standard-prefix install) makes that succeed instead. */
|
||||
static const char* eg_ggml_backend_dir(void) {
|
||||
const char* s = getenv("EL_GGML_BACKEND_PATH");
|
||||
if (s && *s) return s;
|
||||
return "/opt/homebrew/opt/ggml/libexec";
|
||||
}
|
||||
|
||||
/* "<dir>/libggml-metal.so" in a static buffer. Only ever called once, from
|
||||
* eg_ggml_ensure_init(), before any thread could race it. */
|
||||
static const char* eg_ggml_metal_plugin_path(const char* dir) {
|
||||
static char buf[1024];
|
||||
snprintf(buf, sizeof buf, "%s/libggml-metal.so", dir);
|
||||
return buf;
|
||||
}
|
||||
|
||||
/* Largest ne11 (query-batch rows per ggml_mul_mat) that keeps ggml-metal on
|
||||
* its F32 mul_mv kernels instead of the F16-accumulating mul_mm kernel — see
|
||||
* the precision discussion in this file's header. EL_GGML_MULMAT_CHUNK
|
||||
* overrides; a value <= 0 means "use the default". */
|
||||
#define EG_GGML_MULMAT_CHUNK_DEFAULT 8
|
||||
|
||||
static int32_t eg_ggml_mulmat_chunk(void) {
|
||||
static bool resolved = false;
|
||||
static int32_t chunk = EG_GGML_MULMAT_CHUNK_DEFAULT;
|
||||
if (!resolved) {
|
||||
resolved = true;
|
||||
const char* s = getenv("EL_GGML_MULMAT_CHUNK");
|
||||
if (s && *s) {
|
||||
long v = strtol(s, NULL, 10);
|
||||
if (v > 0 && v <= INT32_MAX) chunk = (int32_t)v;
|
||||
}
|
||||
}
|
||||
return chunk;
|
||||
}
|
||||
|
||||
/* Which ggml device this strategy computes on. GPU (Metal) is the default
|
||||
* because offloading is the architectural point — the engram's own graph
|
||||
* traversal and activation spreading are CPU work, and a "GPU" strategy that
|
||||
* quietly saturates the CPU steals from them.
|
||||
*
|
||||
* ACCEL (ggml's BLAS/Accelerate plugin) is reachable here mainly as a
|
||||
* portability fallback and a diagnostic, and it is documented as MEASURED AND
|
||||
* REJECTED rather than as a recommendation. In an isolated probe that timed
|
||||
* only ggml_backend_graph_compute, BLAS looked excellent — 3.4-4.0ms for the
|
||||
* 300-query batch at mean |Δdot| 1.5e-08, i.e. as fast as the old F16 path and
|
||||
* far more accurate. End to end on the real store through vindex_bench it does
|
||||
* not hold up: 0.191 ms/query at id-recall 0.9973, against 0.125-0.142 ms/query
|
||||
* at 0.9987 for the Metal default. It is dominated on BOTH axes, because the
|
||||
* isolated probe was not competing with the rest of the batch for the same CPU
|
||||
* cores and the real call path is. Kept because a machine with no usable Metal
|
||||
* device still wants a working ggml strategy — not because it is faster. */
|
||||
static enum ggml_backend_dev_type eg_ggml_device_type(void) {
|
||||
const char* s = getenv("EL_GGML_DEVICE");
|
||||
if (s && *s) {
|
||||
if (strcmp(s, "accel") == 0) return GGML_BACKEND_DEVICE_TYPE_ACCEL;
|
||||
if (strcmp(s, "cpu") == 0) return GGML_BACKEND_DEVICE_TYPE_CPU;
|
||||
}
|
||||
return GGML_BACKEND_DEVICE_TYPE_GPU;
|
||||
}
|
||||
|
||||
static bool eg_ggml_ensure_init(void) {
|
||||
if (g_init_attempted) return g_init_ok;
|
||||
g_init_attempted = true;
|
||||
|
||||
const char* dir = eg_ggml_backend_dir();
|
||||
const enum ggml_backend_dev_type want = eg_ggml_device_type();
|
||||
|
||||
/* Metal is the only backend this strategy uses by default, so load just
|
||||
* that one plugin rather than dlopening the whole directory (three CPU
|
||||
* micro-arch variants + BLAS + Metal here).
|
||||
*
|
||||
* Be honest about what this buys: almost nothing in wall time. Measured
|
||||
* warm, three runs each — load-everything 58.9/46.9/55.1ms, Metal-only
|
||||
* 52.4/51.2/44.7ms. The cost is dominated by dlopening and initialising
|
||||
* libggml-metal.so itself, not by the four plugins we skip, so the two
|
||||
* overlap inside noise. It is kept because registering four device types
|
||||
* we will never dispatch to is untidy and makes ggml_backend_dev_by_type
|
||||
* ambiguous, not because it is a speedup — do not cite it as one.
|
||||
*
|
||||
* ggml_backend_load() returns NULL for a missing or unloadable path,
|
||||
* which simply falls through to the broader searches below; it is never
|
||||
* fatal. Any non-default device needs the full directory scan to find
|
||||
* its plugin, so skip the fast path there. */
|
||||
if (want == GGML_BACKEND_DEVICE_TYPE_GPU)
|
||||
ggml_backend_load(eg_ggml_metal_plugin_path(dir));
|
||||
|
||||
ggml_backend_dev_t dev = ggml_backend_dev_by_type(want);
|
||||
if (!dev) {
|
||||
/* Non-standard layout, a ggml built with a differently-named Metal
|
||||
* plugin, or a non-default device: dlopen every plugin in `dir`. */
|
||||
ggml_backend_load_all_from_path(dir);
|
||||
dev = ggml_backend_dev_by_type(want);
|
||||
}
|
||||
if (!dev) {
|
||||
/* Fall back to ggml's own default search heuristics only if the
|
||||
* explicit path above found nothing — avoids double-registering the
|
||||
* same plugins (ggml does not dedupe two different paths that
|
||||
* happen to resolve to the same files, e.g. our stable opt-prefix
|
||||
* symlink vs. its own Cellar-relative guess) in the common case
|
||||
* where the explicit path already worked. */
|
||||
ggml_backend_load_all();
|
||||
dev = ggml_backend_dev_by_type(want);
|
||||
}
|
||||
if (!dev) return false;
|
||||
|
||||
ggml_backend_t backend = ggml_backend_dev_init(dev, NULL);
|
||||
if (!backend) return false;
|
||||
|
||||
g_backend = backend;
|
||||
g_init_ok = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ggml_strategy_available(void) {
|
||||
return eg_ggml_ensure_init();
|
||||
}
|
||||
|
||||
/* ── shared core: gather valid rows + norms, matmul, scatter ────────────── */
|
||||
|
||||
/* 4-way partial-sum squared-norm accumulation over `dim` floats — same shape
|
||||
* as eg_cosine_batch.metal's per-thread accumulation and vindex_bench.c's
|
||||
* CPU brute_topk unroll, kept consistent on purpose so the float32 error
|
||||
* profile is comparable across all three strategies. */
|
||||
static float eg_norm_sq_f32(const float* v, int32_t dim) {
|
||||
float s0 = 0, s1 = 0, s2 = 0, s3 = 0;
|
||||
int32_t d = 0, dim4 = dim & ~3;
|
||||
for (; d < dim4; d += 4) {
|
||||
s0 += v[d] * v[d]; s1 += v[d+1] * v[d+1];
|
||||
s2 += v[d+2] * v[d+2]; s3 += v[d+3] * v[d+3];
|
||||
}
|
||||
float s = (s0 + s1) + (s2 + s3);
|
||||
for (; d < dim; d++) s += v[d] * v[d];
|
||||
return s;
|
||||
}
|
||||
|
||||
/* Runs one ggml_mul_mat(node_matrix[dim,n_valid], query_matrix[dim,nq]) and
|
||||
* combines it with CPU-computed norms into cosine scores, scattering into
|
||||
* out_scores at ORIGINAL (ungathered) indices. out_scores must already be
|
||||
* fully sized for n*nq (or n for the single-query case, nq=1) — every entry
|
||||
* gets written (valid rows get a real cosine, invalid rows get -2.0), so
|
||||
* this never leaves a partial result. Returns false only on a genuine
|
||||
* failure (alloc, compute) at which point out_scores is left as whatever a
|
||||
* caller-supplied scratch buffer already contained — callers here always
|
||||
* pass a fresh buffer they discard on false, matching the adapter contract
|
||||
* of "on failure, out_scores is treated as untouched" from the caller's
|
||||
* point of view. */
|
||||
static bool eg_ggml_run(const float* queries, int32_t qdim, int32_t nq,
|
||||
const float* const* node_ptrs, const int32_t* node_dims,
|
||||
int32_t n, double* out_scores) {
|
||||
if (!queries || qdim <= 0 || nq <= 0 || !node_ptrs || !node_dims || n <= 0 || !out_scores)
|
||||
return false;
|
||||
if (!eg_ggml_ensure_init()) return false;
|
||||
|
||||
/* Pass 1 (CPU): gather valid rows (non-NULL ptr, dim == qdim) into a
|
||||
* packed (dim, n_valid) row-major matrix, remembering the original index
|
||||
* of each packed row, and compute each valid row's squared norm in the
|
||||
* same pass. Rows excluded here get -2.0 scattered for every query
|
||||
* below without ever touching the GPU. */
|
||||
int32_t* valid_orig = (int32_t*)malloc((size_t)n * sizeof(int32_t));
|
||||
float* node_norm_sq = (float*)malloc((size_t)n * sizeof(float)); /* indexed by packed position */
|
||||
float* node_matrix = NULL;
|
||||
if (!valid_orig || !node_norm_sq) { free(valid_orig); free(node_norm_sq); return false; }
|
||||
|
||||
int32_t n_valid = 0;
|
||||
for (int32_t i = 0; i < n; i++) {
|
||||
if (node_ptrs[i] && node_dims[i] == qdim) n_valid++;
|
||||
}
|
||||
|
||||
if (n_valid > 0) {
|
||||
node_matrix = (float*)malloc((size_t)n_valid * (size_t)qdim * sizeof(float));
|
||||
if (!node_matrix) { free(valid_orig); free(node_norm_sq); return false; }
|
||||
int32_t w = 0;
|
||||
for (int32_t i = 0; i < n; i++) {
|
||||
if (!node_ptrs[i] || node_dims[i] != qdim) continue;
|
||||
memcpy(node_matrix + (size_t)w * qdim, node_ptrs[i], (size_t)qdim * sizeof(float));
|
||||
node_norm_sq[w] = eg_norm_sq_f32(node_ptrs[i], qdim);
|
||||
valid_orig[w] = i;
|
||||
w++;
|
||||
}
|
||||
}
|
||||
|
||||
/* Query norms — nq is typically small (1 or the size of one batch of
|
||||
* comparison queries), so this loop is cheap regardless. */
|
||||
float* q_norm_sq = (float*)malloc((size_t)nq * sizeof(float));
|
||||
if (!q_norm_sq) { free(valid_orig); free(node_norm_sq); free(node_matrix); return false; }
|
||||
for (int32_t j = 0; j < nq; j++) q_norm_sq[j] = eg_norm_sq_f32(queries + (size_t)j * qdim, qdim);
|
||||
|
||||
/* Nothing valid to compare against: every output is -2.0. Still a fully
|
||||
* and correctly populated result — no GPU dispatch was needed to know
|
||||
* that. */
|
||||
if (n_valid == 0) {
|
||||
for (size_t k = 0; k < (size_t)n * (size_t)nq; k++) out_scores[k] = -2.0;
|
||||
free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq);
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Pass 2 (GPU via ggml): dot[j*n_valid + i] = dot(node_i, query_j),
|
||||
* computed as ceil(nq/chunk) separate ggml_mul_mat ops over ne11<=chunk
|
||||
* ggml_view_2d slices of ONE query tensor, all expanded into ONE graph
|
||||
* and run by ONE ggml_backend_graph_compute. Chunking is what keeps
|
||||
* ggml-metal on its F32 mul_mv kernels rather than the F16-accumulating
|
||||
* mul_mm kernel (see this file's header); sharing one graph and one
|
||||
* t_nodes tensor is what keeps the node matrix uploaded exactly once,
|
||||
* which is the entire reason batch_multi exists. */
|
||||
const int32_t chunk = eg_ggml_mulmat_chunk();
|
||||
const int32_t ngroups = (nq + chunk - 1) / chunk;
|
||||
|
||||
/* Tensors held by the context: t_nodes, t_query, plus one view and one
|
||||
* mul_mat result per group. The graph holds at most one node per view and
|
||||
* one per mul_mat. Slack on both so a ggml that bookkeeps slightly
|
||||
* differently cannot silently overflow the arena. */
|
||||
const size_t n_tensors = (size_t)2 * (size_t)ngroups + 8;
|
||||
const size_t graph_size = (size_t)2 * (size_t)ngroups + 16;
|
||||
struct ggml_init_params gp = {
|
||||
.mem_size = ggml_tensor_overhead() * n_tensors
|
||||
+ ggml_graph_overhead_custom(graph_size, false),
|
||||
.mem_buffer = NULL,
|
||||
.no_alloc = true,
|
||||
};
|
||||
struct ggml_context* ctx = ggml_init(gp);
|
||||
if (!ctx) { free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq); return false; }
|
||||
|
||||
struct ggml_tensor** t_dots = (struct ggml_tensor**)malloc((size_t)ngroups * sizeof(*t_dots));
|
||||
if (!t_dots) { ggml_free(ctx); free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq); return false; }
|
||||
|
||||
struct ggml_tensor* t_nodes = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, qdim, n_valid);
|
||||
struct ggml_tensor* t_query = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, qdim, nq);
|
||||
struct ggml_cgraph* gf = t_nodes && t_query
|
||||
? ggml_new_graph_custom(ctx, graph_size, false) : NULL;
|
||||
if (!gf) { free(t_dots); ggml_free(ctx); free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq); return false; }
|
||||
|
||||
bool built = true;
|
||||
for (int32_t g = 0; g < ngroups; g++) {
|
||||
const int32_t start = g * chunk;
|
||||
const int32_t count = (start + chunk <= nq) ? chunk : (nq - start);
|
||||
struct ggml_tensor* t_qv = ggml_view_2d(ctx, t_query, qdim, count,
|
||||
t_query->nb[1],
|
||||
(size_t)start * t_query->nb[1]);
|
||||
t_dots[g] = t_qv ? ggml_mul_mat(ctx, t_nodes, t_qv) : NULL;
|
||||
if (!t_dots[g]) { built = false; break; }
|
||||
ggml_build_forward_expand(gf, t_dots[g]);
|
||||
}
|
||||
if (!built) { free(t_dots); ggml_free(ctx); free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq); return false; }
|
||||
|
||||
struct ggml_backend_buffer* buf = ggml_backend_alloc_ctx_tensors(ctx, g_backend);
|
||||
if (!buf) { free(t_dots); ggml_free(ctx); free(valid_orig); free(node_norm_sq); free(node_matrix); free(q_norm_sq); return false; }
|
||||
|
||||
ggml_backend_tensor_set(t_nodes, node_matrix, 0, (size_t)n_valid * qdim * sizeof(float));
|
||||
ggml_backend_tensor_set(t_query, queries, 0, (size_t)nq * qdim * sizeof(float));
|
||||
free(node_matrix); /* uploaded; the packed CPU copy is no longer needed */
|
||||
|
||||
enum ggml_status st = ggml_backend_graph_compute(g_backend, gf);
|
||||
if (st != GGML_STATUS_SUCCESS) {
|
||||
free(t_dots); ggml_backend_buffer_free(buf); ggml_free(ctx);
|
||||
free(valid_orig); free(node_norm_sq); free(q_norm_sq);
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Each group's result is [n_valid, count] contiguous, so reading group g
|
||||
* into dot + start*n_valid reconstructs exactly the same flat
|
||||
* dot[j*n_valid + w] layout a single ne11=nq mul_mat would have produced —
|
||||
* Pass 3 below is unchanged by the chunking. */
|
||||
float* dot = (float*)malloc((size_t)n_valid * (size_t)nq * sizeof(float));
|
||||
if (!dot) { free(t_dots); ggml_backend_buffer_free(buf); ggml_free(ctx); free(valid_orig); free(node_norm_sq); free(q_norm_sq); return false; }
|
||||
for (int32_t g = 0; g < ngroups; g++) {
|
||||
const int32_t start = g * chunk;
|
||||
const int32_t count = (start + chunk <= nq) ? chunk : (nq - start);
|
||||
ggml_backend_tensor_get(t_dots[g], dot + (size_t)start * n_valid, 0,
|
||||
(size_t)count * (size_t)n_valid * sizeof(float));
|
||||
}
|
||||
free(t_dots);
|
||||
|
||||
/* Pass 3 (CPU): combine dot/(||a||*||b||) per (query,node) pair, scatter
|
||||
* into out_scores at ORIGINAL node indices; every excluded row gets
|
||||
* -2.0 for every query. out_scores is fully populated either way. */
|
||||
for (int32_t j = 0; j < nq; j++) {
|
||||
double* orow = out_scores + (size_t)j * n;
|
||||
for (int32_t i = 0; i < n; i++) orow[i] = -2.0; /* default: excluded */
|
||||
for (int32_t w = 0; w < n_valid; w++) {
|
||||
float na = node_norm_sq[w], nb = q_norm_sq[j];
|
||||
int32_t oi = valid_orig[w];
|
||||
if (na <= 0.0f || nb <= 0.0f) { orow[oi] = -2.0; continue; }
|
||||
float d = dot[(size_t)j * n_valid + w];
|
||||
orow[oi] = (double)(d / sqrtf(na * nb));
|
||||
}
|
||||
}
|
||||
|
||||
free(dot);
|
||||
ggml_backend_buffer_free(buf);
|
||||
ggml_free(ctx);
|
||||
free(valid_orig); free(node_norm_sq); free(q_norm_sq);
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool ggml_strategy_batch(const float* query, int32_t qdim,
|
||||
const float* const* node_ptrs, const int32_t* node_dims,
|
||||
int32_t n, double* out_scores) {
|
||||
if (!query || qdim <= 0 || !node_ptrs || !node_dims || n <= 0 || !out_scores) return false;
|
||||
/* out_scores here is n doubles (nq=1); eg_ggml_run writes n*nq = n of
|
||||
* them, laid out identically to the single-query contract. */
|
||||
return eg_ggml_run(query, qdim, 1, node_ptrs, node_dims, n, out_scores);
|
||||
}
|
||||
|
||||
static bool ggml_strategy_batch_multi(const float* queries, int32_t qdim, int32_t nq,
|
||||
const float* const* node_ptrs, const int32_t* node_dims,
|
||||
int32_t n, double* out_scores) {
|
||||
return eg_ggml_run(queries, qdim, nq, node_ptrs, node_dims, n, out_scores);
|
||||
}
|
||||
|
||||
static const EgCosineBatchStrategy g_ggml_strategy = {
|
||||
.name = "ggml",
|
||||
.available = ggml_strategy_available,
|
||||
.batch = ggml_strategy_batch,
|
||||
.batch_multi = ggml_strategy_batch_multi,
|
||||
};
|
||||
|
||||
const EgCosineBatchStrategy* eg_cosine_batch_strategy_ggml(void) {
|
||||
return &g_ggml_strategy;
|
||||
}
|
||||
@@ -1,358 +0,0 @@
|
||||
/* eg_cosine_batch_strategy_metal_hand.m — the HAND-ROLLED-METAL Strategy.
|
||||
*
|
||||
* This is PR #114's original Objective-C bridge (formerly eg_metal_cosine.m)
|
||||
* exposing the hand-written Metal compute shader (eg_cosine_batch.metal) as
|
||||
* one concrete EgCosineBatchStrategy. It is preserved here almost verbatim —
|
||||
* real, carefully verified work, not discarded — now living behind the
|
||||
* Adapter/Strategy/Factory restructuring (see eg_cosine_batch.h and
|
||||
* eg_cosine_batch_strategy.h) alongside the new ggml-Metal strategy
|
||||
* (eg_cosine_batch_strategy_ggml.c) and the universal CPU fallback
|
||||
* (eg_cosine_batch_strategy_cpu.c). The factory in eg_cosine_batch.c prefers
|
||||
* ggml by default when both are available; this strategy remains selectable
|
||||
* via EL_COSINE_BATCH_STRATEGY=metal, and is what the factory falls back to
|
||||
* if ggml's backend plugin fails to load/init for any reason.
|
||||
*
|
||||
* Apple-only (Metal has no other platform). This file is excluded from the
|
||||
* build entirely on non-Darwin — see build_vindex_bench.sh, which only
|
||||
* compiles/links this file and defines EG_HAVE_STRATEGY_METAL_HAND when
|
||||
* `uname` is Darwin. On Linux the factory never sees this strategy at all —
|
||||
* callers must always be prepared for the "no real strategy available"
|
||||
* fallback via the CPU strategy, which is also exactly what happens here on
|
||||
* Apple hardware with no usable GPU.
|
||||
*
|
||||
* Design (unchanged from PR #114):
|
||||
* - Device/queue/pipeline are created lazily, once, and cached in static
|
||||
* globals — every call after the first only allocates buffers + submits.
|
||||
* - The Metal shader source is embedded as a C string literal (kMetalSrc
|
||||
* below) rather than loaded from a file at runtime or shipped as a
|
||||
* precompiled .metallib. Chosen over newLibraryWithFile: /a .metallib
|
||||
* because the engram binary can be invoked from an arbitrary working
|
||||
* directory (launchd job, nsbx sandbox, CI) and a file-path shader would
|
||||
* be one relocation away from silently falling back to CPU for reasons
|
||||
* that have nothing to do with Metal availability. Embedding costs one
|
||||
* runtime shader compile (~tens of ms) on first use, amortized over the
|
||||
* process lifetime, in exchange for a genuinely self-contained binary.
|
||||
* Source of truth for review/tooling is eg_cosine_batch.metal — this
|
||||
* string MUST be kept byte-identical to that file (a comment marks both
|
||||
* ends of the copy).
|
||||
* - Buffers use MTLResourceStorageModeShared: on Apple Silicon's unified
|
||||
* memory, CPU and GPU read the same physical pages, so filling a buffer
|
||||
* is a plain memcpy and there is no separate "upload" step.
|
||||
* - ANY failure at ANY step (no device, pipeline compile error, buffer
|
||||
* allocation failure, bad args) returns false and leaves out_scores
|
||||
* untouched. This function is called from the request-handling hot path
|
||||
* of a long-lived daemon — it must never throw, crash, or hang it.
|
||||
*/
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <Metal/Metal.h>
|
||||
#include "eg_cosine_batch_strategy.h"
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
/* ── BEGIN embedded shader source (keep in sync with eg_cosine_batch.metal) ── */
|
||||
static const char* kEgCosineBatchMetalSrc =
|
||||
"#include <metal_stdlib>\n"
|
||||
"using namespace metal;\n"
|
||||
"struct EgCosineParams { uint n; uint dim; };\n"
|
||||
"kernel void eg_cosine_batch_kernel(\n"
|
||||
" device const float* query [[buffer(0)]],\n"
|
||||
" device const float* node_matrix [[buffer(1)]],\n"
|
||||
" device const int* node_dims [[buffer(2)]],\n"
|
||||
" constant EgCosineParams& p [[buffer(3)]],\n"
|
||||
" device float* out_scores [[buffer(4)]],\n"
|
||||
" uint gid [[thread_position_in_grid]])\n"
|
||||
"{\n"
|
||||
" if (gid >= p.n) return;\n"
|
||||
" if (node_dims[gid] != int(p.dim)) { out_scores[gid] = -2.0f; return; }\n"
|
||||
" device const float* row = node_matrix + (uint64_t)gid * (uint64_t)p.dim;\n"
|
||||
" float dot0 = 0.0f, dot1 = 0.0f, dot2 = 0.0f, dot3 = 0.0f;\n"
|
||||
" float na0 = 0.0f, na1 = 0.0f, na2 = 0.0f, na3 = 0.0f;\n"
|
||||
" float nb0 = 0.0f, nb1 = 0.0f, nb2 = 0.0f, nb3 = 0.0f;\n"
|
||||
" uint d = 0;\n"
|
||||
" uint dim4 = p.dim & ~3u;\n"
|
||||
" for (; d < dim4; d += 4) {\n"
|
||||
" float a0 = row[d], b0 = query[d];\n"
|
||||
" float a1 = row[d+1], b1 = query[d+1];\n"
|
||||
" float a2 = row[d+2], b2 = query[d+2];\n"
|
||||
" float a3 = row[d+3], b3 = query[d+3];\n"
|
||||
" dot0 += a0*b0; dot1 += a1*b1; dot2 += a2*b2; dot3 += a3*b3;\n"
|
||||
" na0 += a0*a0; na1 += a1*a1; na2 += a2*a2; na3 += a3*a3;\n"
|
||||
" nb0 += b0*b0; nb1 += b1*b1; nb2 += b2*b2; nb3 += b3*b3;\n"
|
||||
" }\n"
|
||||
" float dot = (dot0 + dot1) + (dot2 + dot3);\n"
|
||||
" float na = (na0 + na1) + (na2 + na3);\n"
|
||||
" float nb = (nb0 + nb1) + (nb2 + nb3);\n"
|
||||
" for (; d < p.dim; d++) {\n"
|
||||
" float a = row[d], b = query[d];\n"
|
||||
" dot += a*b; na += a*a; nb += b*b;\n"
|
||||
" }\n"
|
||||
" if (na <= 0.0f || nb <= 0.0f) { out_scores[gid] = -2.0f; return; }\n"
|
||||
" out_scores[gid] = dot / sqrt(na * nb);\n"
|
||||
"}\n"
|
||||
"struct EgCosineMultiParams { uint n; uint dim; uint nq; };\n"
|
||||
"kernel void eg_cosine_batch_multi_kernel(\n"
|
||||
" device const float* queries [[buffer(0)]],\n"
|
||||
" device const float* node_matrix [[buffer(1)]],\n"
|
||||
" device const int* node_dims [[buffer(2)]],\n"
|
||||
" constant EgCosineMultiParams& p [[buffer(3)]],\n"
|
||||
" device float* out_scores [[buffer(4)]],\n"
|
||||
" uint2 gid [[thread_position_in_grid]])\n"
|
||||
"{\n"
|
||||
" uint nid = gid.x, qid = gid.y;\n"
|
||||
" if (nid >= p.n || qid >= p.nq) return;\n"
|
||||
" uint64_t out_idx = (uint64_t)qid * (uint64_t)p.n + (uint64_t)nid;\n"
|
||||
" if (node_dims[nid] != int(p.dim)) { out_scores[out_idx] = -2.0f; return; }\n"
|
||||
" device const float* row = node_matrix + (uint64_t)nid * (uint64_t)p.dim;\n"
|
||||
" device const float* query = queries + (uint64_t)qid * (uint64_t)p.dim;\n"
|
||||
" float dot0 = 0.0f, dot1 = 0.0f, dot2 = 0.0f, dot3 = 0.0f;\n"
|
||||
" float na0 = 0.0f, na1 = 0.0f, na2 = 0.0f, na3 = 0.0f;\n"
|
||||
" float nb0 = 0.0f, nb1 = 0.0f, nb2 = 0.0f, nb3 = 0.0f;\n"
|
||||
" uint d = 0;\n"
|
||||
" uint dim4 = p.dim & ~3u;\n"
|
||||
" for (; d < dim4; d += 4) {\n"
|
||||
" float a0 = row[d], b0 = query[d];\n"
|
||||
" float a1 = row[d+1], b1 = query[d+1];\n"
|
||||
" float a2 = row[d+2], b2 = query[d+2];\n"
|
||||
" float a3 = row[d+3], b3 = query[d+3];\n"
|
||||
" dot0 += a0*b0; dot1 += a1*b1; dot2 += a2*b2; dot3 += a3*b3;\n"
|
||||
" na0 += a0*a0; na1 += a1*a1; na2 += a2*a2; na3 += a3*a3;\n"
|
||||
" nb0 += b0*b0; nb1 += b1*b1; nb2 += b2*b2; nb3 += b3*b3;\n"
|
||||
" }\n"
|
||||
" float dot = (dot0 + dot1) + (dot2 + dot3);\n"
|
||||
" float na = (na0 + na1) + (na2 + na3);\n"
|
||||
" float nb = (nb0 + nb1) + (nb2 + nb3);\n"
|
||||
" for (; d < p.dim; d++) {\n"
|
||||
" float a = row[d], b = query[d];\n"
|
||||
" dot += a*b; na += a*a; nb += b*b;\n"
|
||||
" }\n"
|
||||
" if (na <= 0.0f || nb <= 0.0f) { out_scores[out_idx] = -2.0f; return; }\n"
|
||||
" out_scores[out_idx] = dot / sqrt(na * nb);\n"
|
||||
"}\n";
|
||||
/* ── END embedded shader source ── */
|
||||
|
||||
typedef struct EgCosineParamsC { uint32_t n; uint32_t dim; } EgCosineParamsC;
|
||||
typedef struct EgCosineMultiParamsC { uint32_t n; uint32_t dim; uint32_t nq; } EgCosineMultiParamsC;
|
||||
|
||||
static id<MTLDevice> g_device = nil;
|
||||
static id<MTLCommandQueue> g_queue = nil;
|
||||
static id<MTLComputePipelineState> g_pipeline = nil; /* single-query kernel */
|
||||
static id<MTLComputePipelineState> g_pipeline_multi = nil; /* multi-query kernel */
|
||||
static bool g_init_attempted = false;
|
||||
static bool g_init_ok = false;
|
||||
|
||||
/* Lazy, one-time setup. Never throws — every Metal call here is the
|
||||
* "returns nil/NSError on failure" flavor, not an exception-throwing one. */
|
||||
static bool eg_metal_ensure_init(void) {
|
||||
if (g_init_attempted) return g_init_ok;
|
||||
g_init_attempted = true;
|
||||
|
||||
@autoreleasepool {
|
||||
id<MTLDevice> dev = MTLCreateSystemDefaultDevice();
|
||||
if (!dev) return false;
|
||||
|
||||
id<MTLCommandQueue> q = [dev newCommandQueue];
|
||||
if (!q) return false;
|
||||
|
||||
NSError* err = nil;
|
||||
NSString* src = [NSString stringWithUTF8String:kEgCosineBatchMetalSrc];
|
||||
MTLCompileOptions* opts = [MTLCompileOptions new];
|
||||
id<MTLLibrary> lib = [dev newLibraryWithSource:src options:opts error:&err];
|
||||
if (!lib) return false;
|
||||
|
||||
id<MTLFunction> fn = [lib newFunctionWithName:@"eg_cosine_batch_kernel"];
|
||||
if (!fn) return false;
|
||||
id<MTLComputePipelineState> pipe = [dev newComputePipelineStateWithFunction:fn error:&err];
|
||||
if (!pipe) return false;
|
||||
|
||||
id<MTLFunction> fnMulti = [lib newFunctionWithName:@"eg_cosine_batch_multi_kernel"];
|
||||
if (!fnMulti) return false;
|
||||
id<MTLComputePipelineState> pipeMulti = [dev newComputePipelineStateWithFunction:fnMulti error:&err];
|
||||
if (!pipeMulti) return false;
|
||||
|
||||
g_device = dev;
|
||||
g_queue = q;
|
||||
g_pipeline = pipe;
|
||||
g_pipeline_multi = pipeMulti;
|
||||
g_init_ok = true;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
static bool mh_available(void) {
|
||||
return eg_metal_ensure_init();
|
||||
}
|
||||
|
||||
static bool mh_batch(const float* query, int32_t qdim,
|
||||
const float* const* node_ptrs,
|
||||
const int32_t* node_dims,
|
||||
int32_t n,
|
||||
double* out_scores) {
|
||||
if (!query || qdim <= 0 || !node_ptrs || !node_dims || n <= 0 || !out_scores) return false;
|
||||
if (!eg_metal_ensure_init()) return false;
|
||||
|
||||
@autoreleasepool {
|
||||
const size_t dim = (size_t)qdim;
|
||||
const size_t nu = (size_t)n;
|
||||
|
||||
/* Gather into a packed row-major matrix — EngramNode.emb is one
|
||||
* malloc per node, not a contiguous array, so this copy is
|
||||
* unavoidable regardless of backend. Rows whose real dim doesn't
|
||||
* match qdim are zero-filled (harmless: the kernel sentinels them
|
||||
* via node_dims before ever reading the row). */
|
||||
float* matrix = (float*)calloc(nu * dim, sizeof(float));
|
||||
int32_t* dims_i32 = (int32_t*)malloc(nu * sizeof(int32_t));
|
||||
if (!matrix || !dims_i32) { free(matrix); free(dims_i32); return false; }
|
||||
|
||||
for (size_t i = 0; i < nu; i++) {
|
||||
dims_i32[i] = node_dims[i];
|
||||
if (node_ptrs[i] && node_dims[i] == qdim) {
|
||||
memcpy(matrix + i * dim, node_ptrs[i], dim * sizeof(float));
|
||||
}
|
||||
/* else: leave zero-filled; node_dims[i] != qdim (or missing)
|
||||
* makes the kernel sentinel it to -2.0 without reading the row. */
|
||||
}
|
||||
|
||||
id<MTLBuffer> bufQuery = [g_device newBufferWithBytes:query
|
||||
length:dim * sizeof(float)
|
||||
options:MTLResourceStorageModeShared];
|
||||
id<MTLBuffer> bufMatrix = [g_device newBufferWithBytes:matrix
|
||||
length:nu * dim * sizeof(float)
|
||||
options:MTLResourceStorageModeShared];
|
||||
id<MTLBuffer> bufDims = [g_device newBufferWithBytes:dims_i32
|
||||
length:nu * sizeof(int32_t)
|
||||
options:MTLResourceStorageModeShared];
|
||||
EgCosineParamsC params = { (uint32_t)nu, (uint32_t)dim };
|
||||
id<MTLBuffer> bufParams = [g_device newBufferWithBytes:¶ms
|
||||
length:sizeof(params)
|
||||
options:MTLResourceStorageModeShared];
|
||||
id<MTLBuffer> bufOut = [g_device newBufferWithLength:nu * sizeof(float)
|
||||
options:MTLResourceStorageModeShared];
|
||||
|
||||
free(matrix); free(dims_i32);
|
||||
|
||||
if (!bufQuery || !bufMatrix || !bufDims || !bufParams || !bufOut) return false;
|
||||
|
||||
id<MTLCommandBuffer> cmd = [g_queue commandBuffer];
|
||||
if (!cmd) return false;
|
||||
id<MTLComputeCommandEncoder> enc = [cmd computeCommandEncoder];
|
||||
if (!enc) return false;
|
||||
|
||||
[enc setComputePipelineState:g_pipeline];
|
||||
[enc setBuffer:bufQuery offset:0 atIndex:0];
|
||||
[enc setBuffer:bufMatrix offset:0 atIndex:1];
|
||||
[enc setBuffer:bufDims offset:0 atIndex:2];
|
||||
[enc setBuffer:bufParams offset:0 atIndex:3];
|
||||
[enc setBuffer:bufOut offset:0 atIndex:4];
|
||||
|
||||
NSUInteger tgSize = g_pipeline.maxTotalThreadsPerThreadgroup;
|
||||
if (tgSize > 256) tgSize = 256;
|
||||
if (tgSize < 1) tgSize = 1;
|
||||
MTLSize gridSize = MTLSizeMake(nu, 1, 1);
|
||||
MTLSize threadgroupSize = MTLSizeMake(tgSize, 1, 1);
|
||||
[enc dispatchThreads:gridSize threadsPerThreadgroup:threadgroupSize];
|
||||
[enc endEncoding];
|
||||
|
||||
[cmd commit];
|
||||
[cmd waitUntilCompleted];
|
||||
|
||||
if (cmd.status != MTLCommandBufferStatusCompleted) return false;
|
||||
|
||||
const float* results = (const float*)bufOut.contents;
|
||||
if (!results) return false;
|
||||
for (size_t i = 0; i < nu; i++) out_scores[i] = (double)results[i];
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
static bool mh_batch_multi(const float* queries, int32_t qdim, int32_t nq,
|
||||
const float* const* node_ptrs,
|
||||
const int32_t* node_dims,
|
||||
int32_t n,
|
||||
double* out_scores) {
|
||||
if (!queries || qdim <= 0 || nq <= 0 || !node_ptrs || !node_dims || n <= 0 || !out_scores) return false;
|
||||
if (!eg_metal_ensure_init()) return false;
|
||||
|
||||
@autoreleasepool {
|
||||
const size_t dim = (size_t)qdim;
|
||||
const size_t nu = (size_t)n;
|
||||
const size_t nqu = (size_t)nq;
|
||||
|
||||
float* matrix = (float*)calloc(nu * dim, sizeof(float));
|
||||
int32_t* dims_i32 = (int32_t*)malloc(nu * sizeof(int32_t));
|
||||
if (!matrix || !dims_i32) { free(matrix); free(dims_i32); return false; }
|
||||
|
||||
for (size_t i = 0; i < nu; i++) {
|
||||
dims_i32[i] = node_dims[i];
|
||||
if (node_ptrs[i] && node_dims[i] == qdim) {
|
||||
memcpy(matrix + i * dim, node_ptrs[i], dim * sizeof(float));
|
||||
}
|
||||
}
|
||||
|
||||
/* This is the ONE upload of node_matrix for the whole nq-query batch —
|
||||
* the fix for the measured re-upload-per-query slowdown. */
|
||||
id<MTLBuffer> bufMatrix = [g_device newBufferWithBytes:matrix
|
||||
length:nu * dim * sizeof(float)
|
||||
options:MTLResourceStorageModeShared];
|
||||
id<MTLBuffer> bufDims = [g_device newBufferWithBytes:dims_i32
|
||||
length:nu * sizeof(int32_t)
|
||||
options:MTLResourceStorageModeShared];
|
||||
id<MTLBuffer> bufQueries = [g_device newBufferWithBytes:queries
|
||||
length:nqu * dim * sizeof(float)
|
||||
options:MTLResourceStorageModeShared];
|
||||
EgCosineMultiParamsC params = { (uint32_t)nu, (uint32_t)dim, (uint32_t)nqu };
|
||||
id<MTLBuffer> bufParams = [g_device newBufferWithBytes:¶ms
|
||||
length:sizeof(params)
|
||||
options:MTLResourceStorageModeShared];
|
||||
id<MTLBuffer> bufOut = [g_device newBufferWithLength:nqu * nu * sizeof(float)
|
||||
options:MTLResourceStorageModeShared];
|
||||
|
||||
free(matrix); free(dims_i32);
|
||||
|
||||
if (!bufMatrix || !bufDims || !bufQueries || !bufParams || !bufOut) return false;
|
||||
|
||||
id<MTLCommandBuffer> cmd = [g_queue commandBuffer];
|
||||
if (!cmd) return false;
|
||||
id<MTLComputeCommandEncoder> enc = [cmd computeCommandEncoder];
|
||||
if (!enc) return false;
|
||||
|
||||
[enc setComputePipelineState:g_pipeline_multi];
|
||||
[enc setBuffer:bufQueries offset:0 atIndex:0];
|
||||
[enc setBuffer:bufMatrix offset:0 atIndex:1];
|
||||
[enc setBuffer:bufDims offset:0 atIndex:2];
|
||||
[enc setBuffer:bufParams offset:0 atIndex:3];
|
||||
[enc setBuffer:bufOut offset:0 atIndex:4];
|
||||
|
||||
/* 2D dispatch: x over nodes, y over queries. Threadgroup width picked
|
||||
* from the pipeline's own limit, height fixed at 1 — nq is typically
|
||||
* small (tens to low hundreds) relative to n (thousands+), so tiling
|
||||
* the wide axis (n) is what matters for occupancy. */
|
||||
NSUInteger tgWidth = g_pipeline_multi.maxTotalThreadsPerThreadgroup;
|
||||
if (tgWidth > 256) tgWidth = 256;
|
||||
if (tgWidth < 1) tgWidth = 1;
|
||||
MTLSize gridSize = MTLSizeMake(nu, nqu, 1);
|
||||
MTLSize threadgroupSize = MTLSizeMake(tgWidth, 1, 1);
|
||||
[enc dispatchThreads:gridSize threadsPerThreadgroup:threadgroupSize];
|
||||
[enc endEncoding];
|
||||
|
||||
[cmd commit];
|
||||
[cmd waitUntilCompleted];
|
||||
|
||||
if (cmd.status != MTLCommandBufferStatusCompleted) return false;
|
||||
|
||||
const float* results = (const float*)bufOut.contents;
|
||||
if (!results) return false;
|
||||
for (size_t i = 0; i < nqu * nu; i++) out_scores[i] = (double)results[i];
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
static const EgCosineBatchStrategy g_metal_hand_strategy = {
|
||||
.name = "metal-hand",
|
||||
.available = mh_available,
|
||||
.batch = mh_batch,
|
||||
.batch_multi = mh_batch_multi,
|
||||
};
|
||||
|
||||
const EgCosineBatchStrategy* eg_cosine_batch_strategy_metal_hand(void) {
|
||||
return &g_metal_hand_strategy;
|
||||
}
|
||||
+17
-470
@@ -105,14 +105,23 @@ static void el_arena_track(char* p) {
|
||||
_tl_arena.ptrs[_tl_arena.count++] = p;
|
||||
}
|
||||
|
||||
/* el_request_start / el_request_end moved to el_seed.c (see its comment at the
|
||||
* definition: "formerly defined in el_runtime.c. Now self-contained in
|
||||
* el_seed.c, delegating to the seed arena."). The copies here were left behind
|
||||
* during that move and made el_seed.o + el_runtime.o fail to link together with
|
||||
* duplicate symbols — which is exactly the link the real product build does.
|
||||
* Declared (not defined) here: el_runtime.c's http_worker still calls them. */
|
||||
void el_request_start(void);
|
||||
void el_request_end(void);
|
||||
/* Called by http_worker before dispatching the El handler. */
|
||||
void el_request_start(void) {
|
||||
_tl_arena.count = 0;
|
||||
_tl_arena_active = 1;
|
||||
_tl_fs_read_len = 0; /* never let a previous request's file length */
|
||||
_tl_fs_read_buf = NULL; /* leak into this response's byte accounting */
|
||||
}
|
||||
|
||||
/* Called by http_worker after the El handler returns and the response is sent.
|
||||
* Frees every intermediate string allocated during the request. */
|
||||
void el_request_end(void) {
|
||||
_tl_arena_active = 0;
|
||||
for (size_t i = 0; i < _tl_arena.count; i++) {
|
||||
free(_tl_arena.ptrs[i]);
|
||||
}
|
||||
_tl_arena.count = 0;
|
||||
}
|
||||
|
||||
/* ── Scoped arena for CLI use ─────────────────────────────────────────────── *
|
||||
* CLI programs never call el_request_start/end, so all strdup allocations are
|
||||
@@ -14080,59 +14089,6 @@ el_val_t engram_wm_top_json(el_val_t n_v) {
|
||||
return el_wrap_str(b.buf);
|
||||
}
|
||||
|
||||
/* op_assert seam (realizer promotion, bl-53/#57).
|
||||
* Gathers the grounded ASSERTION ENVELOPE for a subject node —
|
||||
* { "subject": <node|null>, "grounding": [ {node,edge,hops}... ] }
|
||||
* i.e. the self-geometry a realizer renders as faithful first-person text.
|
||||
* Read-only: realization (geometry->text) stays in the faculty/realizer;
|
||||
* this native primitive produces its structured input from proven paths
|
||||
* (engram_emit_node_json + engram_neighbors_json). arity 2 (node_id, depth). */
|
||||
el_val_t engram_op_assert_json(el_val_t node_id, el_val_t depth) {
|
||||
const char* sid = EL_CSTR(node_id);
|
||||
JsonBuf b; jb_init(&b);
|
||||
jb_puts(&b, "{\"subject\":");
|
||||
EngramNode* n = (sid && *sid) ? engram_find_node(sid) : NULL;
|
||||
if (n) engram_emit_node_json(&b, n, 0); else jb_puts(&b, "null");
|
||||
jb_puts(&b, ",\"grounding\":");
|
||||
el_val_t nb = engram_neighbors_json(node_id, depth, EL_STR("both"));
|
||||
const char* nbs = EL_CSTR(nb);
|
||||
jb_puts(&b, (nbs && *nbs) ? nbs : "[]");
|
||||
jb_putc(&b, '}');
|
||||
return el_wrap_str(b.buf);
|
||||
}
|
||||
|
||||
/* Parametric mutation (purview write-side bounding, keystone 56ecbec6).
|
||||
* The mutation verbs travel with a TARGET MANIFOLD (purview) instead of the
|
||||
* implicit global singleton. purview==0 (EL_NULL) is the DEGENERATE/DEFAULT
|
||||
* case: G = live, behaviour identical to the base op. A non-zero purview is a
|
||||
* bounded target that the engine cannot yet resolve (multi-manifold store is a
|
||||
* promotion item), so we REFUSE rather than silently mutate the live set —
|
||||
* write-side bounding must never leak into G=live. */
|
||||
el_val_t engram_node_full_in(el_val_t purview,
|
||||
el_val_t content, el_val_t node_type, el_val_t label,
|
||||
el_val_t salience, el_val_t importance, el_val_t confidence,
|
||||
el_val_t tier, el_val_t tags) {
|
||||
if (purview == 0) {
|
||||
return engram_node_full(content, node_type, label, salience, importance,
|
||||
confidence, tier, tags);
|
||||
}
|
||||
fprintf(stderr, "[engram] purview write-side not yet resolvable (G != live); "
|
||||
"refusing to append to live store (purview=%lld)\n",
|
||||
(long long)purview);
|
||||
return EL_STR("");
|
||||
}
|
||||
|
||||
void engram_connect_in(el_val_t purview,
|
||||
el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t relation) {
|
||||
if (purview == 0) {
|
||||
engram_connect(from_id, to_id, weight, relation);
|
||||
return;
|
||||
}
|
||||
fprintf(stderr, "[engram] purview write-side not yet resolvable (G != live); "
|
||||
"refusing to connect in live store (purview=%lld)\n",
|
||||
(long long)purview);
|
||||
}
|
||||
|
||||
el_val_t engram_stats_json(void) {
|
||||
EngramStore* g = engram_get();
|
||||
/* embedded_count: how far the lazy backfill has progressed. The single
|
||||
@@ -17795,412 +17751,3 @@ el_val_t emit_event(el_val_t name_v, el_val_t duration_ms_v) {
|
||||
return trace_span_end(h);
|
||||
}
|
||||
|
||||
|
||||
/* ── DHARMA runtime additions ────────────────────────────────────────────────
|
||||
*
|
||||
* Functions required by the dharma registry service. Added here so the
|
||||
* released el_runtime.c includes them without requiring dharma to bundle
|
||||
* its own stubs.
|
||||
*
|
||||
* Functions added:
|
||||
* list_len — alias for el_list_len (used in handlers.el)
|
||||
* list_get — alias for el_list_get (used in handlers.el)
|
||||
* json_array_push — append a pre-encoded JSON element to a JSON array string
|
||||
* now_millis — milliseconds since Unix epoch (alias for time_now)
|
||||
* unix_timestamp_ms — same as now_millis (alias)
|
||||
* time_now_ms — same as now_millis (alias)
|
||||
* log_info — stderr structured log at INFO level
|
||||
* log_warn — stderr structured log at WARN level
|
||||
* config — reads a config value from the environment
|
||||
* http_patch — HTTP PATCH with JSON Content-Type
|
||||
* http_post_engram — HTTP POST with optional X-API-Key header
|
||||
* http_get_engram — HTTP GET with optional X-API-Key header
|
||||
* str_to_bytes — encode a string as a JSON array of byte values
|
||||
* bytes_to_str — decode a JSON array of byte values back to a string
|
||||
* hash_sha256 — SHA-256 hex digest of a string
|
||||
*/
|
||||
|
||||
/* list_len — return the number of elements in a list. */
|
||||
el_val_t list_len(el_val_t list) {
|
||||
return el_list_len(list);
|
||||
}
|
||||
|
||||
/* list_get — return the element at index i in a list. */
|
||||
el_val_t list_get(el_val_t list, el_val_t index) {
|
||||
return el_list_get(list, index);
|
||||
}
|
||||
|
||||
/* json_array_push — append element (a pre-encoded JSON fragment, e.g. "\"foo\""
|
||||
* or "42") to the JSON array string arr. Returns a new JSON array string.
|
||||
* Example: json_array_push("[]", "\"alice\"") -> "[\"alice\"]"
|
||||
* json_array_push("[\"alice\"]", "\"bob\"") -> "[\"alice\",\"bob\"]" */
|
||||
el_val_t json_array_push(el_val_t arr_v, el_val_t elem_v) {
|
||||
const char* arr = EL_CSTR(arr_v);
|
||||
const char* elem = EL_CSTR(elem_v);
|
||||
if (!arr || !*arr) arr = "[]";
|
||||
if (!elem || !*elem) elem = "null";
|
||||
|
||||
/* Trim whitespace, find the closing ']'. */
|
||||
const char* p = arr;
|
||||
while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r') p++;
|
||||
if (*p != '[') {
|
||||
/* Not an array — return a single-element array. */
|
||||
size_t n = strlen(elem) + 4;
|
||||
char* out = el_strbuf(n);
|
||||
snprintf(out, n, "[%s]", elem);
|
||||
return el_wrap_str(out);
|
||||
}
|
||||
size_t arr_len = strlen(arr);
|
||||
size_t elem_len = strlen(elem);
|
||||
|
||||
/* Walk from the end to find the matching ']'. */
|
||||
const char* end = arr + arr_len - 1;
|
||||
while (end > p && (*end == ' ' || *end == '\t' || *end == '\n' || *end == '\r')) end--;
|
||||
if (*end != ']') {
|
||||
/* Malformed — wrap elem in a new array. */
|
||||
size_t n = elem_len + 4;
|
||||
char* out = el_strbuf(n);
|
||||
snprintf(out, n, "[%s]", elem);
|
||||
return el_wrap_str(out);
|
||||
}
|
||||
|
||||
/* Content between '[' and ']'. */
|
||||
const char* inner_start = p + 1;
|
||||
const char* inner_end = end; /* points AT ']' */
|
||||
/* Check if the array is empty (only whitespace between brackets). */
|
||||
const char* q = inner_start;
|
||||
while (q < inner_end && (*q == ' ' || *q == '\t' || *q == '\n' || *q == '\r')) q++;
|
||||
int empty = (q == inner_end);
|
||||
|
||||
/* Build: prefix + (comma if non-empty) + elem + "]" */
|
||||
size_t prefix_len = (size_t)(inner_end - arr); /* up to but not including ']' */
|
||||
size_t sep_len = empty ? 0 : 1; /* "," if non-empty */
|
||||
size_t out_len = prefix_len + sep_len + elem_len + 2; /* +"]" + NUL */
|
||||
char* out = el_strbuf(out_len);
|
||||
memcpy(out, arr, prefix_len);
|
||||
if (!empty) out[prefix_len] = ',';
|
||||
memcpy(out + prefix_len + sep_len, elem, elem_len);
|
||||
out[prefix_len + sep_len + elem_len] = ']';
|
||||
out[prefix_len + sep_len + elem_len + 1] = '\0';
|
||||
return el_wrap_str(out);
|
||||
}
|
||||
|
||||
/* now_millis — milliseconds since Unix epoch. */
|
||||
el_val_t now_millis(void) {
|
||||
return time_now();
|
||||
}
|
||||
|
||||
/* unix_timestamp_ms — same as now_millis. */
|
||||
el_val_t unix_timestamp_ms(void) {
|
||||
return time_now();
|
||||
}
|
||||
|
||||
/* time_now_ms — same as now_millis. */
|
||||
el_val_t time_now_ms(void) {
|
||||
return time_now();
|
||||
}
|
||||
|
||||
/* log_info — write a structured [INFO] line to stderr. */
|
||||
void log_info(el_val_t msg_v) {
|
||||
const char* msg = EL_CSTR(msg_v);
|
||||
fprintf(stderr, "[INFO] %s\n", msg ? msg : "");
|
||||
}
|
||||
|
||||
/* log_warn — write a structured [WARN] line to stderr. */
|
||||
void log_warn(el_val_t msg_v) {
|
||||
const char* msg = EL_CSTR(msg_v);
|
||||
fprintf(stderr, "[WARN] %s\n", msg ? msg : "");
|
||||
}
|
||||
|
||||
/* config — read a configuration value from the environment.
|
||||
* Returns "" if the variable is not set (same as __env_get). */
|
||||
el_val_t config(el_val_t key_v) {
|
||||
const char* key = EL_CSTR(key_v);
|
||||
if (!key || !*key) return EL_STR("");
|
||||
const char* val = getenv(key);
|
||||
if (!val) return EL_STR("");
|
||||
return el_wrap_str(el_strdup(val));
|
||||
}
|
||||
|
||||
#if !defined(_WIN32) || defined(HAVE_CURL)
|
||||
/* http_patch — HTTP PATCH request with Content-Type: application/json.
|
||||
* Returns the response body (same error convention as http_post_json). */
|
||||
el_val_t http_patch(el_val_t url_v, el_val_t body_v) {
|
||||
const char* url = EL_CSTR(url_v);
|
||||
const char* body = EL_CSTR(body_v);
|
||||
if (!url || !*url) return http_error_json("empty url");
|
||||
CURL* c = curl_easy_init();
|
||||
if (!c) return http_error_json("curl_easy_init failed");
|
||||
HttpBuf rb; httpbuf_init(&rb);
|
||||
char errbuf[CURL_ERROR_SIZE]; errbuf[0] = '\0';
|
||||
struct curl_slist* h = NULL;
|
||||
h = curl_slist_append(h, "Content-Type: application/json");
|
||||
curl_easy_setopt(c, CURLOPT_URL, url);
|
||||
curl_easy_setopt(c, CURLOPT_CUSTOMREQUEST, "PATCH");
|
||||
curl_easy_setopt(c, CURLOPT_POSTFIELDS, body ? body : "");
|
||||
curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)(body ? strlen(body) : 0));
|
||||
curl_easy_setopt(c, CURLOPT_HTTPHEADER, h);
|
||||
curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_write_cb);
|
||||
curl_easy_setopt(c, CURLOPT_WRITEDATA, &rb);
|
||||
curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, el_http_timeout_ms());
|
||||
curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L);
|
||||
curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf);
|
||||
curl_easy_setopt(c, CURLOPT_USERAGENT, "el-runtime/1.0");
|
||||
CURLcode rc = curl_easy_perform(c);
|
||||
curl_slist_free_all(h);
|
||||
curl_easy_cleanup(c);
|
||||
if (rc != CURLE_OK) {
|
||||
free(rb.data);
|
||||
const char* m = errbuf[0] ? errbuf : curl_easy_strerror(rc);
|
||||
return http_error_json(m);
|
||||
}
|
||||
return el_wrap_str(rb.data);
|
||||
}
|
||||
|
||||
/* http_post_engram — HTTP POST with optional X-API-Key header.
|
||||
* If key is "" no authentication header is sent. */
|
||||
el_val_t http_post_engram(el_val_t url_v, el_val_t key_v, el_val_t body_v) {
|
||||
const char* url = EL_CSTR(url_v);
|
||||
const char* key = EL_CSTR(key_v);
|
||||
const char* body = EL_CSTR(body_v);
|
||||
if (!url || !*url) return http_error_json("empty url");
|
||||
CURL* c = curl_easy_init();
|
||||
if (!c) return http_error_json("curl_easy_init failed");
|
||||
HttpBuf rb; httpbuf_init(&rb);
|
||||
char errbuf[CURL_ERROR_SIZE]; errbuf[0] = '\0';
|
||||
struct curl_slist* h = NULL;
|
||||
h = curl_slist_append(h, "Content-Type: application/json");
|
||||
if (key && *key) {
|
||||
size_t n = strlen(key) + 32;
|
||||
char* hdr = malloc(n);
|
||||
snprintf(hdr, n, "X-API-Key: %s", key);
|
||||
h = curl_slist_append(h, hdr);
|
||||
free(hdr);
|
||||
}
|
||||
curl_easy_setopt(c, CURLOPT_URL, url);
|
||||
curl_easy_setopt(c, CURLOPT_POST, 1L);
|
||||
curl_easy_setopt(c, CURLOPT_POSTFIELDS, body ? body : "");
|
||||
curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)(body ? strlen(body) : 0));
|
||||
curl_easy_setopt(c, CURLOPT_HTTPHEADER, h);
|
||||
curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_write_cb);
|
||||
curl_easy_setopt(c, CURLOPT_WRITEDATA, &rb);
|
||||
curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, el_http_timeout_ms());
|
||||
curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L);
|
||||
curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf);
|
||||
curl_easy_setopt(c, CURLOPT_USERAGENT, "el-runtime/1.0");
|
||||
CURLcode rc = curl_easy_perform(c);
|
||||
curl_slist_free_all(h);
|
||||
curl_easy_cleanup(c);
|
||||
if (rc != CURLE_OK) {
|
||||
free(rb.data);
|
||||
const char* m = errbuf[0] ? errbuf : curl_easy_strerror(rc);
|
||||
return http_error_json(m);
|
||||
}
|
||||
return el_wrap_str(rb.data);
|
||||
}
|
||||
|
||||
/* http_get_engram — HTTP GET with optional X-API-Key header. */
|
||||
el_val_t http_get_engram(el_val_t url_v, el_val_t key_v) {
|
||||
const char* url = EL_CSTR(url_v);
|
||||
const char* key = EL_CSTR(key_v);
|
||||
if (!url || !*url) return http_error_json("empty url");
|
||||
CURL* c = curl_easy_init();
|
||||
if (!c) return http_error_json("curl_easy_init failed");
|
||||
HttpBuf rb; httpbuf_init(&rb);
|
||||
char errbuf[CURL_ERROR_SIZE]; errbuf[0] = '\0';
|
||||
struct curl_slist* h = NULL;
|
||||
if (key && *key) {
|
||||
size_t n = strlen(key) + 32;
|
||||
char* hdr = malloc(n);
|
||||
snprintf(hdr, n, "X-API-Key: %s", key);
|
||||
h = curl_slist_append(h, hdr);
|
||||
free(hdr);
|
||||
}
|
||||
curl_easy_setopt(c, CURLOPT_URL, url);
|
||||
curl_easy_setopt(c, CURLOPT_HTTPGET, 1L);
|
||||
if (h) curl_easy_setopt(c, CURLOPT_HTTPHEADER, h);
|
||||
curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_write_cb);
|
||||
curl_easy_setopt(c, CURLOPT_WRITEDATA, &rb);
|
||||
curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, el_http_timeout_ms());
|
||||
curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L);
|
||||
curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf);
|
||||
curl_easy_setopt(c, CURLOPT_USERAGENT, "el-runtime/1.0");
|
||||
CURLcode rc = curl_easy_perform(c);
|
||||
if (h) curl_slist_free_all(h);
|
||||
curl_easy_cleanup(c);
|
||||
if (rc != CURLE_OK) {
|
||||
free(rb.data);
|
||||
const char* m = errbuf[0] ? errbuf : curl_easy_strerror(rc);
|
||||
return http_error_json(m);
|
||||
}
|
||||
return el_wrap_str(rb.data);
|
||||
}
|
||||
#endif /* HAVE_CURL */
|
||||
|
||||
/* str_to_bytes — encode a string as a JSON array of unsigned byte values.
|
||||
* "hello" -> "[104,101,108,108,111]"
|
||||
* Used by db.el to store binary content in Engram JSON nodes. */
|
||||
el_val_t str_to_bytes(el_val_t sv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
if (!s || !*s) return el_wrap_str(el_strdup("[]"));
|
||||
size_t n = strlen(s);
|
||||
/* Worst case: each byte is 3 digits + comma = 4 chars, plus "[]" + NUL. */
|
||||
char* out = el_strbuf(n * 4 + 3);
|
||||
size_t pos = 0;
|
||||
out[pos++] = '[';
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
unsigned char b = (unsigned char)s[i];
|
||||
if (i > 0) out[pos++] = ',';
|
||||
/* Write decimal representation of b. */
|
||||
if (b >= 100) {
|
||||
out[pos++] = (char)('0' + b / 100);
|
||||
out[pos++] = (char)('0' + (b / 10) % 10);
|
||||
out[pos++] = (char)('0' + b % 10);
|
||||
} else if (b >= 10) {
|
||||
out[pos++] = (char)('0' + b / 10);
|
||||
out[pos++] = (char)('0' + b % 10);
|
||||
} else {
|
||||
out[pos++] = (char)('0' + b);
|
||||
}
|
||||
}
|
||||
out[pos++] = ']';
|
||||
out[pos] = '\0';
|
||||
return el_wrap_str(out);
|
||||
}
|
||||
|
||||
/* bytes_to_str — decode a JSON array of integer byte values back to a string.
|
||||
* "[104,101,108,108,111]" -> "hello"
|
||||
* Inverse of str_to_bytes. */
|
||||
el_val_t bytes_to_str(el_val_t arr_v) {
|
||||
const char* s = EL_CSTR(arr_v);
|
||||
if (!s) return el_wrap_str(el_strdup(""));
|
||||
/* Skip whitespace, expect '['. */
|
||||
while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++;
|
||||
if (*s != '[') return el_wrap_str(el_strdup(""));
|
||||
s++;
|
||||
|
||||
/* Count elements to size the output buffer. */
|
||||
int64_t n = (int64_t)json_array_len(arr_v);
|
||||
if (n <= 0) return el_wrap_str(el_strdup(""));
|
||||
|
||||
char* out = el_strbuf((size_t)n + 1);
|
||||
size_t pos = 0;
|
||||
|
||||
/* Walk the array, parse each integer, store as a byte. */
|
||||
while (*s) {
|
||||
while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++;
|
||||
if (*s == ']' || *s == '\0') break;
|
||||
/* Parse decimal integer. */
|
||||
char* end_ptr;
|
||||
long v = strtol(s, &end_ptr, 10);
|
||||
if (end_ptr == s) break; /* parse failure */
|
||||
s = end_ptr;
|
||||
if (v >= 0 && v <= 255) out[pos++] = (char)(unsigned char)v;
|
||||
while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++;
|
||||
if (*s == ',') { s++; continue; }
|
||||
if (*s == ']' || *s == '\0') break;
|
||||
}
|
||||
out[pos] = '\0';
|
||||
return el_wrap_str(out);
|
||||
}
|
||||
|
||||
/* hash_sha256 — return the SHA-256 hex digest of a string.
|
||||
* Uses the built-in el_sha256_oneshot implementation (no OpenSSL required). */
|
||||
el_val_t hash_sha256(el_val_t sv) {
|
||||
const char* s = EL_CSTR(sv);
|
||||
if (!s) s = "";
|
||||
unsigned char digest[32];
|
||||
el_sha256_oneshot((const unsigned char*)s, strlen(s), digest);
|
||||
return el_hex_encode(digest, 32);
|
||||
}
|
||||
|
||||
|
||||
/* HTTP client aliases — require curl; defined inside #ifdef HAVE_CURL below
|
||||
* with a matching stub in the #ifndef HAVE_CURL block. */
|
||||
#if !defined(_WIN32) || defined(HAVE_CURL)
|
||||
/* __http_do also lives in el_seed.c; marked weak so el_seed.c's definition
|
||||
* wins when both translation units are linked together (the real product build). */
|
||||
__attribute__((weak)) el_val_t __http_do(el_val_t method, el_val_t url, el_val_t body,
|
||||
el_val_t headers_map, el_val_t timeout_ms) {
|
||||
/* timeout_ms is accepted for API compatibility but ignored here;
|
||||
* el_runtime's http_do uses the EL_HTTP_TIMEOUT_MS env var instead. */
|
||||
(void)timeout_ms;
|
||||
struct curl_slist* h = headers_from_map(headers_map);
|
||||
el_val_t r = http_do(EL_CSTR(method), EL_CSTR(url), EL_CSTR(body), h);
|
||||
if (h) curl_slist_free_all(h);
|
||||
return r;
|
||||
}
|
||||
|
||||
/* __http_do_map — same as __http_do but headers_map arg is a JSON-string
|
||||
* rather than an ElMap. Parse it first, then delegate. */
|
||||
el_val_t __http_do_map(el_val_t method, el_val_t url, el_val_t body,
|
||||
el_val_t headers_json, el_val_t timeout_ms) {
|
||||
(void)timeout_ms;
|
||||
/* Build a curl_slist from a JSON object {"Header":"value",...}. */
|
||||
const char* hj = EL_CSTR(headers_json);
|
||||
struct curl_slist* h = NULL;
|
||||
if (hj && *hj && *hj == '{') {
|
||||
/* Walk the JSON pairs with a simple parser reusing json_get_string logic. */
|
||||
/* For correctness we just call the existing json_get iteration path.
|
||||
* We duplicate the key-extraction loop from headers_from_map but driven
|
||||
* by JSON rather than ElMap. Use json_get_raw to iterate is not easy
|
||||
* without knowing keys, so accept the JSON string and build a tmp map. */
|
||||
el_val_t map = json_parse(EL_STR(hj));
|
||||
h = headers_from_map(map);
|
||||
}
|
||||
el_val_t r = http_do(EL_CSTR(method), EL_CSTR(url), EL_CSTR(body), h);
|
||||
if (h) curl_slist_free_all(h);
|
||||
return r;
|
||||
}
|
||||
|
||||
/* __http_do_map_to_file — same as __http_do_map but streams response body
|
||||
* to a local file path rather than returning it as a string. */
|
||||
el_val_t __http_do_map_to_file(el_val_t method, el_val_t url, el_val_t body,
|
||||
el_val_t headers_json, el_val_t output_path) {
|
||||
const char* hj = EL_CSTR(headers_json);
|
||||
struct curl_slist* h = NULL;
|
||||
if (hj && *hj && *hj == '{') {
|
||||
el_val_t map = json_parse(EL_STR(hj));
|
||||
h = headers_from_map(map);
|
||||
}
|
||||
el_val_t r = http_do_to_file(EL_CSTR(method), EL_CSTR(url), EL_CSTR(body),
|
||||
h, EL_CSTR(output_path));
|
||||
if (h) curl_slist_free_all(h);
|
||||
return r;
|
||||
}
|
||||
#endif /* HAVE_CURL */
|
||||
|
||||
#if defined(_WIN32) && !defined(HAVE_CURL)
|
||||
/* ── HAVE_CURL=0 stubs — compile without -lcurl for the elc CLI binary. ───── *
|
||||
* These return a JSON error string so El programs get a clear message if they
|
||||
* call HTTP/LLM functions in a curl-less build. */
|
||||
static el_val_t _no_curl_err(void) {
|
||||
return el_wrap_str(el_strdup("{\"error\":\"not built with HAVE_CURL\"}"));
|
||||
}
|
||||
el_val_t http_get(el_val_t url) { (void)url; return _no_curl_err(); }
|
||||
el_val_t http_post(el_val_t url, el_val_t body) { (void)url; (void)body; return _no_curl_err(); }
|
||||
el_val_t http_post_json(el_val_t url, el_val_t body) { (void)url; (void)body; return _no_curl_err(); }
|
||||
el_val_t http_get_with_headers(el_val_t url, el_val_t h) { (void)url; (void)h; return _no_curl_err(); }
|
||||
el_val_t http_post_with_headers(el_val_t url, el_val_t b, el_val_t h) { (void)url; (void)b; (void)h; return _no_curl_err(); }
|
||||
el_val_t http_post_json_with_headers(el_val_t url, el_val_t h, el_val_t b) { (void)url; (void)h; (void)b; return _no_curl_err(); }
|
||||
el_val_t http_post_form_auth(el_val_t url, el_val_t b, el_val_t a) { (void)url; (void)b; (void)a; return _no_curl_err(); }
|
||||
el_val_t http_delete(el_val_t url) { (void)url; return _no_curl_err(); }
|
||||
el_val_t http_patch(el_val_t url, el_val_t body) { (void)url; (void)body; return _no_curl_err(); }
|
||||
el_val_t http_get_to_file(el_val_t url, el_val_t h, el_val_t p) { (void)url; (void)h; (void)p; return _no_curl_err(); }
|
||||
el_val_t http_post_to_file(el_val_t url, el_val_t b, el_val_t h, el_val_t p) { (void)url; (void)b; (void)h; (void)p; return _no_curl_err(); }
|
||||
el_val_t http_post_engram(el_val_t url, el_val_t k, el_val_t b) { (void)url; (void)k; (void)b; return _no_curl_err(); }
|
||||
el_val_t http_get_engram(el_val_t url, el_val_t k) { (void)url; (void)k; return _no_curl_err(); }
|
||||
el_val_t llm_call(el_val_t m, el_val_t p) { (void)m; (void)p; return _no_curl_err(); }
|
||||
el_val_t llm_call_system(el_val_t m, el_val_t s, el_val_t u) { (void)m; (void)s; (void)u; return _no_curl_err(); }
|
||||
el_val_t llm_call_agentic(el_val_t m, el_val_t s, el_val_t u, el_val_t t) { (void)m; (void)s; (void)u; (void)t; return _no_curl_err(); }
|
||||
el_val_t llm_vision(el_val_t m, el_val_t s, el_val_t p, el_val_t i) { (void)m; (void)s; (void)p; (void)i; return _no_curl_err(); }
|
||||
el_val_t llm_models(void) { return el_list_empty(); }
|
||||
void llm_register_tool(el_val_t n, el_val_t f) { (void)n; (void)f; }
|
||||
/* __ HTTP stubs (no-curl build) */
|
||||
el_val_t __http_do(el_val_t m, el_val_t u, el_val_t b, el_val_t h, el_val_t t) { (void)m; (void)u; (void)b; (void)h; (void)t; return _no_curl_err(); }
|
||||
el_val_t __http_do_map(el_val_t m, el_val_t u, el_val_t b, el_val_t h, el_val_t t) { (void)m; (void)u; (void)b; (void)h; (void)t; return _no_curl_err(); }
|
||||
el_val_t __http_do_map_to_file(el_val_t m, el_val_t u, el_val_t b, el_val_t h, el_val_t p) { (void)m; (void)u; (void)b; (void)h; (void)p; return _no_curl_err(); }
|
||||
#endif /* !HAVE_CURL */
|
||||
|
||||
@@ -275,10 +275,6 @@ el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value);
|
||||
el_val_t json_array_len(el_val_t json_str);
|
||||
el_val_t json_array_get(el_val_t json_str, el_val_t index);
|
||||
el_val_t json_array_get_string(el_val_t json_str, el_val_t index);
|
||||
el_val_t json_escape_string(el_val_t sv);
|
||||
el_val_t json_build_object(el_val_t kvs);
|
||||
el_val_t json_build_array(el_val_t items);
|
||||
el_val_t json_array_push(el_val_t arr_v, el_val_t elem_v); /* defined in el_runtime.c */
|
||||
|
||||
/* ── Time ────────────────────────────────────────────────────────────────── */
|
||||
|
||||
@@ -305,8 +301,6 @@ el_val_t time_diff(el_val_t ts1, el_val_t ts2, el_val_t unit);
|
||||
|
||||
el_val_t el_now_instant(void);
|
||||
el_val_t now(void);
|
||||
el_val_t now_millis(void); /* wall-clock milliseconds (defined in el_runtime.c) */
|
||||
el_val_t now_ns(void); /* wall-clock nanoseconds (defined in el_runtime.c) */
|
||||
el_val_t unix_seconds(el_val_t n);
|
||||
el_val_t unix_millis(el_val_t n);
|
||||
el_val_t instant_from_iso8601(el_val_t s);
|
||||
@@ -718,14 +712,6 @@ el_val_t engram_label_df(el_val_t term);
|
||||
el_val_t engram_salient_term(el_val_t node_id, el_val_t max_df,
|
||||
el_val_t min_df, el_val_t tabu);
|
||||
el_val_t engram_embed_backfill(el_val_t count);
|
||||
/* op_assert seam: grounded assertion envelope {subject,grounding} for the realizer. */
|
||||
el_val_t engram_op_assert_json(el_val_t node_id, el_val_t depth);
|
||||
/* Parametric mutation (purview write-side): purview==0 => G=live (default), else refuse. */
|
||||
el_val_t engram_node_full_in(el_val_t purview, el_val_t content, el_val_t node_type, el_val_t label,
|
||||
el_val_t salience, el_val_t importance, el_val_t confidence,
|
||||
el_val_t tier, el_val_t tags);
|
||||
void engram_connect_in(el_val_t purview, el_val_t from_id, el_val_t to_id,
|
||||
el_val_t weight, el_val_t relation);
|
||||
el_val_t engram_list_layers_json(void);
|
||||
/* Working memory introspection — count, mean weight, and top-N snapshot.
|
||||
* Ported from runtime on 2026-06-30 self-review. */
|
||||
@@ -906,111 +892,6 @@ el_val_t trace_span_start(el_val_t name);
|
||||
el_val_t trace_span_end(el_val_t span_handle);
|
||||
el_val_t emit_event(el_val_t name, el_val_t duration_ms);
|
||||
|
||||
el_val_t __thread_create(el_val_t fn_name_v, el_val_t arg_v);
|
||||
el_val_t __thread_join(el_val_t tid_v);
|
||||
|
||||
/* Mutex + channel seed primitives (defined in el_runtime.c). Declared here so
|
||||
* that compiled El programs which use runtime/thread.el's with_mutex helper or
|
||||
* runtime/channel.el's Go-style channels see real prototypes instead of an
|
||||
* implicit int-return declaration (which the C11 ABI mis-truncates el_val_t). */
|
||||
el_val_t __mutex_new(void);
|
||||
void __mutex_lock(el_val_t m_v);
|
||||
void __mutex_unlock(el_val_t m_v);
|
||||
el_val_t __channel_new(el_val_t capacity_v);
|
||||
el_val_t __channel_send(el_val_t ch_v, el_val_t msg_v);
|
||||
el_val_t __channel_recv(el_val_t ch_v);
|
||||
el_val_t __channel_try_recv(el_val_t ch_v);
|
||||
el_val_t __channel_close(el_val_t ch_v);
|
||||
|
||||
/* ── __ prefixed aliases (self-hosting compiler ABI) ─────────────────────────
|
||||
* The El self-hosting compiler emits calls to __-prefixed names. These are
|
||||
* forwarding wrappers around the existing el_runtime functions above. */
|
||||
|
||||
/* I/O */
|
||||
el_val_t __println(el_val_t s);
|
||||
el_val_t __print(el_val_t s);
|
||||
el_val_t __readline(void);
|
||||
|
||||
/* String */
|
||||
el_val_t __int_to_str(el_val_t n);
|
||||
el_val_t __str_to_int(el_val_t s);
|
||||
el_val_t __float_to_str(el_val_t f);
|
||||
el_val_t __str_to_float(el_val_t s);
|
||||
el_val_t __str_len(el_val_t s);
|
||||
el_val_t __str_char_at(el_val_t s, el_val_t i);
|
||||
el_val_t __str_cmp(el_val_t a, el_val_t b);
|
||||
el_val_t __str_ncmp(el_val_t a, el_val_t b, el_val_t n);
|
||||
el_val_t __str_concat_raw(el_val_t a, el_val_t b);
|
||||
el_val_t __str_slice_raw(el_val_t s, el_val_t start, el_val_t end);
|
||||
el_val_t __str_alloc(el_val_t n);
|
||||
el_val_t __str_set_char(el_val_t s, el_val_t i, el_val_t c);
|
||||
|
||||
/* URL encoding */
|
||||
el_val_t __url_encode(el_val_t s);
|
||||
el_val_t __url_decode(el_val_t s);
|
||||
|
||||
/* Environment */
|
||||
el_val_t __env_get(el_val_t key);
|
||||
|
||||
/* Subprocess */
|
||||
el_val_t __exec(el_val_t cmd);
|
||||
el_val_t __exec_bg(el_val_t cmd);
|
||||
|
||||
/* Process */
|
||||
el_val_t __exit_program(el_val_t code);
|
||||
|
||||
/* Filesystem */
|
||||
el_val_t __fs_exists(el_val_t path);
|
||||
el_val_t __fs_mkdir(el_val_t path);
|
||||
el_val_t __fs_read(el_val_t path);
|
||||
el_val_t __fs_write(el_val_t path, el_val_t content);
|
||||
el_val_t __fs_write_bytes(el_val_t path, el_val_t bytes, el_val_t n);
|
||||
el_val_t __fs_list_raw(el_val_t path);
|
||||
|
||||
/* HTTP server */
|
||||
el_val_t __http_response(el_val_t status, el_val_t headers_json, el_val_t body);
|
||||
el_val_t __http_serve(el_val_t port, el_val_t handler);
|
||||
el_val_t __http_serve_v2(el_val_t port, el_val_t handler);
|
||||
|
||||
/* HTTP conn fd / SSE (weak; overridden by el_seed.c when linked together) */
|
||||
el_val_t __http_conn_fd(void);
|
||||
el_val_t __http_sse_open(el_val_t conn_id);
|
||||
el_val_t __http_sse_send(el_val_t conn_id, el_val_t data);
|
||||
el_val_t __http_sse_close(el_val_t conn_id);
|
||||
|
||||
/* HTTP client (requires HAVE_CURL; stubs provided for no-curl builds) */
|
||||
el_val_t __http_do(el_val_t method, el_val_t url, el_val_t body,
|
||||
el_val_t headers_map, el_val_t timeout_ms);
|
||||
el_val_t __http_do_map(el_val_t method, el_val_t url, el_val_t body,
|
||||
el_val_t headers_json, el_val_t timeout_ms);
|
||||
el_val_t __http_do_map_to_file(el_val_t method, el_val_t url, el_val_t body,
|
||||
el_val_t headers_json, el_val_t output_path);
|
||||
|
||||
/* JSON */
|
||||
el_val_t __json_array_get(el_val_t json, el_val_t index);
|
||||
el_val_t __json_array_get_string(el_val_t json, el_val_t index);
|
||||
el_val_t __json_array_len(el_val_t json);
|
||||
el_val_t __json_get(el_val_t json, el_val_t key);
|
||||
el_val_t __json_get_raw(el_val_t json, el_val_t key);
|
||||
el_val_t __json_set(el_val_t json, el_val_t key, el_val_t value);
|
||||
el_val_t __json_parse_map(el_val_t json_str);
|
||||
el_val_t __json_stringify_val(el_val_t val);
|
||||
|
||||
/* Hashing */
|
||||
el_val_t __sha256_hex(el_val_t s);
|
||||
|
||||
/* State K/V */
|
||||
el_val_t __state_del(el_val_t key);
|
||||
el_val_t __state_get(el_val_t key);
|
||||
el_val_t __state_keys(void);
|
||||
el_val_t __state_set(el_val_t key, el_val_t val);
|
||||
|
||||
/* UUID */
|
||||
el_val_t __uuid_v4(void);
|
||||
|
||||
/* Args */
|
||||
el_val_t __args_json(void);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -37,82 +37,6 @@
|
||||
#include <dlfcn.h>
|
||||
#include <curl/curl.h>
|
||||
|
||||
/* el_runtime.c bridge prototypes.
|
||||
*
|
||||
* A block of __-prefixed wrappers further down in this file (http serving,
|
||||
* JSON access, key-val state, URL/HTML escaping, and the whole engram_*
|
||||
* node/edge/layer/search surface -- 51 symbols in total) delegate to
|
||||
* unprefixed counterparts that are implemented in el_runtime.c, not here.
|
||||
* Porting them into native el_seed.c or El has not happened yet.
|
||||
* tools/install.sh compiles el_seed.c and el_runtime.c as separate objects
|
||||
* and archives both into libel.a, so the symbols are always present at link
|
||||
* time. el_seed.c alone was just missing the prototypes, which made even a
|
||||
* standalone -c compile of this one file fail on a toolchain that now treats
|
||||
* an implicit function declaration as a hard error under C11.
|
||||
*
|
||||
* A plain include of el_runtime.h was tried first and rejected: it redefines
|
||||
* el_to_float and el_from_float, which el_seed.h already provides. Narrow
|
||||
* prototypes, copied verbatim from el_runtime.h, avoid that collision without
|
||||
* pulling in the rest of the retiring runtime header.
|
||||
*/
|
||||
el_val_t http_response(el_val_t status, el_val_t headers_json, el_val_t body);
|
||||
void http_serve(el_val_t port, el_val_t handler);
|
||||
void http_serve_v2(el_val_t port, el_val_t handler);
|
||||
el_val_t json_get(el_val_t json, el_val_t key);
|
||||
el_val_t json_get_string(el_val_t json_str, el_val_t key);
|
||||
el_val_t json_get_int(el_val_t json_str, el_val_t key);
|
||||
el_val_t json_get_float(el_val_t json_str, el_val_t key);
|
||||
el_val_t json_get_bool(el_val_t json_str, el_val_t key);
|
||||
el_val_t json_get_raw(el_val_t json_str, el_val_t key);
|
||||
el_val_t json_parse(el_val_t s);
|
||||
el_val_t json_set(el_val_t json_str, el_val_t key, el_val_t value);
|
||||
el_val_t json_stringify(el_val_t v);
|
||||
el_val_t json_array_len(el_val_t json_str);
|
||||
el_val_t json_array_get(el_val_t json_str, el_val_t index);
|
||||
el_val_t json_array_get_string(el_val_t json_str, el_val_t index);
|
||||
el_val_t state_set(el_val_t key, el_val_t value);
|
||||
el_val_t state_get(el_val_t key);
|
||||
el_val_t state_del(el_val_t key);
|
||||
el_val_t state_keys(void);
|
||||
el_val_t url_encode(el_val_t s);
|
||||
el_val_t url_decode(el_val_t s);
|
||||
el_val_t el_html_sanitize(el_val_t input_html, el_val_t allowlist_json);
|
||||
el_val_t engram_node(el_val_t content, el_val_t node_type, el_val_t salience);
|
||||
el_val_t engram_node_full(el_val_t content, el_val_t node_type, el_val_t label,
|
||||
el_val_t salience, el_val_t importance, el_val_t confidence,
|
||||
el_val_t tier, el_val_t tags);
|
||||
el_val_t engram_node_layered(el_val_t content, el_val_t node_type, el_val_t label,
|
||||
el_val_t salience, el_val_t certainty, el_val_t confidence,
|
||||
el_val_t status, el_val_t tags, el_val_t layer_id);
|
||||
el_val_t engram_add_layer(el_val_t name, el_val_t priority, el_val_t suppressible,
|
||||
el_val_t transparent, el_val_t injectable);
|
||||
el_val_t engram_remove_layer(el_val_t layer_id);
|
||||
el_val_t engram_list_layers(void);
|
||||
el_val_t engram_list_layers_json(void);
|
||||
el_val_t engram_get_node(el_val_t id);
|
||||
el_val_t engram_get_node_json(el_val_t id);
|
||||
el_val_t engram_get_node_by_label(el_val_t label);
|
||||
void engram_strengthen(el_val_t node_id);
|
||||
void engram_forget(el_val_t node_id);
|
||||
el_val_t engram_node_count(void);
|
||||
el_val_t engram_edge_count(void);
|
||||
el_val_t engram_scan_nodes(el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset);
|
||||
el_val_t engram_search(el_val_t query, el_val_t limit);
|
||||
el_val_t engram_search_json(el_val_t query, el_val_t limit);
|
||||
el_val_t engram_activate(el_val_t query, el_val_t depth);
|
||||
el_val_t engram_activate_json(el_val_t query, el_val_t depth);
|
||||
el_val_t engram_compile_layered_json(el_val_t intent, el_val_t depth);
|
||||
el_val_t engram_stats_json(void);
|
||||
void engram_connect(el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t relation);
|
||||
el_val_t engram_edge_between(el_val_t from_id, el_val_t to_id);
|
||||
el_val_t engram_neighbors(el_val_t node_id);
|
||||
el_val_t engram_neighbors_filtered(el_val_t node_id, el_val_t max_depth, el_val_t direction);
|
||||
el_val_t engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction);
|
||||
el_val_t engram_load(el_val_t path);
|
||||
el_val_t engram_save(el_val_t path);
|
||||
|
||||
/* ── Private allocator ───────────────────────────────────────────────────── */
|
||||
/*
|
||||
* el_seed.c carries its own arena for per-request allocation tracking.
|
||||
@@ -907,219 +831,6 @@ void __mutex_unlock(el_val_t m) {
|
||||
pthread_mutex_unlock(&_el_mutexes[slot]);
|
||||
}
|
||||
|
||||
/* ── Channels ─────────────────────────────────────────────────────────────── *
|
||||
* Buffered MPMC channel backed by a mutex + condvar + circular buffer.
|
||||
* Ported from the pre-restructure el_runtime.c (b2aac4b) — runtime/channel.el
|
||||
* has always called these five primitives, but they were never carried
|
||||
* forward into el_seed.c when el_runtime.c was consolidated onto the
|
||||
* canonical release copy. Native channels were silently unlinkable on dev
|
||||
* until this port.
|
||||
*
|
||||
* __channel_new(capacity) -> Int (handle)
|
||||
* __channel_send(ch, msg) — blocks if full (capacity > 0) or never (unbounded)
|
||||
* __channel_recv(ch) -> String — blocks until a message is available
|
||||
* __channel_try_recv(ch) -> String — non-blocking, returns "" if empty
|
||||
* __channel_close(ch) — signal no more sends; recv drains remaining
|
||||
*
|
||||
* Bounded channels (cap > 0): circular buffer, sender blocks when full.
|
||||
* Unbounded channels (cap == 0): dynamic array, sender never blocks.
|
||||
*/
|
||||
#define EL_CHANNEL_MAX 64
|
||||
#define EL_CHANNEL_BUF 1024
|
||||
|
||||
typedef struct {
|
||||
char** buf;
|
||||
int cap; /* 0 = unbounded (grows dynamically) */
|
||||
int head, tail, count;
|
||||
int dyn_cap; /* allocated slots for unbounded mode */
|
||||
int closed;
|
||||
pthread_mutex_t mu;
|
||||
pthread_cond_t not_empty;
|
||||
pthread_cond_t not_full;
|
||||
} ElChannel;
|
||||
|
||||
static ElChannel _channels[EL_CHANNEL_MAX];
|
||||
static int _channel_count = 0;
|
||||
static pthread_mutex_t _channel_alloc_mu = PTHREAD_MUTEX_INITIALIZER;
|
||||
|
||||
el_val_t __channel_new(el_val_t capacity_v) {
|
||||
int cap = (int)(int64_t)capacity_v;
|
||||
if (cap < 0) cap = 0;
|
||||
|
||||
pthread_mutex_lock(&_channel_alloc_mu);
|
||||
if (_channel_count >= EL_CHANNEL_MAX) {
|
||||
pthread_mutex_unlock(&_channel_alloc_mu);
|
||||
fprintf(stderr, "[__channel_new] channel table full\n");
|
||||
return EL_INT(-1);
|
||||
}
|
||||
int slot = _channel_count++;
|
||||
pthread_mutex_unlock(&_channel_alloc_mu);
|
||||
|
||||
ElChannel* ch = &_channels[slot];
|
||||
memset(ch, 0, sizeof(*ch));
|
||||
ch->cap = cap;
|
||||
ch->closed = 0;
|
||||
ch->head = 0;
|
||||
ch->tail = 0;
|
||||
ch->count = 0;
|
||||
|
||||
if (cap > 0) {
|
||||
/* Bounded: fixed circular buffer. */
|
||||
ch->buf = (char**)malloc((size_t)cap * sizeof(char*));
|
||||
ch->dyn_cap = cap;
|
||||
} else {
|
||||
/* Unbounded: start with EL_CHANNEL_BUF slots, grow as needed. */
|
||||
ch->buf = (char**)malloc(EL_CHANNEL_BUF * sizeof(char*));
|
||||
ch->dyn_cap = EL_CHANNEL_BUF;
|
||||
}
|
||||
if (!ch->buf) {
|
||||
fprintf(stderr, "[__channel_new] out of memory\n");
|
||||
return EL_INT(-1);
|
||||
}
|
||||
|
||||
pthread_mutex_init(&ch->mu, NULL);
|
||||
pthread_cond_init(&ch->not_empty, NULL);
|
||||
pthread_cond_init(&ch->not_full, NULL);
|
||||
|
||||
return EL_INT(slot);
|
||||
}
|
||||
|
||||
el_val_t __channel_send(el_val_t ch_v, el_val_t msg_v) {
|
||||
int slot = (int)(int64_t)ch_v;
|
||||
if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR("");
|
||||
ElChannel* ch = &_channels[slot];
|
||||
|
||||
const char* msg = EL_CSTR(msg_v);
|
||||
if (!msg) msg = "";
|
||||
char* copy = strdup(msg); /* channel owns the string */
|
||||
|
||||
pthread_mutex_lock(&ch->mu);
|
||||
|
||||
if (ch->closed) {
|
||||
/* Send on closed channel is a no-op (drop the message). */
|
||||
pthread_mutex_unlock(&ch->mu);
|
||||
free(copy);
|
||||
return EL_STR("");
|
||||
}
|
||||
|
||||
if (ch->cap > 0) {
|
||||
/* Bounded: block while full. */
|
||||
while (ch->count >= ch->cap && !ch->closed) {
|
||||
pthread_cond_wait(&ch->not_full, &ch->mu);
|
||||
}
|
||||
if (ch->closed) {
|
||||
pthread_mutex_unlock(&ch->mu);
|
||||
free(copy);
|
||||
return EL_STR("");
|
||||
}
|
||||
ch->buf[ch->tail] = copy;
|
||||
ch->tail = (ch->tail + 1) % ch->cap;
|
||||
ch->count++;
|
||||
} else {
|
||||
/* Unbounded: grow the buffer if needed. */
|
||||
if (ch->count >= ch->dyn_cap) {
|
||||
int new_cap = ch->dyn_cap * 2;
|
||||
char** grown = (char**)realloc(ch->buf, (size_t)new_cap * sizeof(char*));
|
||||
if (!grown) {
|
||||
pthread_mutex_unlock(&ch->mu);
|
||||
free(copy);
|
||||
fprintf(stderr, "[__channel_send] out of memory growing channel\n");
|
||||
return EL_STR("");
|
||||
}
|
||||
/* The circular buffer may have wrapped. Linearise it first.
|
||||
* In unbounded mode head is always 0 (we append at tail, drain
|
||||
* from head), so a simple memmove isn't needed — but if the
|
||||
* buffer did wrap (tail < head after growth), we need to fix up.
|
||||
* Simplest safe path: if tail wrapped, move the head..old_cap
|
||||
* segment to new_cap..new_cap+(old_cap-head). */
|
||||
if (ch->tail < ch->head) {
|
||||
/* Wrapped: [head..old_cap) is the front, [0..tail) is the back. */
|
||||
int front = ch->dyn_cap - ch->head;
|
||||
memmove(grown + ch->dyn_cap, grown + ch->head, (size_t)front * sizeof(char*));
|
||||
ch->head = ch->dyn_cap;
|
||||
}
|
||||
ch->buf = grown;
|
||||
ch->dyn_cap = new_cap;
|
||||
}
|
||||
ch->buf[ch->tail] = copy;
|
||||
ch->tail = (ch->tail + 1) % ch->dyn_cap;
|
||||
ch->count++;
|
||||
}
|
||||
|
||||
pthread_cond_signal(&ch->not_empty);
|
||||
pthread_mutex_unlock(&ch->mu);
|
||||
return EL_STR("");
|
||||
}
|
||||
|
||||
el_val_t __channel_recv(el_val_t ch_v) {
|
||||
int slot = (int)(int64_t)ch_v;
|
||||
if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR("");
|
||||
ElChannel* ch = &_channels[slot];
|
||||
|
||||
pthread_mutex_lock(&ch->mu);
|
||||
|
||||
/* Block until there is a message or the channel is closed and drained. */
|
||||
while (ch->count == 0 && !ch->closed) {
|
||||
pthread_cond_wait(&ch->not_empty, &ch->mu);
|
||||
}
|
||||
|
||||
if (ch->count == 0) {
|
||||
/* Closed and empty — signal EOF. */
|
||||
pthread_mutex_unlock(&ch->mu);
|
||||
return EL_STR("");
|
||||
}
|
||||
|
||||
int buf_cap = (ch->cap > 0) ? ch->cap : ch->dyn_cap;
|
||||
char* msg = ch->buf[ch->head];
|
||||
ch->head = (ch->head + 1) % buf_cap;
|
||||
ch->count--;
|
||||
|
||||
pthread_cond_signal(&ch->not_full);
|
||||
pthread_mutex_unlock(&ch->mu);
|
||||
|
||||
/* Hand the string to the arena so it is freed after the request. */
|
||||
seed_arena_track(msg);
|
||||
return EL_STR(msg);
|
||||
}
|
||||
|
||||
el_val_t __channel_try_recv(el_val_t ch_v) {
|
||||
int slot = (int)(int64_t)ch_v;
|
||||
if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR("");
|
||||
ElChannel* ch = &_channels[slot];
|
||||
|
||||
pthread_mutex_lock(&ch->mu);
|
||||
|
||||
if (ch->count == 0) {
|
||||
pthread_mutex_unlock(&ch->mu);
|
||||
return EL_STR("");
|
||||
}
|
||||
|
||||
int buf_cap = (ch->cap > 0) ? ch->cap : ch->dyn_cap;
|
||||
char* msg = ch->buf[ch->head];
|
||||
ch->head = (ch->head + 1) % buf_cap;
|
||||
ch->count--;
|
||||
|
||||
pthread_cond_signal(&ch->not_full);
|
||||
pthread_mutex_unlock(&ch->mu);
|
||||
|
||||
seed_arena_track(msg);
|
||||
return EL_STR(msg);
|
||||
}
|
||||
|
||||
el_val_t __channel_close(el_val_t ch_v) {
|
||||
int slot = (int)(int64_t)ch_v;
|
||||
if (slot < 0 || slot >= EL_CHANNEL_MAX) return EL_STR("");
|
||||
ElChannel* ch = &_channels[slot];
|
||||
|
||||
pthread_mutex_lock(&ch->mu);
|
||||
ch->closed = 1;
|
||||
/* Wake all blocked recvers and senders so they can observe the close. */
|
||||
pthread_cond_broadcast(&ch->not_empty);
|
||||
pthread_cond_broadcast(&ch->not_full);
|
||||
pthread_mutex_unlock(&ch->mu);
|
||||
return EL_STR("");
|
||||
}
|
||||
|
||||
/* ── Subprocess ──────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t __exec(el_val_t cmd) {
|
||||
@@ -1383,27 +1094,7 @@ el_val_t __engram_activate_json(el_val_t query, el_val_t depth) {
|
||||
return engram_activate_json(query, depth);
|
||||
}
|
||||
|
||||
/* Forward decls for el_runtime.c symbols this file wraps. el_seed.c does not
|
||||
* include el_runtime.h (documented in lang/AGENTS.md), so each wrapped symbol
|
||||
* needs a prototype here or clang treats it as an implicit declaration (error
|
||||
* under C99+) and the ABI mis-truncates the el_val_t return. */
|
||||
el_val_t engram_op_assert_json(el_val_t node_id, el_val_t depth);
|
||||
el_val_t engram_node_full_in(el_val_t purview, el_val_t content, el_val_t node_type, el_val_t label,
|
||||
el_val_t salience, el_val_t importance, el_val_t confidence,
|
||||
el_val_t tier, el_val_t tags);
|
||||
void engram_connect_in(el_val_t purview, el_val_t from_id, el_val_t to_id,
|
||||
el_val_t weight, el_val_t relation);
|
||||
|
||||
el_val_t __engram_stats_json(void) { return engram_stats_json(); }
|
||||
el_val_t __engram_op_assert_json(el_val_t node_id, el_val_t depth) { return engram_op_assert_json(node_id, depth); }
|
||||
el_val_t __engram_node_full_in(el_val_t purview, el_val_t content, el_val_t node_type, el_val_t label,
|
||||
el_val_t salience, el_val_t importance, el_val_t confidence,
|
||||
el_val_t tier, el_val_t tags) {
|
||||
return engram_node_full_in(purview, content, node_type, label, salience, importance, confidence, tier, tags);
|
||||
}
|
||||
void __engram_connect_in(el_val_t purview, el_val_t from_id, el_val_t to_id, el_val_t weight, el_val_t relation) {
|
||||
engram_connect_in(purview, from_id, to_id, weight, relation);
|
||||
}
|
||||
el_val_t __engram_list_layers_json(void) { return engram_list_layers_json(); }
|
||||
|
||||
el_val_t __engram_compile_layered_json(el_val_t intent, el_val_t depth) {
|
||||
|
||||
@@ -139,13 +139,6 @@ el_val_t __mutex_new(void);
|
||||
void __mutex_lock(el_val_t m);
|
||||
void __mutex_unlock(el_val_t m);
|
||||
|
||||
/* Buffered MPMC channel (runtime/channel.el). capacity=0 means unbounded. */
|
||||
el_val_t __channel_new(el_val_t capacity);
|
||||
el_val_t __channel_send(el_val_t ch, el_val_t msg); /* blocks if bounded+full */
|
||||
el_val_t __channel_recv(el_val_t ch); /* blocks until available */
|
||||
el_val_t __channel_try_recv(el_val_t ch); /* non-blocking, "" if empty */
|
||||
el_val_t __channel_close(el_val_t ch);
|
||||
|
||||
/* ── Subprocess ──────────────────────────────────────────────────────────── */
|
||||
|
||||
el_val_t __exec(el_val_t cmd); /* popen, capture all stdout, return String */
|
||||
@@ -240,12 +233,6 @@ el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, e
|
||||
el_val_t __engram_neighbors_json(el_val_t node_id, el_val_t max_depth, el_val_t direction);
|
||||
el_val_t __engram_activate_json(el_val_t query, el_val_t depth);
|
||||
el_val_t __engram_stats_json(void);
|
||||
el_val_t __engram_op_assert_json(el_val_t node_id, el_val_t depth);
|
||||
el_val_t __engram_node_full_in(el_val_t purview, el_val_t content, el_val_t node_type, el_val_t label,
|
||||
el_val_t salience, el_val_t importance, el_val_t confidence,
|
||||
el_val_t tier, el_val_t tags);
|
||||
void __engram_connect_in(el_val_t purview, el_val_t from_id, el_val_t to_id,
|
||||
el_val_t weight, el_val_t relation);
|
||||
el_val_t __engram_list_layers_json(void);
|
||||
el_val_t __engram_compile_layered_json(el_val_t intent, el_val_t depth);
|
||||
|
||||
|
||||
+5
-147
@@ -7,29 +7,11 @@
|
||||
*
|
||||
* Read-only: never opens a socket, never writes the store. Safe on an nsbx clone.
|
||||
*
|
||||
* Also runs the brute-force oracle a second (and third) way, through the
|
||||
* batch-cosine Strategies behind eg_cosine_batch_strategy.h — the ggml
|
||||
* strategy and the hand-rolled-Metal strategy (Apple/Metal only; see
|
||||
* eg_cosine_batch.h/eg_cosine_batch_strategy.h) — and reports each one's
|
||||
* latency + a correctness check against the CPU oracle side-by-side with the
|
||||
* existing CPU-vs-HNSW numbers. This harness deliberately reaches past the
|
||||
* single-selection Factory (eg_cosine_batch.c) to instantiate every
|
||||
* compiled-in strategy directly, so it can compare all of them against the
|
||||
* SAME dataset in one run — that is the harness's whole job; a real call
|
||||
* site (el_runtime.c) never does this, it only ever calls the plain
|
||||
* eg_cosine_batch()/eg_cosine_batch_multi() adapter functions.
|
||||
* EL_METAL_COSINE=0 forces CPU-only (skips every strategy comparison).
|
||||
*
|
||||
* Build (macOS, ggml + hand-rolled Metal): see build_vindex_bench.sh.
|
||||
* Build (Linux / no Metal): omit every eg_cosine_batch_strategy_*.{c,m} file
|
||||
* except eg_cosine_batch_strategy_cpu.c — this file never references
|
||||
* ggml/Metal directly except through the plain-C strategy header, guarded
|
||||
* by the same EG_HAVE_STRATEGY_* build macros the Factory itself uses.
|
||||
* Build: cc -O2 -std=c11 vindex_bench.c engram_vindex.c -lm -o vindex_bench
|
||||
* Usage: vindex_bench store <neuron.egm> <dim> [nqueries] [k] [ef_csv]
|
||||
* vindex_bench synth <N> [dim] [clusters] [nqueries] [k] [ef_csv]
|
||||
*/
|
||||
#include "engram_vindex.h"
|
||||
#include "eg_cosine_batch_strategy.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
@@ -92,106 +74,6 @@ static double recall_at_k(const int* gt, const uint64_t* ann, int nann, int k){
|
||||
return (double)hit / (double)k;
|
||||
}
|
||||
|
||||
/* EL_METAL_COSINE: 0/off/false disables EVERY strategy comparison outright
|
||||
* (falls back to brute_topk() only), matching el_runtime.c's own gate for
|
||||
* the same env var (back-compat name kept from PR #114; it now gates all
|
||||
* GPU-backed strategies, not just the hand-rolled Metal one). Unset or any
|
||||
* other value = try every compiled-in strategy, report each that's
|
||||
* available, skip (without failing the run) any that isn't. */
|
||||
static bool g_strategy_env_checked = false;
|
||||
static bool g_strategy_disabled_by_env = false;
|
||||
static void eg_strategy_check_env_once(void){
|
||||
if (g_strategy_env_checked) return;
|
||||
g_strategy_env_checked = true;
|
||||
const char* v = getenv("EL_METAL_COSINE");
|
||||
if (v && (v[0]=='0' || v[0]=='n' || v[0]=='N' || v[0]=='f' || v[0]=='F'))
|
||||
g_strategy_disabled_by_env = true;
|
||||
}
|
||||
|
||||
/* Batched sibling of brute_topk, generalized over ANY EgCosineBatchStrategy:
|
||||
* computes top-k for ALL nq queries in ONE strategy->batch_multi() call,
|
||||
* uploading/preparing the node population exactly once instead of once per
|
||||
* query. out_ids/out_d are nq*k, row-major (query i's results at
|
||||
* out_ids+i*k / out_d+i*k). Returns false (nothing written) on any
|
||||
* failure/unavailability; caller treats that as "skip this strategy in the
|
||||
* report", never as a hard error. */
|
||||
static bool batch_topk_strategy(const EgCosineBatchStrategy* strat,
|
||||
const float* data, int n, int dim,
|
||||
const float* queries, int nq,
|
||||
int k, int* out_ids, float* out_d){
|
||||
if (!strat || !strat->available()) return false;
|
||||
|
||||
const float** row_ptrs = malloc((size_t)n * sizeof(float*));
|
||||
int32_t* dims = malloc((size_t)n * sizeof(int32_t));
|
||||
double* scores = malloc((size_t)nq * (size_t)n * sizeof(double));
|
||||
if (!row_ptrs || !dims || !scores) { free(row_ptrs); free(dims); free(scores); return false; }
|
||||
|
||||
for (int i = 0; i < n; i++) { row_ptrs[i] = data + (size_t)i * dim; dims[i] = dim; }
|
||||
|
||||
bool ok = strat->batch_multi(queries, dim, nq, row_ptrs, dims, n, scores);
|
||||
free(row_ptrs); free(dims);
|
||||
if (!ok) { free(scores); return false; }
|
||||
|
||||
for (int qi = 0; qi < nq; qi++) {
|
||||
int* ids = out_ids + (size_t)qi * k;
|
||||
float* ds = out_d + (size_t)qi * k;
|
||||
const double* srow = scores + (size_t)qi * n;
|
||||
for (int i = 0; i < k; i++) { ids[i] = -1; ds[i] = 3.0f; }
|
||||
for (int i = 0; i < n; i++) {
|
||||
float d = 1.0f - (float)srow[i]; /* same distance convention as brute_topk */
|
||||
if (d >= ds[k-1]) continue;
|
||||
int p = k - 1;
|
||||
while (p > 0 && ds[p-1] > d) { ds[p] = ds[p-1]; ids[p] = ids[p-1]; p--; }
|
||||
ds[p] = d; ids[p] = i;
|
||||
}
|
||||
}
|
||||
free(scores);
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Runs batch_topk_strategy for one named strategy over ALL nq queries, diffs
|
||||
* against the CPU ground truth (gt/gd, both nq*k), and prints a report line
|
||||
* in the same shape PR #114 established for BRUTE-METAL — id-recall over
|
||||
* every query plus the actual max/mean same-rank distance delta across
|
||||
* every (query,rank) pair that was compared, never fabricated or assumed. */
|
||||
static void report_strategy_vs_oracle(const char* label, const EgCosineBatchStrategy* strat,
|
||||
const float* data, int n, int dim,
|
||||
const float* qv, int nq, int k,
|
||||
const int* gt, const float* gd, double brute_ms){
|
||||
if (g_strategy_disabled_by_env) { printf("%-13s: disabled via EL_METAL_COSINE\n", label); return; }
|
||||
if (!strat || !strat->available()) { printf("%-13s: not available on this build/host — skipped\n", label); return; }
|
||||
|
||||
int* gtm = malloc((size_t)nq*k*sizeof(int));
|
||||
float* gdm = malloc((size_t)nq*k*sizeof(float));
|
||||
double tm0 = now_s();
|
||||
bool ok = batch_topk_strategy(strat, data, n, dim, qv, nq, k, gtm, gdm);
|
||||
double strat_ms = (now_s()-tm0)*1000.0/nq;
|
||||
if (ok) {
|
||||
double rec_sum = 0; double max_ddiff = 0; double sum_ddiff = 0; int compared = 0;
|
||||
for (int i=0;i<nq;i++) {
|
||||
const int* ids_gt = gt+(size_t)i*k;
|
||||
const float* d_gt = gd+(size_t)i*k;
|
||||
const int* ids_m = gtm+(size_t)i*k;
|
||||
const float* d_m = gdm+(size_t)i*k;
|
||||
uint64_t idset[512]; int m = (k<512)?k:512;
|
||||
for (int j=0;j<m;j++) idset[j] = (uint64_t)ids_m[j];
|
||||
rec_sum += recall_at_k(ids_gt, idset, m, k);
|
||||
for (int j=0;j<k;j++) {
|
||||
if (ids_gt[j] == ids_m[j]) {
|
||||
double diff = fabs((double)d_gt[j]-(double)d_m[j]);
|
||||
if (diff>max_ddiff) max_ddiff=diff;
|
||||
sum_ddiff += diff; compared++;
|
||||
}
|
||||
}
|
||||
}
|
||||
printf("%-13s: %8.3f ms/query (%.1fx vs CPU brute; id-recall %.4f vs CPU oracle over %d queries; same-rank |Δdist|: max %.2e, mean %.2e over %d compared)\n",
|
||||
label, strat_ms, brute_ms/strat_ms, rec_sum/nq, nq, max_ddiff, compared?sum_ddiff/compared:0.0, compared);
|
||||
} else {
|
||||
printf("%-13s: batch call failed mid-run — skipped\n", label);
|
||||
}
|
||||
free(gtm); free(gdm);
|
||||
}
|
||||
|
||||
/* Parse "64,128,256" into an int array; returns count. */
|
||||
static int parse_csv(const char* s, int* out, int maxo){
|
||||
int n=0; if(!s||!*s) return 0;
|
||||
@@ -260,37 +142,13 @@ static void run_bench(const char* label, float* data, int n, int dim,
|
||||
l2norm(dst, dim);
|
||||
}
|
||||
|
||||
/* ground truth: brute-force top-k for every query (also the oracle latency).
|
||||
* gd is nq*k (one real slot per query, not a shared scratch buffer) so the
|
||||
* strategy comparisons below can diff against every query's actual
|
||||
* distances, not just whichever query happened to run last. */
|
||||
/* ground truth: brute-force top-k for every query (also the oracle latency). */
|
||||
int* gt = malloc((size_t)nq*k*sizeof(int));
|
||||
float* gd = malloc((size_t)nq*k*sizeof(float));
|
||||
float* gd = malloc((size_t)k*sizeof(float));
|
||||
double tb0 = now_s();
|
||||
for (int i=0;i<nq;i++) brute_topk(data, n, dim, qv+(size_t)i*dim, k, gt+(size_t)i*k, gd+(size_t)i*k);
|
||||
for (int i=0;i<nq;i++) brute_topk(data, n, dim, qv+(size_t)i*dim, k, gt+(size_t)i*k, gd);
|
||||
double brute_ms = (now_s()-tb0)*1000.0/nq;
|
||||
printf("BRUTE-FORCE : %8.3f ms/query (oracle; O(N*D), CPU)\n", brute_ms);
|
||||
|
||||
/* GPU-backed oracles: SAME nq queries, SAME top-k contract, via each
|
||||
* compiled-in Strategy's batch_multi() (uploads/prepares the node
|
||||
* population once, not once per query). Run only for strategies that
|
||||
* are actually available (checked internally) — never fabricated, never
|
||||
* assumed. Verified against the CPU ground truth computed above:
|
||||
* id-recall across ALL nq queries, plus the actual max/mean distance
|
||||
* delta across every (query,rank) pair that was compared. */
|
||||
eg_strategy_check_env_once();
|
||||
#ifdef EG_HAVE_STRATEGY_GGML
|
||||
report_strategy_vs_oracle("BRUTE-GGML", eg_cosine_batch_strategy_ggml(),
|
||||
data, n, dim, qv, nq, k, gt, gd, brute_ms);
|
||||
#else
|
||||
printf("BRUTE-GGML : strategy not compiled into this build\n");
|
||||
#endif
|
||||
#ifdef EG_HAVE_STRATEGY_METAL_HAND
|
||||
report_strategy_vs_oracle("BRUTE-METAL", eg_cosine_batch_strategy_metal_hand(),
|
||||
data, n, dim, qv, nq, k, gt, gd, brute_ms);
|
||||
#else
|
||||
printf("BRUTE-METAL : strategy not compiled into this build\n");
|
||||
#endif
|
||||
printf("BRUTE-FORCE : %8.3f ms/query (oracle; O(N*D))\n", brute_ms);
|
||||
|
||||
/* HNSW at each ef. */
|
||||
uint64_t* aid = malloc((size_t)k*sizeof(uint64_t));
|
||||
|
||||
@@ -1,184 +0,0 @@
|
||||
# Swarm + CCR + Work-Tracking — Neuron's bounded parallel execution, in native El
|
||||
|
||||
Bounded parallel agent execution on El's **native** concurrency — no external
|
||||
orchestrator. Grounded directly in two of Will's frameworks:
|
||||
|
||||
- **Swarm Architecture** (*Bounded Parallel Agent Execution*, Mar 2026)
|
||||
- **Compiled Context Runtime / CCR** (*Process-Driven Agent Execution with
|
||||
Unbounded Local Memory*, Mar 2026)
|
||||
|
||||
A swarm is a **coordinator** (the main thread) that mints a correlation identity,
|
||||
compiles a **bounded per-worker context (CCR)**, dispatches workers as **native
|
||||
pthreads** (`thread.el` `spawn`/`join`), tracks every unit of work durably, and
|
||||
**converges** results before returning control to the parent step.
|
||||
|
||||
```
|
||||
Parent step
|
||||
└─ swarm_run(blueprint, knowledge_refs, inputs, config)
|
||||
fan-out ──▶ worker_1 (CCR ctx_1) ─┐ native
|
||||
worker_2 (CCR ctx_2) ─┤ pthreads,
|
||||
worker_k (CCR ctx_k) ─┘ bounded by `concurrency`
|
||||
converge ─▶ collect | merge | vote | reduce ──▶ merged result
|
||||
```
|
||||
|
||||
## Why it runs on El natively
|
||||
|
||||
El is natively agentic. This capability composes El's shipped primitives — it
|
||||
adds no bespoke runtime:
|
||||
|
||||
| Primitive | Source | Role in the swarm |
|
||||
|-----------|--------|-------------------|
|
||||
| `spawn(fn,arg)` / `join(tid)` | `runtime/thread.el` → `__thread_create` (pthread + dlsym) | fan-out / rejoin |
|
||||
| `parallel_map`, `with_mutex` | `runtime/thread.el` | reference concurrency patterns |
|
||||
| Go-style channels | `runtime/channel.el` → `__channel_*` | available for vertical event streams |
|
||||
| `engram_*`, `http_*`, `fs_*`, `json_*` | `el_runtime.c` builtins | retrieval, tracking, I/O |
|
||||
|
||||
Every El fn compiles to a global C symbol, so any top-level `(String)->String`
|
||||
fn is directly threadable — the worker entry is exactly such a fn.
|
||||
|
||||
## Modules
|
||||
|
||||
| File | Framework grounding | What it does |
|
||||
|------|--------------------|--------------|
|
||||
| `worktrack.el` | Swarm §6 (correlation IDs, audit) | Durable, single-writer **JSONL journal** keyed by correlation ID; reconstructable status report; opt-in engram mirror (`SWARM_MIRROR=1`). |
|
||||
| `containment.el` | Swarm §3 + the single-writer invariant | Scope tokens w/ capabilities; **Rule 1** (no join), **Rule 2** (no open), **Rule 3** (no lateral edge), **Rule 4** (engram-write is @manager-only, by capability) enforced as checks. |
|
||||
| `ccr.el` | CCR §5 + Swarm §9.3 | Per-worker **Compiled Context Routing**: retrieve → scope → compact into a **bounded, minimal** package. The compiled-context boundary *is* the security boundary. |
|
||||
| `primitives.el` | CCR §2 (Five Primitives) | `attend / think / intend / act / learn` seam the swarm composes over. Engram-backed; explicit binding point for the API-surface reshape. |
|
||||
| `swarm.el` | Swarm §2, §4, §5 | The coordinator: fan-out/converge on native threads, bounded concurrency, four convergence strategies, integer failure threshold, full tracking. |
|
||||
|
||||
## Invariant: only the orchestrator mutates global engram state
|
||||
|
||||
**Only the orchestrator (@manager) writes to the engram / mutates global state.
|
||||
Workers are read-only against the full engram and may write only their own local
|
||||
geometry (their returned result + the journal). A worker is STRUCTURALLY UNABLE
|
||||
to mutate global engram state.**
|
||||
|
||||
This is **Rule 4** — an **authority gate, not a health gate**. Scope tokens carry
|
||||
a capability set: the orchestrator's token holds `engram:write` + `dharma:emit`
|
||||
(@manager-only, the VBD rule that only the manager mutates global state); a
|
||||
worker's token holds **only** `engram:read`. Every engram mutation
|
||||
(`op_write`/`op_relate`/`op_supersede` → `POST /api/nodes`, `/api/edges`,
|
||||
`DELETE`) flows through `swarm_engram_write`, which checks the caller's capability
|
||||
via the **same scope-token mechanism as the live Rule-2 denial** and rejects any
|
||||
worker **before any HTTP is issued**. Capability is fixed at mint time and cannot
|
||||
be acquired at runtime — so the guarantee holds regardless of engram health
|
||||
(distinct from the `SWARM_WRITE_HEALTHY` *health* gate).
|
||||
|
||||
The **curated merge is the only write path**: workers return geometry; the
|
||||
orchestrator, and only the orchestrator, commits the approved/verified geometry
|
||||
back (`commit=1`). Workers keep full-engram **read** access (`op_think`/`op_read`).
|
||||
|
||||
Proven in `harness_real_cognition.el` (§G): a worker `swarm_engram_write` is
|
||||
DENIED by capability with no node created and the violation journalled; the
|
||||
orchestrator passes the gate as the sole authorized writer.
|
||||
|
||||
## Containment → distribution
|
||||
|
||||
The three containment rules make workers **location-independent** (Swarm §9): a
|
||||
worker reads only its compiled context, shares no state with siblings, and its
|
||||
only outward edge is the returned result. The same coordinator can run workers
|
||||
as local threads today or dispatch them across machines later — the mechanism is
|
||||
identical; only the topology changes. Enforced here:
|
||||
|
||||
- **Rule 2** — `swarm_run` rejects any swarm opened under a worker token.
|
||||
- **Rules 1 + 3** — each worker gets a *closed* worker token; the coordinator is
|
||||
the only journal writer, so workers share no mutable state.
|
||||
|
||||
## Usage
|
||||
|
||||
```el
|
||||
// one process step fans out; results converge before the next step
|
||||
let inputs: String = "[\"billing\",\"payments\",\"ledger\"]"
|
||||
let refs: String = "[\"Volatility-Based Decomposition\"]" // CCR knowledge refs
|
||||
let cfg: String = "{\"concurrency\":\"4\",\"strategy\":\"collect\",\"min_success_ratio\":\"1.0\"}"
|
||||
let result: String = swarm_run("analyze_item", refs, inputs, cfg)
|
||||
// result: { corr_id, status, merged, report }
|
||||
```
|
||||
|
||||
Build any program that uses the swarm:
|
||||
|
||||
```bash
|
||||
lang/swarm/build.sh myprog.el ./myprog # concat + elc + cc (el_runtime.c)
|
||||
```
|
||||
|
||||
Config keys: `concurrency` (max workers at once), `strategy`
|
||||
(`collect|merge|vote|reduce`), `min_success_ratio` (decimal string, e.g. `0.8`),
|
||||
`caller_token` (containment). Env: `SWARM_TRACK_DIR` (journal dir),
|
||||
`CCR_TOKEN_BUDGET`, `ENGRAM_URL`/`ENGRAM_API_KEY` (retrieval + mirror),
|
||||
`SWARM_MIRROR=1`.
|
||||
|
||||
## Tests
|
||||
|
||||
```bash
|
||||
lang/swarm/build.sh lang/swarm/tests/test_swarm.el /tmp/t && SWARM_TRACK_DIR=/tmp/trk /tmp/t # 12/12
|
||||
lang/swarm/build.sh lang/swarm/tests/test_convergence.el /tmp/c && SWARM_TRACK_DIR=/tmp/trk /tmp/c # 8/8
|
||||
# integration against an isolated engram clone (never live):
|
||||
source <sandbox>/.nsbx-env
|
||||
lang/swarm/build.sh lang/swarm/tests/integ_engram.el /tmp/i && /tmp/i
|
||||
```
|
||||
|
||||
## Local-swarm integration harness (the one flip)
|
||||
|
||||
`tests/harness_local_swarm.el` proves the **full local-swarm mechanics today** on
|
||||
the isolated clone with the primitive seam pointed at the hermetic stub — 17/17
|
||||
green: 8 native-thread workers at concurrency 4, reduce + vote convergence, CCR
|
||||
scoping + non-leak, all three containment rules (incl. live Rule-2 denial),
|
||||
durable work-tracking, and **afferent telemetry** observed by the @manager.
|
||||
|
||||
Binding to the reshape's decorated primitives is **one flip and a run**:
|
||||
|
||||
```
|
||||
# in primitive_binding.el — change one line each:
|
||||
fn bound_think(ctx, instruction) { return think(ctx, instruction) } # decorated, dharma bus
|
||||
# then:
|
||||
SWARM_PRIMITIVE_SEAM=decorated lang/swarm/build.sh tests/harness_local_swarm.el ./h && ./h
|
||||
```
|
||||
|
||||
Nothing else in the swarm changes. `primitive_seam.el` (`seam_think/attend/learn`)
|
||||
already routes every worker primitive call through this one switch, and the same
|
||||
harness runs the bound path. Today `SWARM_PRIMITIVE_SEAM=decorated` still runs
|
||||
green because the binding falls back to the stub — proving the flip path executes.
|
||||
|
||||
## Real cognition — the seam is BOUND
|
||||
|
||||
`primitive_binding.el` is bound to the api-reshape agent's proven primitives
|
||||
(`wt/api-reshape@d4f401d`): `bound_think -> op_think` (GET `/api/think`), real
|
||||
768-dim gradients over the engram geometry. `reshape_surface.el` composes those
|
||||
read/cognition primitives verbatim (`op_think/read/attend/learn`).
|
||||
|
||||
`tests/harness_real_cognition.el` runs the **local swarm on real cognition**,
|
||||
17/17 green with `SWARM_PRIMITIVE_SEAM=decorated` against the `:8901` clone: 8
|
||||
native-thread workers, each a real `think` over its CCR-scoped **node-id anchor**
|
||||
(free-text anchors return "geometry unavailable"), `@manager` reduce+vote, all
|
||||
three containment rules, afferent telemetry, durable tracking. Per-anchor support
|
||||
counts (e.g. 6 / 16 / 87) drive a genuine, cognition-derived vote.
|
||||
|
||||
> **Build note (load-bearing):** the swarm build **must** define `HAVE_CURL`
|
||||
> (`build.sh` does). Without it every `http_*` builtin is a
|
||||
> `{"error":"not built with HAVE_CURL"}` stub — real HTTP silently disappears.
|
||||
|
||||
Writes (`attend`/`learn`, `POST`) are gated behind `SWARM_WRITE_HEALTHY=1` and the
|
||||
api-reshape agent's gate-1 write-healthy clone; the proven run is read-cognition.
|
||||
|
||||
## Built vs stubbed (honest)
|
||||
|
||||
**Real, tested:**
|
||||
- Native-thread fan-out/converge, bounded concurrency, order-preserving rejoin.
|
||||
- All three containment rules enforced (scope tokens + lateral-edge check).
|
||||
- CCR per-worker context: retrieval → scoping → compaction, bounded, non-leaking
|
||||
(a worker never receives sibling inputs) — verified against the live isolated mind.
|
||||
- Full durable work-tracking (JSONL journal, reconstructable report).
|
||||
- Four convergence strategies + integer failure threshold / partial-abort.
|
||||
|
||||
**Seam / not yet bound:**
|
||||
- `primitives.el` `think` is a deterministic, hermetic transform (no model call).
|
||||
Binding point is marked `PRIMITIVE_BINDING`; wire to the API-surface reshape's
|
||||
`think/act/attend/intend/learn` when it lands.
|
||||
- Blueprints are dispatched by name in `swarm_run_blueprint` (default +
|
||||
`classify`/`faildemo` demos). A YAML process-definition loader (Swarm §5) is
|
||||
future work — the runtime contract is in place.
|
||||
- Distributed placement (cloud/edge/federated topologies, Swarm §9.2) is
|
||||
structurally enabled by containment but not yet wired to a placement layer;
|
||||
today all workers are local native threads.
|
||||
- Engram work-tracking mirror is opt-in; the durable substrate is the journal.
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# build.sh — compile an El program that uses the swarm capability.
|
||||
#
|
||||
# Concatenates the El native-concurrency stdlib (thread.el, channel.el) and the
|
||||
# swarm capability modules in dependency order, then the user program, compiles
|
||||
# with the canonical elc, and links against the shared C runtime.
|
||||
#
|
||||
# Usage:
|
||||
# swarm/build.sh <program.el> <out-binary>
|
||||
#
|
||||
# The swarm modules use only el_runtime.c builtins plus thread.el/channel.el,
|
||||
# so nothing else needs concatenating (engram_*, json_*, str_*, fs_*, http_*,
|
||||
# uuid_v4, now_millis are all C builtins in el_runtime.c).
|
||||
|
||||
set -uo pipefail
|
||||
cd "$(dirname "$0")/.." # -> lang/
|
||||
LANG_DIR="$(pwd)"
|
||||
ELC="${ELC:-${LANG_DIR}/dist/platform/elc}"
|
||||
RT="${LANG_DIR}/el-compiler/runtime"
|
||||
|
||||
PROG="${1:?usage: build.sh <program.el> <out-binary>}"
|
||||
OUT="${2:?usage: build.sh <program.el> <out-binary>}"
|
||||
|
||||
# swarm module load order (each may depend on those before it):
|
||||
# worktrack — durable work-tracking journal (no swarm deps)
|
||||
# containment — the three containment rules (no swarm deps)
|
||||
# primitives — think/act/attend/intend/learn seam (no swarm deps)
|
||||
# ccr — per-worker compiled bounded context (depends: primitives)
|
||||
# swarm — orchestrator: fan-out/converge (depends: all above + thread)
|
||||
SWARM_MODULES="
|
||||
swarm/worktrack.el
|
||||
swarm/containment.el
|
||||
swarm/primitives.el
|
||||
swarm/reshape_surface.el
|
||||
swarm/primitive_binding.el
|
||||
swarm/primitive_seam.el
|
||||
swarm/ccr.el
|
||||
swarm/swarm.el
|
||||
"
|
||||
|
||||
TMP_C="$(mktemp -t swarm_build.XXXXXX).c"
|
||||
COMBINED="$(mktemp -t swarm_combined.XXXXXX).el"
|
||||
|
||||
cat runtime/thread.el runtime/channel.el $SWARM_MODULES "$PROG" > "$COMBINED"
|
||||
|
||||
if ! "$ELC" "$COMBINED" > "$TMP_C" 2>/tmp/swarm.elc.err; then
|
||||
echo "elc FAILED:" >&2
|
||||
sed 's/^/ /' /tmp/swarm.elc.err >&2
|
||||
rm -f "$TMP_C" "$COMBINED"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! cc -O2 -DHAVE_CURL -I "$RT" "$TMP_C" "$RT/el_runtime.c" -lcurl -lpthread -lm -o "$OUT" 2>/tmp/swarm.cc.err; then
|
||||
echo "cc FAILED:" >&2
|
||||
sed 's/^/ /' /tmp/swarm.cc.err >&2
|
||||
rm -f "$TMP_C" "$COMBINED"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rm -f "$TMP_C" "$COMBINED"
|
||||
echo "built: $OUT"
|
||||
@@ -1,153 +0,0 @@
|
||||
// ccr.el — Compiled Context Routing for work distribution.
|
||||
//
|
||||
// The same spine as the API's vantage-read, applied per worker. Instead of
|
||||
// handing every worker the coordinator's full memory, CCR compiles a MINIMAL,
|
||||
// BOUNDED context package scoped to exactly one worker's input (CCR §5, "Compiled
|
||||
// Context Injection"; Swarm §9.3, "The Compiled Context Boundary as Security
|
||||
// Boundary").
|
||||
//
|
||||
// The pipeline is CCR §5.1: Retrieval -> Scoping -> Compilation -> (Injection,
|
||||
// which here is placing the package into the worker's task envelope).
|
||||
//
|
||||
// 1. Retrieval — resolve the blueprint's knowledge refs + the input's salient
|
||||
// terms against the mind (primitive_attend).
|
||||
// 2. Scoping — keep only what THIS input needs; drop everything else. A
|
||||
// worker never receives sibling inputs or unrelated memory.
|
||||
// 3. Compilation— compact to a CTX string within a token budget (lossless of
|
||||
// meaning, smaller in tokens): collapse blank runs, dedupe
|
||||
// lines, then bound to the budget.
|
||||
//
|
||||
// The package a worker receives is therefore (a) sufficient for its task and
|
||||
// (b) incapable of leaking what it was never given — the containment boundary
|
||||
// and the security boundary are the same object.
|
||||
|
||||
// ── token budget helpers ─────────────────────────────────────────────────────
|
||||
|
||||
// ccr_est_tokens — cheap token estimate (~4 chars/token).
|
||||
fn ccr_est_tokens(s: String) -> Int {
|
||||
return str_len(s) / 4
|
||||
}
|
||||
|
||||
// ccr_default_budget — default per-worker context budget in tokens.
|
||||
// Override with CCR_TOKEN_BUDGET.
|
||||
fn ccr_default_budget() -> Int {
|
||||
let b: String = env("CCR_TOKEN_BUDGET")
|
||||
if str_eq(b, "") {
|
||||
return 1200
|
||||
}
|
||||
return str_to_int(b)
|
||||
}
|
||||
|
||||
// ── stage 3: compaction ──────────────────────────────────────────────────────
|
||||
|
||||
// ccr_compact — collapse blank-line runs and drop exact duplicate lines, then
|
||||
// bound the result to `budget` tokens (truncate on a line boundary). Meaning is
|
||||
// preserved; token count falls (CCR §5.2).
|
||||
fn ccr_compact(text: String, budget: Int) -> String {
|
||||
let lines: [String] = str_split_lines(text)
|
||||
let n: Int = el_list_len(lines)
|
||||
let seen: String = "\n"
|
||||
let out: String = ""
|
||||
let out_tokens = 0
|
||||
let i = 0
|
||||
while i < n {
|
||||
let ln: String = str_trim(el_list_get(lines, i))
|
||||
if str_eq(ln, "") {
|
||||
let i = i + 1
|
||||
} else {
|
||||
let marker: String = "\n" + ln + "\n"
|
||||
if str_contains(seen, marker) {
|
||||
// duplicate line — skip
|
||||
let i = i + 1
|
||||
} else {
|
||||
let seen = seen + ln + "\n"
|
||||
let line_tokens: Int = ccr_est_tokens(ln) + 1
|
||||
if out_tokens + line_tokens > budget {
|
||||
// budget exhausted — stop (bounded)
|
||||
let i = n
|
||||
} else {
|
||||
let out = out + ln + "\n"
|
||||
let out_tokens = out_tokens + line_tokens
|
||||
let i = i + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ── stages 1+2: retrieve + scope ─────────────────────────────────────────────
|
||||
|
||||
// ccr_retrieve_scoped — pull context relevant to this input and its blueprint
|
||||
// knowledge refs, scoped to a fraction of the budget so no single source floods
|
||||
// the package. Returns compacted retrieved text (may be empty if the mind is
|
||||
// unreachable — the input alone is still a valid minimal context).
|
||||
fn ccr_retrieve_scoped(blueprint: String, knowledge_refs: String, input_item: String, budget: Int) -> String {
|
||||
let acc: String = ""
|
||||
// knowledge_refs is a JSON array of query strings.
|
||||
let m: Int = json_array_len(knowledge_refs)
|
||||
let i = 0
|
||||
while i < m {
|
||||
let ref: String = json_array_get_string(knowledge_refs, i)
|
||||
let hit: String = primitive_attend(ref, 3)
|
||||
let acc = acc + "# ref:" + ref + "\n" + hit + "\n"
|
||||
let i = i + 1
|
||||
}
|
||||
// the input's own salient text also seeds retrieval
|
||||
let hit2: String = primitive_attend(input_item, 3)
|
||||
let acc = acc + "# input-context\n" + hit2 + "\n"
|
||||
// scope retrieval to ~60% of budget; the input itself gets the rest
|
||||
let retr_budget: Int = (budget * 6) / 10
|
||||
return ccr_compact(acc, retr_budget)
|
||||
}
|
||||
|
||||
// ── ccr_compile — assemble the bounded per-worker context package ─────────────
|
||||
//
|
||||
// blueprint : task blueprint name
|
||||
// knowledge_refs : JSON array of retrieval queries from the blueprint
|
||||
// input_item : THIS worker's single input (and nothing else)
|
||||
// corr_id : swarm correlation ID
|
||||
// worker_id : this worker's ID
|
||||
// scope_token : the worker's containment token (closed boundary)
|
||||
//
|
||||
// Returns a JSON package: { blueprint, corr_id, worker_id, scope_token,
|
||||
// input, knowledge, budget_tokens, compiled_tokens }. `knowledge` is compiled
|
||||
// and bounded; the package as a whole is bounded by budget.
|
||||
fn ccr_compile(blueprint: String, knowledge_refs: String, input_item: String,
|
||||
corr_id: String, worker_id: String, scope_token: String) -> String {
|
||||
let budget: Int = ccr_default_budget()
|
||||
let knowledge: String = ccr_retrieve_scoped(blueprint, knowledge_refs, input_item, budget)
|
||||
|
||||
let kv: [String] = el_list_empty()
|
||||
let kv = el_list_append(kv, "blueprint")
|
||||
let kv = el_list_append(kv, blueprint)
|
||||
let kv = el_list_append(kv, "corr_id")
|
||||
let kv = el_list_append(kv, corr_id)
|
||||
let kv = el_list_append(kv, "worker_id")
|
||||
let kv = el_list_append(kv, worker_id)
|
||||
let kv = el_list_append(kv, "input")
|
||||
let kv = el_list_append(kv, input_item)
|
||||
let kv = el_list_append(kv, "knowledge")
|
||||
let kv = el_list_append(kv, knowledge)
|
||||
let kv = el_list_append(kv, "budget_tokens")
|
||||
let kv = el_list_append(kv, int_to_str(budget))
|
||||
let pkg: String = json_build_object(kv)
|
||||
// stamp the scope token as a nested object, and the measured size
|
||||
let pkg2: String = json_set(pkg, "scope_token", scope_token)
|
||||
let compiled_tokens: Int = ccr_est_tokens(pkg2)
|
||||
let pkg3: String = json_set(pkg2, "compiled_tokens", int_to_str(compiled_tokens))
|
||||
return pkg3
|
||||
}
|
||||
|
||||
// ccr_within_budget — did the compiled package stay within its budget?
|
||||
// (Retrieval is bounded to 60% and the input is small; this asserts the whole
|
||||
// package is bounded — the property distribution relies on.)
|
||||
fn ccr_within_budget(pkg: String) -> Bool {
|
||||
let budget: Int = str_to_int(json_get_string(pkg, "budget_tokens"))
|
||||
let compiled: Int = str_to_int(json_get_string(pkg, "compiled_tokens"))
|
||||
// allow a small envelope for JSON framing overhead
|
||||
if compiled <= budget + 200 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,181 +0,0 @@
|
||||
// containment.el — the Swarm Architecture containment rules, enforced.
|
||||
//
|
||||
// "These rules are not conventions. They are enforced by the runtime."
|
||||
// (Swarm Architecture §3.2). The three rules that make bounded parallelism —
|
||||
// and therefore location-independent distribution — safe:
|
||||
//
|
||||
// Rule 1: a worker may NOT join another swarm.
|
||||
// Rule 2: a worker may NOT initiate a new swarm.
|
||||
// Rule 3: a worker may NOT communicate laterally with sibling workers.
|
||||
//
|
||||
// Enforcement is by SCOPE TOKEN. When a swarm fans out, the coordinator mints a
|
||||
// swarm scope token and stamps a distinct worker scope token into each worker's
|
||||
// task envelope. Any attempt to create or join a swarm checks the caller's
|
||||
// token: if the caller already holds a WORKER token, the operation is rejected.
|
||||
// Rule 3 is enforced structurally elsewhere — workers share no mutable state and
|
||||
// the only channels they hold are the vertical result path — but this module
|
||||
// provides the explicit lateral-edge check for the execution tree.
|
||||
//
|
||||
// A scope token is a JSON object: {"kind":"coordinator|worker","swarm":"<corr>",
|
||||
// "worker":"<id-or-empty>","depth":"<n>"}.
|
||||
|
||||
// ── Token minting ────────────────────────────────────────────────────────────
|
||||
|
||||
// CAPABILITIES. A scope token carries a `caps` set — the authority it holds.
|
||||
// This is an AUTHORITY gate, not a health gate: capability is decided at mint
|
||||
// time and cannot be acquired at runtime. Engram-WRITE (op_write/op_relate/
|
||||
// op_supersede -> POST /api/nodes, /api/edges, DELETE) and dharma_emit are
|
||||
// @manager-ONLY capabilities — exactly the VBD rule that only the orchestrator
|
||||
// mutates global state. The orchestrator's token carries them; a worker's token
|
||||
// NEVER does. A worker is therefore STRUCTURALLY UNABLE to mutate global engram
|
||||
// state, regardless of engram health.
|
||||
fn cap_orchestrator() -> String { return "engram:read,engram:write,dharma:emit,state:write" }
|
||||
fn cap_worker() -> String { return "engram:read" }
|
||||
|
||||
// containment_coordinator_token — the token the orchestrator (@manager) holds.
|
||||
// Depth 0. Carries the engram-WRITE + dharma-emit capabilities (@manager-only).
|
||||
fn containment_coordinator_token(corr_id: String) -> String {
|
||||
let kv: [String] = el_list_empty()
|
||||
let kv = el_list_append(kv, "kind")
|
||||
let kv = el_list_append(kv, "coordinator")
|
||||
let kv = el_list_append(kv, "swarm")
|
||||
let kv = el_list_append(kv, corr_id)
|
||||
let kv = el_list_append(kv, "worker")
|
||||
let kv = el_list_append(kv, "")
|
||||
let kv = el_list_append(kv, "depth")
|
||||
let kv = el_list_append(kv, "0")
|
||||
let kv = el_list_append(kv, "caps")
|
||||
let kv = el_list_append(kv, cap_orchestrator())
|
||||
return json_build_object(kv)
|
||||
}
|
||||
|
||||
// containment_worker_token — the token stamped into a worker's envelope. Depth 1.
|
||||
// A closed boundary: forbids opening/joining swarms AND carries ONLY the
|
||||
// engram:READ capability — no engram:write, no dharma:emit. Read-only against the
|
||||
// full engram; may write only its own local geometry (its returned result).
|
||||
fn containment_worker_token(corr_id: String, worker_id: String) -> String {
|
||||
let kv: [String] = el_list_empty()
|
||||
let kv = el_list_append(kv, "kind")
|
||||
let kv = el_list_append(kv, "worker")
|
||||
let kv = el_list_append(kv, "swarm")
|
||||
let kv = el_list_append(kv, corr_id)
|
||||
let kv = el_list_append(kv, "worker")
|
||||
let kv = el_list_append(kv, worker_id)
|
||||
let kv = el_list_append(kv, "depth")
|
||||
let kv = el_list_append(kv, "1")
|
||||
let kv = el_list_append(kv, "caps")
|
||||
let kv = el_list_append(kv, cap_worker())
|
||||
return json_build_object(kv)
|
||||
}
|
||||
|
||||
// containment_has_cap — does this token carry capability `cap`?
|
||||
fn containment_has_cap(token: String, cap: String) -> Bool {
|
||||
return str_contains(json_get_string(token, "caps"), cap)
|
||||
}
|
||||
|
||||
// ── Rule checks (return "" on allow, or a rejection reason string) ───────────
|
||||
|
||||
// containment_check_open — may the holder of `token` OPEN a new swarm?
|
||||
// Enforces Rule 2 (a worker may not initiate a new swarm). Only a coordinator
|
||||
// token, or an absent token (top-level process), may open one.
|
||||
fn containment_check_open(token: String) -> String {
|
||||
if str_eq(token, "") {
|
||||
return ""
|
||||
}
|
||||
let kind: String = json_get_string(token, "kind")
|
||||
if str_eq(kind, "worker") {
|
||||
return "CONTAINMENT rule 2: a swarm worker may not initiate a new swarm (worker=" + json_get_string(token, "worker") + " swarm=" + json_get_string(token, "swarm") + ")"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// containment_check_join — may the holder of `token` JOIN swarm `target_corr`?
|
||||
// Enforces Rule 1 (a worker may not join another swarm). A worker already bound
|
||||
// to swarm A may not register into swarm B; and a worker may not re-join at all.
|
||||
fn containment_check_join(token: String, target_corr: String) -> String {
|
||||
if str_eq(token, "") {
|
||||
return ""
|
||||
}
|
||||
let kind: String = json_get_string(token, "kind")
|
||||
if str_eq(kind, "worker") {
|
||||
return "CONTAINMENT rule 1: a swarm worker may not join another swarm (worker=" + json_get_string(token, "worker") + " bound-swarm=" + json_get_string(token, "swarm") + " attempted-swarm=" + target_corr + ")"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// containment_check_lateral — may `from_token` open a communication edge to a
|
||||
// sibling worker `to_worker_id`? Enforces Rule 3 (no lateral communication).
|
||||
// The only permitted edges are vertical: worker->coordinator and
|
||||
// coordinator->worker. Any worker->worker edge is rejected.
|
||||
fn containment_check_lateral(from_token: String, to_worker_id: String) -> String {
|
||||
let kind: String = json_get_string(from_token, "kind")
|
||||
if str_eq(kind, "worker") {
|
||||
if str_eq(to_worker_id, "") {
|
||||
// empty target = the coordinator (vertical) — allowed
|
||||
return ""
|
||||
}
|
||||
return "CONTAINMENT rule 3: a swarm worker may not communicate laterally with sibling workers (from=" + json_get_string(from_token, "worker") + " to=" + to_worker_id + ")"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// containment_check_engram_write — RULE 4: only a token carrying the
|
||||
// engram:write capability (the orchestrator's) may mutate global engram state.
|
||||
// A worker token (engram:read only) is REJECTED — the authority gate. Reuses the
|
||||
// exact scope-token mechanism as Rule 2's open-denial. Returns "" on allow, or a
|
||||
// rejection reason. This is an AUTHORITY gate: it does not consult engram health.
|
||||
fn containment_check_engram_write(token: String, op: String) -> String {
|
||||
if containment_has_cap(token, "engram:write") {
|
||||
return ""
|
||||
}
|
||||
return "CONTAINMENT rule 4: engram-write is @manager-only — a worker is read-only against the engram and may not mutate global state (op=" + op + " kind=" + json_get_string(token, "kind") + " worker=" + json_get_string(token, "worker") + " caps=" + json_get_string(token, "caps") + ")"
|
||||
}
|
||||
|
||||
// containment_check_dharma_emit — the same @manager-only rule for dharma_emit,
|
||||
// grounding Rule 4 in VBD: global-state mutations (engram-write, dharma-emit) are
|
||||
// orchestrator-only, checked by the one capability mechanism.
|
||||
fn containment_check_dharma_emit(token: String) -> String {
|
||||
if containment_has_cap(token, "dharma:emit") {
|
||||
return ""
|
||||
}
|
||||
return "CONTAINMENT rule 4: dharma_emit is @manager-only (kind=" + json_get_string(token, "kind") + ")"
|
||||
}
|
||||
|
||||
// ── Enforcement helpers ──────────────────────────────────────────────────────
|
||||
|
||||
// containment_allows_open — Bool convenience over containment_check_open.
|
||||
fn containment_allows_open(token: String) -> Bool {
|
||||
return str_eq(containment_check_open(token), "")
|
||||
}
|
||||
|
||||
// containment_is_worker — is this a worker-scoped (closed-boundary) token?
|
||||
fn containment_is_worker(token: String) -> Bool {
|
||||
return str_eq(json_get_string(token, "kind"), "worker")
|
||||
}
|
||||
|
||||
// containment_guard_open — assert a swarm may be opened under this token.
|
||||
// Returns "" if allowed, or records a CONTAINMENT violation to the work-tracking
|
||||
// journal and returns the reason. Callers must abort on a non-empty return.
|
||||
fn containment_guard_open(token: String, corr_id: String) -> String {
|
||||
let reason: String = containment_check_open(token)
|
||||
if str_eq(reason, "") {
|
||||
return ""
|
||||
}
|
||||
let p: String = json_set_str("{}", "reason", reason)
|
||||
worktrack_append("containment.violation", corr_id, "open", p)
|
||||
return reason
|
||||
}
|
||||
|
||||
// containment_guard_engram_write — assert a token may mutate global engram state
|
||||
// (Rule 4). Returns "" if allowed; otherwise journals a containment.violation and
|
||||
// returns the reason. The write path MUST abort on a non-empty return.
|
||||
fn containment_guard_engram_write(token: String, corr_id: String, op: String) -> String {
|
||||
let reason: String = containment_check_engram_write(token, op)
|
||||
if str_eq(reason, "") {
|
||||
return ""
|
||||
}
|
||||
let p0: String = json_set_str("{}", "reason", reason)
|
||||
let p1: String = json_set_str(p0, "op", op)
|
||||
worktrack_append("containment.violation", corr_id, "engram-write", p1)
|
||||
return reason
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
// primitive_binding.el — THE ONE FLIP POINT.
|
||||
//
|
||||
// This file is the single seam between the swarm and the real agentic
|
||||
// primitives. Binding the reshape's decorated primitives is a one-line change
|
||||
// HERE and nothing else changes anywhere in the swarm.
|
||||
//
|
||||
// The api-reshape agent (wt/api-reshape) is wiring the primitives as DECORATED
|
||||
// El on the dharma_* event bus over the engram — think/attend/learn/ground/assert
|
||||
// become decorated fns that emit afferent events onto the bus. The moment they
|
||||
// land, flip `bound_think` (and its siblings) to call them.
|
||||
//
|
||||
// TODAY (stub fallback, compiles + runs now against :8901):
|
||||
// fn bound_think(...) { return primitive_think(ctx, instruction) }
|
||||
//
|
||||
// THE FLIP (when reshape's decorated primitives land — one line each):
|
||||
// fn bound_think(...) { return think(ctx, instruction) } // decorated, on dharma bus
|
||||
//
|
||||
// Keep the stub as fallback: `bound_think` is only reached when the seam mode is
|
||||
// "decorated" (SWARM_PRIMITIVE_SEAM=decorated). Until you flip these bodies AND
|
||||
// set that env, the harness runs entirely on the hermetic stub.
|
||||
|
||||
// bound_think — BOUND to the reshape's proven decorated `think` (op_think),
|
||||
// real cognition over the engram geometry. The worker's CCR slice carries a
|
||||
// NODE-ID anchor in ctx.input (free-text anchors return "geometry unavailable");
|
||||
// think re-origins at that node's region under the faculty and returns a real
|
||||
// 768-dim gradient.
|
||||
fn bound_think(ctx: String, instruction: String) -> String {
|
||||
let anchor: String = json_get_string(ctx, "input")
|
||||
let faculty: String = json_get_string(ctx, "faculty")
|
||||
return op_think(anchor, faculty)
|
||||
}
|
||||
|
||||
// bound_attend — BOUND to the reshape's op_attend (POST /api/attend). Needs the
|
||||
// gate-1 write-healthy clone; falls back to the read-side attend otherwise.
|
||||
fn bound_attend(query: String, limit: Int) -> String {
|
||||
if str_eq(env("SWARM_WRITE_HEALTHY"), "1") {
|
||||
return op_attend(query, "self")
|
||||
}
|
||||
return primitive_attend(query, limit)
|
||||
}
|
||||
|
||||
// bound_learn — BOUND to the reshape's op_learn (correspondence-beat). Needs the
|
||||
// gate-1 write-healthy clone; falls back to the opt-in journal-only learn.
|
||||
fn bound_learn(corr_id: String, observation: String) -> String {
|
||||
if str_eq(env("SWARM_WRITE_HEALTHY"), "1") {
|
||||
return op_learn(observation, "induce")
|
||||
}
|
||||
return primitive_learn(corr_id, observation)
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
// primitive_seam.el — the configurable primitive seam + telemetry.
|
||||
//
|
||||
// One switch selects where a worker's primitive invocation goes:
|
||||
// SWARM_PRIMITIVE_SEAM=stub (default) — hermetic in-process think.
|
||||
// SWARM_PRIMITIVE_SEAM=decorated — the reshape's decorated
|
||||
// primitives on the dharma bus
|
||||
// (see primitive_binding.el).
|
||||
//
|
||||
// Every seam invocation is an AFFERENT signal — a primitive call travelling
|
||||
// toward the manager. The seam stamps telemetry onto each thought (seam_mode +
|
||||
// one afferent tick) so the coordinator can aggregate afferent counters across
|
||||
// the swarm without any shared mutable state (containment-safe: counts ride the
|
||||
// vertical result path, not a shared bus register).
|
||||
|
||||
// seam_mode — "stub" (default) or "decorated".
|
||||
fn seam_mode() -> String {
|
||||
let m: String = env("SWARM_PRIMITIVE_SEAM")
|
||||
if str_eq(m, "decorated") {
|
||||
return "decorated"
|
||||
}
|
||||
return "stub"
|
||||
}
|
||||
|
||||
// seam_think — route a worker's `think` through the configured seam and stamp
|
||||
// telemetry. Returns the thought JSON augmented with:
|
||||
// seam_mode : which side of the seam served this call
|
||||
// afferent : "1" — one afferent primitive signal was emitted
|
||||
fn seam_think(ctx: String, instruction: String) -> String {
|
||||
let mode: String = seam_mode()
|
||||
let thought: String = ""
|
||||
if str_eq(mode, "decorated") {
|
||||
let thought = bound_think(ctx, instruction)
|
||||
} else {
|
||||
let thought = primitive_think(ctx, instruction)
|
||||
}
|
||||
let t1: String = json_set_str(thought, "seam_mode", mode)
|
||||
let t2: String = json_set_str(t1, "afferent", "1")
|
||||
return t2
|
||||
}
|
||||
|
||||
// seam_attend / seam_learn — same seam for the other primitives (used when a
|
||||
// blueprint retrieves or writes through the bus).
|
||||
fn seam_attend(query: String, limit: Int) -> String {
|
||||
if str_eq(seam_mode(), "decorated") {
|
||||
return bound_attend(query, limit)
|
||||
}
|
||||
return primitive_attend(query, limit)
|
||||
}
|
||||
|
||||
fn seam_learn(corr_id: String, observation: String) -> String {
|
||||
if str_eq(seam_mode(), "decorated") {
|
||||
return bound_learn(corr_id, observation)
|
||||
}
|
||||
return primitive_learn(corr_id, observation)
|
||||
}
|
||||
@@ -1,94 +0,0 @@
|
||||
// primitives.el — the agentic primitive SEAM the swarm composes over.
|
||||
//
|
||||
// The swarm is orchestration OVER the five CCR primitives, not a replacement for
|
||||
// them (CCR §2, "The Five Primitives / The Execution Cycle"): a worker executes
|
||||
// its task blueprint as attend -> think -> intend -> act -> learn against its
|
||||
// compiled, bounded context.
|
||||
//
|
||||
// This file is the SEAM. The parallel API-surface reshape exposes the canonical
|
||||
// primitive tools; when it lands, bind each primitive below to the reshaped
|
||||
// implementation (see PRIMITIVE_BINDING). Until then these are thin, engram-
|
||||
// backed fallbacks so the swarm — its fan-out, containment, CCR context
|
||||
// compilation, convergence, and work-tracking — is fully exercisable today.
|
||||
//
|
||||
// Contract: every primitive takes and returns String (JSON where structured), so
|
||||
// any primitive is directly threadable via thread.el's spawn (which runs
|
||||
// top-level (String)->String El fns).
|
||||
//
|
||||
// PRIMITIVE_BINDING: to bind the reshape's real tools, replace each fallback body
|
||||
// with a call to the reshaped El fn / API endpoint. Signatures here are the
|
||||
// stable contract the swarm depends on; keep them.
|
||||
|
||||
// ── attend — retrieve the minimal relevant context for a focus ───────────────
|
||||
// Vantage-read: pull only what this focus needs from the mind. Backed by the
|
||||
// engram's spreading-activation retrieval.
|
||||
fn primitive_attend(query: String, limit: Int) -> String {
|
||||
if str_eq(query, "") {
|
||||
return "[]"
|
||||
}
|
||||
// Location-independent worker model: when an engram daemon is configured,
|
||||
// retrieve over HTTP (the worker may run anywhere). POST /api/search
|
||||
// {query,limit,_auth}. Falls back to the in-process store otherwise.
|
||||
let url: String = env("ENGRAM_URL")
|
||||
if str_eq(url, "") {
|
||||
return engram_activate(query, limit)
|
||||
}
|
||||
let kv: [String] = el_list_empty()
|
||||
let kv = el_list_append(kv, "query")
|
||||
let kv = el_list_append(kv, query)
|
||||
let body0: String = json_build_object(kv)
|
||||
let body1: String = json_set(body0, "limit", int_to_str(limit))
|
||||
let body2: String = json_set_str(body1, "_auth", env("ENGRAM_API_KEY"))
|
||||
return http_post(url + "/api/search", body2)
|
||||
}
|
||||
|
||||
// ── think — reason over the compiled context ─────────────────────────────────
|
||||
// In production this routes to a model (CCR dynamic model selection). Here it is
|
||||
// a deterministic, hermetic transform so swarm behaviour is testable without an
|
||||
// external model: it echoes a structured verdict derived from the context. The
|
||||
// binding point for a real model is explicit.
|
||||
fn primitive_think(compiled_ctx: String, instruction: String) -> String {
|
||||
// PRIMITIVE_BINDING: replace with the reshape's think() (model inference).
|
||||
let kv: [String] = el_list_empty()
|
||||
let kv = el_list_append(kv, "instruction")
|
||||
let kv = el_list_append(kv, instruction)
|
||||
let kv = el_list_append(kv, "ctx_bytes")
|
||||
let kv = el_list_append(kv, int_to_str(str_len(compiled_ctx)))
|
||||
let kv = el_list_append(kv, "conclusion")
|
||||
let kv = el_list_append(kv, "reasoned:" + instruction)
|
||||
return json_build_object(kv)
|
||||
}
|
||||
|
||||
// ── intend — form a bounded plan/decision from a thought ─────────────────────
|
||||
fn primitive_intend(thought: String) -> String {
|
||||
let concl: String = json_get_string(thought, "conclusion")
|
||||
let kv: [String] = el_list_empty()
|
||||
let kv = el_list_append(kv, "intent")
|
||||
let kv = el_list_append(kv, concl)
|
||||
return json_build_object(kv)
|
||||
}
|
||||
|
||||
// ── act — execute a bounded effect and return its result ─────────────────────
|
||||
// Workers defer real side-effects to the coordinator (idempotency requirement,
|
||||
// Swarm §7.3). Here act produces an artifact-shaped result the coordinator
|
||||
// collects during convergence.
|
||||
fn primitive_act(intent: String, input_item: String) -> String {
|
||||
let kv: [String] = el_list_empty()
|
||||
let kv = el_list_append(kv, "acted_on")
|
||||
let kv = el_list_append(kv, input_item)
|
||||
let kv = el_list_append(kv, "via")
|
||||
let kv = el_list_append(kv, json_get_string(intent, "intent"))
|
||||
return json_build_object(kv)
|
||||
}
|
||||
|
||||
// ── learn — record an observation into the mind, tagged by correlation ID ────
|
||||
// Append-only, naturally idempotent (Swarm §7.3). Best-effort: a worker that
|
||||
// cannot reach the mind still returns its result.
|
||||
fn primitive_learn(corr_id: String, observation: String) -> String {
|
||||
let url: String = env("ENGRAM_URL")
|
||||
if str_eq(url, "") {
|
||||
return ""
|
||||
}
|
||||
let content: String = "swarm-worker-obs corr=" + corr_id + " :: " + observation
|
||||
return engram_node(content, "Memory", 0.4)
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
// reshape_surface.el — the api-reshape agent's PROVEN decorated primitives,
|
||||
// composed into the swarm build to bind real cognition.
|
||||
//
|
||||
// PROVENANCE: these fns are the reshape's surface at wt/api-reshape @ d4f401d
|
||||
// ("reshape: decorator-as-seam — port @route codegen, prove decorate->serve,
|
||||
// rewrite surface as decorated El"), verified live against
|
||||
// engram.cognition-20260814. Copied verbatim (read/cognition ops only) so the
|
||||
// swarm binds the REAL primitives, not a reimplementation. The write ops
|
||||
// (op_write/op_relate/op_supersede/op_ground) are intentionally NOT composed
|
||||
// here — they exercise the persist_node write path that needs the gate-1
|
||||
// write-healthy clone; the swarm's proven run is read-cognition (think/read).
|
||||
//
|
||||
// Ops route to the ENGRAM over ENGRAM_URL — pinned by THIS worktree's .nsbx-env
|
||||
// to the :8901 swarm clone (never the reshape agent's :8900). Separate clones,
|
||||
// no collision.
|
||||
|
||||
fn engram_url() -> String {
|
||||
let u: String = env("ENGRAM_URL")
|
||||
if str_eq(u, "") { return "http://127.0.0.1:8900" }
|
||||
return u
|
||||
}
|
||||
fn engram_key() -> String {
|
||||
let k: String = env("ENGRAM_API_KEY")
|
||||
if str_eq(k, "") { return "sbx-dev-api-reshape" }
|
||||
return k
|
||||
}
|
||||
fn SELF_KEY() -> String { return "kn-efeb4a5b-5aff-4759-8a97-7233099be6ee" }
|
||||
fn VALUES_KEY() -> String { return "kn-5b606390-a52d-4ca2-8e0e-eba141d13440" }
|
||||
|
||||
// self/values name -> keystone id; anything else passes through unchanged.
|
||||
fn resolve_named(v: String) -> String {
|
||||
if str_eq(v, "self") { return SELF_KEY() }
|
||||
if str_eq(v, "neuron") { return SELF_KEY() }
|
||||
if str_eq(v, "values") { return VALUES_KEY() }
|
||||
if str_eq(v, "values_hub") { return VALUES_KEY() }
|
||||
return v
|
||||
}
|
||||
|
||||
// read — THE VANTAGE-READ. Re-origin at a point + aperture -> a BOUNDED slice.
|
||||
fn op_read(vantage: String, typ: String, k: Int) -> String {
|
||||
let vid: String = resolve_named(vantage)
|
||||
if str_eq(typ, "edges") {
|
||||
return http_get(engram_url() + "/api/neighbors/" + vid)
|
||||
}
|
||||
if str_starts_with(vid, "kn-") {
|
||||
return http_get(engram_url() + "/api/neighbors/" + vid)
|
||||
}
|
||||
return http_get(engram_url() + "/api/search?q=" + url_encode(vid) + "&limit=" + int_to_str(k))
|
||||
}
|
||||
|
||||
// think — THE ONE OPERATION. anchor (node ids) steered by faculty -> gradient.
|
||||
fn op_think(seeds: String, faculty: String) -> String {
|
||||
let s: String = resolve_named(seeds)
|
||||
let f: String = if str_eq(faculty, "") { "reason" } else { faculty }
|
||||
return http_get(engram_url() + "/api/think?seeds=" + url_encode(s) + "&faculty=" + f)
|
||||
}
|
||||
|
||||
// attend — aim attention at a region. (POST — needs a write-healthy clone.)
|
||||
fn op_attend(node: String, observer: String) -> String {
|
||||
let n: String = resolve_named(node)
|
||||
let o: String = if str_eq(observer, "") { SELF_KEY() } else { resolve_named(observer) }
|
||||
let body: String = "{\"_auth\":\"" + engram_key() + "\",\"node\":\"" + n
|
||||
+ "\",\"observer\":\"" + o + "\",\"salience\":\"0.6\"}"
|
||||
return http_post_json(engram_url() + "/api/attend", body)
|
||||
}
|
||||
|
||||
fn identity_typed(t: String) -> Bool {
|
||||
if str_eq(t, "self") { return true }
|
||||
if str_eq(t, "values") { return true }
|
||||
return false
|
||||
}
|
||||
fn type_to_node_type(t: String) -> String {
|
||||
if str_eq(t, "knowledge") { return "Knowledge" }
|
||||
if str_eq(t, "artifact") { return "Artifact" }
|
||||
if str_eq(t, "backlog") { return "WorkItem" }
|
||||
if str_eq(t, "process") { return "Process" }
|
||||
if str_eq(t, "state") { return "InternalStateEvent" }
|
||||
return "Memory"
|
||||
}
|
||||
|
||||
// write — add a node (POST /api/nodes). Identity types refused. This is a
|
||||
// global-engram MUTATION — @manager-only (Rule 4); never called on a worker path.
|
||||
// (Reshape's op_write, with json_escape -> the available json_escape_string.)
|
||||
fn op_write(content: String, typ: String, importance: Float) -> String {
|
||||
if str_eq(content, "") { return "{\"error\":\"write: content required\"}" }
|
||||
if identity_typed(typ) {
|
||||
return "{\"error\":\"write type=" + typ + " is write-protected -> intentional-cultivation\"}"
|
||||
}
|
||||
let body: String = "{\"_auth\":\"" + engram_key() + "\",\"content\":\"" + json_escape_string(content)
|
||||
+ "\",\"node_type\":\"" + type_to_node_type(typ) + "\",\"tier\":\"Working\",\"importance\":"
|
||||
+ float_to_str(importance) + "}"
|
||||
return http_post_json(engram_url() + "/api/nodes", body)
|
||||
}
|
||||
|
||||
// learn — the reflexive correspondence-beat: calibrate the steering-prior.
|
||||
// (POST — needs a write-healthy clone.)
|
||||
fn op_learn(seeds: String, faculty: String) -> String {
|
||||
let s: String = resolve_named(seeds)
|
||||
let f: String = if str_eq(faculty, "") { "induce" } else { faculty }
|
||||
let body: String = "{\"_auth\":\"" + engram_key() + "\",\"seeds\":\"" + s
|
||||
+ "\",\"faculty\":\"" + f + "\",\"keystone\":\"false\"}"
|
||||
return http_post_json(engram_url() + "/api/correspondence-beat", body)
|
||||
}
|
||||
@@ -1,483 +0,0 @@
|
||||
// swarm.el — the swarm orchestrator: bounded parallel agent execution.
|
||||
//
|
||||
// Implements Swarm Architecture's single pattern — fan out, execute independently,
|
||||
// converge — on El's NATIVE concurrency (thread.el spawn/join). No external
|
||||
// orchestrator: a swarm is a coordinator (this file, the main thread) that mints
|
||||
// a correlation identity, compiles a bounded CCR context per worker, dispatches
|
||||
// workers as native pthreads, tracks every unit of work, and converges the
|
||||
// results before returning control to the parent step.
|
||||
//
|
||||
// The five properties of every swarm (Swarm §2.1) are all present:
|
||||
// parent step -> swarm_run is called from one process step
|
||||
// task blueprint -> `blueprint` name + knowledge refs, run by every worker
|
||||
// input set -> `inputs_json`, one item per worker
|
||||
// convergence -> `strategy` in config (collect|merge|vote|reduce)
|
||||
// correlation ID -> minted here, threaded through tracking + every worker
|
||||
//
|
||||
// Containment (Swarm §3) is enforced: the caller must hold a coordinator/absent
|
||||
// token to open a swarm (Rule 2), each worker is stamped a closed worker token
|
||||
// (Rules 1+3), and workers share no mutable state (the coordinator is the only
|
||||
// journal writer).
|
||||
|
||||
// ── worker entry — the top-level (String)->String fn native threads run ──────
|
||||
//
|
||||
// Every El fn compiles to a global C symbol; spawn() resolves this by name via
|
||||
// dlsym and runs it in a pthread. The envelope carries everything the worker is
|
||||
// permitted to see — its compiled context and nothing else (§9.3).
|
||||
//
|
||||
// Returns a result JSON: {worker_id, status:"completed"|"failed", output|error}.
|
||||
fn swarm_worker_entry(envelope_json: String) -> String {
|
||||
let worker_id: String = json_get_string(envelope_json, "worker_id")
|
||||
let ctx: String = json_get_raw(envelope_json, "ctx")
|
||||
|
||||
// The worker holds a CLOSED worker token (Rules 1+3): it shares no state
|
||||
// with siblings and may not open/join a swarm. That boundary is enforced at
|
||||
// the point of attempt — swarm_run rejects any swarm opened under a worker
|
||||
// token (Rule 2). A worker simply executing its blueprint is not opening a
|
||||
// swarm, so it proceeds. Its only outward edge is this returned result
|
||||
// (the vertical worker->coordinator path).
|
||||
let out: String = swarm_run_blueprint(ctx)
|
||||
// A worker reports failed iff its blueprint signalled failure. This is the
|
||||
// vertical status edge the coordinator reads during convergence (§4.3, §7).
|
||||
let bstatus: String = json_get_string(out, "blueprint_status")
|
||||
let status: String = "completed"
|
||||
if str_eq(bstatus, "failed") {
|
||||
let status = "failed"
|
||||
}
|
||||
let kv: [String] = el_list_empty()
|
||||
let kv = el_list_append(kv, "worker_id")
|
||||
let kv = el_list_append(kv, worker_id)
|
||||
let kv = el_list_append(kv, "status")
|
||||
let kv = el_list_append(kv, status)
|
||||
let res: String = json_build_object(kv)
|
||||
return json_set(res, "output", out)
|
||||
}
|
||||
|
||||
// swarm_run_blueprint — execute the task blueprint over a compiled context.
|
||||
// The default blueprint is the CCR execution cycle: think -> intend -> act over
|
||||
// the worker's bounded context. Specialise by dispatching on
|
||||
// json_get_string(ctx,"blueprint"). Idempotent: reads ctx, writes only its
|
||||
// returned output (§7.3).
|
||||
fn swarm_run_blueprint(ctx: String) -> String {
|
||||
let blueprint: String = json_get_string(ctx, "blueprint")
|
||||
let input_item: String = json_get_string(ctx, "input")
|
||||
let knowledge: String = json_get_string(ctx, "knowledge")
|
||||
|
||||
// classify — deterministic verdict for the `vote` convergence strategy:
|
||||
// verdict is "long" if the input has >4 chars, else "short".
|
||||
if str_eq(blueprint, "classify") {
|
||||
let verdict: String = "short"
|
||||
if str_len(input_item) > 4 {
|
||||
let verdict = "long"
|
||||
}
|
||||
let kv: [String] = el_list_empty()
|
||||
let kv = el_list_append(kv, "verdict")
|
||||
let kv = el_list_append(kv, verdict)
|
||||
let kv = el_list_append(kv, "blueprint_status")
|
||||
let kv = el_list_append(kv, "ok")
|
||||
return json_build_object(kv)
|
||||
}
|
||||
|
||||
// faildemo — a worker that fails on inputs beginning with "x" (exercises the
|
||||
// failure threshold + partial convergence path). Idempotent, side-effect-free.
|
||||
if str_eq(blueprint, "faildemo") {
|
||||
let st: String = "ok"
|
||||
if str_starts_with(input_item, "x") {
|
||||
let st = "failed"
|
||||
}
|
||||
return json_set_str("{}", "blueprint_status", st)
|
||||
}
|
||||
|
||||
// cognize — REAL-COGNITION blueprint. Routes think through the seam (bound to
|
||||
// op_think in decorated mode) over the worker's NODE-ID anchor, then derives a
|
||||
// vote verdict from the gradient's confidence. In stub mode there is no
|
||||
// gradient, so the verdict falls back to a deterministic slice hash — the
|
||||
// same blueprint runs green on either side of the seam.
|
||||
if str_eq(blueprint, "cognize") {
|
||||
let thought: String = seam_think(ctx, "reason over " + input_item)
|
||||
// Derive the vote verdict from the REAL gradient's support count
|
||||
// (json_get_int, since n_support is numeric). Different anchors have
|
||||
// different support -> genuine, cognition-driven vote diversity. In stub
|
||||
// mode there is no gradient (n_support -> 0) -> "uncertain".
|
||||
let nsup: Int = json_get_int(thought, "n_support")
|
||||
let verdict: String = "uncertain"
|
||||
if nsup >= 10 {
|
||||
let verdict = "confident"
|
||||
}
|
||||
let ck: [String] = el_list_empty()
|
||||
let ck = el_list_append(ck, "verdict")
|
||||
let ck = el_list_append(ck, verdict)
|
||||
let ck = el_list_append(ck, "blueprint_status")
|
||||
let ck = el_list_append(ck, "ok")
|
||||
let cout0: String = json_build_object(ck)
|
||||
let cout1: String = json_set_str(cout0, "n_support", int_to_str(nsup))
|
||||
let cout2: String = json_set_str(cout1, "seam_mode", json_get_string(thought, "seam_mode"))
|
||||
return json_set_str(cout2, "afferent", json_get_string(thought, "afferent"))
|
||||
}
|
||||
|
||||
// default (analyze_item): the CCR execution cycle think -> intend -> act,
|
||||
// with `think` routed through the CONFIGURABLE PRIMITIVE SEAM. Telemetry
|
||||
// (seam_mode + afferent tick) rides the worker's returned output.
|
||||
let instruction: String = "process input: " + input_item
|
||||
let thought: String = seam_think(ctx, instruction)
|
||||
let intent: String = primitive_intend(thought)
|
||||
let effect: String = primitive_act(intent, input_item)
|
||||
let e1: String = json_set_str(effect, "blueprint_status", "ok")
|
||||
let e2: String = json_set_str(e1, "seam_mode", json_get_string(thought, "seam_mode"))
|
||||
let e3: String = json_set_str(e2, "afferent", json_get_string(thought, "afferent"))
|
||||
return e3
|
||||
}
|
||||
|
||||
// ── native-thread fan-out, bounded by concurrency, order-preserving ──────────
|
||||
//
|
||||
// parallel_map (thread.el) spawns ALL threads at once. The swarm honours the
|
||||
// blueprint's `concurrency` cap (§5.1: a resource constraint, not a parallelism
|
||||
// constraint — all items are processed, at most N at a time) by dispatching in
|
||||
// waves of N native threads, joining each wave before the next. Results are
|
||||
// returned in input order.
|
||||
fn swarm_fanout(worker_fn: String, envelopes: [String], concurrency: Int) -> [String] {
|
||||
let n: Int = el_list_len(envelopes)
|
||||
let cap: Int = concurrency
|
||||
if cap < 1 {
|
||||
let cap = 1
|
||||
}
|
||||
let results: [String] = el_list_empty()
|
||||
let base = 0
|
||||
while base < n {
|
||||
// spawn a wave of up to `cap` workers
|
||||
let tids: [String] = el_list_empty()
|
||||
let k = 0
|
||||
while k < cap {
|
||||
let idx: Int = base + k
|
||||
if idx < n {
|
||||
let env_item: String = el_list_get(envelopes, idx)
|
||||
let tid: Int = spawn(worker_fn, env_item)
|
||||
let tids = el_list_append(tids, int_to_str(tid))
|
||||
}
|
||||
let k = k + 1
|
||||
}
|
||||
// join the wave in order
|
||||
let j = 0
|
||||
let jn: Int = el_list_len(tids)
|
||||
while j < jn {
|
||||
let tid: Int = str_to_int(el_list_get(tids, j))
|
||||
let r: String = join(tid)
|
||||
let results = el_list_append(results, r)
|
||||
let j = j + 1
|
||||
}
|
||||
let base = base + cap
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
// ── convergence strategies (Swarm §4.2) ──────────────────────────────────────
|
||||
|
||||
// swarm_converge_collect — ordered list, no transformation.
|
||||
fn swarm_converge_collect(results: [String]) -> String {
|
||||
let n: Int = el_list_len(results)
|
||||
let arr: String = "[]"
|
||||
let i = 0
|
||||
while i < n {
|
||||
let arr = json_array_push(arr, el_list_get(results, i))
|
||||
let i = i + 1
|
||||
}
|
||||
return arr
|
||||
}
|
||||
|
||||
// swarm_converge_merge — combine worker outputs into a single joined string.
|
||||
fn swarm_converge_merge(results: [String]) -> String {
|
||||
let n: Int = el_list_len(results)
|
||||
let merged: String = ""
|
||||
let i = 0
|
||||
while i < n {
|
||||
let out: String = json_get_raw(el_list_get(results, i), "output")
|
||||
if i > 0 {
|
||||
let merged = merged + " | "
|
||||
}
|
||||
let merged = merged + out
|
||||
let i = i + 1
|
||||
}
|
||||
return json_set_str("{}", "merged", merged)
|
||||
}
|
||||
|
||||
// swarm_converge_vote — tally a field across worker outputs, pick the majority.
|
||||
// Each worker output is expected to carry a "verdict" string field.
|
||||
fn swarm_converge_vote(results: [String]) -> String {
|
||||
let n: Int = el_list_len(results)
|
||||
// Collect verdicts (no mutable tally: json_set can't update an existing key
|
||||
// and there is no el_list_set). Then count each verdict by rescanning.
|
||||
let verdicts: [String] = el_list_empty()
|
||||
let i = 0
|
||||
while i < n {
|
||||
let out: String = json_get_raw(el_list_get(results, i), "output")
|
||||
let v: String = json_get_string(out, "verdict")
|
||||
if str_eq(v, "") {
|
||||
let i = i + 1
|
||||
} else {
|
||||
let verdicts = el_list_append(verdicts, v)
|
||||
let i = i + 1
|
||||
}
|
||||
}
|
||||
// pick the verdict with the highest count (first-past-the-post)
|
||||
let vn: Int = el_list_len(verdicts)
|
||||
let best: String = ""
|
||||
let bestc = 0
|
||||
let a = 0
|
||||
while a < vn {
|
||||
let cand: String = el_list_get(verdicts, a)
|
||||
// count occurrences of cand
|
||||
let c = 0
|
||||
let b = 0
|
||||
while b < vn {
|
||||
if str_eq(el_list_get(verdicts, b), cand) {
|
||||
let c = c + 1
|
||||
}
|
||||
let b = b + 1
|
||||
}
|
||||
if c > bestc {
|
||||
let bestc = c
|
||||
let best = cand
|
||||
}
|
||||
let a = a + 1
|
||||
}
|
||||
let kv: [String] = el_list_empty()
|
||||
let kv = el_list_append(kv, "winner")
|
||||
let kv = el_list_append(kv, best)
|
||||
let kv = el_list_append(kv, "votes")
|
||||
let kv = el_list_append(kv, int_to_str(bestc))
|
||||
return json_build_object(kv)
|
||||
}
|
||||
|
||||
// swarm_converge_reduce — fold outputs into an accumulator (count + concat).
|
||||
fn swarm_converge_reduce(results: [String]) -> String {
|
||||
let n: Int = el_list_len(results)
|
||||
let acc: String = ""
|
||||
let i = 0
|
||||
while i < n {
|
||||
let out: String = json_get_raw(el_list_get(results, i), "output")
|
||||
let acc = acc + out
|
||||
let i = i + 1
|
||||
}
|
||||
let kv: [String] = el_list_empty()
|
||||
let kv = el_list_append(kv, "count")
|
||||
let kv = el_list_append(kv, int_to_str(n))
|
||||
let kv = el_list_append(kv, "accumulated")
|
||||
let kv = el_list_append(kv, acc)
|
||||
return json_build_object(kv)
|
||||
}
|
||||
|
||||
// ratio_to_permille — parse a decimal ratio string ("1.0", "0.8") into an
|
||||
// integer per-mille (1000, 800) so failure thresholds use exact integer math.
|
||||
// (El float division is unreliable in this runtime — int_to_float(n)/int_to_float(n)
|
||||
// does not equal 1.0 — so the swarm deliberately avoids floats.)
|
||||
fn ratio_to_permille(s: String) -> Int {
|
||||
if str_eq(s, "") {
|
||||
return 1000
|
||||
}
|
||||
let parts: [String] = str_split(s, ".")
|
||||
let whole: Int = str_to_int(el_list_get(parts, 0))
|
||||
let permille: Int = whole * 1000
|
||||
if el_list_len(parts) > 1 {
|
||||
let frac_raw: String = el_list_get(parts, 1)
|
||||
let frac3: String = str_slice(str_pad_right(frac_raw, 3, "0"), 0, 3)
|
||||
let permille = permille + str_to_int(frac3)
|
||||
}
|
||||
return permille
|
||||
}
|
||||
|
||||
// swarm_converge — dispatch on strategy name.
|
||||
fn swarm_converge(strategy: String, results: [String]) -> String {
|
||||
if str_eq(strategy, "merge") {
|
||||
return swarm_converge_merge(results)
|
||||
}
|
||||
if str_eq(strategy, "vote") {
|
||||
return swarm_converge_vote(results)
|
||||
}
|
||||
if str_eq(strategy, "reduce") {
|
||||
return swarm_converge_reduce(results)
|
||||
}
|
||||
// default: collect
|
||||
return swarm_converge_collect(results)
|
||||
}
|
||||
|
||||
// ── the ONLY global-engram write path (Rule 4, @manager-only) ────────────────
|
||||
//
|
||||
// Every engram mutation flows through here and is gated by the caller's token
|
||||
// capability. Only the orchestrator's token carries engram:write, so a worker
|
||||
// (engram:read only) calling this is DENIED by capability before any HTTP is
|
||||
// issued — structurally unable to mutate global engram state, regardless of
|
||||
// engram health. This is the curated-merge write: the orchestrator committing
|
||||
// the geometry it approved. Workers never reach a successful branch here.
|
||||
fn swarm_engram_write(token: String, corr_id: String, content: String, typ: String, importance: Float) -> String {
|
||||
let deny: String = containment_guard_engram_write(token, corr_id, "engram.write")
|
||||
if str_eq(deny, "") {
|
||||
// authorized (orchestrator) — perform the write
|
||||
let res: String = op_write(content, typ, importance)
|
||||
let new_id: String = json_get_string(res, "id")
|
||||
let cp: String = json_set_str("{}", "node_id", new_id)
|
||||
worktrack_append("swarm.committed", corr_id, "orchestrator", cp)
|
||||
return res
|
||||
}
|
||||
// denied by capability — return the rejection, no engram mutation performed
|
||||
return json_set_str("{}", "denied", deny)
|
||||
}
|
||||
|
||||
// ── the coordinator: fan out -> track -> converge ────────────────────────────
|
||||
//
|
||||
// blueprint : task blueprint name run by every worker
|
||||
// knowledge_refs : JSON array of retrieval queries for CCR compilation
|
||||
// inputs_json : JSON array of input items (one per worker)
|
||||
// config_json : { concurrency, strategy, min_success_ratio,
|
||||
// failure_action, caller_token }
|
||||
//
|
||||
// Returns: { corr_id, status:"completed"|"aborted", merged, report }.
|
||||
fn swarm_run(blueprint: String, knowledge_refs: String, inputs_json: String, config_json: String) -> String {
|
||||
let corr_id: String = "swarm-" + uuid_v4()
|
||||
let caller_token: String = json_get_raw(config_json, "caller_token")
|
||||
let concurrency: Int = str_to_int(json_get_string(config_json, "concurrency"))
|
||||
if concurrency < 1 {
|
||||
let concurrency = 4
|
||||
}
|
||||
let strategy: String = json_get_string(config_json, "strategy")
|
||||
|
||||
// ── Containment Rule 2: only a coordinator/absent token may open a swarm ──
|
||||
let deny: String = containment_guard_open(caller_token, corr_id)
|
||||
if str_eq(deny, "") {
|
||||
// allowed — proceed
|
||||
let n: Int = json_array_len(inputs_json)
|
||||
|
||||
// swarm.created
|
||||
let cp: String = json_set_str("{}", "blueprint", blueprint)
|
||||
let cp2: String = json_set(cp, "input_count", int_to_str(n))
|
||||
worktrack_append("swarm.created", corr_id, corr_id, cp2)
|
||||
|
||||
// build per-worker envelopes: worker token + CCR-compiled bounded context
|
||||
let envelopes: [String] = el_list_empty()
|
||||
let i = 0
|
||||
while i < n {
|
||||
let worker_id: String = corr_id + "/worker-" + int_to_str(i)
|
||||
let input_item: String = json_array_get_string(inputs_json, i)
|
||||
let wtoken: String = containment_worker_token(corr_id, worker_id)
|
||||
let ctx: String = ccr_compile(blueprint, knowledge_refs, input_item, corr_id, worker_id, wtoken)
|
||||
// envelope: only this worker's compiled context + its closed token
|
||||
let ekv: [String] = el_list_empty()
|
||||
let ekv = el_list_append(ekv, "worker_id")
|
||||
let ekv = el_list_append(ekv, worker_id)
|
||||
let ekv = el_list_append(ekv, "corr_id")
|
||||
let ekv = el_list_append(ekv, corr_id)
|
||||
let env0: String = json_build_object(ekv)
|
||||
let env1: String = json_set(env0, "scope_token", wtoken)
|
||||
let env2: String = json_set(env1, "ctx", ctx)
|
||||
let envelopes = el_list_append(envelopes, env2)
|
||||
|
||||
let sp: String = json_set_str("{}", "input", input_item)
|
||||
worktrack_append("worker.started", corr_id, worker_id, sp)
|
||||
let i = i + 1
|
||||
}
|
||||
|
||||
// ── native-thread fan-out (bounded) ──
|
||||
let results: [String] = swarm_fanout("swarm_worker_entry", envelopes, concurrency)
|
||||
|
||||
// record per-worker terminal status + aggregate AFFERENT telemetry.
|
||||
// Afferent counters (primitive signals travelling toward the @manager)
|
||||
// are summed from the vertical result path — no shared bus register,
|
||||
// so the aggregation is containment-safe.
|
||||
let succ = 0
|
||||
let afferent = 0
|
||||
let seam_mode_seen: String = "stub"
|
||||
let rn: Int = el_list_len(results)
|
||||
let r = 0
|
||||
while r < rn {
|
||||
let res: String = el_list_get(results, r)
|
||||
let wid: String = json_get_string(res, "worker_id")
|
||||
let st: String = json_get_string(res, "status")
|
||||
let out: String = json_get_raw(res, "output")
|
||||
let aff: Int = str_to_int(json_get_string(out, "afferent"))
|
||||
let afferent = afferent + aff
|
||||
let sm: String = json_get_string(out, "seam_mode")
|
||||
if str_eq(sm, "") {
|
||||
let seam_mode_seen = seam_mode_seen
|
||||
} else {
|
||||
let seam_mode_seen = sm
|
||||
}
|
||||
if str_eq(st, "completed") {
|
||||
let succ = succ + 1
|
||||
worktrack_append("worker.completed", corr_id, wid, json_set_str("{}", "status", "completed"))
|
||||
} else {
|
||||
worktrack_append("worker.failed", corr_id, wid, json_set_str("{}", "error", json_get_string(res, "error")))
|
||||
}
|
||||
let r = r + 1
|
||||
}
|
||||
|
||||
// swarm.converging
|
||||
let vg: String = json_set("{}", "success_count", int_to_str(succ))
|
||||
worktrack_append("swarm.converging", corr_id, corr_id, vg)
|
||||
|
||||
// swarm.telemetry — afferent counters observed by the @manager.
|
||||
let tkv: [String] = el_list_empty()
|
||||
let tkv = el_list_append(tkv, "seam_mode")
|
||||
let tkv = el_list_append(tkv, seam_mode_seen)
|
||||
let telem0: String = json_build_object(tkv)
|
||||
let telem1: String = json_set_str(telem0, "afferent_think", int_to_str(afferent))
|
||||
let telemetry: String = json_set_str(telem1, "results_received", int_to_str(rn))
|
||||
worktrack_append("swarm.telemetry", corr_id, corr_id, telemetry)
|
||||
|
||||
// ── failure threshold (Swarm §4.3), integer per-mille math ──
|
||||
// require succ/n >= min_success_ratio <=> succ*1000 >= permille*n
|
||||
let permille: Int = ratio_to_permille(json_get_string(config_json, "min_success_ratio"))
|
||||
let status: String = "completed"
|
||||
if succ * 1000 < permille * n {
|
||||
let status = "aborted"
|
||||
}
|
||||
|
||||
if str_eq(status, "aborted") {
|
||||
let ap: String = json_set_str("{}", "reason", "success ratio below min_success_ratio")
|
||||
worktrack_append("swarm.aborted", corr_id, corr_id, ap)
|
||||
let rep: String = worktrack_swarm_report(corr_id)
|
||||
let ok: [String] = el_list_empty()
|
||||
let ok = el_list_append(ok, "corr_id")
|
||||
let ok = el_list_append(ok, corr_id)
|
||||
let ok = el_list_append(ok, "status")
|
||||
let ok = el_list_append(ok, "aborted")
|
||||
let out0: String = json_build_object(ok)
|
||||
return json_set(out0, "report", rep)
|
||||
}
|
||||
|
||||
// ── converge ──
|
||||
let merged: String = swarm_converge(strategy, results)
|
||||
let dp: String = json_set_str("{}", "strategy", strategy)
|
||||
worktrack_append("swarm.completed", corr_id, corr_id, dp)
|
||||
|
||||
// ── curated merge = the ONLY engram write path (Rule 4) ──
|
||||
// With "commit":"1", the ORCHESTRATOR (its token carries engram:write)
|
||||
// commits the approved merged geometry back to the engram. This is the
|
||||
// single writer. Workers returned geometry; only the orchestrator writes.
|
||||
let commit_id: String = ""
|
||||
if str_eq(json_get_string(config_json, "commit"), "1") {
|
||||
let orch_token: String = containment_coordinator_token(corr_id)
|
||||
let cres: String = swarm_engram_write(orch_token, corr_id, "swarm-merge " + corr_id + " :: " + merged, "memory", 0.5)
|
||||
let commit_id = json_get_string(cres, "id")
|
||||
}
|
||||
|
||||
let rep2: String = worktrack_swarm_report(corr_id)
|
||||
let ok2: [String] = el_list_empty()
|
||||
let ok2 = el_list_append(ok2, "corr_id")
|
||||
let ok2 = el_list_append(ok2, corr_id)
|
||||
let ok2 = el_list_append(ok2, "status")
|
||||
let ok2 = el_list_append(ok2, "completed")
|
||||
let out1: String = json_build_object(ok2)
|
||||
let out2: String = json_set(out1, "report", rep2)
|
||||
let out3: String = json_set(out2, "merged", merged)
|
||||
let out4: String = json_set(out3, "telemetry", telemetry)
|
||||
return json_set_str(out4, "committed_node", commit_id)
|
||||
}
|
||||
// ── denied: caller was a worker trying to open a swarm (Rule 2) ──
|
||||
let dkv: [String] = el_list_empty()
|
||||
let dkv = el_list_append(dkv, "corr_id")
|
||||
let dkv = el_list_append(dkv, corr_id)
|
||||
let dkv = el_list_append(dkv, "status")
|
||||
let dkv = el_list_append(dkv, "denied")
|
||||
let dkv = el_list_append(dkv, "error")
|
||||
let dkv = el_list_append(dkv, deny)
|
||||
return json_build_object(dkv)
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
// harness_local_swarm.el — LOCAL-SWARM INTEGRATION HARNESS.
|
||||
//
|
||||
// Proves the FULL local-swarm mechanics end-to-end, TODAY, on the isolated
|
||||
// engram clone (:8901), with the primitive seam pointed at the hermetic stub.
|
||||
// The moment the api-reshape agent lands the decorated primitives on the
|
||||
// dharma bus, binding is ONE flip (primitive_binding.el) + SWARM_PRIMITIVE_SEAM=
|
||||
// decorated — this same harness then runs the bound path with no other change.
|
||||
//
|
||||
// The @manager (the coordinator) fans out N native El worker threads at real
|
||||
// concurrency, each given a CCR-scoped engram slice, each invoking the primitive
|
||||
// seam (think over its slice), enforces all three containment rules, converges
|
||||
// (vote AND reduce), work-tracks durably, and observes afferent telemetry.
|
||||
//
|
||||
// Run with the sandbox env sourced (ENGRAM_URL=:8901) to also exercise CCR
|
||||
// retrieval against the real (isolated) mind; runs fully without it too.
|
||||
|
||||
fn ok(label: String, cond: Bool, fails: Int) -> Int {
|
||||
if cond { print(" ok " + label); return fails }
|
||||
print(" FAIL " + label); return fails + 1
|
||||
}
|
||||
|
||||
fn main() -> Int {
|
||||
let fails = 0
|
||||
print("== LOCAL-SWARM INTEGRATION HARNESS (seam=" + seam_mode() + ") ==")
|
||||
|
||||
// 8 independent slices, real concurrency of 4 (2 waves of native pthreads).
|
||||
let inputs: String = "[\"billing\",\"payments\",\"ledger\",\"invoicing\",\"tax\",\"payroll\",\"audit\",\"fx\"]"
|
||||
let refs: String = "[\"Volatility-Based Decomposition\"]"
|
||||
|
||||
// ── A) fan-out / converge at real concurrency (reduce) ──
|
||||
let cfg_r: String = "{\"concurrency\":\"4\",\"strategy\":\"reduce\",\"min_success_ratio\":\"1.0\"}"
|
||||
let rr: String = swarm_run("analyze_item", refs, inputs, cfg_r)
|
||||
let fails = ok("swarm completed at concurrency=4 over 8 native-thread workers", str_eq(json_get_string(rr, "status"), "completed"), fails)
|
||||
let corr: String = json_get_string(rr, "corr_id")
|
||||
let merged_r: String = json_get_raw(rr, "merged")
|
||||
let fails = ok("reduce converged all 8 worker outputs", str_to_int(json_get_string(merged_r, "count")) == 8, fails)
|
||||
|
||||
// ── B) afferent telemetry observed by the @manager ──
|
||||
let telem: String = json_get_raw(rr, "telemetry")
|
||||
let aff: Int = str_to_int(json_get_string(telem, "afferent_think"))
|
||||
let seen_mode: String = json_get_string(telem, "seam_mode")
|
||||
let fails = ok("afferent think-signals counted = 8 (one per worker)", aff == 8, fails)
|
||||
let fails = ok("telemetry records the active seam mode", str_eq(seen_mode, seam_mode()), fails)
|
||||
let telem_recs: Int = worktrack_count_kind(corr, "swarm.telemetry")
|
||||
let fails = ok("telemetry durably journalled", telem_recs == 1, fails)
|
||||
|
||||
// ── C) CCR scoping + non-leak per worker ──
|
||||
let wt: String = containment_worker_token(corr, corr + "/worker-3")
|
||||
let ctx3: String = ccr_compile("analyze_item", refs, "invoicing", corr, corr + "/worker-3", wt)
|
||||
let fails = ok("CCR context bounded within token budget", ccr_within_budget(ctx3), fails)
|
||||
let fails = ok("CCR context carries THIS slice", str_eq(json_get_string(ctx3, "input"), "invoicing"), fails)
|
||||
let leaks: Bool = str_contains(ctx3, "payroll") || str_contains(ctx3, "audit")
|
||||
let fails = ok("CCR context does NOT leak sibling slices (security boundary)", !leaks, fails)
|
||||
|
||||
// ── D) all three containment rules ──
|
||||
let deny: String = containment_check_open(wt)
|
||||
let fails = ok("Rule 2: worker token may not OPEN a swarm", !str_eq(deny, ""), fails)
|
||||
let denyj: String = containment_check_join(wt, "other-swarm")
|
||||
let fails = ok("Rule 1: worker token may not JOIN another swarm", !str_eq(denyj, ""), fails)
|
||||
let lat: String = containment_check_lateral(wt, "sibling-9")
|
||||
let fails = ok("Rule 3: worker->worker lateral edge rejected", !str_eq(lat, ""), fails)
|
||||
let ver: String = containment_check_lateral(wt, "")
|
||||
let fails = ok("Rule 3: worker->manager vertical edge allowed", str_eq(ver, ""), fails)
|
||||
// enforced live: a worker-token caller is denied opening a real swarm
|
||||
let wcfg: String = json_set(cfg_r, "caller_token", wt)
|
||||
let denied: String = swarm_run("analyze_item", refs, inputs, wcfg)
|
||||
let fails = ok("Rule 2 enforced live: worker-caller swarm denied", str_eq(json_get_string(denied, "status"), "denied"), fails)
|
||||
|
||||
// ── E) vote convergence strategy at concurrency ──
|
||||
let cfg_v: String = "{\"concurrency\":\"8\",\"strategy\":\"vote\",\"min_success_ratio\":\"1.0\"}"
|
||||
let rv: String = swarm_run("classify", refs, inputs, cfg_v)
|
||||
let winner: String = json_get_string(json_get_raw(rv, "merged"), "winner")
|
||||
// billing/payments/ledger/invoicing/payroll/audit = long(>4); tax/fx = short -> long wins
|
||||
let fails = ok("vote converged (winner=long)", str_eq(winner, "long"), fails)
|
||||
|
||||
// ── F) durable, inspectable work-tracking ──
|
||||
let started: Int = worktrack_count_kind(corr, "worker.started")
|
||||
let completed: Int = worktrack_count_kind(corr, "worker.completed")
|
||||
let fails = ok("work-tracking journal: 8 started + 8 completed", (started == 8) && (completed == 8), fails)
|
||||
|
||||
print("")
|
||||
if fails == 0 {
|
||||
print("HARNESS GREEN — full local-swarm mechanics proven with seam=" + seam_mode())
|
||||
return 0
|
||||
}
|
||||
print("HARNESS FAIL (" + int_to_str(fails) + ")")
|
||||
return 1
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
// harness_real_cognition.el — the LOCAL SWARM running REAL cognition.
|
||||
//
|
||||
// Run with: SWARM_PRIMITIVE_SEAM=decorated + the sandbox env sourced
|
||||
// (ENGRAM_URL=:8901). Each worker's `think` is BOUND to the reshape's proven
|
||||
// op_think (GET /api/think) over its NODE-ID anchor — real 768-dim gradients from
|
||||
// the live (isolated) geometry, not the stub. The @manager fans out N native-El
|
||||
// worker threads at real concurrency, converges (reduce + vote) over the real
|
||||
// cognition, enforces all three containment rules, observes afferent telemetry,
|
||||
// and work-tracks durably.
|
||||
//
|
||||
// Anchors are real self-neighbourhood node ids on the :8901 clone (free-text
|
||||
// anchors return "geometry unavailable", so these must be node ids).
|
||||
|
||||
fn ok(label: String, cond: Bool, fails: Int) -> Int {
|
||||
if cond { print(" ok " + label); return fails }
|
||||
print(" FAIL " + label); return fails + 1
|
||||
}
|
||||
|
||||
fn main() -> Int {
|
||||
let fails = 0
|
||||
print("== REAL-COGNITION LOCAL SWARM (seam=" + seam_mode() + ", engram=" + env("ENGRAM_URL") + ") ==")
|
||||
|
||||
// ── 0) direct proof the bound primitive returns REAL cognition ──
|
||||
let g: String = op_think("self", "plan")
|
||||
let dim: Int = json_get_int(g, "dim")
|
||||
let nsup: Int = json_get_int(g, "n_support")
|
||||
let fails = ok("bound op_think returns a real 768-dim gradient", dim == 768, fails)
|
||||
let fails = ok("real gradient has support (n_support>0)", nsup > 0, fails)
|
||||
let gfree: String = op_think("this-is-free-text-not-a-node", "reason")
|
||||
let fails = ok("free-text anchor correctly refused (geometry unavailable)", str_contains(gfree, "geometry unavailable"), fails)
|
||||
|
||||
// ── the input set: 8 real NODE-ID anchors from self's neighbourhood ──
|
||||
let anchors: String = "[\"a1000001-0000-0000-0000-000000000001\",\"5f011441-fa43-4fe7-a9c0-c78a584ef11d\",\"kn-5adecd7e-d6db-4576-87fe-6ef8a935cea6\",\"76d7fd0b-0672-4511-a2f5-a095cf9c60ae\",\"7027e302-593f-441d-8fd6-9c400c163108\",\"2a730b18-6566-46ee-a21e-4f4dd0380908\",\"46b0e4dd-2c19-48d2-bcbc-19f61d6c79ae\",\"9162cde8-8739-4f00-bfc9-2850ed612e50\"]"
|
||||
let refs: String = "[\"self\"]"
|
||||
|
||||
// ── A) fan-out real cognition at concurrency, converge with REDUCE ──
|
||||
let cfg_r: String = "{\"concurrency\":\"4\",\"strategy\":\"reduce\",\"min_success_ratio\":\"1.0\"}"
|
||||
let rr: String = swarm_run("cognize", refs, anchors, cfg_r)
|
||||
let fails = ok("swarm completed: 8 workers each a real think, concurrency=4", str_eq(json_get_string(rr, "status"), "completed"), fails)
|
||||
let corr: String = json_get_string(rr, "corr_id")
|
||||
let merged_r: String = json_get_raw(rr, "merged")
|
||||
let fails = ok("reduce converged all 8 real-cognition outputs", str_to_int(json_get_string(merged_r, "count")) == 8, fails)
|
||||
let acc: String = json_get_string(merged_r, "accumulated")
|
||||
let fails = ok("converged output carries real gradient support (n_support)", str_contains(acc, "n_support"), fails)
|
||||
|
||||
// ── B) afferent telemetry: 8 real think-signals, decorated seam ──
|
||||
let telem: String = json_get_raw(rr, "telemetry")
|
||||
let aff: Int = str_to_int(json_get_string(telem, "afferent_think"))
|
||||
let fails = ok("afferent counters = 8 real think invocations", aff == 8, fails)
|
||||
let fails = ok("telemetry records seam_mode=decorated", str_eq(json_get_string(telem, "seam_mode"), "decorated"), fails)
|
||||
let fails = ok("telemetry durably journalled", worktrack_count_kind(corr, "swarm.telemetry") == 1, fails)
|
||||
|
||||
// ── C) converge with VOTE over real cognition ──
|
||||
let cfg_v: String = "{\"concurrency\":\"8\",\"strategy\":\"vote\",\"min_success_ratio\":\"1.0\"}"
|
||||
let rv: String = swarm_run("cognize", refs, anchors, cfg_v)
|
||||
let winner: String = json_get_string(json_get_raw(rv, "merged"), "winner")
|
||||
let fails = ok("vote converged over real cognition (winner=" + winner + ")", !str_eq(winner, ""), fails)
|
||||
|
||||
// ── D) all three containment rules still enforced ──
|
||||
let wt: String = containment_worker_token(corr, corr + "/worker-2")
|
||||
let fails = ok("Rule 2: worker may not open a swarm", !str_eq(containment_check_open(wt), ""), fails)
|
||||
let fails = ok("Rule 1: worker may not join another swarm", !str_eq(containment_check_join(wt, "s2"), ""), fails)
|
||||
let fails = ok("Rule 3: worker->worker lateral edge rejected", !str_eq(containment_check_lateral(wt, "sib"), ""), fails)
|
||||
let wcfg: String = json_set(cfg_r, "caller_token", wt)
|
||||
let denied: String = swarm_run("cognize", refs, anchors, wcfg)
|
||||
let fails = ok("Rule 2 enforced LIVE: worker-caller swarm denied", str_eq(json_get_string(denied, "status"), "denied"), fails)
|
||||
|
||||
// ── E) CCR scoping + non-leak over node-id anchors ──
|
||||
let ctx: String = ccr_compile("cognize", refs, "a1000001-0000-0000-0000-000000000001", corr, corr + "/worker-0", wt)
|
||||
let fails = ok("CCR context bounded within budget", ccr_within_budget(ctx), fails)
|
||||
let leaks: Bool = str_contains(ctx, "9162cde8")
|
||||
let fails = ok("CCR context does NOT leak sibling anchors", !leaks, fails)
|
||||
|
||||
// ── F) durable work-tracking ──
|
||||
let started: Int = worktrack_count_kind(corr, "worker.started")
|
||||
let completed: Int = worktrack_count_kind(corr, "worker.completed")
|
||||
let fails = ok("work-tracking: 8 started + 8 completed", (started == 8) && (completed == 8), fails)
|
||||
|
||||
// ── G) RULE 4 — engram-write is @manager-ONLY (authority gate) ──
|
||||
// A worker token (engram:read only) is STRUCTURALLY denied any engram write.
|
||||
let worker_tok: String = containment_worker_token(corr, corr + "/worker-1")
|
||||
let orch_tok: String = containment_coordinator_token(corr)
|
||||
let fails = ok("worker token carries engram:read", containment_has_cap(worker_tok, "engram:read"), fails)
|
||||
let fails = ok("worker token does NOT carry engram:write", !containment_has_cap(worker_tok, "engram:write"), fails)
|
||||
let fails = ok("orchestrator token carries engram:write", containment_has_cap(orch_tok, "engram:write"), fails)
|
||||
// a worker attempting an engram write is DENIED BY CAPABILITY (no HTTP issued)
|
||||
let wdeny: String = swarm_engram_write(worker_tok, corr, "worker tries to mutate global state", "memory", 0.5)
|
||||
let denied_reason: String = json_get_string(wdeny, "denied")
|
||||
let fails = ok("worker engram-write DENIED by capability (Rule 4)", str_contains(denied_reason, "rule 4"), fails)
|
||||
let fails = ok("denied worker write performed NO engram mutation (no node id)", str_eq(json_get_string(wdeny, "id"), ""), fails)
|
||||
let fails = ok("Rule-4 violation journalled", worktrack_count_kind(corr, "containment.violation") >= 1, fails)
|
||||
// the orchestrator passes the capability gate (sole authorized writer)
|
||||
let odeny: String = containment_check_engram_write(orch_tok, "engram.write")
|
||||
let fails = ok("orchestrator PASSES the engram-write capability gate (sole writer)", str_eq(odeny, ""), fails)
|
||||
|
||||
// ── H) curated merge = the only write path (orchestrator commits) ──
|
||||
// The AUTHORITY gate above is already proven (worker denied, orchestrator
|
||||
// authorized) WITHOUT issuing a write. The actual persisting commit exercises
|
||||
// the engram write path, which needs the gate-1 write-healthy clone — so it
|
||||
// runs only under SWARM_WRITE_HEALTHY=1 (else it would hit the known daemon
|
||||
// write-crash). Authority != health: the gate holds either way.
|
||||
if str_eq(env("SWARM_WRITE_HEALTHY"), "1") {
|
||||
let cfg_commit: String = "{\"concurrency\":\"4\",\"strategy\":\"reduce\",\"min_success_ratio\":\"1.0\",\"commit\":\"1\"}"
|
||||
let rc: String = swarm_run("cognize", refs, anchors, cfg_commit)
|
||||
let committed: String = json_get_string(rc, "committed_node")
|
||||
let fails2: Int = ok("orchestrator (sole writer) committed the merge to the engram", !str_eq(committed, ""), fails)
|
||||
let fails = fails2
|
||||
} else {
|
||||
print(" note curated-merge commit deferred to the gate-1 write-healthy clone (set SWARM_WRITE_HEALTHY=1); authority gate already proven above")
|
||||
}
|
||||
|
||||
print("")
|
||||
if fails == 0 {
|
||||
print("REAL-COGNITION SWARM GREEN — Neuron thinking in parallel over its own geometry.")
|
||||
return 0
|
||||
}
|
||||
print("REAL-COGNITION SWARM FAIL (" + int_to_str(fails) + ")")
|
||||
return 1
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
// integ_engram.el — integration proof against a LIVE (isolated) engram.
|
||||
//
|
||||
// Run with the sandbox env sourced (ENGRAM_URL=http://127.0.0.1:8901,
|
||||
// ENGRAM_API_KEY=sbx-dev-swarm-ccr). Proves:
|
||||
// (a) CCR retrieval pulls REAL content from the mind over HTTP;
|
||||
// (b) a full swarm runs and converges against the live mind;
|
||||
// (c) work-tracking mirrors records into the engram as SwarmTrack nodes.
|
||||
|
||||
fn main() -> Int {
|
||||
let url: String = env("ENGRAM_URL")
|
||||
if str_eq(url, "") {
|
||||
print("SKIP integ_engram (ENGRAM_URL not set)")
|
||||
return 0
|
||||
}
|
||||
|
||||
// (a) CCR compiles a bounded context whose retrieval hit the real mind.
|
||||
let refs: String = "[\"Volatility-Based Decomposition\",\"Swarm Architecture containment\"]"
|
||||
let wt: String = containment_worker_token("integ", "integ/w0")
|
||||
let ctx: String = ccr_compile("analyze_item", refs, "decompose the billing module", "integ", "integ/w0", wt)
|
||||
let knowledge: String = json_get_string(ctx, "knowledge")
|
||||
let pulled_real: Bool = str_contains(knowledge, "olatility") || str_contains(knowledge, "Anderson") || str_contains(knowledge, "VBD")
|
||||
if pulled_real {
|
||||
print(" ok CCR retrieval pulled real mind content (" + int_to_str(str_len(knowledge)) + " bytes, bounded)")
|
||||
} else {
|
||||
print(" FAIL CCR retrieval returned no mind content")
|
||||
}
|
||||
let bounded: Bool = ccr_within_budget(ctx)
|
||||
if bounded { print(" ok compiled context stayed within budget") } else { print(" FAIL context over budget") }
|
||||
|
||||
// (b) a real swarm over the live mind.
|
||||
let inputs: String = "[\"billing\",\"payments\",\"ledger\"]"
|
||||
let cfg: String = "{\"concurrency\":\"3\",\"strategy\":\"collect\",\"min_success_ratio\":\"1.0\"}"
|
||||
let res: String = swarm_run("analyze_item", refs, inputs, cfg)
|
||||
let status: String = json_get_string(res, "status")
|
||||
if str_eq(status, "completed") { print(" ok swarm completed against live engram") } else { print(" FAIL swarm status=" + status) }
|
||||
let corr: String = json_get_string(res, "corr_id")
|
||||
|
||||
// (c) work-tracking mirrored into the mind: search for this swarm's records.
|
||||
let hits: String = primitive_attend(corr, 5)
|
||||
let mirrored: Bool = str_contains(hits, "swarm-track") || str_contains(hits, corr)
|
||||
if mirrored { print(" ok work-tracking mirrored into the engram (queryable)") } else { print(" note mirror not yet visible to search (async index)") }
|
||||
|
||||
print("DONE integ_engram corr=" + corr)
|
||||
return 0
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
// test_convergence.el — convergence strategies + failure threshold / abort.
|
||||
|
||||
fn assert_true(label: String, cond: Bool, fails: Int) -> Int {
|
||||
if cond { print(" ok " + label); return fails }
|
||||
print(" FAIL " + label); return fails + 1
|
||||
}
|
||||
|
||||
fn main() -> Int {
|
||||
let fails = 0
|
||||
let refs: String = "[]"
|
||||
|
||||
// ── vote: classify 5 inputs; 3 "long" (>4 chars) vs 2 "short" -> winner long ──
|
||||
let inputs: String = "[\"alpha\",\"bravo\",\"hi\",\"charlie\",\"ok\"]"
|
||||
let cfg_v: String = "{\"concurrency\":\"3\",\"strategy\":\"vote\",\"min_success_ratio\":\"1.0\"}"
|
||||
let rv: String = swarm_run("classify", refs, inputs, cfg_v)
|
||||
let merged_v: String = json_get_raw(rv, "merged")
|
||||
let winner: String = json_get_string(merged_v, "winner")
|
||||
let votes: Int = str_to_int(json_get_string(merged_v, "votes"))
|
||||
let fails = assert_true("vote winner = long", str_eq(winner, "long"), fails)
|
||||
let fails = assert_true("vote count = 3", votes == 3, fails)
|
||||
|
||||
// ── merge: outputs joined ──
|
||||
let cfg_m: String = "{\"concurrency\":\"2\",\"strategy\":\"merge\",\"min_success_ratio\":\"1.0\"}"
|
||||
let rm: String = swarm_run("analyze_item", refs, "[\"a\",\"b\",\"c\"]", cfg_m)
|
||||
let merged_m: String = json_get_raw(rm, "merged")
|
||||
let joined: String = json_get_string(merged_m, "merged")
|
||||
let fails = assert_true("merge produced a joined string", str_contains(joined, "|"), fails)
|
||||
|
||||
// ── reduce: count accumulates ──
|
||||
let cfg_r: String = "{\"concurrency\":\"4\",\"strategy\":\"reduce\",\"min_success_ratio\":\"1.0\"}"
|
||||
let rr: String = swarm_run("analyze_item", refs, "[\"a\",\"b\",\"c\",\"d\"]", cfg_r)
|
||||
let merged_r: String = json_get_raw(rr, "merged")
|
||||
let rcount: Int = str_to_int(json_get_string(merged_r, "count"))
|
||||
let fails = assert_true("reduce count = 4", rcount == 4, fails)
|
||||
|
||||
// ── failure threshold: 2 of 5 fail (x-prefixed); ratio 3/5=0.6 < 0.8 -> aborted ──
|
||||
let fin: String = "[\"a\",\"xb\",\"c\",\"xd\",\"e\"]"
|
||||
let cfg_f: String = "{\"concurrency\":\"5\",\"strategy\":\"collect\",\"min_success_ratio\":\"0.8\"}"
|
||||
let rf: String = swarm_run("faildemo", refs, fin, cfg_f)
|
||||
let fstatus: String = json_get_string(rf, "status")
|
||||
let fails = assert_true("swarm aborted below min_success_ratio (0.6<0.8)", str_eq(fstatus, "aborted"), fails)
|
||||
let corr_f: String = json_get_string(rf, "corr_id")
|
||||
let failed_n: Int = worktrack_count_kind(corr_f, "worker.failed")
|
||||
let aborted_n: Int = worktrack_count_kind(corr_f, "swarm.aborted")
|
||||
let fails = assert_true("tracked 2 worker.failed", failed_n == 2, fails)
|
||||
let fails = assert_true("tracked swarm.aborted", aborted_n == 1, fails)
|
||||
|
||||
// ── same failures tolerated when min_success_ratio=0.5 (0.6>=0.5) -> completed ──
|
||||
let cfg_ok: String = "{\"concurrency\":\"5\",\"strategy\":\"collect\",\"min_success_ratio\":\"0.5\"}"
|
||||
let rok: String = swarm_run("faildemo", refs, fin, cfg_ok)
|
||||
let fails = assert_true("swarm completes when failures within tolerance", str_eq(json_get_string(rok, "status"), "completed"), fails)
|
||||
|
||||
if fails == 0 { print("PASS test_convergence"); return 0 }
|
||||
print("FAIL test_convergence (" + int_to_str(fails) + ")"); return 1
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
// test_swarm.el — end-to-end proof of the swarm capability on native El threads.
|
||||
//
|
||||
// Proves: native-thread fan-out/converge, bounded concurrency, per-worker CCR
|
||||
// bounded context (with the security-boundary property), containment Rule 2
|
||||
// enforcement, and durable work-tracking.
|
||||
|
||||
fn assert_true(label: String, cond: Bool, fails: Int) -> Int {
|
||||
if cond {
|
||||
print(" ok " + label)
|
||||
return fails
|
||||
}
|
||||
print(" FAIL " + label)
|
||||
return fails + 1
|
||||
}
|
||||
|
||||
fn main() -> Int {
|
||||
let fails = 0
|
||||
|
||||
// ── 1) fan-out / converge (collect) over native threads ──
|
||||
let inputs: String = "[\"alpha\",\"bravo\",\"charlie\",\"delta\",\"echo\"]"
|
||||
let refs: String = "[]"
|
||||
let cfg: String = "{\"concurrency\":\"2\",\"strategy\":\"collect\",\"min_success_ratio\":\"1.0\"}"
|
||||
let res: String = swarm_run("analyze_item", refs, inputs, cfg)
|
||||
let status: String = json_get_string(res, "status")
|
||||
let fails = assert_true("swarm completed", str_eq(status, "completed"), fails)
|
||||
|
||||
let merged: String = json_get_raw(res, "merged")
|
||||
let count: Int = json_array_len(merged)
|
||||
let fails = assert_true("collect returned 5 results (bounded concurrency=2)", count == 5, fails)
|
||||
|
||||
// ── 2) work-tracking is durable + complete ──
|
||||
let corr: String = json_get_string(res, "corr_id")
|
||||
let started: Int = worktrack_count_kind(corr, "worker.started")
|
||||
let completed: Int = worktrack_count_kind(corr, "worker.completed")
|
||||
let created: Int = worktrack_count_kind(corr, "swarm.created")
|
||||
let done: Int = worktrack_count_kind(corr, "swarm.completed")
|
||||
let fails = assert_true("tracked 5 worker.started", started == 5, fails)
|
||||
let fails = assert_true("tracked 5 worker.completed", completed == 5, fails)
|
||||
let fails = assert_true("tracked swarm.created + swarm.completed", (created == 1) && (done == 1), fails)
|
||||
|
||||
// ── 3) CCR: bounded, minimal, non-leaking per-worker context ──
|
||||
let wtoken: String = containment_worker_token(corr, corr + "/worker-0")
|
||||
let ctx: String = ccr_compile("analyze_item", refs, "alpha", corr, corr + "/worker-0", wtoken)
|
||||
let in_budget: Bool = ccr_within_budget(ctx)
|
||||
let fails = assert_true("CCR context within token budget", in_budget, fails)
|
||||
let this_input: String = json_get_string(ctx, "input")
|
||||
let fails = assert_true("CCR context contains THIS worker's input", str_eq(this_input, "alpha"), fails)
|
||||
// security boundary: a worker's compiled context must not carry a sibling input
|
||||
let leaks_sibling: Bool = str_contains(ctx, "charlie")
|
||||
let fails = assert_true("CCR context does NOT leak sibling inputs", !leaks_sibling, fails)
|
||||
|
||||
// ── 4) containment Rule 2: a worker may not open a swarm ──
|
||||
let worker_caller_cfg: String = json_set(cfg, "caller_token", wtoken)
|
||||
let denied: String = swarm_run("analyze_item", refs, inputs, worker_caller_cfg)
|
||||
let dstatus: String = json_get_string(denied, "status")
|
||||
let fails = assert_true("worker-token caller denied opening a swarm (Rule 2)", str_eq(dstatus, "denied"), fails)
|
||||
|
||||
// coordinator token IS allowed
|
||||
let coord: String = containment_coordinator_token("some-corr")
|
||||
let allow_reason: String = containment_check_open(coord)
|
||||
let fails = assert_true("coordinator token allowed to open a swarm", str_eq(allow_reason, ""), fails)
|
||||
|
||||
// ── 5) containment Rule 3: no lateral worker->worker edge ──
|
||||
let lateral: String = containment_check_lateral(wtoken, "some-sibling")
|
||||
let fails = assert_true("lateral worker->worker edge rejected (Rule 3)", !str_eq(lateral, ""), fails)
|
||||
let vertical: String = containment_check_lateral(wtoken, "")
|
||||
let fails = assert_true("vertical worker->coordinator edge allowed", str_eq(vertical, ""), fails)
|
||||
|
||||
if fails == 0 {
|
||||
print("PASS test_swarm")
|
||||
return 0
|
||||
}
|
||||
print("FAIL test_swarm (" + int_to_str(fails) + " failures)")
|
||||
return 1
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
// test_worktrack.el — durability + inspectability of the work-tracking journal.
|
||||
|
||||
fn main() -> Int {
|
||||
let corr: String = "test-" + uuid_v4()
|
||||
|
||||
// record a swarm lifecycle
|
||||
let p1: String = json_set("{}", "input_count", "3")
|
||||
worktrack_append("swarm.created", corr, "swarm-1", p1)
|
||||
worktrack_append("worker.started", corr, "worker-001", "{}")
|
||||
worktrack_append("worker.started", corr, "worker-002", "{}")
|
||||
worktrack_append("worker.completed", corr, "worker-001", "{}")
|
||||
worktrack_append("worker.failed", corr, "worker-002", "{}")
|
||||
worktrack_append("swarm.completed", corr, "swarm-1", "{}")
|
||||
|
||||
// inspect: reconstruct the report from the durable journal
|
||||
let report: String = worktrack_swarm_report(corr)
|
||||
print("report=" + report)
|
||||
|
||||
let recs_n: Int = el_list_len(worktrack_records(corr))
|
||||
print("records=" + int_to_str(recs_n))
|
||||
|
||||
let state: String = json_get_string(report, "state")
|
||||
let completed: Int = str_to_int(json_get_string(report, "workers_completed"))
|
||||
let failed: Int = str_to_int(json_get_string(report, "workers_failed"))
|
||||
|
||||
if str_eq(state, "completed") {
|
||||
if completed == 1 {
|
||||
if failed == 1 {
|
||||
if recs_n == 6 {
|
||||
print("PASS worktrack")
|
||||
return 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
print("FAIL worktrack")
|
||||
return 1
|
||||
}
|
||||
@@ -1,217 +0,0 @@
|
||||
// worktrack.el — full work-tracking for the swarm.
|
||||
//
|
||||
// "Intent all the way up, orchestrator at the top." Every unit of parallel
|
||||
// work a swarm fans out is recorded here: the swarm itself, each worker, its
|
||||
// status, its result summary, the convergence, and the final merged output —
|
||||
// all threaded by a single correlation ID so the entire execution graph can be
|
||||
// reconstructed and audited (Swarm Architecture §6.1).
|
||||
//
|
||||
// DURABILITY. Records are appended to a JSON-lines journal on disk. The journal
|
||||
// is append-only and single-writer: only the coordinator (the main thread, before
|
||||
// and after each fan-out and during convergence) writes to it. Workers never
|
||||
// touch it — they return structured results and the coordinator records them.
|
||||
// This is deliberate: it makes the tracking store race-free and, not
|
||||
// coincidentally, enforces Swarm containment rule 3 (no lateral worker state).
|
||||
//
|
||||
// INSPECTABILITY. The journal is plain JSONL — greppable, tailable, replayable.
|
||||
// worktrack_read() loads it back; worktrack_swarm_report() reconstructs a
|
||||
// swarm's full record from its correlation ID.
|
||||
//
|
||||
// ENGRAM MIRROR (optional). When ENGRAM_URL is set, each record is also mirrored
|
||||
// into the engram as a node (POST /api/node) tagged with the correlation ID, so
|
||||
// the swarm's execution becomes part of the durable mind, queryable by memory.
|
||||
//
|
||||
// Depends on: el_runtime.c builtins (fs_*, http_post, env, json_*, uuid_v4,
|
||||
// now_millis, str_*). No El-module concat dependencies of its own.
|
||||
|
||||
// ── JSON helper ──────────────────────────────────────────────────────────────
|
||||
// json_set inserts its value as a RAW JSON fragment (objects/arrays/numbers).
|
||||
// json_set_str sets a plain STRING value, correctly quoted and escaped. Use
|
||||
// json_set for nested JSON, json_set_str for strings.
|
||||
fn json_set_str(j: String, key: String, val: String) -> String {
|
||||
return json_set(j, key, "\"" + json_escape_string(val) + "\"")
|
||||
}
|
||||
|
||||
// ── Journal location ─────────────────────────────────────────────────────────
|
||||
|
||||
// worktrack_dir — directory holding the swarm journals.
|
||||
// Override with SWARM_TRACK_DIR; defaults to ./.swarm-track (relative to CWD).
|
||||
fn worktrack_dir() -> String {
|
||||
let d: String = env("SWARM_TRACK_DIR")
|
||||
if str_eq(d, "") {
|
||||
return ".swarm-track"
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// worktrack_journal_path — the JSONL journal file for one correlation ID.
|
||||
fn worktrack_journal_path(corr_id: String) -> String {
|
||||
return worktrack_dir() + "/" + corr_id + ".jsonl"
|
||||
}
|
||||
|
||||
// worktrack_init — ensure the journal directory exists. Idempotent.
|
||||
fn worktrack_init() -> Bool {
|
||||
let d: String = worktrack_dir()
|
||||
if fs_exists(d) {
|
||||
return true
|
||||
}
|
||||
return fs_mkdir(d)
|
||||
}
|
||||
|
||||
// ── Record construction ──────────────────────────────────────────────────────
|
||||
|
||||
// worktrack_record — build one journal record as a JSON object string.
|
||||
// kind: the record kind (swarm.created, worker.started, ...)
|
||||
// corr_id: the swarm correlation ID (links every record)
|
||||
// subject: the entity the record is about (swarm id, worker id, "")
|
||||
// payload: a JSON object string with kind-specific fields
|
||||
fn worktrack_record(kind: String, corr_id: String, subject: String, payload: String) -> String {
|
||||
let kv: [String] = el_list_empty()
|
||||
let kv = el_list_append(kv, "kind")
|
||||
let kv = el_list_append(kv, kind)
|
||||
let kv = el_list_append(kv, "corr_id")
|
||||
let kv = el_list_append(kv, corr_id)
|
||||
let kv = el_list_append(kv, "subject")
|
||||
let kv = el_list_append(kv, subject)
|
||||
let kv = el_list_append(kv, "ts_ms")
|
||||
let kv = el_list_append(kv, int_to_str(now_millis()))
|
||||
let rec: String = json_build_object(kv)
|
||||
// Attach the payload as a nested raw JSON field.
|
||||
let rec2: String = json_set(rec, "data", payload)
|
||||
return rec2
|
||||
}
|
||||
|
||||
// ── Journal append (single-writer, durable) ──────────────────────────────────
|
||||
|
||||
// worktrack_append — append one record to the correlation journal (durable),
|
||||
// and mirror it to the engram if ENGRAM_URL is configured. Returns the record.
|
||||
//
|
||||
// fs_write here is used in append semantics: we read-modify-write the file. The
|
||||
// coordinator is the only writer, so this is safe and race-free.
|
||||
fn worktrack_append(kind: String, corr_id: String, subject: String, payload: String) -> String {
|
||||
worktrack_init()
|
||||
let rec: String = worktrack_record(kind, corr_id, subject, payload)
|
||||
let path: String = worktrack_journal_path(corr_id)
|
||||
let prior: String = ""
|
||||
if fs_exists(path) {
|
||||
let prior = fs_read(path)
|
||||
}
|
||||
let next: String = prior + rec + "\n"
|
||||
fs_write(path, next)
|
||||
worktrack_mirror_engram(rec, corr_id, kind, subject)
|
||||
return rec
|
||||
}
|
||||
|
||||
// worktrack_mirror_engram — best-effort mirror of a record into the engram.
|
||||
// No-op unless ENGRAM_URL is set. Failures are swallowed (tracking must not
|
||||
// depend on the mind being reachable).
|
||||
fn worktrack_mirror_engram(rec: String, corr_id: String, kind: String, subject: String) -> Bool {
|
||||
// Opt-in: the durable substrate is the JSONL journal (always written). The
|
||||
// engram mirror is an additional convenience, enabled with SWARM_MIRROR=1,
|
||||
// so a swarm never depends on — or loads — the mind just to track its work.
|
||||
if str_eq(env("SWARM_MIRROR"), "1") {
|
||||
// enabled — fall through to the mirror POST
|
||||
let _go: Int = 1
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
let url: String = env("ENGRAM_URL")
|
||||
if str_eq(url, "") {
|
||||
return false
|
||||
}
|
||||
let content: String = "swarm-track " + kind + " " + subject + " :: " + rec
|
||||
let body_kv: [String] = el_list_empty()
|
||||
let body_kv = el_list_append(body_kv, "content")
|
||||
let body_kv = el_list_append(body_kv, content)
|
||||
let body_kv = el_list_append(body_kv, "node_type")
|
||||
let body_kv = el_list_append(body_kv, "SwarmTrack")
|
||||
let body_kv = el_list_append(body_kv, "salience")
|
||||
let body_kv = el_list_append(body_kv, "0.5")
|
||||
let body: String = json_build_object(body_kv)
|
||||
let key: String = env("ENGRAM_API_KEY")
|
||||
let body2: String = json_set_str(body, "_auth", key)
|
||||
let resp: String = http_post(url + "/api/nodes", body2)
|
||||
return true
|
||||
}
|
||||
|
||||
// ── Read / inspect ───────────────────────────────────────────────────────────
|
||||
|
||||
// worktrack_read — read the raw JSONL journal for a correlation ID.
|
||||
fn worktrack_read(corr_id: String) -> String {
|
||||
let path: String = worktrack_journal_path(corr_id)
|
||||
if fs_exists(path) {
|
||||
return fs_read(path)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// worktrack_records — the journal as a [String] of record JSON objects, in order.
|
||||
fn worktrack_records(corr_id: String) -> [String] {
|
||||
let raw: String = worktrack_read(corr_id)
|
||||
let out: [String] = el_list_empty()
|
||||
if str_eq(raw, "") {
|
||||
return out
|
||||
}
|
||||
let lines: [String] = str_split_lines(raw)
|
||||
let n: Int = el_list_len(lines)
|
||||
let i = 0
|
||||
while i < n {
|
||||
let ln: String = el_list_get(lines, i)
|
||||
if str_eq(ln, "") {
|
||||
let i = i + 1
|
||||
} else {
|
||||
let out = el_list_append(out, ln)
|
||||
let i = i + 1
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// worktrack_count_kind — how many records of a given kind exist for a swarm.
|
||||
// Powers assertions and live status ("how many workers completed").
|
||||
fn worktrack_count_kind(corr_id: String, kind: String) -> Int {
|
||||
let recs: [String] = worktrack_records(corr_id)
|
||||
let n: Int = el_list_len(recs)
|
||||
let c = 0
|
||||
let i = 0
|
||||
while i < n {
|
||||
let r: String = el_list_get(recs, i)
|
||||
let k: String = json_get_string(r, "kind")
|
||||
if str_eq(k, kind) {
|
||||
let c = c + 1
|
||||
}
|
||||
let i = i + 1
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// worktrack_swarm_report — reconstruct a compact status report for a swarm from
|
||||
// its journal: counts of started/completed/failed workers and terminal state.
|
||||
// Inspectable, durable, derived purely from the append-only record.
|
||||
fn worktrack_swarm_report(corr_id: String) -> String {
|
||||
let started: Int = worktrack_count_kind(corr_id, "worker.started")
|
||||
let completed: Int = worktrack_count_kind(corr_id, "worker.completed")
|
||||
let failed: Int = worktrack_count_kind(corr_id, "worker.failed")
|
||||
let done: Int = worktrack_count_kind(corr_id, "swarm.completed")
|
||||
let aborted: Int = worktrack_count_kind(corr_id, "swarm.aborted")
|
||||
let state: String = "running"
|
||||
if aborted > 0 {
|
||||
let state = "aborted"
|
||||
} else {
|
||||
if done > 0 {
|
||||
let state = "completed"
|
||||
}
|
||||
}
|
||||
let kv: [String] = el_list_empty()
|
||||
let kv = el_list_append(kv, "corr_id")
|
||||
let kv = el_list_append(kv, corr_id)
|
||||
let kv = el_list_append(kv, "state")
|
||||
let kv = el_list_append(kv, state)
|
||||
let kv = el_list_append(kv, "workers_started")
|
||||
let kv = el_list_append(kv, int_to_str(started))
|
||||
let kv = el_list_append(kv, "workers_completed")
|
||||
let kv = el_list_append(kv, int_to_str(completed))
|
||||
let kv = el_list_append(kv, "workers_failed")
|
||||
let kv = el_list_append(kv, int_to_str(failed))
|
||||
return json_build_object(kv)
|
||||
}
|
||||
@@ -173,7 +173,4 @@ Each ad-hoc harness becomes `nsbx run <name> …` (or `--source` build) against
|
||||
## Env knobs
|
||||
|
||||
`NSBX_ROOT`, `NSBX_PORT_BASE`, `NSBX_RSS_BOUND_MB`, `NSBX_REMERGE_THRESHOLD`,
|
||||
`NSBX_READY_TIMEOUT_SECS` (default 15 — how long `up`/`create`/`build` wait for a
|
||||
daemon to answer `/api/stats` before reporting failure; raise it if a boot is
|
||||
legitimately slow under concurrent sandbox/CPU load rather than actually broken),
|
||||
`EL_REPO` (for `elc` + runtime sources), `ENGRAM_LIVE_DATA_DIR`, `ENGRAM_LIVE_PLIST`.
|
||||
|
||||
+29
-134
@@ -32,12 +32,6 @@ EL_REPO="${EL_REPO:-$HOME/Development/neuron-technologies/foundation/el}"
|
||||
PORT_BASE="${NSBX_PORT_BASE:-8900}"
|
||||
RSS_BOUND_MB="${NSBX_RSS_BOUND_MB:-550}" # from store-fix reboot-proof (aaf13f88)
|
||||
REMERGE_THRESHOLD="${NSBX_REMERGE_THRESHOLD:-40000}"
|
||||
# readiness-poll window for start_daemon (0.5s ticks). Default unchanged (15s) —
|
||||
# but a cold boot against the full live store, under concurrent CPU contention
|
||||
# from other running sandboxes, has been observed live to take well past that.
|
||||
# Bump per-invocation with NSBX_READY_TIMEOUT_SECS if `up`/`create` reports a
|
||||
# not-ready failure but the daemon looks otherwise fine (see its logs/daemon.log).
|
||||
READY_TICKS=$(( ${NSBX_READY_TIMEOUT_SECS:-15} * 2 ))
|
||||
KEYSTONES=( "kn-efeb4a5b-5aff-4759-8a97-7233099be6ee" "kn-5b606390-a52d-4ca2-8e0e-eba141d13440" )
|
||||
# fixed probe set for retrieval-parity (stable, identity-anchored)
|
||||
PARITY_QUERIES=( "who am I" "self identity core" "engram store durability" "keystone self anchor" "grounding honesty" )
|
||||
@@ -83,7 +77,7 @@ _port_claimed(){ # is another sandbox already assigned this port?
|
||||
live_stats(){ curl -s -m5 "$LIVE_URL/api/stats" 2>/dev/null; }
|
||||
api(){ # api <name> <path> [json-body]
|
||||
local name="$1" path="$2" body="${3:-}"
|
||||
local port; port="$(mget "$name" "['port']")"; [ -n "$port" ] || die "unknown sandbox: $name (run: nsbx list)"
|
||||
local port; port="$(mget "$name" "['port']")"; [ -n "$port" ] || die "unknown sandbox: $name"
|
||||
local url="http://127.0.0.1:${port}${path}"
|
||||
if [ -n "$body" ]; then curl -s -m30 -X POST -H 'Content-Type: application/json' -d "$body" "$url"
|
||||
else curl -s -m30 "$url"; fi
|
||||
@@ -94,19 +88,6 @@ stat_field(){ printf '%s' "$1" | sed -n "s/.*\"$2\":\([0-9]*\).*/\1/p"; }
|
||||
daemon_pid(){ local f; f="$(sdir "$1")/daemon.pid"; [ -f "$f" ] && cat "$f" || true; }
|
||||
daemon_alive(){ local p; p="$(daemon_pid "$1")"; [ -n "$p" ] && kill -0 "$p" 2>/dev/null; }
|
||||
|
||||
# daemon_health <name> : prints "stopped" | "running" | "unresponsive" (to stdout).
|
||||
# "running" means the pid is alive AND /api/stats actually answered — process
|
||||
# liveness alone (daemon_alive) is not proof the HTTP server is serving; a pegged
|
||||
# or hung process still passes kill -0. Short timeout (2s) since this runs per-row
|
||||
# in `nsbx list`.
|
||||
daemon_health(){
|
||||
local name="$1"
|
||||
daemon_alive "$name" || { echo "stopped"; return 0; }
|
||||
local port; port="$(mget "$name" "['port']")"
|
||||
local s; s="$(curl -s -m2 "http://127.0.0.1:${port}/api/stats" 2>/dev/null)"
|
||||
[ -n "$s" ] && echo "running" || echo "unresponsive"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------- elc/build ----
|
||||
find_elc(){
|
||||
command -v elc 2>/dev/null && return 0
|
||||
@@ -142,49 +123,6 @@ _build_binary(){
|
||||
ok "built: $out ($(ls -lh "$out" | awk '{print $5}'), sha $(sha "$out" | cut -c1-12))"
|
||||
}
|
||||
|
||||
# bin_built_at <path> : human-readable build timestamp. `cp -p` preserves mtime,
|
||||
# so this is the ORIGINAL build time even for binaries copied stock-prod into a
|
||||
# sandbox — not the copy time.
|
||||
bin_built_at(){ stat -f '%Sm' -t '%Y-%m-%d %H:%M:%S' "$1" 2>/dev/null || echo "unknown"; }
|
||||
|
||||
# _binary_freshness <name> : best-effort staleness note (empty string if fresh/
|
||||
# unknown — never guesses). Two cases:
|
||||
# - stock-prod: compares the sha recorded at create time against the CURRENTLY
|
||||
# configured live real binary's sha (recomputed now, not cached) — catches
|
||||
# "live prod moved on since this sandbox was cloned".
|
||||
# - branch/source/rebuilt: compares the recorded source_commit against the
|
||||
# LOCAL origin/dev ref (no fetch — reads whatever the repo already has) —
|
||||
# catches "built from a commit that predates current dev tip".
|
||||
_binary_freshness(){
|
||||
local name="$1" src; src="$(mget "$name" "['source']")"
|
||||
case "$src" in
|
||||
stock-prod:*)
|
||||
local live_bin cur_sha rec_sha
|
||||
live_bin="$(_live_real_bin)"
|
||||
[ -n "$live_bin" ] && [ -x "$live_bin" ] || return 0
|
||||
cur_sha="$(sha "$live_bin")"; rec_sha="$(mget "$name" "['binary_sha256']")"
|
||||
[ -n "$cur_sha" ] && [ -n "$rec_sha" ] && [ "$cur_sha" != "$rec_sha" ] \
|
||||
&& printf 'stale: live prod binary has moved on since this sandbox was cloned (live is now %s, sha %s) — nsbx build %s --binary %s to catch up' \
|
||||
"$(basename "$live_bin")" "${cur_sha:0:12}" "$name" "$live_bin"
|
||||
;;
|
||||
branch:*|source:*|rebuilt:*)
|
||||
local commit cur behind
|
||||
commit="$(mget "$name" "['source_commit']")"
|
||||
[ -n "$commit" ] || return 0
|
||||
cur="$(git -C "$EL_REPO" rev-parse origin/dev 2>/dev/null)" || return 0
|
||||
[ -n "$cur" ] && [ "$commit" != "$cur" ] || return 0
|
||||
git -C "$EL_REPO" merge-base --is-ancestor "$commit" "$cur" 2>/dev/null || return 0
|
||||
behind="$(git -C "$EL_REPO" rev-list --count "$commit..$cur" 2>/dev/null)"
|
||||
printf 'stale: built from %s, %s commit(s) behind local origin/dev (%s) — nsbx build %s --branch origin/dev' \
|
||||
"${commit:0:12}" "${behind:-?}" "${cur:0:12}" "$name"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# _source_commit <dir> : best-effort git HEAD of a source tree used to build a
|
||||
# sandbox binary, empty if not a git repo (e.g. a prebuilt --binary path has none).
|
||||
_source_commit(){ git -C "$1" rev-parse HEAD 2>/dev/null || true; }
|
||||
|
||||
# ---------------------------------------------------------------- daemon -------
|
||||
# start_daemon <name> : boots the sandbox's real engram binary on its isolated
|
||||
# port against its cloned data dir, with the SAME auto-remerge net the live soul
|
||||
@@ -195,9 +133,9 @@ start_daemon(){
|
||||
local port bin data export key
|
||||
port="$(mget "$name" "['port']")"; bin="$d/bin/engram"; data="$d/data"
|
||||
key="sbx-$name"; export="$data/.scan-export.reseed-clean.json"
|
||||
[ -x "$bin" ] || die "sandbox binary missing: $bin (run: nsbx build $name --source DIR | --branch REF, or nsbx destroy $name && nsbx create $name to reclone stock-prod)"
|
||||
[ -x "$bin" ] || die "sandbox binary missing: $bin"
|
||||
[ "$port" != "$LIVE_BIND_PORT" ] && [ "$port" != "$SOUL_PORT" ] || die "refusing forbidden port $port"
|
||||
[ -f "$data/neuron.egm" ] || die "sandbox has no cloned store: $data/neuron.egm (data dir is corrupt/incomplete — run: nsbx destroy $name && nsbx create $name)"
|
||||
[ -f "$data/neuron.egm" ] || die "sandbox has no cloned store: $data/neuron.egm"
|
||||
# HARD guard: never point a sandbox daemon at the live data dir.
|
||||
[ "$(cd "$data" && pwd -P)" != "$(cd "$LIVE_DATA_DIR" && pwd -P)" ] || die "refusing: sandbox data dir resolves to LIVE store"
|
||||
|
||||
@@ -212,11 +150,11 @@ start_daemon(){
|
||||
echo "$pid" > "$d/daemon.pid"
|
||||
# readiness poll
|
||||
local url="http://127.0.0.1:$port" i s
|
||||
for i in $(seq 1 "$READY_TICKS"); do
|
||||
for i in $(seq 1 30); do
|
||||
s="$(curl -s -m3 "$url/api/stats" 2>/dev/null)"
|
||||
[ -n "$s" ] && break; sleep 0.5
|
||||
done
|
||||
[ -n "$s" ] || { warn "daemon did not become ready within ${NSBX_READY_TIMEOUT_SECS:-15}s (see $d/logs/daemon.log). pid $pid may still be alive and slow to boot under load — check: lsof -iTCP:$port -P, or retry with NSBX_READY_TIMEOUT_SECS=45"; return 1; }
|
||||
[ -n "$s" ] || { warn "daemon did not become ready (see $d/logs/daemon.log)"; return 1; }
|
||||
ok "ready pid=$pid boot-stats: $s"
|
||||
# auto-remerge net (idempotent): match live edge population if the export is present
|
||||
if [ -f "$export" ]; then
|
||||
@@ -293,35 +231,33 @@ cmd_create(){
|
||||
info "live baseline stats: ${lstats:-<unavailable>}"
|
||||
|
||||
# ---- determine + place the runtime binary (versioned into the snapshot) ----
|
||||
local source_desc live_bin source_commit=""
|
||||
local source_desc live_bin
|
||||
live_bin="$(_live_real_bin)"
|
||||
if [ -n "$binpath" ]; then
|
||||
[ -x "$binpath" ] || die "not an executable binary: $binpath"
|
||||
cp -p "$binpath" "$d/bin/engram"; source_desc="prebuilt:$binpath"
|
||||
elif [ -n "$src" ]; then
|
||||
_build_binary "$src" "$d/bin/engram" "$d/build"; source_desc="source:$src"
|
||||
source_commit="$(_source_commit "$src")"
|
||||
elif [ -n "$branch" ]; then
|
||||
log "worktree: $repo @ $branch -> $d/build/worktree"
|
||||
git -C "$repo" worktree add --detach "$d/build/worktree" "$branch" >/dev/null 2>&1 \
|
||||
|| die "git worktree add failed ($repo @ $branch)"
|
||||
_build_binary "$d/build/worktree" "$d/bin/engram" "$d/build"; source_desc="branch:$branch@$repo"
|
||||
source_commit="$(_source_commit "$d/build/worktree")"
|
||||
else
|
||||
[ -x "$live_bin" ] || die "cannot resolve live ENGRAM_REAL_BIN: $live_bin"
|
||||
cp -p "$live_bin" "$d/bin/engram"; source_desc="stock-prod:$live_bin"
|
||||
fi
|
||||
local bin_sha; bin_sha="$(sha "$d/bin/engram")"
|
||||
info "runtime: $source_desc (sha ${bin_sha:0:12}, built $(bin_built_at "$d/bin/engram"))"
|
||||
info "runtime: $source_desc (sha ${bin_sha:0:12})"
|
||||
|
||||
# ---- write manifest ----
|
||||
python3 - "$name" "$port" "$source_desc" "$bin_sha" "$egm_sha" "$base_nodes" "$base_edges" "$(sha "$live_bin" 2>/dev/null)" "$source_commit" <<'PY' > "$(manifest "$name")"
|
||||
python3 - "$name" "$port" "$source_desc" "$bin_sha" "$egm_sha" "$base_nodes" "$base_edges" "$(sha "$live_bin" 2>/dev/null)" <<'PY' > "$(manifest "$name")"
|
||||
import json,sys,datetime
|
||||
name,port,src,binsha,egmsha,bn,be,livebinsha,source_commit=sys.argv[1:10]
|
||||
name,port,src,binsha,egmsha,bn,be,livebinsha=sys.argv[1:9]
|
||||
json.dump({
|
||||
"name":name,"port":int(port),"created_at":datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"source":src,"binary_sha256":binsha,"clone_egm_sha256":egmsha,
|
||||
"live_binary_sha256":livebinsha,"source_commit":source_commit,
|
||||
"live_binary_sha256":livebinsha,
|
||||
"live_baseline":{"node_count":int(bn or 0),"edge_count":int(be or 0)},
|
||||
"keystones":["kn-efeb4a5b-5aff-4759-8a97-7233099be6ee","kn-5b606390-a52d-4ca2-8e0e-eba141d13440"]
|
||||
}, sys.stdout, indent=2)
|
||||
@@ -332,17 +268,6 @@ PY
|
||||
start_daemon "$name" || die "daemon failed to start"
|
||||
local sstats; sstats="$(sbx_stats "$name")"
|
||||
local sbn sbe; sbn="$(stat_field "$sstats" node_count)"; sbe="$(stat_field "$sstats" edge_count)"
|
||||
# a boot immediately followed by an auto-remerge can leave the daemon briefly
|
||||
# busy — retry rather than silently folding a 0/0 baseline into the manifest.
|
||||
# `validate`'s zero-loss/reboot-prove checks compare current counts >= baseline,
|
||||
# so a 0/0 baseline would make them trivially PASS regardless of real data loss.
|
||||
local _bi
|
||||
for _bi in 1 2 3 4 5; do
|
||||
[ -n "$sbn" ] && [ "$sbn" != "0" ] && break
|
||||
sleep 1
|
||||
sstats="$(sbx_stats "$name")"; sbn="$(stat_field "$sstats" node_count)"; sbe="$(stat_field "$sstats" edge_count)"
|
||||
done
|
||||
[ -z "$sbn" ] || [ "$sbn" = "0" ] && warn "sandbox stats still empty/zero after retries — recording sbx_baseline 0/0. This makes 'nsbx validate $name' zero-loss checks trivially pass; investigate before trusting a validate PASS: nsbx status $name"
|
||||
_capture_retrieval "$name" "$d/baseline/retrieval.json"
|
||||
# fold sandbox baseline into manifest
|
||||
python3 - "$(manifest "$name")" "$sbn" "$sbe" <<'PY'
|
||||
@@ -395,12 +320,7 @@ except Exception: print("[]")' 2>/dev/null)"
|
||||
# thereafter. Prod on :$LIVE_BIND_PORT/:$SOUL_PORT is unreachable from here by design.
|
||||
cmd_up(){
|
||||
local name; if [ $# -gt 0 ] && [ "${1#-}" = "$1" ]; then name="$1"; shift; else name="${USER:-dev}-dev"; fi
|
||||
if mexists "$name"; then
|
||||
daemon_alive "$name" || start_daemon "$name" \
|
||||
|| die "daemon did not become ready — see $(sdir "$name")/logs/daemon.log (try: nsbx up $name again once you've checked the log)"
|
||||
else
|
||||
cmd_create "$name" "$@"
|
||||
fi
|
||||
if mexists "$name"; then daemon_alive "$name" || start_daemon "$name"; else cmd_create "$name" "$@"; fi
|
||||
local port; port="$(mget "$name" "['port']")"
|
||||
echo >&2
|
||||
ok "your sandbox '$name' is ready at http://127.0.0.1:$port (a private copy of the mind — prod is untouchable)"
|
||||
@@ -414,37 +334,34 @@ cmd_up(){
|
||||
# it on the SAME clone + port (the code-change dev loop, in place).
|
||||
cmd_build(){
|
||||
local name="$1"; shift || true
|
||||
mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)"
|
||||
mexists "$name" || die "no such sandbox: $name"
|
||||
local src="" branch="" repo="$EL_REPO"
|
||||
while [ $# -gt 0 ]; do case "$1" in
|
||||
--source) src="$2"; shift 2;; --branch) branch="$2"; shift 2;; --repo) repo="$2"; shift 2;;
|
||||
*) die "unknown flag: $1";; esac; done
|
||||
local d; d="$(sdir "$name")"
|
||||
stop_daemon "$name"
|
||||
local source_commit=""
|
||||
if [ -n "$src" ]; then _build_binary "$src" "$d/bin/engram" "$d/build"; source_commit="$(_source_commit "$src")"
|
||||
if [ -n "$src" ]; then _build_binary "$src" "$d/bin/engram" "$d/build"
|
||||
elif [ -n "$branch" ]; then
|
||||
rm -rf "$d/build/worktree" 2>/dev/null; git -C "$repo" worktree prune 2>/dev/null
|
||||
git -C "$repo" worktree add --detach "$d/build/worktree" "$branch" >/dev/null 2>&1 || die "worktree add failed"
|
||||
_build_binary "$d/build/worktree" "$d/bin/engram" "$d/build"
|
||||
source_commit="$(_source_commit "$d/build/worktree")"
|
||||
else die "usage: nsbx build <name> --source DIR | --branch REF [--repo R]"; fi
|
||||
# record new binary sha
|
||||
python3 - "$(manifest "$name")" "$(sha "$d/bin/engram")" "${src:-branch:$branch}" "$source_commit" <<'PY'
|
||||
import json,sys; mf,s,src,source_commit=sys.argv[1:5]
|
||||
d=json.load(open(mf)); d["binary_sha256"]=s; d["source"]="rebuilt:"+src; d["source_commit"]=source_commit
|
||||
python3 - "$(manifest "$name")" "$(sha "$d/bin/engram")" "${src:-branch:$branch}" <<'PY'
|
||||
import json,sys; mf,s,src=sys.argv[1:4]
|
||||
d=json.load(open(mf)); d["binary_sha256"]=s; d["source"]="rebuilt:"+src
|
||||
json.dump(d,open(mf,'w'),indent=2)
|
||||
PY
|
||||
start_daemon "$name" \
|
||||
|| die "rebuilt binary did not become ready — see $d/logs/daemon.log (the old binary is gone; fix the code and re-run nsbx build $name ...)"
|
||||
start_daemon "$name"
|
||||
ok "rebuilt + restarted on :$(mget "$name" "['port']")"
|
||||
}
|
||||
|
||||
# ================================================================ run ==========
|
||||
cmd_run(){
|
||||
local name="$1"; shift || true
|
||||
mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)"
|
||||
daemon_alive "$name" || start_daemon "$name" || die "daemon not running and failed to start — see $(sdir "$name")/logs/daemon.log"
|
||||
mexists "$name" || die "no such sandbox: $name"
|
||||
daemon_alive "$name" || start_daemon "$name"
|
||||
local d port; d="$(sdir "$name")"; port="$(mget "$name" "['port']")"
|
||||
# direct API form: nsbx run <name> api <path> [json]
|
||||
if [ "${1:-}" = "api" ]; then
|
||||
@@ -478,8 +395,8 @@ cmd_run(){
|
||||
# RSS bound; retrieval parity; keystone integrity.
|
||||
cmd_validate(){
|
||||
local name="$1"; shift || true
|
||||
mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)"
|
||||
daemon_alive "$name" || start_daemon "$name" || die "daemon not running and failed to start — see $(sdir "$name")/logs/daemon.log"
|
||||
mexists "$name" || die "no such sandbox: $name"
|
||||
daemon_alive "$name" || start_daemon "$name"
|
||||
local d port key; d="$(sdir "$name")"; port="$(mget "$name" "['port']")"; key="sbx-$name"
|
||||
local url="http://127.0.0.1:$port"
|
||||
local bn be; bn="$(mget "$name" "['sbx_baseline']['node_count']")"; be="$(mget "$name" "['sbx_baseline']['edge_count']")"
|
||||
@@ -572,7 +489,7 @@ PY
|
||||
# Default is a DRY-RUN plan; requires --i-approve-prod-cutover to actually cut over.
|
||||
cmd_promote(){
|
||||
local name="$1"; shift || true
|
||||
mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)"
|
||||
mexists "$name" || die "no such sandbox: $name"
|
||||
local approve=0 do_data=0
|
||||
while [ $# -gt 0 ]; do case "$1" in
|
||||
--i-approve-prod-cutover) approve=1; shift;;
|
||||
@@ -671,7 +588,7 @@ PY
|
||||
# ================================================================ destroy ======
|
||||
cmd_destroy(){
|
||||
local name="$1"; shift || true
|
||||
mexists "$name" || die "no such sandbox: $name (run: nsbx list — or nsbx create $name to make it)"
|
||||
mexists "$name" || die "no such sandbox: $name"
|
||||
local d; d="$(sdir "$name")"
|
||||
stop_daemon "$name"
|
||||
if [ -d "$d/build/worktree" ]; then
|
||||
@@ -687,44 +604,22 @@ cmd_destroy(){
|
||||
# ================================================================ list/status ==
|
||||
cmd_list(){
|
||||
[ -d "$SBX_ROOT" ] || { echo "no sandboxes"; return 0; }
|
||||
printf '%-16s %-6s %-13s %-9s %-19s %s\n' NAME PORT STATE PID "BUILT" SOURCE
|
||||
printf '%-16s %-6s %-8s %-9s %s\n' NAME PORT STATE PID SOURCE
|
||||
local m
|
||||
for m in "$SBX_ROOT"/*/manifest.json; do
|
||||
[ -f "$m" ] || continue
|
||||
local n p src pid state bpath built fresh
|
||||
local n p src pid state
|
||||
n="$(python3 -c "import json;print(json.load(open('$m'))['name'])")"
|
||||
p="$(python3 -c "import json;print(json.load(open('$m'))['port'])")"
|
||||
src="$(python3 -c "import json;print(json.load(open('$m'))['source'])")"
|
||||
pid="$(daemon_pid "$n")"
|
||||
case "$(daemon_health "$n")" in
|
||||
running) state="running";;
|
||||
unresponsive) state="running(!resp)";;
|
||||
*) state="stopped";;
|
||||
esac
|
||||
bpath="$(sdir "$n")/bin/engram"; built="$([ -f "$bpath" ] && bin_built_at "$bpath" || echo unknown)"
|
||||
fresh="$(_binary_freshness "$n")"; [ -n "$fresh" ] && src="[STALE] $src"
|
||||
printf '%-16s %-6s %-13s %-9s %-19s %s\n' "$n" "$p" "$state" "${pid:-–}" "$built" "$src"
|
||||
pid="$(daemon_pid "$n")"; state="stopped"; daemon_alive "$n" && state="running"
|
||||
printf '%-16s %-6s %-8s %-9s %s\n' "$n" "$p" "$state" "${pid:-–}" "$src"
|
||||
done
|
||||
info "state 'running(!resp)' = process alive but /api/stats didn't answer — see: nsbx status <name>"
|
||||
}
|
||||
cmd_status(){
|
||||
local name="$1"; mexists "$name" || die "no such sandbox: $name (run: nsbx list to see what exists)"
|
||||
local name="$1"; mexists "$name" || die "no such sandbox: $name"
|
||||
python3 -m json.tool "$(manifest "$name")"
|
||||
local bpath; bpath="$(sdir "$name")/bin/engram"
|
||||
if [ -f "$bpath" ]; then
|
||||
echo "binary: sha=$(sha "$bpath" | cut -c1-12) built=$(bin_built_at "$bpath")"
|
||||
local fresh; fresh="$(_binary_freshness "$name")"
|
||||
[ -n "$fresh" ] && printf '%s%s%s\n' "$C_YEL" "$fresh" "$C_0"
|
||||
fi
|
||||
case "$(daemon_health "$name")" in
|
||||
running)
|
||||
echo "state: running (pid $(daemon_pid "$name")) stats: $(sbx_stats "$name")";;
|
||||
unresponsive)
|
||||
printf '%sstate: running but NOT RESPONDING%s (pid %s) — process alive, /api/stats returned nothing.\n' "$C_RED" "$C_0" "$(daemon_pid "$name")"
|
||||
info "check: tail -50 $(sdir "$name")/logs/daemon.log | next: kill -9 $(daemon_pid "$name") && nsbx up $name"
|
||||
;;
|
||||
*) echo "state: stopped";;
|
||||
esac
|
||||
daemon_alive "$name" && echo "state: running (pid $(daemon_pid "$name")) stats: $(sbx_stats "$name")" || echo "state: stopped"
|
||||
[ -f "$(sdir "$name")/validate.json" ] && { echo "--- last validation ---"; python3 -m json.tool "$(sdir "$name")/validate.json"; }
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user