From 0a0a2bcb4485ec9f85c2ed3ddde0e40f8f66935e Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Tue, 14 Jul 2026 14:21:39 -0500 Subject: [PATCH 1/7] parser: bound token reads to Eof so malformed input errors instead of OOMing Out-of-range tok_kind/tok_value reads returned runtime null (el_list_get OOB -> 0) rather than the Eof sentinel, so the inner parse loops (parse_block, call-arg, array-literal, match-arm) that terminate only on their close delimiter or k=="Eof" never saw Eof once the cursor ran past the single trailing Eof token. On unclosed-delimiter input the parser then appended AST nodes forever -> unbounded allocation -> ~700GB -> OOM (observed compiling neuron/sessions.el). Fix at the choke point: tok_kind returns "Eof" and tok_value returns "" for out-of-range positions, restoring the parser-wide contract that reads at/after the end yield Eof. expect() no longer steps past the Eof sentinel on mismatch. This terminates every overrun loop simultaneously; a malformed program now surfaces as a normal (best-effort) parse end instead of exhausting memory. Requires a self-hosted bootstrap rebuild of elc to take effect. --- lang/el-compiler/src/parser.el | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/lang/el-compiler/src/parser.el b/lang/el-compiler/src/parser.el index e4de3b9..d93fcba 100644 --- a/lang/el-compiler/src/parser.el +++ b/lang/el-compiler/src/parser.el @@ -23,10 +23,29 @@ fn tok_at(tokens: [Any], pos: Int) -> Map { } fn tok_kind(tokens: [Any], pos: Int) -> String { + // Out-of-range reads must report the Eof sentinel so every `== "Eof"` + // termination guard in the parser fires. Without this, reading past the + // single trailing Eof token returns runtime null (el_list_get OOB -> 0), + // which matches no delimiter, letting inner parse loops append AST nodes + // forever on malformed input -> unbounded allocation -> OOM. + let n: Int = native_list_len(tokens) / 2 + if pos < 0 { + return "Eof" + } + if pos >= n { + return "Eof" + } native_list_get(tokens, pos * 2) } fn tok_value(tokens: [Any], pos: Int) -> String { + let n: Int = native_list_len(tokens) / 2 + if pos < 0 { + return "" + } + if pos >= n { + return "" + } native_list_get(tokens, pos * 2 + 1) } @@ -35,7 +54,12 @@ fn expect(tokens: [Any], pos: Int, kind: String) -> Int { if k == kind { return pos + 1 } - // On mismatch just advance; error recovery is best-effort + // On mismatch, error recovery is best-effort. But never step PAST the Eof + // sentinel: once at Eof a mismatch means the input ended early, and + // advancing would run the cursor off the token list. + if k == "Eof" { + return pos + } pos + 1 } From e3dabe3e082cb95d143a2084995dbc3ae40d4d28 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Tue, 14 Jul 2026 18:39:07 -0500 Subject: [PATCH 2/7] fix(engram): tokenized + ranked lexical search, not whole-query Ctrl-F MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit engram search/activate/goal-bias matched the ENTIRE raw query string as a single case-insensitive substring (istr_contains(field, q)). Multi-word queries like "windows msi signing" only matched a node containing that exact contiguous run, so real multi-word queries returned ZERO on a graph saturated with the answer. This is Ctrl-F, not search — and search is the core of the engram being useful. Fix: split the query on whitespace into distinct tokens; a node matches if it contains ANY token in content/label/tags. Rank by distinct tokens matched (desc) then salience (desc). istr_contains is kept unchanged as the per-token primitive. Single-token queries are a strict special case (score 0 or 1) so the many single-word callers do not regress. Sites changed (all in el_runtime.c): - new helpers engram_tokenize_query / engram_node_match_score / engram_rank_cmp - engram_search (internal el_val_t path) - engram_search_json (HTTP /api/search path) - engram_activate seed loop (HTTP /api/activate path; seed activation scaled by token coverage so full-query matches seed more strongly) - engram_goal_bias overlap bonus upgraded to graded token coverage Proof (6591-node snapshot copy, rebuilt binary on :8799, POST JSON path): windows msi signing 0 -> 20 Will Anderson 0 -> 20 windows msi 0 -> 20 tokenized search fix 0 -> 20 Single-word parity preserved (VBD/volatility/elc capped at limit; unkey = all matching nodes). Top hits are relevant (e.g. "Will Anderson" surfaces the Project Design and VBD whitepapers). Note: GET ?q=a%20b still returns 0 because query_param (server.el) does not URL-decode — a separate EL-layer bug; the soul's POST-JSON path is fixed here. --- lang/el-compiler/runtime/el_runtime.c | 164 +++++++++++++++++++++----- 1 file changed, 137 insertions(+), 27 deletions(-) diff --git a/lang/el-compiler/runtime/el_runtime.c b/lang/el-compiler/runtime/el_runtime.c index 68e6dca..1d4d9ac 100644 --- a/lang/el-compiler/runtime/el_runtime.c +++ b/lang/el-compiler/runtime/el_runtime.c @@ -6826,6 +6826,75 @@ static int istr_contains(const char* hay, const char* needle) { return 0; } +/* ── Tokenized query matching ─────────────────────────────────────────── + * The engram query surface (search / activate / goal-bias) historically + * matched the ENTIRE raw query string as a single case-insensitive + * substring via istr_contains(field, q). That is Ctrl-F, not search: + * a multi-word query like "windows msi signing" only matched a node whose + * text contained that exact contiguous run, so real multi-word queries + * returned zero. istr_contains stays as the per-TOKEN primitive; these + * helpers split the query on whitespace and match ANY token, then rank by + * how many DISTINCT tokens a node covers. Single-token queries are a strict + * special case (score is 0 or 1) so single-word callers never regress. */ +#define ENGRAM_MAX_QTOKENS 32 +#define ENGRAM_QTOK_LEN 256 + +/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct + * (case-insensitive) tokens. Returns the token count. Over-long tokens are + * truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */ +static int engram_tokenize_query(const char* q, + char toks[][ENGRAM_QTOK_LEN], int maxtok) { + int n = 0; + if (!q) return 0; + const char* p = q; + while (*p && n < maxtok) { + while (*p && isspace((unsigned char)*p)) p++; + if (!*p) break; + char buf[ENGRAM_QTOK_LEN]; + size_t tl = 0; + while (*p && !isspace((unsigned char)*p)) { + if (tl < sizeof(buf) - 1) buf[tl++] = *p; + p++; + } + buf[tl] = '\0'; + if (tl == 0) continue; + int dup = 0; + for (int s = 0; s < n; s++) { + if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; } + } + if (dup) continue; + memcpy(toks[n], buf, tl + 1); + n++; + } + return n; +} + +/* Count how many of the ntok distinct query tokens appear (case-insensitive) + * in the node's content, label, or tags. 0 == no match. */ +static int engram_node_match_score(const EngramNode* n, + char toks[][ENGRAM_QTOK_LEN], int ntok) { + int score = 0; + for (int t = 0; t < ntok; t++) { + if (istr_contains(n->content, toks[t]) || + istr_contains(n->label, toks[t]) || + istr_contains(n->tags, toks[t])) + score++; + } + return score; +} + +/* Rank entry: distinct-token match count (primary, desc) then salience + * (tiebreak, desc). */ +typedef struct { int64_t idx; int score; double salience; } EngramRankEntry; +static int engram_rank_cmp(const void* a, const void* b) { + const EngramRankEntry* ea = (const EngramRankEntry*)a; + const EngramRankEntry* eb = (const EngramRankEntry*)b; + if (ea->score != eb->score) return eb->score - ea->score; /* desc */ + if (ea->salience < eb->salience) return 1; + if (ea->salience > eb->salience) return -1; + return 0; +} + el_val_t engram_search(el_val_t query, el_val_t limit) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); @@ -6833,21 +6902,34 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { if (lim <= 0) lim = 100; el_val_t lst = el_list_empty(); if (!q || !*q) return lst; - int64_t found = 0; - for (int64_t i = 0; i < g->node_count && found < lim; i++) { + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS); + if (ntok == 0) return lst; + EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry)); + if (!hits) return lst; + int64_t nhits = 0; + for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; /* Filter transparent layers: nodes whose layer is `transparent=1` * shape output but are invisible to introspection ("what do you * know about yourself"). They still surface via engram_activate * + engram_compile_layered_json — that's the legitimate path. */ if (engram_layer_is_transparent(n->layer_id)) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { - lst = el_list_append(lst, engram_node_to_map(n)); - found++; + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0) { + hits[nhits].idx = i; + hits[nhits].score = sc; + hits[nhits].salience = n->salience; + nhits++; } } + /* Rank by distinct tokens matched (desc) then salience (desc), then cap. */ + qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); + int64_t end = nhits < lim ? nhits : lim; + for (int64_t k = 0; k < end; k++) { + lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx])); + } + free(hits); return lst; } @@ -7124,10 +7206,14 @@ static double engram_temporal_proximity_bonus(int64_t node_created, static double engram_goal_bias(const EngramNode* n, const char* query) { if (!query || !*query) return 1.0; double bias = 1.0; - /* Direct lexical overlap: node content/label/tags share text with query. */ - if (istr_contains(n->content, query) || istr_contains(n->label, query) || - istr_contains(n->tags, query)) { - bias += 0.5; + /* Direct lexical overlap, graded by token coverage: a node covering all + * query tokens gets the full +0.5; partial coverage gets a proportional + * share. Single-token queries → full +0.5 on match, identical to before. */ + { + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(query, toks, ENGRAM_MAX_QTOKENS); + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0 && ntok > 0) bias += 0.5 * ((double)sc / (double)ntok); } /* Node-type resonance with query intent. */ int technical_query = istr_contains(query, "code") || @@ -7193,14 +7279,21 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { if (!seeds) { free(best_bg); free(best_hops); free(reached); return out; } + /* Tokenize once: a node seeds if it matches ANY query token, and its seed + * activation is scaled by token coverage (fraction of distinct query + * tokens it contains) so a node matching all words seeds more strongly + * than one matching a single word. Single-word queries → coverage 1.0, + * identical to the prior whole-query behavior. */ + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS); for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0) { double tdecay = engram_temporal_decay(n, now_ms); double dampen = engram_activation_dampen(n); - double act = n->salience * tdecay * dampen; + double cover = ntok > 0 ? (double)sc / (double)ntok : 1.0; + double act = n->salience * tdecay * dampen * cover; seeds[seed_count].idx = i; seeds[seed_count].act = act; seeds[seed_count].created_at = n->created_at; @@ -7769,19 +7862,36 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { JsonBuf b; jb_init(&b); jb_putc(&b, '['); int first = 1; - int64_t found = 0; if (q && *q) { - for (int64_t i = 0; i < g->node_count && found < lim; i++) { - EngramNode* n = &g->nodes[i]; - /* Filter transparent layers — same as engram_search. */ - if (engram_layer_is_transparent(n->layer_id)) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { - if (!first) jb_putc(&b, ','); - engram_emit_node_json(&b, n); - first = 0; - found++; + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS); + if (ntok > 0) { + EngramRankEntry* hits = + malloc((size_t)g->node_count * sizeof(EngramRankEntry)); + if (hits) { + int64_t nhits = 0; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + /* Filter transparent layers — same as engram_search. */ + if (engram_layer_is_transparent(n->layer_id)) continue; + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0) { + hits[nhits].idx = i; + hits[nhits].score = sc; + hits[nhits].salience = n->salience; + nhits++; + } + } + /* Rank by distinct tokens matched (desc) then salience (desc). */ + qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), + engram_rank_cmp); + int64_t end = nhits < lim ? nhits : lim; + for (int64_t k = 0; k < end; k++) { + if (!first) jb_putc(&b, ','); + engram_emit_node_json(&b, &g->nodes[hits[k].idx]); + first = 0; + } + free(hits); } } } From b4967af13e966bf4eaa048df1a60f498b85dbe46 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Tue, 14 Jul 2026 18:48:16 -0500 Subject: [PATCH 3/7] =?UTF-8?q?feat(engram):=20semantic=20search=20layer?= =?UTF-8?q?=20via=20nomic-embed-text=20(cosine=20=E2=88=AA=20lexical)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lexical istr_contains alone can't surface a node whose words don't appear in the query. This adds an optional dense-vector layer: node content and the query are embedded through Ollama (nomic-embed-text), and nodes are ranked by cosine similarity unioned with lexical hits, so a paraphrase query reaches the right node. Wired into all three query entry points in el_runtime.c: - engram_search_json (HTTP /api/search): collect lexical ∪ semantic candidates, score (lexical base 1.0 + cosine; pure-semantic = cosine), rank, emit top-N. Stable sort preserves old order when semantic is off. - engram_search (internal el_val twin): lexical ∪ semantic union. - engram_activate seed loop (HTTP /api/activate): a node seeds if it lexically matches OR clears the cosine threshold; pure-semantic seeds enter scaled by cosine so paraphrase spreads without overpowering. Degradable by design: the whole layer is gated on HAVE_CURL plus a one-shot runtime probe. If curl is compiled out, Ollama is unreachable, or ENGRAM_SEMANTIC=0, every entry point yields zero semantic signal and callers fall back byte-for-byte to the pre-existing lexical search. Node embeddings are cached in process memory keyed by node id with an FNV-1a content hash for invalidation; the query is embedded once per call — so the graph is not re-embedded on every query. nomic task prefixes (search_query:/search_document:) are applied for retrieval separation. Build steps gain -DHAVE_CURL so the engram artifact compiles the layer in (-lcurl was already linked). Env: ENGRAM_SEMANTIC, ENGRAM_EMBED_URL, ENGRAM_EMBED_MODEL, ENGRAM_SEMANTIC_MIN (cosine threshold, default 0.6). --- engram/.gitea/workflows/ci-dev.yaml | 2 +- engram/.gitea/workflows/ci-stage.yaml | 2 +- engram/.gitea/workflows/engram-release.yaml | 2 +- lang/el-compiler/runtime/el_runtime.c | 331 ++++++++++++++++++-- 4 files changed, 314 insertions(+), 23 deletions(-) diff --git a/engram/.gitea/workflows/ci-dev.yaml b/engram/.gitea/workflows/ci-dev.yaml index 7463aa8..a432869 100644 --- a/engram/.gitea/workflows/ci-dev.yaml +++ b/engram/.gitea/workflows/ci-dev.yaml @@ -81,7 +81,7 @@ jobs: # Link to produce the engram binary - name: Link engram binary run: | - cc -std=c11 -O2 \ + cc -std=c11 -O2 -DHAVE_CURL \ -I /usr/local/lib/el \ -o dist/engram \ dist/engram.c \ diff --git a/engram/.gitea/workflows/ci-stage.yaml b/engram/.gitea/workflows/ci-stage.yaml index 69ed162..a134117 100644 --- a/engram/.gitea/workflows/ci-stage.yaml +++ b/engram/.gitea/workflows/ci-stage.yaml @@ -88,7 +88,7 @@ jobs: # Link to produce the engram binary - name: Link engram binary run: | - cc -std=c11 -O2 \ + cc -std=c11 -O2 -DHAVE_CURL \ -I /usr/local/lib/el \ -o dist/engram \ dist/engram.c \ diff --git a/engram/.gitea/workflows/engram-release.yaml b/engram/.gitea/workflows/engram-release.yaml index bdf3258..aa02e9f 100644 --- a/engram/.gitea/workflows/engram-release.yaml +++ b/engram/.gitea/workflows/engram-release.yaml @@ -62,7 +62,7 @@ jobs: # Link to produce the engram binary - name: Link engram binary run: | - cc -std=c11 -O2 \ + cc -std=c11 -O2 -DHAVE_CURL \ -I /usr/local/lib/el \ -o dist/engram \ dist/engram.c \ diff --git a/lang/el-compiler/runtime/el_runtime.c b/lang/el-compiler/runtime/el_runtime.c index 68e6dca..4e41611 100644 --- a/lang/el-compiler/runtime/el_runtime.c +++ b/lang/el-compiler/runtime/el_runtime.c @@ -6826,6 +6826,243 @@ static int istr_contains(const char* hay, const char* needle) { return 0; } +/* ══════════════════════════════════════════════════════════════════════════ + * SEMANTIC SEARCH LAYER — nomic-embed-text via Ollama /api/embeddings + * ────────────────────────────────────────────────────────────────────────── + * Augments the lexical (istr_contains) matcher with dense-vector retrieval. + * Node content and the query are embedded through a local Ollama server; + * nodes are ranked by cosine similarity and UNIONED with lexical hits. This + * lets a paraphrase query surface a node whose words never appear in it. + * + * DEGRADABLE BY DESIGN. The whole layer is gated on HAVE_CURL plus a one-shot + * runtime probe of the embedding endpoint. If curl is not compiled in, or + * Ollama is unreachable, or ENGRAM_SEMANTIC=0, every entry point returns + * "no semantic signal" and callers fall back to pure lexical behaviour — + * byte-for-byte the pre-existing search. + * + * CACHE. Node embeddings are computed lazily on first use and cached in + * process memory keyed by node id, with an FNV-1a content hash for + * invalidation (edited content re-embeds). The query is embedded once per + * search call. This is what "avoid re-embedding the whole graph every query" + * buys us: a warm cache serves cosine from RAM. (A cold process still pays + * O(N) embed calls the first time each node is scanned — persisting the cache + * to a snapshot sidecar is the documented next step, not done here.) + * + * nomic task prefixes ("search_query:" / "search_document:") are applied + * because nomic-embed-text is trained with them; they materially improve + * retrieval separation (empirically: paraphrase 0.72 vs distractors <0.48). + * + * ENV: + * ENGRAM_SEMANTIC "0" disables; unset/other = auto-probe + * ENGRAM_EMBED_URL default http://localhost:11434/api/embeddings + * ENGRAM_EMBED_MODEL default nomic-embed-text + * ENGRAM_SEMANTIC_MIN cosine threshold for a pure-semantic match (def 0.6) + * ════════════════════════════════════════════════════════════════════════ */ + +static double engram_semantic_min(void) { + static double v = -1.0; + if (v >= 0.0) return v; + const char* s = getenv("ENGRAM_SEMANTIC_MIN"); + double d = 0.6; + if (s && *s) { char* e = NULL; double t = strtod(s, &e); + if (e != s && t >= 0.0 && t <= 1.0) d = t; } + v = d; return v; +} + +#ifdef HAVE_CURL + +typedef struct { char* id; uint64_t hash; float* vec; int dim; } EngramEmbEntry; +static EngramEmbEntry* g_emb_items = NULL; +static int64_t g_emb_count = 0, g_emb_cap = 0; +static int g_emb_state = 0; /* 0=unprobed, 1=available, -1=disabled */ + +static uint64_t engram_fnv1a(const char* s) { + uint64_t h = 1469598103934665603ULL; + if (s) for (const unsigned char* p = (const unsigned char*)s; *p; p++) { + h ^= *p; h *= 1099511628211ULL; + } + return h; +} + +/* Parse "embedding":[f,f,...] from an Ollama response. malloc'd vec, or NULL. */ +static float* engram_parse_embedding(const char* json, int* out_dim) { + if (!json) return NULL; + const char* p = strstr(json, "\"embedding\""); + if (!p) return NULL; + p = strchr(p, '['); + if (!p) return NULL; + p++; + int cap = 1024, n = 0; + float* v = malloc((size_t)cap * sizeof(float)); + if (!v) return NULL; + while (*p && *p != ']') { + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r' || *p == ',') p++; + if (*p == ']' || !*p) break; + char* e = NULL; + double d = strtod(p, &e); + if (e == p) break; + if (n >= cap) { cap *= 2; float* nv = realloc(v, (size_t)cap * sizeof(float)); + if (!nv) { free(v); return NULL; } v = nv; } + v[n++] = (float)d; + p = e; + } + if (n == 0) { free(v); return NULL; } + *out_dim = n; + return v; +} + +/* JSON-escape src into a malloc'd buffer (no surrounding quotes). */ +static char* engram_json_escape(const char* src) { + if (!src) src = ""; + size_t n = strlen(src); + char* out = malloc(n * 2 + 1); + if (!out) return NULL; + size_t j = 0; + for (size_t i = 0; i < n; i++) { + unsigned char c = (unsigned char)src[i]; + if (c == '"') { out[j++] = '\\'; out[j++] = '"'; } + else if (c == '\\') { out[j++] = '\\'; out[j++] = '\\'; } + else if (c == '\n') { out[j++] = '\\'; out[j++] = 'n'; } + else if (c == '\r') { out[j++] = '\\'; out[j++] = 'r'; } + else if (c == '\t') { out[j++] = '\\'; out[j++] = 't'; } + else if (c < 0x20) { /* drop other control bytes */ } + else { out[j++] = (char)c; } + } + out[j] = '\0'; + return out; +} + +/* Embed `prefix+text` via Ollama. Returns malloc'd vec (caller frees), or NULL. */ +static float* engram_embed_raw(const char* prefix, const char* text, int* out_dim) { + if (!text) return NULL; + const char* url = getenv("ENGRAM_EMBED_URL"); + if (!url || !*url) url = "http://localhost:11434/api/embeddings"; + const char* model = getenv("ENGRAM_EMBED_MODEL"); + if (!model || !*model) model = "nomic-embed-text"; + /* Bound content length to keep latency/memory sane on huge nodes. */ + char* trunc = NULL; + size_t maxlen = 8192; + if (strlen(text) > maxlen) { + trunc = malloc(maxlen + 1); + if (trunc) { memcpy(trunc, text, maxlen); trunc[maxlen] = '\0'; text = trunc; } + } + char* esc_prefix = engram_json_escape(prefix ? prefix : ""); + char* esc = engram_json_escape(text); + free(trunc); + if (!esc || !esc_prefix) { free(esc); free(esc_prefix); return NULL; } + size_t blen = strlen(esc) + strlen(esc_prefix) + strlen(model) + 64; + char* body = malloc(blen); + if (!body) { free(esc); free(esc_prefix); return NULL; } + snprintf(body, blen, "{\"model\":\"%s\",\"prompt\":\"%s%s\"}", model, esc_prefix, esc); + free(esc); free(esc_prefix); + + CURL* c = curl_easy_init(); + if (!c) { free(body); return NULL; } + HttpBuf rb; httpbuf_init(&rb); + struct curl_slist* h = curl_slist_append(NULL, "Content-Type: application/json"); + char errbuf[CURL_ERROR_SIZE]; errbuf[0] = '\0'; + curl_easy_setopt(c, CURLOPT_URL, url); + curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, http_write_cb); + curl_easy_setopt(c, CURLOPT_WRITEDATA, &rb); + curl_easy_setopt(c, CURLOPT_POST, 1L); + curl_easy_setopt(c, CURLOPT_POSTFIELDS, body); + curl_easy_setopt(c, CURLOPT_POSTFIELDSIZE, (long)strlen(body)); + curl_easy_setopt(c, CURLOPT_HTTPHEADER, h); + curl_easy_setopt(c, CURLOPT_TIMEOUT_MS, el_http_timeout_ms()); + curl_easy_setopt(c, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(c, CURLOPT_ERRORBUFFER, errbuf); + CURLcode rc = curl_easy_perform(c); + curl_slist_free_all(h); + curl_easy_cleanup(c); + free(body); + if (rc != CURLE_OK) { free(rb.data); return NULL; } + float* v = engram_parse_embedding(rb.data, out_dim); + free(rb.data); + return v; +} + +/* One-shot probe: is semantic search available? Caches the verdict. */ +static int engram_semantic_enabled(void) { + if (g_emb_state != 0) return g_emb_state == 1; + const char* s = getenv("ENGRAM_SEMANTIC"); + if (s && strcmp(s, "0") == 0) { g_emb_state = -1; return 0; } + int dim = 0; + float* v = engram_embed_raw("search_query: ", "probe", &dim); + if (v && dim > 0) { free(v); g_emb_state = 1; return 1; } + free(v); + g_emb_state = -1; return 0; +} + +/* Embed the query. Returns malloc'd vec (caller frees), or NULL if semantic off. */ +static float* engram_embed_query(const char* q, int* dim) { + if (!engram_semantic_enabled()) return NULL; + if (!q || !*q) return NULL; + return engram_embed_raw("search_query: ", q, dim); +} + +/* Cached node embedding. Returns a pointer OWNED BY THE CACHE — do not free. */ +static const float* engram_node_vec(EngramNode* n, int* out_dim) { + if (!n || !n->id) return NULL; + uint64_t h = engram_fnv1a(n->content); + for (int64_t i = 0; i < g_emb_count; i++) { + if (g_emb_items[i].id && strcmp(g_emb_items[i].id, n->id) == 0) { + if (g_emb_items[i].hash == h && g_emb_items[i].vec) { + *out_dim = g_emb_items[i].dim; return g_emb_items[i].vec; + } + /* content changed → re-embed in place */ + int dim = 0; + float* v = engram_embed_raw("search_document: ", n->content ? n->content : "", &dim); + if (!v) return NULL; + free(g_emb_items[i].vec); + g_emb_items[i].vec = v; g_emb_items[i].dim = dim; g_emb_items[i].hash = h; + *out_dim = dim; return v; + } + } + int dim = 0; + float* v = engram_embed_raw("search_document: ", n->content ? n->content : "", &dim); + if (!v) return NULL; + if (g_emb_count >= g_emb_cap) { + int64_t nc = g_emb_cap ? g_emb_cap * 2 : 256; + EngramEmbEntry* ni = realloc(g_emb_items, (size_t)nc * sizeof(EngramEmbEntry)); + if (!ni) { free(v); return NULL; } + g_emb_items = ni; g_emb_cap = nc; + } + g_emb_items[g_emb_count].id = strdup(n->id); + g_emb_items[g_emb_count].hash = h; + g_emb_items[g_emb_count].vec = v; + g_emb_items[g_emb_count].dim = dim; + g_emb_count++; + *out_dim = dim; return v; +} + +static double engram_cosine(const float* a, const float* b, int dim) { + double dot = 0, na = 0, nb = 0; + for (int i = 0; i < dim; i++) { dot += (double)a[i] * b[i]; + na += (double)a[i] * a[i]; + nb += (double)b[i] * b[i]; } + if (na <= 0 || nb <= 0) return 0.0; + return dot / (sqrt(na) * sqrt(nb)); +} + +/* Cosine of node n against the query vector; 0 if unavailable / dim mismatch. */ +static double engram_node_cosine(EngramNode* n, const float* qvec, int qdim) { + if (!qvec || qdim <= 0) return 0.0; + int ndim = 0; + const float* nv = engram_node_vec(n, &ndim); + if (!nv || ndim != qdim) return 0.0; + return engram_cosine(qvec, nv, qdim); +} + +#else /* !HAVE_CURL — semantic layer compiled out; callers stay pure-lexical. + * Only the two boundary functions the always-compiled search/activate + * code calls are stubbed; the query embed always yields NULL so every + * cosine is 0 and every caller collapses to lexical-only. */ +static float* engram_embed_query(const char* q, int* dim) { (void)q; (void)dim; return NULL; } +static double engram_node_cosine(EngramNode* n, const float* qvec, int qdim) { + (void)n; (void)qvec; (void)qdim; return 0.0; +} +#endif /* HAVE_CURL */ + el_val_t engram_search(el_val_t query, el_val_t limit) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); @@ -6833,6 +7070,12 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { if (lim <= 0) lim = 100; el_val_t lst = el_list_empty(); if (!q || !*q) return lst; + /* Semantic augmentation: embed the query once; a node matches if it is a + * lexical hit OR its cosine similarity clears the threshold. qvec is NULL + * (and cosine 0) whenever semantic search is unavailable → pure lexical. */ + int qdim = 0; + float* qvec = engram_embed_query(q, &qdim); + double sem_min = engram_semantic_min(); int64_t found = 0; for (int64_t i = 0; i < g->node_count && found < lim; i++) { EngramNode* n = &g->nodes[i]; @@ -6841,13 +7084,16 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { * know about yourself"). They still surface via engram_activate * + engram_compile_layered_json — that's the legitimate path. */ if (engram_layer_is_transparent(n->layer_id)) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { + int lex = istr_contains(n->content, q) || + istr_contains(n->label, q) || + istr_contains(n->tags, q); + double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0; + if (lex || sem >= sem_min) { lst = el_list_append(lst, engram_node_to_map(n)); found++; } } + free(qvec); return lst; } @@ -7193,14 +7439,28 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { if (!seeds) { free(best_bg); free(best_hops); free(reached); return out; } + /* Semantic seed augmentation: embed the query once; a node becomes a seed + * if it lexically matches OR its cosine clears the threshold. Semantic-only + * seeds enter at reduced strength (scaled by cosine) so pure paraphrase + * matches spread activation without overpowering exact lexical seeds. + * qvec is NULL (cosine 0) when semantic search is unavailable → the seed + * set is exactly the pre-existing lexical one. qvec is freed right after + * this loop so the many downstream early-returns need no cleanup change. */ + int q_dim = 0; + float* q_vec = engram_embed_query(q, &q_dim); + double q_sem_min = engram_semantic_min(); for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { + int lex = istr_contains(n->content, q) || + istr_contains(n->label, q) || + istr_contains(n->tags, q); + double sem = q_vec ? engram_node_cosine(n, q_vec, q_dim) : 0.0; + if (lex || sem >= q_sem_min) { double tdecay = engram_temporal_decay(n, now_ms); double dampen = engram_activation_dampen(n); double act = n->salience * tdecay * dampen; + /* Down-weight pure-semantic seeds by their cosine strength. */ + if (!lex) act *= sem; seeds[seed_count].idx = i; seeds[seed_count].act = act; seeds[seed_count].created_at = n->created_at; @@ -7210,6 +7470,7 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { reached[i] = 1; } } + free(q_vec); /* Compute mean seed created_at for temporal proximity bonus. */ int64_t seed_epoch = 0; if (seed_count > 0) { @@ -7768,22 +8029,52 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { if (lim <= 0) lim = 100; JsonBuf b; jb_init(&b); jb_putc(&b, '['); - int first = 1; - int64_t found = 0; - if (q && *q) { - for (int64_t i = 0; i < g->node_count && found < lim; i++) { - EngramNode* n = &g->nodes[i]; - /* Filter transparent layers — same as engram_search. */ - if (engram_layer_is_transparent(n->layer_id)) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { - if (!first) jb_putc(&b, ','); - engram_emit_node_json(&b, n); - first = 0; - found++; + if (q && *q && g->node_count > 0) { + /* Collect candidates from the UNION of lexical and semantic matches, + * score each, rank by score, then emit the top `lim`. A node is a + * candidate if it lexically matches OR its query cosine clears the + * threshold. Lexical hits get a base of 1.0 (so they always outrank a + * pure-semantic hit, whose cosine is in [0,1)), refined by cosine; + * pure-semantic hits are scored by cosine alone. + * + * When semantic search is unavailable, qvec is NULL, sem is 0, only + * lexical hits are collected (score 1.0), and the stable insertion + * sort preserves node order — identical to the pre-existing search. */ + int qdim = 0; + float* qvec = engram_embed_query(q, &qdim); + double sem_min = engram_semantic_min(); + typedef struct { int64_t idx; double score; } Cand; + Cand* cand = malloc((size_t)g->node_count * sizeof(Cand)); + if (cand) { + int64_t nc = 0; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + if (engram_layer_is_transparent(n->layer_id)) continue; + int lex = istr_contains(n->content, q) || + istr_contains(n->label, q) || + istr_contains(n->tags, q); + double sem = qvec ? engram_node_cosine(n, qvec, qdim) : 0.0; + if (lex || sem >= sem_min) { + cand[nc].idx = i; + cand[nc].score = (lex ? 1.0 : 0.0) + sem; + nc++; + } } + /* Insertion sort by score desc; stable for equal scores. */ + for (int64_t i = 1; i < nc; i++) { + Cand k = cand[i]; int64_t j = i - 1; + while (j >= 0 && cand[j].score < k.score) { cand[j + 1] = cand[j]; j--; } + cand[j + 1] = k; + } + int first = 1; + for (int64_t i = 0; i < nc && i < lim; i++) { + if (!first) jb_putc(&b, ','); + engram_emit_node_json(&b, &g->nodes[cand[i].idx]); + first = 0; + } + free(cand); } + free(qvec); } jb_putc(&b, ']'); return el_wrap_str(jb_finish(&b)); From 6b9d9e6c4af4fa902deebbd5b49ec46fa5d97d55 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Wed, 15 Jul 2026 04:07:33 -0500 Subject: [PATCH 4/7] Add engram_get_node_by_label runtime native to unblock soul link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit chat.el calls the runtime native engram_get_node_by_label to fetch well-known nodes (conv:history, session:summary) by stable label rather than by ID — immune to vector-index drift across restarts. The current runtime never defined it, so the regenerated dist/soul.c fails to link. Backport the function verbatim (idiom-adapted to jb_finish) from release runtime v1.0.0-20260501 and register it as an EL builtin exactly like its siblings: runtime definition + prototype, __-prefixed seed wrapper + prototype, and codegen arity entry. No search-site code is touched. --- lang/el-compiler/runtime/el_runtime.c | 29 +++++++++++++++++++++++++++ lang/el-compiler/runtime/el_runtime.h | 1 + lang/el-compiler/runtime/el_seed.c | 1 + lang/el-compiler/runtime/el_seed.h | 1 + lang/el-compiler/src/codegen.el | 1 + 5 files changed, 33 insertions(+) diff --git a/lang/el-compiler/runtime/el_runtime.c b/lang/el-compiler/runtime/el_runtime.c index 68e6dca..57a9042 100644 --- a/lang/el-compiler/runtime/el_runtime.c +++ b/lang/el-compiler/runtime/el_runtime.c @@ -7761,6 +7761,35 @@ el_val_t engram_get_node_json(el_val_t id) { return el_wrap_str(jb_finish(&b)); } +/* engram_get_node_by_label — find the first node whose label field exactly + * matches the given string. Returns the node as a JSON object string, or "{}" + * if no match is found. + * + * Used by chat.el to retrieve well-known nodes (e.g. "conv:history", + * "session:summary") by their stable label rather than by ID, which is immune + * to vector index drift across restarts. + * + * Exact match (strcmp, not istr_contains) because labels like "conv:history" + * must not collide with nodes whose content happens to contain that substring. + * + * Backported verbatim (idiom-adapted to jb_finish) from release runtime + * v1.0.0-20260501 to unblock the soul regen link: chat.el references this + * native but the current runtime lacked its definition. */ +el_val_t engram_get_node_by_label(el_val_t label) { + const char* lbl = EL_CSTR(label); + if (!lbl || !*lbl) return el_wrap_str(el_strdup("{}")); + EngramStore* g = engram_get(); + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + if (n->label && strcmp(n->label, lbl) == 0) { + JsonBuf b; jb_init(&b); + engram_emit_node_json(&b, n); + return el_wrap_str(jb_finish(&b)); + } + } + return el_wrap_str(el_strdup("{}")); +} + el_val_t engram_search_json(el_val_t query, el_val_t limit) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); diff --git a/lang/el-compiler/runtime/el_runtime.h b/lang/el-compiler/runtime/el_runtime.h index f64bf63..87348f5 100644 --- a/lang/el-compiler/runtime/el_runtime.h +++ b/lang/el-compiler/runtime/el_runtime.h @@ -632,6 +632,7 @@ el_val_t engram_load(el_val_t path); * can pass results straight through without round-tripping ElList/ElMap * through json_stringify. */ el_val_t engram_get_node_json(el_val_t id); +el_val_t engram_get_node_by_label(el_val_t label); el_val_t engram_search_json(el_val_t query, el_val_t limit); el_val_t engram_scan_nodes_json(el_val_t limit, el_val_t offset); el_val_t engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset); diff --git a/lang/el-compiler/runtime/el_seed.c b/lang/el-compiler/runtime/el_seed.c index b509e48..cda893c 100644 --- a/lang/el-compiler/runtime/el_seed.c +++ b/lang/el-compiler/runtime/el_seed.c @@ -1072,6 +1072,7 @@ el_val_t __engram_save(el_val_t path) { return engram_save el_val_t __engram_load(el_val_t path) { return engram_load(path); } el_val_t __engram_get_node_json(el_val_t id) { return engram_get_node_json(id); } +el_val_t __engram_get_node_by_label(el_val_t label) { return engram_get_node_by_label(label); } el_val_t __engram_search_json(el_val_t query, el_val_t limit) { return engram_search_json(query, limit); diff --git a/lang/el-compiler/runtime/el_seed.h b/lang/el-compiler/runtime/el_seed.h index c502f10..7597511 100644 --- a/lang/el-compiler/runtime/el_seed.h +++ b/lang/el-compiler/runtime/el_seed.h @@ -226,6 +226,7 @@ el_val_t __engram_activate(el_val_t query, el_val_t depth); el_val_t __engram_save(el_val_t path); el_val_t __engram_load(el_val_t path); el_val_t __engram_get_node_json(el_val_t id); +el_val_t __engram_get_node_by_label(el_val_t label); el_val_t __engram_search_json(el_val_t query, el_val_t limit); el_val_t __engram_scan_nodes_json(el_val_t limit, el_val_t offset); el_val_t __engram_scan_nodes_by_type_json(el_val_t node_type, el_val_t limit, el_val_t offset); diff --git a/lang/el-compiler/src/codegen.el b/lang/el-compiler/src/codegen.el index 917fb31..a20569a 100644 --- a/lang/el-compiler/src/codegen.el +++ b/lang/el-compiler/src/codegen.el @@ -2670,6 +2670,7 @@ fn builtin_arity(name: String) -> Int { if str_eq(name, "engram_save") { return 1 } if str_eq(name, "engram_load") { return 1 } if str_eq(name, "engram_get_node_json") { return 1 } + if str_eq(name, "engram_get_node_by_label") { return 1 } if str_eq(name, "engram_search_json") { return 2 } if str_eq(name, "engram_scan_nodes_json") { return 2 } if str_eq(name, "engram_neighbors_json") { return 3 } From 4773dd0aa2ed868330349f0021c812948f79e9e8 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Wed, 15 Jul 2026 04:24:08 -0500 Subject: [PATCH 5/7] runtime: make Windows soul reproducible from a clean el checkout Two el_runtime portability defects only ever lived in staged local copies used to hand-build neuron-ui PR #136's curl-enabled Windows neuron.exe. gcc 15 promotes both to hard errors, so a clean el checkout cannot rebuild that soul. Upstream the minimal fixes so the build is reproducible: - http_serve_async: cast setsockopt optval to (const char*). Win32/mingw setsockopt wants const char*, not int*; the cast is a no-op on POSIX and matches the four already-cast sites elsewhere in this file. - engram_save persist path: map fsync -> _commit in the _WIN32-only el_platform_win.h (io.h already included). Windows has no fsync(); the POSIX path is untouched. --- lang/el-compiler/runtime/el_platform_win.h | 1 + lang/el-compiler/runtime/el_runtime.c | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lang/el-compiler/runtime/el_platform_win.h b/lang/el-compiler/runtime/el_platform_win.h index 88204a0..df42c2b 100644 --- a/lang/el-compiler/runtime/el_platform_win.h +++ b/lang/el-compiler/runtime/el_platform_win.h @@ -75,6 +75,7 @@ static inline void* el_win_dlsym(void* handle, const char* name) { #include /* _mkdir */ #define mkdir(path, mode) _mkdir(path) /* POSIX mkdir(path,mode) → _mkdir(path) */ #define timegm _mkgmtime /* UTC tm → time_t */ +#define fsync(fd) _commit(fd) /* no fsync() on Windows; _commit() () is the equiv */ /* setenv/unsetenv: not in the Windows CRT; map to _putenv_s / SetEnvironmentVariable. */ static inline int setenv(const char* name, const char* value, int overwrite) { diff --git a/lang/el-compiler/runtime/el_runtime.c b/lang/el-compiler/runtime/el_runtime.c index 68e6dca..5c9a066 100644 --- a/lang/el-compiler/runtime/el_runtime.c +++ b/lang/el-compiler/runtime/el_runtime.c @@ -1963,8 +1963,9 @@ void http_serve_async(el_val_t port, el_val_t handler) { int sock = socket(AF_INET6, SOCK_STREAM, 0); if (sock < 0) { perror("socket"); return; } int yes = 1; int no = 0; - setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); - setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &no, sizeof(no)); + /* Win32/mingw setsockopt takes optval as (const char*); the cast is portable on POSIX too. */ + setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (const char*)&yes, sizeof(yes)); + setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, (const char*)&no, sizeof(no)); struct sockaddr_in6 addr; memset(&addr, 0, sizeof(addr)); addr.sin6_family = AF_INET6; From 8ce8656de2ede82eacd91351b71f98e7fd7dd339 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Wed, 15 Jul 2026 10:14:43 -0500 Subject: [PATCH 6/7] epm: declare cross-module callees as extern fn so strict compilers accept generated C epm's sibling modules (registry/install/update) call functions defined in other modules and in the El runtime (config, read_installed, registry_find, manifest_deps, manifest_name, registry_latest_version, registry_token, install_vessel, installed_version) without importing them, so elc emits no C prototype for those calls. gcc<=13 treated the resulting implicit declarations as warnings; gcc>=14 and clang reject them as hard errors, which is why the "Build epm" CI step fails and blocks the whole dev/stage pipeline. Add `extern fn` forward declarations -- El's own separate-compilation mechanism -- for each cross-module callee at the top of registry/install/update. This gives elc the correct C prototype in every generated translation unit, so the calls compile cleanly and still resolve at link time. Simply suppressing -Wimplicit-function-declaration would be unsafe: an implicit int return truncates the 64-bit pointer returns of config/registry_find into a latent crash, so declaring the true signatures is the correct fix. Localized to epm; touches neither elc nor the runtime. --- epm/src/install.el | 10 ++++++++++ epm/src/registry.el | 9 +++++++++ epm/src/update.el | 9 +++++++++ 3 files changed, 28 insertions(+) diff --git a/epm/src/install.el b/epm/src/install.el index 882fc90..1561cf8 100644 --- a/epm/src/install.el +++ b/epm/src/install.el @@ -17,6 +17,16 @@ // 4. Append dep to order after all its transitive deps // 5. Deduplicate: skip already-ordered vessels +// ── Cross-module forward declarations ───────────────────────────────────────── +// Defined in sibling epm modules; resolved at link time. The `extern fn` decls +// give elc the C prototypes so generated install.c compiles cleanly under strict +// compilers (gcc>=14 / clang) that reject implicit function declarations. +extern fn manifest_name(src: String) -> String // manifest.el +extern fn manifest_deps(src: String) -> String // manifest.el +extern fn registry_token() -> String // registry.el +extern fn registry_find(name: String, version: String) -> String // registry.el +extern fn registry_latest_version(name: String) -> String // registry.el + // ── Install paths ───────────────────────────────────────────────────────────── // packages_dir returns the root directory for installed vessels. diff --git a/epm/src/registry.el b/epm/src/registry.el index 8e1b2b5..d6f9730 100644 --- a/epm/src/registry.el +++ b/epm/src/registry.el @@ -14,6 +14,15 @@ // EPM_REGISTRY_ORG — org name that hosts vessel repos (default: neuron-technologies) // EPM_TOKEN — Gitea personal access token (required for publish) +// ── Cross-module forward declarations ───────────────────────────────────────── +// These symbols are defined in sibling epm modules or the El runtime and are +// resolved at link time. The `extern fn` decls give elc the C prototype so the +// generated registry.c compiles cleanly under strict compilers (gcc>=14 / clang) +// that reject implicit function declarations. Signature arity must match the +// definition; return/param types are informational (all lower to el_val_t). +extern fn config(key: String) -> String // El runtime builtin +extern fn read_installed() -> String // install.el + // ── Config helpers ──────────────────────────────────────────────────────────── // registry_api_url returns the Gitea API base URL with no trailing slash. diff --git a/epm/src/update.el b/epm/src/update.el index 42f60a5..3804ab2 100644 --- a/epm/src/update.el +++ b/epm/src/update.el @@ -6,6 +6,15 @@ // Depends on: registry.el (registry_latest_version, registry_find), // install.el (read_installed, install_vessel, installed_version) +// ── Cross-module forward declarations ───────────────────────────────────────── +// Defined in sibling epm modules; resolved at link time. The `extern fn` decls +// give elc the C prototypes so generated update.c compiles cleanly under strict +// compilers (gcc>=14 / clang) that reject implicit function declarations. +extern fn read_installed() -> String // install.el +extern fn installed_version(name: String) -> String // install.el +extern fn install_vessel(name: String, version: String) -> Bool // install.el +extern fn registry_latest_version(name: String) -> String // registry.el + // ── Semver helpers ──────────────────────────────────────────────────────────── // semver_part extracts the Nth dot-separated component from a semver string. From 4696fd68337f20b5b2d3ef38bd68a559820eea70 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Wed, 15 Jul 2026 11:33:37 -0500 Subject: [PATCH 7/7] ci(dev): make SDK publish fail loudly, decouple ci-base rebuild The dev push build went green-then-red while nothing published: the Publish step had no set -e, so an auth/upload failure exited 0 (silent no-publish), while the ci-base rebuild (set -euo pipefail + Docker on the host-mode runner) hard-failed the job. Add set -euo pipefail + an empty-key guard + active-account echo to the Publish step so failures surface with a retrievable log, and mark the ci-base cache rebuild continue-on-error so the fragile Docker step can never block the actual SDK artifact publish. --- .gitea/workflows/ci-dev.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.gitea/workflows/ci-dev.yaml b/.gitea/workflows/ci-dev.yaml index f788d8b..092a80e 100644 --- a/.gitea/workflows/ci-dev.yaml +++ b/.gitea/workflows/ci-dev.yaml @@ -214,9 +214,18 @@ jobs: env: GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} run: | + # Fail loudly: previously this step had no `set -e`, so an auth or + # upload failure was swallowed (step exited 0 on the trailing echo) + # and the SDK silently never published. Surface failures now. + set -euo pipefail + if [ -z "${GCP_SA_KEY:-}" ]; then + echo "FATAL: GCP_SA_KEY secret is empty — cannot authenticate to publish" >&2 + exit 1 + fi echo "${GCP_SA_KEY}" > /tmp/gcp-key.json gcloud auth activate-service-account --key-file=/tmp/gcp-key.json gcloud config set project neuron-785695 + echo "Publishing as active account: $(gcloud config get-value account 2>/dev/null)" VERSION="${GITHUB_SHA:0:8}" @@ -268,6 +277,12 @@ jobs: # Patches ci-base:dev in-place: pulls the existing image (which has all # system deps — Node, Go, gcloud, Docker CLI, etc.) and overlays the freshly # built El SDK on top. Keeps the full ci-base rebuild fast and incremental. + # + # continue-on-error: this is a CI-cache optimization, NOT the release + # artifact. It runs Docker (pull/build/push ~600MB) on the host-mode GCE + # runner where DinD/Docker availability is fragile. A failure here must + # never block or redden the job — the SDK publish above is the deliverable. + continue-on-error: true if: github.event_name == 'push' env: GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}