self-review 2026-07-19: port stranded fixes to the release runtime (production copy)

Three fixes that existed elsewhere but never reached the runtime the engram
binary actually builds against:

- tokenized + ranked query matching (search/search_json/activate seeds/
  goal_bias) ported from the el-compiler copy (e3dabe3, 2026-07-14) — the
  production engram kept whole-query Ctrl-F for 5 days after the fix
  'shipped'. Multi-word curiosity seeds went 0 -> 36 activated. Kept the
  ISE seed exclusion the el-compiler copy dropped.
- Knowledge -> 0.20 WM threshold after tier checks (dev-line 4bf7716):
  Semantic/Episodic Knowledge nodes fell to the 0.40 note default and only
  entered WM via breakthrough.
- goal_bias: Knowledge in is_knowledge + curiosity-seed technical terms
  (dev-line d53516b).

Also: seed_epoch was a running pairwise average, not the mean it claimed —
exponentially over-weighted later seeds in the temporal-proximity bonus.
Fixed to a true int64-sum mean. Stale INHIBITION_FACTOR comment corrected.

Root cause captured as knowledge: two runtime copies + branch-per-fix
without merge discipline stranded the entire dev semantic layer (cosine
activation, embeddings) out of production. Reconciliation planned as P1.
This commit is contained in:
2026-07-19 08:46:47 -05:00
parent ab6b52a0b4
commit eba9eac8a8
2 changed files with 189 additions and 34 deletions
BIN
View File
Binary file not shown.
+189 -34
View File
@@ -5591,7 +5591,9 @@ void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal,
* ENGRAM_BREAKTHROUGH_WEIGHT: the reduced working_memory_weight assigned * ENGRAM_BREAKTHROUGH_WEIGHT: the reduced working_memory_weight assigned
* when a suppressed node breaks through. * when a suppressed node breaks through.
* ENGRAM_INHIBITION_FACTOR: multiplier applied to working_memory_weight when * ENGRAM_INHIBITION_FACTOR: multiplier applied to working_memory_weight when
* an inhibitory edge fires against a node (0 = full suppress, 0.3 = partial). */ * an inhibitory edge fires against a node (0 = full suppress; current value
* 0.1 = near-full suppression comment previously said 0.3, which drifted
* from the actual constant below). */
#define ENGRAM_WM_THRESHOLD 0.15 #define ENGRAM_WM_THRESHOLD 0.15
#define ENGRAM_WM_DECAY 0.7 #define ENGRAM_WM_DECAY 0.7
#define ENGRAM_SUPPRESSION_BREAKTHROUGH 5 #define ENGRAM_SUPPRESSION_BREAKTHROUGH 5
@@ -5699,6 +5701,15 @@ static double engram_type_threshold(const char* node_type, const char* tier) {
if (node_type) { if (node_type) {
if (strcmp(node_type, "Belief") == 0) return 0.30; if (strcmp(node_type, "Belief") == 0) return 0.30;
if (strcmp(node_type, "Entity") == 0) return 0.30; if (strcmp(node_type, "Entity") == 0) return 0.30;
/* Knowledge nodes (captureKnowledge, world-ingestor) at non-Canonical/
* non-Lesson tiers (Semantic/Episodic/Procedural) previously fell
* through to the 0.40 note default same bar as ephemeral notes
* so curated knowledge mostly entered WM via breakthrough suppression
* (visible as wm weights pinned near the breakthrough floor) instead
* of natural promotion. Placed AFTER the tier checks so Canonical
* (0.15) and Lesson (0.25) still win. Ported from the dev-line fix
* (2026-06-13 self-review). (2026-07-19 self-review) */
if (strcmp(node_type, "Knowledge") == 0) return 0.20;
} }
return 0.40; /* Note / Memory / Working (most nodes) */ return 0.40; /* Note / Memory / Working (most nodes) */
} }
@@ -6728,6 +6739,78 @@ static int istr_contains(const char* hay, const char* needle) {
return 0; return 0;
} }
/* ── Tokenized query matching ───────────────────────────────────────────
* The engram query surface (search / activate / goal-bias) historically
* matched the ENTIRE raw query string as a single case-insensitive
* substring via istr_contains(field, q). That is Ctrl-F, not search:
* a multi-word query like "windows msi signing" only matched a node whose
* text contained that exact contiguous run, so real multi-word queries
* returned zero. istr_contains stays as the per-TOKEN primitive; these
* helpers split the query on whitespace and match ANY token, then rank by
* how many DISTINCT tokens a node covers. Single-token queries are a strict
* special case (score is 0 or 1) so single-word callers never regress.
* (Ported 2026-07-19 from the el-compiler runtime copy, where the 2026-07-14
* fix landed but never reached this release runtime the copy the engram
* binary actually builds against.) */
#define ENGRAM_MAX_QTOKENS 32
#define ENGRAM_QTOK_LEN 256
/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct
* (case-insensitive) tokens. Returns the token count. Over-long tokens are
* truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */
static int engram_tokenize_query(const char* q,
char toks[][ENGRAM_QTOK_LEN], int maxtok) {
int n = 0;
if (!q) return 0;
const char* p = q;
while (*p && n < maxtok) {
while (*p && isspace((unsigned char)*p)) p++;
if (!*p) break;
char buf[ENGRAM_QTOK_LEN];
size_t tl = 0;
while (*p && !isspace((unsigned char)*p)) {
if (tl < sizeof(buf) - 1) buf[tl++] = *p;
p++;
}
buf[tl] = '\0';
if (tl == 0) continue;
int dup = 0;
for (int s = 0; s < n; s++) {
if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; }
}
if (dup) continue;
memcpy(toks[n], buf, tl + 1);
n++;
}
return n;
}
/* Count how many of the ntok distinct query tokens appear (case-insensitive)
* in the node's content, label, or tags. 0 == no match. */
static int engram_node_match_score(const EngramNode* n,
char toks[][ENGRAM_QTOK_LEN], int ntok) {
int score = 0;
for (int t = 0; t < ntok; t++) {
if (istr_contains(n->content, toks[t]) ||
istr_contains(n->label, toks[t]) ||
istr_contains(n->tags, toks[t]))
score++;
}
return score;
}
/* Rank entry: distinct-token match count (primary, desc) then salience
* (tiebreak, desc). */
typedef struct { int64_t idx; int score; double salience; } EngramRankEntry;
static int engram_rank_cmp(const void* a, const void* b) {
const EngramRankEntry* ea = (const EngramRankEntry*)a;
const EngramRankEntry* eb = (const EngramRankEntry*)b;
if (ea->score != eb->score) return eb->score - ea->score; /* desc */
if (ea->salience < eb->salience) return 1;
if (ea->salience > eb->salience) return -1;
return 0;
}
el_val_t engram_search(el_val_t query, el_val_t limit) { el_val_t engram_search(el_val_t query, el_val_t limit) {
EngramStore* g = engram_get(); EngramStore* g = engram_get();
const char* q = EL_CSTR(query); const char* q = EL_CSTR(query);
@@ -6735,21 +6818,34 @@ el_val_t engram_search(el_val_t query, el_val_t limit) {
if (lim <= 0) lim = 100; if (lim <= 0) lim = 100;
el_val_t lst = el_list_empty(); el_val_t lst = el_list_empty();
if (!q || !*q) return lst; if (!q || !*q) return lst;
int64_t found = 0; char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
for (int64_t i = 0; i < g->node_count && found < lim; i++) { int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
if (ntok == 0) return lst;
EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry));
if (!hits) return lst;
int64_t nhits = 0;
for (int64_t i = 0; i < g->node_count; i++) {
EngramNode* n = &g->nodes[i]; EngramNode* n = &g->nodes[i];
/* Filter transparent layers: nodes whose layer is `transparent=1` /* Filter transparent layers: nodes whose layer is `transparent=1`
* shape output but are invisible to introspection ("what do you * shape output but are invisible to introspection ("what do you
* know about yourself"). They still surface via engram_activate * know about yourself"). They still surface via engram_activate
* + engram_compile_layered_json that's the legitimate path. */ * + engram_compile_layered_json that's the legitimate path. */
if (engram_layer_is_transparent(n->layer_id)) continue; if (engram_layer_is_transparent(n->layer_id)) continue;
if (istr_contains(n->content, q) || int sc = engram_node_match_score(n, toks, ntok);
istr_contains(n->label, q) || if (sc > 0) {
istr_contains(n->tags, q)) { hits[nhits].idx = i;
lst = el_list_append(lst, engram_node_to_map(n)); hits[nhits].score = sc;
found++; hits[nhits].salience = n->salience;
nhits++;
} }
} }
/* Rank by distinct tokens matched (desc) then salience (desc), then cap. */
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
int64_t end = nhits < lim ? nhits : lim;
for (int64_t k = 0; k < end; k++) {
lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx]));
}
free(hits);
return lst; return lst;
} }
@@ -7027,10 +7123,15 @@ static double engram_temporal_proximity_bonus(int64_t node_created,
static double engram_goal_bias(const EngramNode* n, const char* query) { static double engram_goal_bias(const EngramNode* n, const char* query) {
if (!query || !*query) return 1.0; if (!query || !*query) return 1.0;
double bias = 1.0; double bias = 1.0;
/* Direct lexical overlap: node content/label/tags share text with query. */ /* Direct lexical overlap, graded by token coverage: a node covering all
if (istr_contains(n->content, query) || istr_contains(n->label, query) || * query tokens gets the full +0.5; partial coverage gets a proportional
istr_contains(n->tags, query)) { * share. Single-token queries full +0.5 on match, identical to before.
bias += 0.5; * (2026-07-19 port of the 2026-07-14 tokenized-search fix) */
{
char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
int ntok = engram_tokenize_query(query, toks, ENGRAM_MAX_QTOKENS);
int sc = engram_node_match_score(n, toks, ntok);
if (sc > 0 && ntok > 0) bias += 0.5 * ((double)sc / (double)ntok);
} }
/* Node-type resonance with query intent. */ /* Node-type resonance with query intent. */
int technical_query = istr_contains(query, "code") || int technical_query = istr_contains(query, "code") ||
@@ -7041,7 +7142,17 @@ static double engram_goal_bias(const EngramNode* n, const char* query) {
istr_contains(query, "build") || istr_contains(query, "build") ||
istr_contains(query, "system") || istr_contains(query, "system") ||
istr_contains(query, "design") || istr_contains(query, "design") ||
istr_contains(query, "architecture"); istr_contains(query, "architecture") ||
/* Curiosity-scan seeds: without these, idle-loop
* activation queries ("decision pattern lesson",
* "memory knowledge context") produce no goal-bias
* differentiation at all. Ported from dev-line fix
* d53516b (2026-06-14). (2026-07-19 self-review) */
istr_contains(query, "knowledge") ||
istr_contains(query, "pattern") ||
istr_contains(query, "decision") ||
istr_contains(query, "memory") ||
istr_contains(query, "lesson");
int personal_query = istr_contains(query, "feel") || int personal_query = istr_contains(query, "feel") ||
istr_contains(query, "emotion") || istr_contains(query, "emotion") ||
istr_contains(query, "remember") || istr_contains(query, "remember") ||
@@ -7051,7 +7162,14 @@ static double engram_goal_bias(const EngramNode* n, const char* query) {
if (n->node_type) { if (n->node_type) {
int is_knowledge = (strcmp(n->node_type, "Belief") == 0) || int is_knowledge = (strcmp(n->node_type, "Belief") == 0) ||
(strcmp(n->node_type, "DharmaSelf") == 0) || (strcmp(n->node_type, "DharmaSelf") == 0) ||
(strcmp(n->node_type, "Safety") == 0); (strcmp(n->node_type, "Safety") == 0) ||
/* The primary knowledge-capture type was absent
* from its own bias class: captureKnowledge() and
* the world ingestor write node_type "Knowledge",
* which competed at neutral bias on technical
* queries. Ported from dev-line fix d53516b.
* (2026-07-19 self-review) */
(strcmp(n->node_type, "Knowledge") == 0);
int is_personal = (strcmp(n->node_type, "Memory") == 0) || int is_personal = (strcmp(n->node_type, "Memory") == 0) ||
(strcmp(n->node_type, "Entity") == 0); (strcmp(n->node_type, "Entity") == 0);
if (technical_query && is_knowledge) bias += 0.3; if (technical_query && is_knowledge) bias += 0.3;
@@ -7102,6 +7220,18 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
if (!seeds) { if (!seeds) {
free(best_bg); free(best_hops); free(reached); return out; free(best_bg); free(best_hops); free(reached); return out;
} }
/* Tokenize once: a node seeds if it matches ANY query token, and its seed
* activation is scaled by token coverage (fraction of distinct query
* tokens it contains) so a node matching all words seeds more strongly
* than one matching a single word. Single-word queries coverage 1.0,
* identical to the prior whole-query behavior. Before this, the soul's
* rotating 3-word curiosity seeds ("working project active") activated
* ZERO nodes almost every scan idle cognition firing blanks.
* (2026-07-19 port of the 2026-07-14 tokenized-search fix; NOTE the
* el-compiler copy of this fix dropped the ISE seed exclusion below
* kept here deliberately, do not "sync" it away.) */
char qtoks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
int qntok = engram_tokenize_query(q, qtoks, ENGRAM_MAX_QTOKENS);
for (int64_t i = 0; i < g->node_count; i++) { for (int64_t i = 0; i < g->node_count; i++) {
EngramNode* n = &g->nodes[i]; EngramNode* n = &g->nodes[i];
/* InternalStateEvent nodes are observability-only telemetry — never /* InternalStateEvent nodes are observability-only telemetry — never
@@ -7112,12 +7242,12 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
* seeding here too. */ * seeding here too. */
if (n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0) if (n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0)
continue; continue;
if (istr_contains(n->content, q) || int msc = engram_node_match_score(n, qtoks, qntok);
istr_contains(n->label, q) || if (msc > 0) {
istr_contains(n->tags, q)) {
double tdecay = engram_temporal_decay(n, now_ms); double tdecay = engram_temporal_decay(n, now_ms);
double dampen = engram_activation_dampen(n); double dampen = engram_activation_dampen(n);
double act = n->salience * tdecay * dampen; double cover = qntok > 0 ? (double)msc / (double)qntok : 1.0;
double act = n->salience * tdecay * dampen * cover;
seeds[seed_count].idx = i; seeds[seed_count].idx = i;
seeds[seed_count].act = act; seeds[seed_count].act = act;
seeds[seed_count].created_at = n->created_at; seeds[seed_count].created_at = n->created_at;
@@ -7127,12 +7257,20 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) {
reached[i] = 1; reached[i] = 1;
} }
} }
/* Compute mean seed created_at for temporal proximity bonus. */ /* Compute mean seed created_at for temporal proximity bonus.
* Was a running pairwise average seed_epoch = (seed_epoch + t_s)/2
* which is NOT the arithmetic mean: it exponentially over-weights the
* later seeds (last seed gets weight 1/2, second-to-last 1/4, ...), so
* the temporal-proximity bonus skewed toward whichever seeds happened
* to sit later in the scan order. True mean via int64 sum: ms epochs
* (~1.8e12) times any plausible seed_count stays far below INT64_MAX.
* (2026-07-19 self-review) */
int64_t seed_epoch = 0; int64_t seed_epoch = 0;
if (seed_count > 0) { if (seed_count > 0) {
seed_epoch = seeds[0].created_at; int64_t epoch_sum = 0;
for (int64_t s = 1; s < seed_count; s++) for (int64_t s = 0; s < seed_count; s++)
seed_epoch = (seed_epoch + seeds[s].created_at) / 2; epoch_sum += seeds[s].created_at;
seed_epoch = epoch_sum / seed_count;
} }
typedef struct { int64_t idx; int64_t hops; double act; } Frontier; typedef struct { int64_t idx; int64_t hops; double act; } Frontier;
Frontier* fr = malloc((size_t)(g->node_count * (max_depth + 1)) * sizeof(Frontier) + 16 * sizeof(Frontier)); Frontier* fr = malloc((size_t)(g->node_count * (max_depth + 1)) * sizeof(Frontier) + 16 * sizeof(Frontier));
@@ -8116,19 +8254,36 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) {
JsonBuf b; jb_init(&b); JsonBuf b; jb_init(&b);
jb_putc(&b, '['); jb_putc(&b, '[');
int first = 1; int first = 1;
int64_t found = 0;
if (q && *q) { if (q && *q) {
for (int64_t i = 0; i < g->node_count && found < lim; i++) { /* Tokenized + ranked, same scheme as engram_search: match ANY query
EngramNode* n = &g->nodes[i]; * token, rank by distinct-token coverage then salience, cap at lim.
/* Filter transparent layers — same as engram_search. */ * (2026-07-19 port of the 2026-07-14 tokenized-search fix) */
if (engram_layer_is_transparent(n->layer_id)) continue; char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN];
if (istr_contains(n->content, q) || int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS);
istr_contains(n->label, q) || if (ntok > 0) {
istr_contains(n->tags, q)) { EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry));
if (!first) jb_putc(&b, ','); if (hits) {
engram_emit_node_json(&b, n); int64_t nhits = 0;
first = 0; for (int64_t i = 0; i < g->node_count; i++) {
found++; EngramNode* n = &g->nodes[i];
/* Filter transparent layers — same as engram_search. */
if (engram_layer_is_transparent(n->layer_id)) continue;
int sc = engram_node_match_score(n, toks, ntok);
if (sc > 0) {
hits[nhits].idx = i;
hits[nhits].score = sc;
hits[nhits].salience = n->salience;
nhits++;
}
}
qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp);
int64_t end = nhits < lim ? nhits : lim;
for (int64_t k = 0; k < end; k++) {
if (!first) jb_putc(&b, ',');
engram_emit_node_json(&b, &g->nodes[hits[k].idx]);
first = 0;
}
free(hits);
} }
} }
} }