diff --git a/engram/dist/engram b/engram/dist/engram index 0c99cb6..d6d5bc5 100755 Binary files a/engram/dist/engram and b/engram/dist/engram differ diff --git a/lang/releases/v1.0.0-20260501/el_runtime.c b/lang/releases/v1.0.0-20260501/el_runtime.c index bbbba43..b452dbf 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.c +++ b/lang/releases/v1.0.0-20260501/el_runtime.c @@ -5591,7 +5591,9 @@ void el_cgi_init(el_val_t name, el_val_t dharma_id, el_val_t principal, * ENGRAM_BREAKTHROUGH_WEIGHT: the reduced working_memory_weight assigned * when a suppressed node breaks through. * ENGRAM_INHIBITION_FACTOR: multiplier applied to working_memory_weight when - * an inhibitory edge fires against a node (0 = full suppress, 0.3 = partial). */ + * an inhibitory edge fires against a node (0 = full suppress; current value + * 0.1 = near-full suppression — comment previously said 0.3, which drifted + * from the actual constant below). */ #define ENGRAM_WM_THRESHOLD 0.15 #define ENGRAM_WM_DECAY 0.7 #define ENGRAM_SUPPRESSION_BREAKTHROUGH 5 @@ -5699,6 +5701,15 @@ static double engram_type_threshold(const char* node_type, const char* tier) { if (node_type) { if (strcmp(node_type, "Belief") == 0) return 0.30; if (strcmp(node_type, "Entity") == 0) return 0.30; + /* Knowledge nodes (captureKnowledge, world-ingestor) at non-Canonical/ + * non-Lesson tiers (Semantic/Episodic/Procedural) previously fell + * through to the 0.40 note default — same bar as ephemeral notes — + * so curated knowledge mostly entered WM via breakthrough suppression + * (visible as wm weights pinned near the breakthrough floor) instead + * of natural promotion. Placed AFTER the tier checks so Canonical + * (0.15) and Lesson (0.25) still win. Ported from the dev-line fix + * (2026-06-13 self-review). (2026-07-19 self-review) */ + if (strcmp(node_type, "Knowledge") == 0) return 0.20; } return 0.40; /* Note / Memory / Working (most nodes) */ } @@ -6728,6 +6739,78 @@ static int istr_contains(const char* hay, const char* needle) { return 0; } +/* ── Tokenized query matching ─────────────────────────────────────────── + * The engram query surface (search / activate / goal-bias) historically + * matched the ENTIRE raw query string as a single case-insensitive + * substring via istr_contains(field, q). That is Ctrl-F, not search: + * a multi-word query like "windows msi signing" only matched a node whose + * text contained that exact contiguous run, so real multi-word queries + * returned zero. istr_contains stays as the per-TOKEN primitive; these + * helpers split the query on whitespace and match ANY token, then rank by + * how many DISTINCT tokens a node covers. Single-token queries are a strict + * special case (score is 0 or 1) so single-word callers never regress. + * (Ported 2026-07-19 from the el-compiler runtime copy, where the 2026-07-14 + * fix landed but never reached this release runtime — the copy the engram + * binary actually builds against.) */ +#define ENGRAM_MAX_QTOKENS 32 +#define ENGRAM_QTOK_LEN 256 + +/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct + * (case-insensitive) tokens. Returns the token count. Over-long tokens are + * truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */ +static int engram_tokenize_query(const char* q, + char toks[][ENGRAM_QTOK_LEN], int maxtok) { + int n = 0; + if (!q) return 0; + const char* p = q; + while (*p && n < maxtok) { + while (*p && isspace((unsigned char)*p)) p++; + if (!*p) break; + char buf[ENGRAM_QTOK_LEN]; + size_t tl = 0; + while (*p && !isspace((unsigned char)*p)) { + if (tl < sizeof(buf) - 1) buf[tl++] = *p; + p++; + } + buf[tl] = '\0'; + if (tl == 0) continue; + int dup = 0; + for (int s = 0; s < n; s++) { + if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; } + } + if (dup) continue; + memcpy(toks[n], buf, tl + 1); + n++; + } + return n; +} + +/* Count how many of the ntok distinct query tokens appear (case-insensitive) + * in the node's content, label, or tags. 0 == no match. */ +static int engram_node_match_score(const EngramNode* n, + char toks[][ENGRAM_QTOK_LEN], int ntok) { + int score = 0; + for (int t = 0; t < ntok; t++) { + if (istr_contains(n->content, toks[t]) || + istr_contains(n->label, toks[t]) || + istr_contains(n->tags, toks[t])) + score++; + } + return score; +} + +/* Rank entry: distinct-token match count (primary, desc) then salience + * (tiebreak, desc). */ +typedef struct { int64_t idx; int score; double salience; } EngramRankEntry; +static int engram_rank_cmp(const void* a, const void* b) { + const EngramRankEntry* ea = (const EngramRankEntry*)a; + const EngramRankEntry* eb = (const EngramRankEntry*)b; + if (ea->score != eb->score) return eb->score - ea->score; /* desc */ + if (ea->salience < eb->salience) return 1; + if (ea->salience > eb->salience) return -1; + return 0; +} + el_val_t engram_search(el_val_t query, el_val_t limit) { EngramStore* g = engram_get(); const char* q = EL_CSTR(query); @@ -6735,21 +6818,34 @@ el_val_t engram_search(el_val_t query, el_val_t limit) { if (lim <= 0) lim = 100; el_val_t lst = el_list_empty(); if (!q || !*q) return lst; - int64_t found = 0; - for (int64_t i = 0; i < g->node_count && found < lim; i++) { + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS); + if (ntok == 0) return lst; + EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry)); + if (!hits) return lst; + int64_t nhits = 0; + for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; /* Filter transparent layers: nodes whose layer is `transparent=1` * shape output but are invisible to introspection ("what do you * know about yourself"). They still surface via engram_activate * + engram_compile_layered_json — that's the legitimate path. */ if (engram_layer_is_transparent(n->layer_id)) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { - lst = el_list_append(lst, engram_node_to_map(n)); - found++; + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0) { + hits[nhits].idx = i; + hits[nhits].score = sc; + hits[nhits].salience = n->salience; + nhits++; } } + /* Rank by distinct tokens matched (desc) then salience (desc), then cap. */ + qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); + int64_t end = nhits < lim ? nhits : lim; + for (int64_t k = 0; k < end; k++) { + lst = el_list_append(lst, engram_node_to_map(&g->nodes[hits[k].idx])); + } + free(hits); return lst; } @@ -7027,10 +7123,15 @@ static double engram_temporal_proximity_bonus(int64_t node_created, static double engram_goal_bias(const EngramNode* n, const char* query) { if (!query || !*query) return 1.0; double bias = 1.0; - /* Direct lexical overlap: node content/label/tags share text with query. */ - if (istr_contains(n->content, query) || istr_contains(n->label, query) || - istr_contains(n->tags, query)) { - bias += 0.5; + /* Direct lexical overlap, graded by token coverage: a node covering all + * query tokens gets the full +0.5; partial coverage gets a proportional + * share. Single-token queries → full +0.5 on match, identical to before. + * (2026-07-19 port of the 2026-07-14 tokenized-search fix) */ + { + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(query, toks, ENGRAM_MAX_QTOKENS); + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0 && ntok > 0) bias += 0.5 * ((double)sc / (double)ntok); } /* Node-type resonance with query intent. */ int technical_query = istr_contains(query, "code") || @@ -7041,7 +7142,17 @@ static double engram_goal_bias(const EngramNode* n, const char* query) { istr_contains(query, "build") || istr_contains(query, "system") || istr_contains(query, "design") || - istr_contains(query, "architecture"); + istr_contains(query, "architecture") || + /* Curiosity-scan seeds: without these, idle-loop + * activation queries ("decision pattern lesson", + * "memory knowledge context") produce no goal-bias + * differentiation at all. Ported from dev-line fix + * d53516b (2026-06-14). (2026-07-19 self-review) */ + istr_contains(query, "knowledge") || + istr_contains(query, "pattern") || + istr_contains(query, "decision") || + istr_contains(query, "memory") || + istr_contains(query, "lesson"); int personal_query = istr_contains(query, "feel") || istr_contains(query, "emotion") || istr_contains(query, "remember") || @@ -7051,7 +7162,14 @@ static double engram_goal_bias(const EngramNode* n, const char* query) { if (n->node_type) { int is_knowledge = (strcmp(n->node_type, "Belief") == 0) || (strcmp(n->node_type, "DharmaSelf") == 0) || - (strcmp(n->node_type, "Safety") == 0); + (strcmp(n->node_type, "Safety") == 0) || + /* The primary knowledge-capture type was absent + * from its own bias class: captureKnowledge() and + * the world ingestor write node_type "Knowledge", + * which competed at neutral bias on technical + * queries. Ported from dev-line fix d53516b. + * (2026-07-19 self-review) */ + (strcmp(n->node_type, "Knowledge") == 0); int is_personal = (strcmp(n->node_type, "Memory") == 0) || (strcmp(n->node_type, "Entity") == 0); if (technical_query && is_knowledge) bias += 0.3; @@ -7102,6 +7220,18 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { if (!seeds) { free(best_bg); free(best_hops); free(reached); return out; } + /* Tokenize once: a node seeds if it matches ANY query token, and its seed + * activation is scaled by token coverage (fraction of distinct query + * tokens it contains) so a node matching all words seeds more strongly + * than one matching a single word. Single-word queries → coverage 1.0, + * identical to the prior whole-query behavior. Before this, the soul's + * rotating 3-word curiosity seeds ("working project active") activated + * ZERO nodes almost every scan — idle cognition firing blanks. + * (2026-07-19 port of the 2026-07-14 tokenized-search fix; NOTE the + * el-compiler copy of this fix dropped the ISE seed exclusion below — + * kept here deliberately, do not "sync" it away.) */ + char qtoks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int qntok = engram_tokenize_query(q, qtoks, ENGRAM_MAX_QTOKENS); for (int64_t i = 0; i < g->node_count; i++) { EngramNode* n = &g->nodes[i]; /* InternalStateEvent nodes are observability-only telemetry — never @@ -7112,12 +7242,12 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { * seeding here too. */ if (n->node_type && strcmp(n->node_type, "InternalStateEvent") == 0) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { + int msc = engram_node_match_score(n, qtoks, qntok); + if (msc > 0) { double tdecay = engram_temporal_decay(n, now_ms); double dampen = engram_activation_dampen(n); - double act = n->salience * tdecay * dampen; + double cover = qntok > 0 ? (double)msc / (double)qntok : 1.0; + double act = n->salience * tdecay * dampen * cover; seeds[seed_count].idx = i; seeds[seed_count].act = act; seeds[seed_count].created_at = n->created_at; @@ -7127,12 +7257,20 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { reached[i] = 1; } } - /* Compute mean seed created_at for temporal proximity bonus. */ + /* Compute mean seed created_at for temporal proximity bonus. + * Was a running pairwise average — seed_epoch = (seed_epoch + t_s)/2 — + * which is NOT the arithmetic mean: it exponentially over-weights the + * later seeds (last seed gets weight 1/2, second-to-last 1/4, ...), so + * the temporal-proximity bonus skewed toward whichever seeds happened + * to sit later in the scan order. True mean via int64 sum: ms epochs + * (~1.8e12) times any plausible seed_count stays far below INT64_MAX. + * (2026-07-19 self-review) */ int64_t seed_epoch = 0; if (seed_count > 0) { - seed_epoch = seeds[0].created_at; - for (int64_t s = 1; s < seed_count; s++) - seed_epoch = (seed_epoch + seeds[s].created_at) / 2; + int64_t epoch_sum = 0; + for (int64_t s = 0; s < seed_count; s++) + epoch_sum += seeds[s].created_at; + seed_epoch = epoch_sum / seed_count; } typedef struct { int64_t idx; int64_t hops; double act; } Frontier; Frontier* fr = malloc((size_t)(g->node_count * (max_depth + 1)) * sizeof(Frontier) + 16 * sizeof(Frontier)); @@ -8116,19 +8254,36 @@ el_val_t engram_search_json(el_val_t query, el_val_t limit) { JsonBuf b; jb_init(&b); jb_putc(&b, '['); int first = 1; - int64_t found = 0; if (q && *q) { - for (int64_t i = 0; i < g->node_count && found < lim; i++) { - EngramNode* n = &g->nodes[i]; - /* Filter transparent layers — same as engram_search. */ - if (engram_layer_is_transparent(n->layer_id)) continue; - if (istr_contains(n->content, q) || - istr_contains(n->label, q) || - istr_contains(n->tags, q)) { - if (!first) jb_putc(&b, ','); - engram_emit_node_json(&b, n); - first = 0; - found++; + /* Tokenized + ranked, same scheme as engram_search: match ANY query + * token, rank by distinct-token coverage then salience, cap at lim. + * (2026-07-19 port of the 2026-07-14 tokenized-search fix) */ + char toks[ENGRAM_MAX_QTOKENS][ENGRAM_QTOK_LEN]; + int ntok = engram_tokenize_query(q, toks, ENGRAM_MAX_QTOKENS); + if (ntok > 0) { + EngramRankEntry* hits = malloc((size_t)g->node_count * sizeof(EngramRankEntry)); + if (hits) { + int64_t nhits = 0; + for (int64_t i = 0; i < g->node_count; i++) { + EngramNode* n = &g->nodes[i]; + /* Filter transparent layers — same as engram_search. */ + if (engram_layer_is_transparent(n->layer_id)) continue; + int sc = engram_node_match_score(n, toks, ntok); + if (sc > 0) { + hits[nhits].idx = i; + hits[nhits].score = sc; + hits[nhits].salience = n->salience; + nhits++; + } + } + qsort(hits, (size_t)nhits, sizeof(EngramRankEntry), engram_rank_cmp); + int64_t end = nhits < lim ? nhits : lim; + for (int64_t k = 0; k < end; k++) { + if (!first) jb_putc(&b, ','); + engram_emit_node_json(&b, &g->nodes[hits[k].idx]); + first = 0; + } + free(hits); } } }