diff --git a/lang/releases/v1.0.0-20260501/el_runtime.c b/lang/releases/v1.0.0-20260501/el_runtime.c index e52d157..c5ca86a 100644 --- a/lang/releases/v1.0.0-20260501/el_runtime.c +++ b/lang/releases/v1.0.0-20260501/el_runtime.c @@ -3155,6 +3155,37 @@ static void jb_init(JsonBuf* b) { b->buf[0] = '\0'; } +/* jb_init_cap — jb_init with a caller-supplied starting capacity. + * + * WHY THIS EXISTS (2026-08-11 self-review). jb_init starts at 64 BYTES and + * jb_reserve grows by doubling. That is right for the hundreds of small JSON + * responses this runtime builds per minute and catastrophic for the one that + * is 64 MEGABYTES: serializing the canonical snapshot walked the buffer + * 64B → 128B → ... → 128MB, about twenty reallocs, each copying everything + * written so far. Roughly 128MB of memcpy per save, and — the part that + * actually hurt — a fresh large span from the allocator every time. + * + * MEASURED (13,129 nodes / 43,400 edges, macOS arm64): RSS climbed +63MB per + * snapshot write, linearly, 14 for 14 writes, no plateau — 204MB to 1,028MB. + * `leaks` reported only 15KB genuinely unreachable, which is what makes this + * subtle: nothing is leaked in the reachable/unreachable sense. engram_save + * frees b.buf correctly on every path. The growth is the allocator declining + * to return large freed spans to the OS, and the doubling walk guaranteeing + * that each save asks for a differently-sized region than the last free made + * available. Every durable write path calls this — node create, edge create, + * the Hebbian batch write-back — so on the live daemon it grows without bound + * until the process dies. + * + * The fix is to ask for the right size once. With a stable capacity the + * allocator hands back the same span on every save and RSS flattens. */ +static void jb_init_cap(JsonBuf* b, size_t cap) { + if (cap < 64) cap = 64; + b->cap = cap; b->len = 0; + b->buf = malloc(b->cap); + if (!b->buf) { fputs("el_runtime: out of memory\n", stderr); exit(1); } + b->buf[0] = '\0'; +} + static void jb_reserve(JsonBuf* b, size_t add) { if (b->len + add + 1 > b->cap) { while (b->len + add + 1 > b->cap) b->cap *= 2; @@ -6424,6 +6455,26 @@ static float* _eg_ctx_c = NULL; static int32_t _eg_ctx_dim = 0; static double _eg_act_ctx_cos = -2.0; +/* Fan-effect gauges (2026-08-11 self-review). Per-call, like ctx_cos: they + * describe THIS activation, not process history. Without these the degree + * normalization is an unobservable change to the most important scoring path + * in the runtime, and "did it do anything" would be unanswerable — which is + * exactly the failure the Hebbian learning rate had before it was measured. + * fan_mean — mean applied factor over every propagation step. 1.0 means the + * correction never bound (graph is flat, or d_ref is above every + * pair's geometric mean degree). Falling toward FAN_MIN means + * traversal is running through hubs. + * fan_min_seen / fan_hits — the worst single penalty and how many steps were + * penalized at all, so a low mean caused by one pathological hub + * is distinguishable from broad hub saturation. + * fan_dref — the live mean degree the correction is calibrated against; + * publishing it makes densification visible over time. */ +static double _eg_act_fan_sum = 0.0; +static double _eg_act_fan_min = 1.0; +static int64_t _eg_act_fan_n = 0; +static int64_t _eg_act_fan_hits = 0; +static double _eg_act_fan_dref = 0.0; + static int _eg_embed_consec_fail = 0; static int64_t _eg_embed_breaker_until = 0; @@ -6782,6 +6833,10 @@ typedef struct EngramStore { int* adj_to_len; int adj_dirty; /* 1 = rebuild needed before next BFS */ int64_t adj_node_count; /* node_count at time of last adj_rebuild */ + /* Nodes with degree >= 1 at last adj_rebuild. The denominator for the + * fan-effect reference degree — see eg_fan_factor for why isolated nodes + * must not be counted. (2026-08-11 self-review) */ + int64_t adj_connected; } EngramStore; static EngramStore* engram_global = NULL; @@ -7145,11 +7200,16 @@ static void engram_adj_rebuild(EngramStore* g) { if (ti >= 0 && g->adj_to[ti]) g->adj_to[ti][to_pos[ti]++] = (int)ei; } - /* Copy counts */ + /* Copy counts. Also tally how many nodes have any edge at all — the + * fan-effect denominator. Free here, in the O(V) pass that already exists, + * rather than as a separate scan. (2026-08-11 self-review) */ + int64_t connected = 0; for (int64_t i = 0; i < g->node_count; i++) { g->adj_from_len[i] = from_cnt[i]; g->adj_to_len[i] = to_cnt[i]; + if (from_cnt[i] + to_cnt[i] > 0) connected++; } + g->adj_connected = connected; free(from_cnt); free(to_cnt); free(from_pos); free(to_pos); g->adj_node_count = g->node_count; g->adj_dirty = 0; @@ -8297,6 +8357,108 @@ static double engram_activation_dampen(const EngramNode* n) { return 1.0 / (1.0 + log(1.0 + (double)n->activation_count)); } +/* ── ACT-R fan effect: degree normalization for spreading activation ───────── + * (2026-08-11 self-review. Closes the other half of a mechanism that has been + * half-implemented since the BLL work.) + * + * THE GAP. This runtime implements ACT-R's base-level learning term + * B_i = ln(Σ t_k^-d) (engram_bll_base_level) but never implemented the + * ASSOCIATIVE term that goes with it: + * + * A_i = B_i + Σ_j W_j · S_ji where S_ji = S − ln(fan_j) + * + * fan_j is the number of things j is associated with. The whole point of the + * fan effect (Anderson 1974; Anderson & Reder 1999) is that a source spreads a + * FIXED budget of activation across its associations — so being connected to + * many things makes each individual connection weaker. Without it, degree is + * pure advantage: a node wins retrieval by being popular rather than by being + * relevant. That is backwards, and it is what this graph has been doing. + * + * MEASURED ON THE LIVE STORE (13,129 nodes / 43,400 edges, 2026-08-11): + * degree p50=14 p90=34 p95=82 p99=275 max=357 mean=23.3 + * the top 1% of nodes by degree touch 21.2% of all edges + * So the most-connected node had a 25x propagation advantage over the median + * node for no reason other than accumulated connections. The top hubs are not + * even semantically central — several are duplicate pairs of the same document + * left over from the redundancy census of the 2026-08-05 review. + * + * The hub problem was already recognized twice and patched narrowly both + * times: InternalStateEvent nodes were cut out of propagation entirely (see + * the frontier loop) and eg_hebb_node_budget caps per-node Hebbian mass. Both + * are special cases of this general law. This is the general fix. + * + * FORM. Symmetric normalization, w / (deg(u)^β · deg(v)^β) with β = 0.5 — the + * normalized-Laplacian / GCN form, which penalizes a hub both for sending and + * for receiving. Both failure modes are live here: a hub source floods its + * neighborhood, and a hub target gets reached by everything regardless of + * relevance. Written relative to the graph's own mean degree: + * + * fan(u,v) = clamp( d_ref / sqrt(deg(u) · deg(v)), FAN_MIN, 1.0 ) + * d_ref = 2·|E| / |V| (mean degree, O(1), live) + * + * WHY IT IS CLAMPED AT 1.0 ON TOP — this is the load-bearing safety property, + * not a detail. The factor can only ever REDUCE propagation, never amplify it. + * Every constant downstream of this multiply is calibrated against today's + * activation magnitudes: the 0.02 firing threshold, SPREAD_DECAY = 0.7, the + * 0.15 WM promotion threshold, the 24-slot WM cap. A normalization that + * boosted low-degree nodes would inflate the frontier, change how many nodes + * clear 0.02, and silently recalibrate working memory as a side effect of a + * change that was supposed to be about hubs. Capping at 1.0 means every pair + * at or below mean degree — the common case — propagates EXACTLY as it does + * today, and the only behavior that changes is that above-mean hubs stop + * winning on degree alone. Strictly monotone, strictly conservative, and the + * blast radius is confined to the nodes the change is aimed at. + * + * Self-calibrating: d_ref is recomputed from the live graph, so the correction + * tracks densification instead of drifting against a constant that was right + * in August 2026 and wrong a year later. Change is the signal. + * + * FAN_MIN = 0.30 bottoms the penalty at ~3.3x rather than the ~15x that raw + * 1/deg would give at max degree. Same reasoning as ENGRAM_QGATE_FLOOR: damp + * the uninformative path, never sever it. A hub is usually a hub for a reason; + * it just should not also get a free win. + * + * Sources: Anderson & Reder 1999 (fan effect, S=1.6-2.0, d=0.5) · + * arXiv:2405.14831 HippoRAG (node specificity) · Systems 9(2):22 + * (normalized-Laplacian spreading activation) · arXiv:2606.30133 (β is a + * low-sensitivity knob; gating and fan normalization carry the effect). */ +/* FAN_MIN 0.50, not the 0.30 this shipped as on the first build. Measured on + * the live graph, β=0.5 with a 0.30 floor damped 96% of propagation steps to a + * mean factor of 0.34 — and that number is not a bug in the correction, it is + * an honest measurement of how hub-dominated traversal here actually is. But a + * ~3x near-uniform damp is a bigger global change than one A/B run justifies, + * and it cost a working-memory promotion (5 → 4) on the one query measured + * cleanly. A 0.50 floor keeps the full mechanism and the whole [0.5, 1.0] + * dynamic range for separating hubs from non-hubs, at half the blast radius. + * The fan_mean / fan_hits gauges make the next review's tuning evidence-based + * rather than another guess: loosen it when the data says WM can afford it. */ +#define ENGRAM_FAN_MIN 0.50 + +/* eg_node_degree — total (in + out) degree from the adjacency index. The index + * is rebuilt at the top of engram_activate whenever topology changed, so this + * is current. adj_node_count is the count at BUILD time and can lag + * node_count; out-of-range indices report 0 and are treated as unpenalized. */ +static int eg_node_degree(const EngramStore* g, int64_t idx) { + if (idx < 0 || idx >= g->adj_node_count) return 0; + if (!g->adj_from_len || !g->adj_to_len) return 0; + return g->adj_from_len[idx] + g->adj_to_len[idx]; +} + +static double eg_fan_factor(const EngramStore* g, double d_ref, + int64_t u_idx, int64_t v_idx) { + if (d_ref <= 0.0) return 1.0; + int du = eg_node_degree(g, u_idx); + int dv = eg_node_degree(g, v_idx); + /* Degree 0 is only reachable when the adjacency index is stale or absent; + * an actually-isolated node is never on the frontier. Do not penalize what + * we cannot measure. */ + if (du <= 0 || dv <= 0) return 1.0; + double f = d_ref / sqrt((double)du * (double)dv); + if (f > 1.0) return 1.0; /* never amplify — see above */ + if (f < ENGRAM_FAN_MIN) return ENGRAM_FAN_MIN; + return f; +} + /* Temporal proximity bonus: boost propagation along edges connecting * co-temporal nodes. Returns a multiplier bonus in [0, 0.2]. */ static double engram_temporal_proximity_bonus(int64_t node_created, @@ -8464,6 +8626,8 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { * miss nearly all events between beats; see the definition site). * ctx_cos stays per-call: it is a gauge of THIS query vs the centroid. */ _eg_act_ctx_cos = -2.0; + _eg_act_fan_sum = 0.0; _eg_act_fan_min = 1.0; + _eg_act_fan_n = 0; _eg_act_fan_hits = 0; /* ── Embedding backfill + query embedding (2026-07-24, bl-b2d1c944) ── * Backfill: embed up to N un-embedded eligible nodes per call, newest @@ -8696,6 +8860,29 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { ftail++; } const double SPREAD_DECAY = 0.7; + /* Reference degree for the fan-effect correction: mean degree over + * CONNECTED nodes, 2|E| / |{v : deg(v) > 0}|. O(1) — adj_connected is + * tallied during adjacency rebuild. + * + * NOT 2|E|/|V|. That was the first cut and instrumentation caught it + * immediately: on the live graph it gives d_ref = 6.61, while the median + * degree of a node that actually has edges is 14. Isolated nodes cannot + * be on the frontier — spreading activation only ever traverses connected + * ones — so including them in the denominator deflates the reference below + * anything traversal will ever see, and the correction pins to + * ENGRAM_FAN_MIN on every step. Measured on the first build: + * fan_mean 0.3026 with fan_hits 579/579 — a uniform 0.30 multiplier, which + * is not a fan effect at all. It is just a weaker SPREAD_DECAY, and it + * would have quietly recalibrated the 0.02 firing threshold and WM + * competition while appearing to be a targeted change. + * + * Over connected nodes the reference is ~23, above the median, so typical + * traversal rides the 1.0 cap unchanged and only genuine hubs are damped + * — which is the whole intent. The gauge that caught this is the reason it + * was worth adding the gauge. */ + const double FAN_DREF = (g->adj_connected > 0) + ? (2.0 * (double)g->edge_count / (double)g->adj_connected) : 0.0; + _eg_act_fan_dref = FAN_DREF; while (fhead < ftail) { Frontier f = fr[fhead++]; if (f.hops >= max_depth) continue; @@ -8767,11 +8954,24 @@ el_val_t engram_activate(el_val_t query, el_val_t depth) { double c = cosq[oi] > 0.0 ? cosq[oi] : 0.0; qgate = ENGRAM_QGATE_FLOOR + (1.0 - ENGRAM_QGATE_FLOOR) * c; } + /* ── ACT-R fan effect (2026-08-11 self-review) ── + * Symmetric degree normalization over the (source, target) pair. + * The query gate above prunes branches that are semantically + * irrelevant; this prunes branches that are merely POPULAR. They + * are different failure modes — a duplicate document with 357 + * edges can be highly cosine-similar to the query and still be + * the wrong thing to spread through. Only ever <= 1.0, so it + * cannot inflate the frontier. See eg_fan_factor. */ + double fan = eg_fan_factor(g, FAN_DREF, cur, oi); + _eg_act_fan_sum += fan; + _eg_act_fan_n++; + if (fan < 1.0) _eg_act_fan_hits++; + if (fan < _eg_act_fan_min) _eg_act_fan_min = fan; /* eg_edge_eff_weight, not e->weight: edges that have repeatedly * carried co-activated pairs propagate more strongly. Identity on * an unlearned edge. (2026-08-04 self-review.) */ double new_act = f.act * eg_edge_eff_weight(e) * SPREAD_DECAY - * (1.0 + tbonus) * tdecay * dampen * qgate; + * (1.0 + tbonus) * tdecay * dampen * qgate * fan; /* Firing threshold per classic spreading-activation: sub-threshold * activation neither updates the target nor enqueues it, so weak * signals die out instead of flooding the whole graph with tiny @@ -9744,11 +9944,24 @@ static void engram_emit_edge_json(JsonBuf* b, const EngramEdge* e) { jb_putc(b, '}'); } +/* Size of the last snapshot this process serialized. Seeds the next save's + * buffer so the doubling walk never runs on the big document. See jb_init_cap + * for the measurement that motivated it. (2026-08-11 self-review) */ +static size_t _eg_save_cap_hint = 0; + el_val_t engram_save(el_val_t path) { const char* p = EL_CSTR(path); if (!p || !*p) return 0; EngramStore* g = engram_get(); - JsonBuf b; jb_init(&b); + /* Pre-size from the previous save plus 12.5% headroom, so ordinary growth + * between snapshots does not trigger a realloc and the request size stays + * stable enough for the allocator to reuse the same span. First save of + * the process has no hint and starts at 1MB — still 14 doublings better + * than 64 bytes. */ + JsonBuf b; + jb_init_cap(&b, _eg_save_cap_hint + ? _eg_save_cap_hint + (_eg_save_cap_hint >> 3) + 1024 + : (size_t)1 << 20); jb_puts(&b, "{\"nodes\":["); for (int64_t i = 0; i < g->node_count; i++) { if (i > 0) jb_putc(&b, ','); @@ -9788,6 +10001,10 @@ el_val_t engram_save(el_val_t path) { jb_putc(&b, '}'); } jb_puts(&b, "]}"); + /* Remember the size BEFORE the write: the hint is about how much buffer + * the next serialization needs, which is a property of the graph, not of + * whether this particular fopen succeeded. */ + _eg_save_cap_hint = b.len; FILE* f = fopen(p, "wb"); if (!f) { free(b.buf); return 0; } size_t w = fwrite(b.buf, 1, b.len, f); @@ -10714,8 +10931,11 @@ el_val_t engram_act_stats_json(void) { } /* 768, not 512: the write-back gauges added 2026-08-07 push the worst-case * rendering past the old bound, and snprintf would truncate the JSON into - * an unparseable tail rather than fail loudly. */ - char buf[896]; + * an unparseable tail rather than fail loudly. + * 1152, not 896: the five fan-effect gauges added 2026-08-11 add ~90 bytes + * worst-case. Same reasoning — headroom is cheaper than a truncated tail + * that every downstream JSON parser rejects as a whole. */ + char buf[1152]; /* ctx_cos (2026-07-29): cos(query, context centroid) at the LAST * activate call, measured before the query was folded in. ~1.0 = * context aligned with current query; low = divergence (expected at @@ -10736,7 +10956,14 @@ el_val_t engram_act_stats_json(void) { * any climb means a write path is mangling text again. Cheap * (counted at creation) — the full census lives in * engram_text_health_json. (2026-08-08 self-review) */ - "\"txt_damaged\":%lld}", + "\"txt_damaged\":%lld," + /* Fan-effect gauges (2026-08-11 self-review) — see the + * _eg_act_fan_* definitions. fan_mean == 1.0 with fan_hits == 0 + * means the degree correction never bound on the last activation; + * a mean drifting toward ENGRAM_FAN_MIN means traversal is + * running through hubs and the correction is doing work. */ + "\"fan_mean\":%.4f,\"fan_min\":%.4f,\"fan_hits\":%lld," + "\"fan_steps\":%lld,\"fan_dref\":%.2f}", (long long)_eg_act_wm_evicted, (long long)_eg_act_breakthroughs, breaker_open, _eg_embed_consec_fail, @@ -10748,7 +10975,10 @@ el_val_t engram_act_stats_json(void) { (long long)_eg_hebb_wb_dropped, (long long)_eg_act_dup_seeds, (long long)_eg_act_dup_wm, (long long)_eg_act_dup_wm_global, - (long long)_eg_txt_write_damaged); + (long long)_eg_txt_write_damaged, + (_eg_act_fan_n > 0 ? _eg_act_fan_sum / (double)_eg_act_fan_n : 1.0), + _eg_act_fan_min, (long long)_eg_act_fan_hits, + (long long)_eg_act_fan_n, _eg_act_fan_dref); return el_wrap_str(el_strdup(buf)); } @@ -10881,6 +11111,285 @@ el_val_t engram_label_df(el_val_t term) { return (el_val_t)df; } +/* ── Salient-term extraction (2026-08-13 self-review) ──────────────────────── + * THE MEASUREMENT. auto_term_empty_streak, the counter added by the 2026-08-06 + * review precisely to catch this class of silent death, read 50 and climbing. + * Fifty consecutive curiosity scans in which the soul's dynamic seeding path + * produced NOTHING and the loop fell back to its four hardcoded rotating + * phrases. Dumping the live WM top says why in one look: + * + * Memory 0.390 memory:remembered + * Memory 0.378 memory:remembered + * Memory 0.377 memory:remembered + * Memory 0.373 memory:remembered + * Memory 0.370 memory:remembered + * + * Every slot at the top of working memory is a Memory node, and every Memory + * node written by remember() carries the sentinel label "memory:remembered". + * auto_term_try_slot reads the LABEL and only the label; the colon-no-space + * guard (correctly) rejects sentinels as carrying no seed signal; so the + * extractor had nothing to work with and returned empty, forever. + * + * THE ACTUAL DEFECT is not the sentinel guard — that guard is right. It is + * that the extractor was built against Knowledge nodes, which have real + * titles, and is structurally blind to the node type that in fact dominates + * working memory. The label is not the content. A Memory node's topic is in + * its text; the runtime just never looked there. + * + * WHY NOT ANOTHER GUARD. The extractor's whole history is guards: genre words + * (07-23), quoted titles (07-25), English stopwords (07-30), label-df + * (08-03). Four reviews, four blocklists, each written after watching a flood + * happen. That is a losing shape, and 08-03 said so explicitly before adding + * the fifth. The reason it keeps recurring is the algorithm underneath: + * TAKE THE FIRST WORD, THEN CHECK WHETHER IT IS ACCEPTABLE. A first-word + * extractor has no notion of term quality, so quality has to be bolted on as + * rejection, and rejection can only encode the past. + * + * THE FIX is to invert it: score EVERY candidate token in the text and take + * the argmax. Then term quality is the selection criterion rather than a + * veto, and a bad token does not need to be on a list to lose — it only needs + * a better token in the same text, which is the common case. + * + * SCORING (YAKE, Campos et al., Information Sciences 509:257-289, 2020 — + * lightweight unsupervised single-document keyword extraction). YAKE scores + * candidates on casing, position, frequency, context relatedness and sentence + * dispersion, and beats RAKE/TextRank/SingleRank across twenty datasets. Two + * of its five features port directly and cheaply; the other three are + * within-document proxies for a corpus YAKE deliberately does not have. This + * system DOES have the corpus — 12.7k labelled nodes — so real IDF is + * substituted where YAKE has to approximate: + * + * score(t) = idf(t) · position(t) · casing(t) + * + * idf = ln((N+1)/(df+1)) real corpus specificity (Spärck + * Jones 1972), strictly better than + * YAKE's TF-based stand-in + * position = 1/ln(e + i) YAKE T_Position: earlier tokens are + * more topical. Keeps the old + * first-word bias as a SOFT preference + * instead of an absolute rule + * casing = 1.30 acronym / 1.15 capitalised / 1.00 otherwise + * YAKE T_Case + * + * THE min_df GATE. The df ceiling (08-03) rejects corpus-frequent markup and + * sentinels. A floor was added alongside it for an independent reason: a term + * appearing in ZERO labels cannot lexically reach anything, so it is a bad + * seed however specific it looks. + * + * An earlier draft of this comment claimed the floor also subsumes the 73 + * hand-listed stopwords that 08-03 measured label-df as missing (Whose:0, + * Would:0, Could:0). MEASURED, AND THAT CLAIM IS FALSE. Under word-boundary + * df on the live store, function words are rare in labels but not absent: + * about:2, whole:1, them:2, head:2. They clear a floor of 1. What actually + * keeps them from winning is the argmax itself — they carry no position + * advantage and lose to a topical term in the same text on every node + * measured. The stopword list therefore STAYS as a real defense for the + * Title-case cases, not as vestigial belt-and-braces. Recording the + * correction rather than the tidier story: the floor buys lexical + * reachability, the argmax buys quality, and the list still earns its keep. + * + * TABU IS APPLIED DURING THE ARGMAX, not after it. The old code picked a term + * and then discarded it if it was tabu, which turned inhibition-of-return + * into another source of empty scans. Excluding tabu terms from the candidate + * set instead yields the best NON-TABU term, so rotation costs quality rather + * than costing the whole scan. + * + * COST. One pass over g->nodes scoring all candidates at once (12.7k labels × + * <=32 candidates, short strings, good locality), twice per 30 s scan. + * + * POLICY LIVES IN THE SOUL. Thresholds arrive as arguments; the runtime + * measures and ranks, awareness.el decides. Same split as engram_label_df. + * + * Returns the winning token, or "" when the node is missing, has no usable + * text, or every candidate is gated out — "" remains the honest signal that + * this slot yielded no seed, and auto_term_empty_streak still counts it. */ +#define ENGRAM_ST_MAXCAND 32 +#define ENGRAM_ST_TOKLEN 64 +#define ENGRAM_ST_SCANCHARS 400 + +/* Trim leading/trailing non-alphanumerics, then accept only tokens whose core + * is alphanumeric plus '-' and '_' with at least 3 letters. This subsumes the + * quoted-title guard (2026-07-25) and the "