From afc92f4e33742aa8bc2a55a0f0c7b4659cd886f4 Mon Sep 17 00:00:00 2001
From: Will Anderson <will.anderson@neurontechnologies.ai>
Date: Mon, 3 Aug 2026 08:38:58 -0500
Subject: [PATCH] self-review 2026-08-03: add engram_label_df term-specificity
 measure

The soul's curiosity auto-term extractor takes the first word of a top-WM
node label. It has no term-quality scoring, so three prior self-reviews each
bolted on another hand-curated blocklist (genre words 07-23, quoted titles
07-25, stopwords 07-30). Every one was written reactively, after a flood was
already observed. A list can only contain floods that already happened.

Two were in flight and unfixed when this review ran:
  "<!--"  label df 220 -> 252 nodes activated
  "SELF"  label df 175 -> 541 nodes activated (list has "Self" Title-case;
           str_eq is case-sensitive, so the uppercase token sailed through)

engram_label_df(term) counts nodes whose label contains term. Low-specificity
tokens are corpus-frequent by definition, so this catches the flood class
prospectively and tracks the corpus as the world-ingestor changes it. This is
Sparck Jones (1972), which introduced IDF under the name 'term specificity';
automatic stopword compilation from it is the textbook application.

NOT a replacement for the stopword list -- verified against all 86 listed
terms, not assumed. Catches 13 (Will:306, Self:175, Over:116, Knowledge:112),
misses 73 (Whose:0, Would:0, Could:0, This:9). Labels are terse titles, so
English function words are genuinely rare in them. The gates cover disjoint
failure modes; both are required.

Policy lives in awareness.el, not here: the runtime measures, the soul decides.
---
 lang/releases/v1.0.0-20260501/el_runtime.c | 61 ++++++++++++++++++++++
 lang/releases/v1.0.0-20260501/el_runtime.h |  3 ++
 2 files changed, 64 insertions(+)

diff --git a/lang/releases/v1.0.0-20260501/el_runtime.c b/lang/releases/v1.0.0-20260501/el_runtime.c
index b83afb1..4cc00dd 100644
--- a/lang/releases/v1.0.0-20260501/el_runtime.c
+++ b/lang/releases/v1.0.0-20260501/el_runtime.c
@@ -9548,6 +9548,67 @@ el_val_t engram_cosine_sim(el_val_t id_a, el_val_t id_b) {
     return el_from_float(eg_cosine(a->emb, b->emb, a->emb_dim));
 }
 
+/* engram_label_df — document frequency of `term` across node LABELS.
+ * Returns the count of nodes whose label contains term (case-insensitive),
+ * or the total node count for an empty/NULL term so callers treat "no term"
+ * as maximally unspecific (i.e. reject it).
+ *
+ * WHY THIS EXISTS (2026-08-03 self-review). The soul's auto-term extractor
+ * (awareness.el:auto_term_try_slot) picks a curiosity seed by taking the
+ * FIRST WORD of a top-WM node label. A first-word extractor has no notion of
+ * term quality, so three consecutive self-reviews each bolted another
+ * hand-curated blocklist onto it — genre words (07-23), quoted titles
+ * (07-25), English stopwords (07-30). Every one of those was written
+ * REACTIVELY, after observing a flood in the live ISE stream. The mechanism
+ * is whack-a-mole: the list can only ever contain floods that already
+ * happened.
+ *
+ * Measured live on this store (13,370 nodes) while two unanticipated floods
+ * were in flight and unfixed:
+ *     "<!--"             df 220   → 252 nodes activated
+ *     "SELF"             df 175   → 541 nodes activated  (list has "Self",
+ *                                    Title-case — the guard is case-SENSITIVE
+ *                                    and the live token was uppercase)
+ *     "Context"          df  53
+ *     ── an order-of-magnitude gap ──
+ *     "Dual"             df  12
+ *     "Sparse"           df   8
+ *     "engram_goal_bias" df   1   → 73 activated
+ *     "Clin-JEPA"        df   1   → 66 activated
+ * Document frequency separates flood tokens from topical tokens cleanly, and
+ * does it PROSPECTIVELY — no list maintenance, and it tracks the corpus as
+ * the world-ingestor changes what the store is made of.
+ *
+ * This is Spärck Jones (1972), "A statistical interpretation of term
+ * specificity and its application in retrieval" — the paper that introduced
+ * IDF, originally under the name *term specificity*. Automatic stopword-list
+ * compilation from IDF is the standard application; that is exactly the job
+ * here.
+ *
+ * NOT A REPLACEMENT FOR THE STOPWORD LIST — verified, not assumed. Tested
+ * all 86 hand-listed terms against this measure at threshold node_count/400:
+ * it catches 13 (Will:306, Self:175, Over:116, Knowledge:112, From:90,
+ * With:69, Test:61, Value:56, Context:53, Paper:48, Very:45, Must:42,
+ * What:37) and MISSES 73 (Whose:0, Would:0, Could:0, Might:0, This:9,
+ * Some:8 …). The reason is structural: labels are terse titles, so English
+ * function words are genuinely RARE in them — low df, high noise. The two
+ * gates cover disjoint failure modes and both are required:
+ *   - stopword list → function words (rare in labels, no topical signal)
+ *   - label df      → corpus-frequent tokens (markup, sentinels, genre tags)
+ * Policy (the threshold) deliberately lives in awareness.el, not here: the
+ * runtime measures, the soul decides. */
+el_val_t engram_label_df(el_val_t term) {
+    EngramStore* g = engram_get();
+    const char* t = EL_CSTR(term);
+    if (!t || !*t) return (el_val_t)g->node_count;
+    int64_t df = 0;
+    for (int64_t i = 0; i < g->node_count; i++) {
+        const char* lbl = g->nodes[i].label;
+        if (lbl && istr_contains(lbl, t)) df++;
+    }
+    return (el_val_t)df;
+}
+
 /* engram_embed_backfill — explicitly drive the lazy embedding backfill.
  * (2026-07-25 self-review.) The per-activate backfill (8 nodes/call) only
  * runs inside engram_activate, and on the authoritative HTTP store nothing
diff --git a/lang/releases/v1.0.0-20260501/el_runtime.h b/lang/releases/v1.0.0-20260501/el_runtime.h
index de43bd2..88da992 100644
--- a/lang/releases/v1.0.0-20260501/el_runtime.h
+++ b/lang/releases/v1.0.0-20260501/el_runtime.h
@@ -619,6 +619,9 @@ el_val_t  engram_activate_json(el_val_t query, el_val_t depth);
 el_val_t  engram_stats_json(void);
 el_val_t  engram_act_stats_json(void);
 el_val_t  engram_cosine_sim(el_val_t id_a, el_val_t id_b);
+/* Document frequency of a term across node labels — term-specificity signal
+ * for curiosity seed selection. (2026-08-03 self-review.) */
+el_val_t  engram_label_df(el_val_t term);
 el_val_t  engram_embed_backfill(el_val_t count);
 el_val_t  engram_list_layers_json(void);
 /* Working memory introspection — count, mean weight, and top-N snapshot.