el/lang/runtime/engram_text.c

/* engram_text.c — see engram_text.h.
 *
 * Moved verbatim out of el_runtime.c (2026-08-16). Bodies are unchanged; only
 * `static` was dropped so they link from this translation unit, and each
 * function's doc comment travelled with it.
 */
#include "engram_text.h"

#include <ctype.h>
#include <string.h>

/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct
 * (case-insensitive) tokens. Returns the token count. Over-long tokens are
 * truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */
int engram_tokenize_query(const char* q,
                                 char toks[][ENGRAM_QTOK_LEN], int maxtok) {
    int n = 0;
    if (!q) return 0;
    const char* p = q;
    while (*p && n < maxtok) {
        while (*p && isspace((unsigned char)*p)) p++;
        if (!*p) break;
        char buf[ENGRAM_QTOK_LEN];
        size_t tl = 0;
        while (*p && !isspace((unsigned char)*p)) {
            if (tl < sizeof(buf) - 1) buf[tl++] = *p;
            p++;
        }
        buf[tl] = '\0';
        if (tl == 0) continue;
        int dup = 0;
        for (int s = 0; s < n; s++) {
            if (strcasecmp(toks[s], buf) == 0) { dup = 1; break; }
        }
        if (dup) continue;
        memcpy(toks[n], buf, tl + 1);
        n++;
    }
    return n;
}

/* Trim leading/trailing non-alphanumerics, then accept only tokens whose core
 * is alphanumeric plus '-' and '_' with at least 3 letters. This subsumes the
 * quoted-title guard (2026-07-25) and the "<!--" flood (2026-08-03)
 * structurally: markup and punctuation-bearing tokens never become
 * candidates, rather than being blocklisted after the fact. */
int eg_st_clean_token(const char* raw, size_t rawlen,
                             char* out, size_t outcap) {
    size_t s = 0, e = rawlen;
    while (s < e && !isalnum((unsigned char)raw[s])) s++;
    while (e > s && !isalnum((unsigned char)raw[e - 1])) e--;
    size_t len = e - s;
    if (len < 4 || len >= outcap) return 0;
    int alpha = 0;
    for (size_t i = 0; i < len; i++) {
        unsigned char c = (unsigned char)raw[s + i];
        if (isalpha(c)) alpha++;
        else if (!isdigit(c) && c != '-' && c != '_') return 0;
    }
    if (alpha < 3) return 0;
    memcpy(out, raw + s, len);
    out[len] = '\0';
    return 1;
}

/* Word-boundary document frequency. engram_label_df uses istr_contains, i.e.
 * SUBSTRING matching, and that is the wrong estimator for term specificity on
 * short tokens: "them" hits inside "theme" and "anthem", "about" and "whole"
 * come back with df 2 and 1 rather than 0. That matters here specifically
 * because the min_df floor is what rejects English function words, and it can
 * only do that job if their df is honestly zero. Substring df quietly handed
 * them a survival ticket. Measured on the live store before this fix, "whole"
 * (df=1, idf=8.76) and "about" (df=2, idf=8.36) were outscoring real topical
 * terms and losing only on position — one node whose text happened to open
 * with a function word would have seeded on it.
 *
 * engram_label_df keeps substring semantics: it is a separate published
 * measure with existing callers, and changing it underneath them is not this
 * change's business. */
int eg_st_label_has_word(const char* hay, const char* word) {
    size_t wl = strlen(word);
    for (const char* p = hay; *p; p++) {
        if (strncasecmp(p, word, wl) != 0) continue;
        char before = (p == hay) ? '\0' : p[-1];
        char after  = p[wl];
        if (before && (isalnum((unsigned char)before) || before == '_')) continue;
        if (after  && (isalnum((unsigned char)after)  || after  == '_')) continue;
        return 1;
    }
    return 0;
}

/* Text-damage signature. Extracted with the function from el_runtime.c's
 * "Text-integrity instrumentation" block; the stock/flow gauges that use it
 * (engram_text_health_json, _eg_txt_write_damaged) stay there because they
 * touch store and EL value types.
 *
 * SIGNATURE. Conservative on purpose — a false alarm that cries corruption
 * over ordinary punctuation is worse than useless. Two patterns, both of
 * which are essentially absent from well-formed English prose:
 *   (a) alnum '?' alnum   — "na?ve", "caf?s", "don?t". A real question mark
 *                           never sits between two word characters.
 *   (b) ' ? ' followed by a lowercase letter — a lost em/en dash. A real
 *                           question mark is not preceded by a space, and
 *                           what follows one starts a new sentence.
 * Deliberately NOT flagged: a trailing '?' after a word, '? ' before a
 * capital, or '?' at end of string — all legitimate. This under-counts (it
 * cannot see a mangled 'café ' where the '?' landed before a space), so the
 * census is a floor on the damage, never an exaggeration of it. */
int eg_text_loss_signature(const char* s) {
    if (!s) return 0;
    for (const char* p = s; *p; p++) {
        if (*p != '?') continue;
        unsigned char prev = (p == s) ? 0 : (unsigned char)p[-1];
        unsigned char next = (unsigned char)p[1];
        /* (a) sandwiched between word characters. */
        if (isalnum(prev) && isalnum(next)) return 1;
        /* (b) spaced, with lowercase continuation — a lost dash. */
        if (prev == ' ' && next == ' ' && islower((unsigned char)p[2])) return 1;
    }
    return 0;
}