/* engram_text.h — text handling for the engram: query tokenization, candidate
 * token hygiene, word-boundary matching, and the text-damage signature.
 *
 * WHY THIS FILE EXISTS
 * --------------------
 * These functions lived in el_runtime.c, which is a 2026-05-03 build shim that
 * was scheduled for deletion, never retired, and grew to 20,527 lines. They do
 * not belong there: they touch no EL value type and no engram store type. They
 * are plain C over <ctype.h>/<string.h> operating on char buffers, and they are
 * a concern of their own — so they get a translation unit of their own.
 *
 * Adding a new text helper?  Add it HERE, not to el_runtime.c. A new .c costs
 * exactly one line in lang/runtime/SOURCES, and every build path picks it up.
 * Placement is a LINK-TIME concern: the compiler cannot tell which .c a symbol
 * came from (builtin_arity is an arity guard, not a dispatch table), so a
 * function defined here is exactly as linkable as one defined in el_runtime.c.
 */
#ifndef ENGRAM_TEXT_H
#define ENGRAM_TEXT_H

#include <stddef.h>

#ifdef __cplusplus
extern "C" {
#endif

/* Max bytes per query token, including the NUL. */
#define ENGRAM_QTOK_LEN 256

/* Split q on whitespace into up to ENGRAM_MAX_QTOKENS distinct
 * (case-insensitive) tokens. Returns the token count. Over-long tokens are
 * truncated to ENGRAM_QTOK_LEN-1; over-count tokens are ignored. */
int engram_tokenize_query(const char* q, char toks[][ENGRAM_QTOK_LEN], int maxtok);

/* Trim leading/trailing non-alphanumerics, then accept only tokens whose core
 * is alphanumeric plus '-' and '_' with at least 3 letters. This subsumes the
 * quoted-title guard (2026-07-25) and the "<!--" flood (2026-08-03)
 * structurally: markup and punctuation-bearing tokens never become
 * candidates, rather than being blocklisted after the fact. */
int eg_st_clean_token(const char* raw, size_t rawlen, char* out, size_t outcap);

/* Word-boundary document frequency. engram_label_df uses istr_contains, i.e.
 * SUBSTRING matching, and that is the wrong estimator for term specificity on
 * short tokens: "them" hits inside "theme" and "anthem", "about" and "whole"
 * come back with df 2 and 1 rather than 0. That matters here specifically
 * because the min_df floor is what rejects English function words, and it can
 * only do that job if their df is honestly zero. Substring df quietly handed
 * them a survival ticket. Measured on the live store before this fix, "whole"
 * (df=1, idf=8.76) and "about" (df=2, idf=8.36) were outscoring real topical
 * terms and losing only on position — one node whose text happened to open
 * with a function word would have seeded on it.
 *
 * engram_label_df keeps substring semantics: it is a separate published
 * measure with existing callers, and changing it underneath them is not this
 * change's business. */
int eg_st_label_has_word(const char* hay, const char* word);

/* Whether s carries the text-loss signature left by the \uXXXX -> '?' parser
 * defect. Conservative by design; see engram_text.c for the full rationale. */
int eg_text_loss_signature(const char* s);

#ifdef __cplusplus
}
#endif

#endif /* ENGRAM_TEXT_H */