diff --git a/chat.el b/chat.el index c101aa8..263f222 100644 --- a/chat.el +++ b/chat.el @@ -12,47 +12,107 @@ fn chat_default_model() -> String { return "claude-sonnet-4-5" } +// parse_salience_100 — convert a %g-serialized float to integer * 100. +// The C runtime serializes floats with %g which trims trailing zeros: +// 0.70 → "0.7", 0.60 → "0.6", 0.50 → "0.5", 1.0 → "1" +// The naive str_replace(".", "") approach breaks for single-decimal strings: +// "0.7" → "07" → str_to_int → 7 (WRONG, should be 70) +// "0.5" → "05" → str_to_int → 5 (WRONG, should be 50) +// "0.85" → "085" → str_to_int → 85 (accidentally correct — two decimal digits) +// Fix: use str_index_of to find the decimal point and scale accordingly: +// No decimal ("1"): multiply raw by 100 +// One decimal digit ("0.7"): multiply stripped value by 10 +// Two+ decimal digits ("0.85"): stripped value is already in hundredths +fn parse_salience_100(s: String) -> Int { + if str_eq(s, "") { return 70 } + let dot_pos: Int = str_index_of(s, ".") + let raw: Int = if dot_pos < 0 { + // No decimal point — integer like "1" means 100% + str_to_int(s) * 100 + } else { + let after_dot: String = str_slice(s, dot_pos + 1, str_len(s)) + let decimal_digits: Int = str_len(after_dot) + let stripped: Int = str_to_int(str_replace(s, ".", "")) + if decimal_digits == 1 { stripped * 10 } else { stripped } + } + if raw > 100 { 100 } else { if raw < 0 { 0 } else { raw } } +} + // engram_score_node — compute a recency x relevance score for a single engram -// node JSON object. Higher is better. Score = salience * importance * recency_factor. -// recency_factor decays linearly over 30 days: nodes updated today score 1.0, -// nodes 30+ days old score 0.1 (floor). Nodes with no created_at score 0.5. -// This keeps fresh, high-salience nodes at the top and pushes stale low-signal -// nodes to the bottom so they get trimmed when we cap context size. +// node JSON object. Higher is better. +// +// Bugs fixed vs original implementation: +// 1. FLOAT PARSING: parse_salience_100 correctly handles %g single-decimal output. +// "0.7" → 70, "0.6" → 60, "0.5" → 50 (was: 7, 6, 5 — scored near zero and +// were filtered by threshold=25, making the function broken for the majority +// of the graph where conv/utterance nodes have salience/importance ≈ 0.6/0.7). +// 2. RECENCY USES LAST TOUCH: uses max(created_at, updated_at, last_activated) so +// nodes strengthened by engram_strengthen() after chat turns are not penalised +// for a stale created_at. A node referenced yesterday but created 25 days ago +// now correctly scores as fresh rather than borderline-filtered. +// 3. COMPRESSED RECENCY RANGE: old formula (sal * imp * recency / 10000) gave +// recency a 10x dynamic range (10-100) vs 1.9x for salience/importance. A +// canonical high-importance node at 30 days scored the same as a fresh noise +// node. New formula compresses recency to 1.54x via (50 + recency/2) weight. +// 4. SOFTER FLOOR: recency floor raised from 10 to 30 with tier-aware decay windows +// so canonical identity/persona nodes never bottom out to near-zero. fn engram_score_node(node_json: String) -> Int { let salience_str: String = json_get(node_json, "salience") let importance_str: String = json_get(node_json, "importance") let created_str: String = json_get(node_json, "created_at") + let updated_str: String = json_get(node_json, "updated_at") + let activated_str: String = json_get(node_json, "last_activated") + let tier_str: String = json_get(node_json, "tier") - // Parse as floats via * 100 integer arithmetic (el has no float math) - let salience_100: Int = if str_eq(salience_str, "") { 70 } else { - let s: Int = str_to_int(str_replace(salience_str, ".", "")) - // Clamp to 0-100 range (value was e.g. "0.85" -> parsed "085" = 85) - if s > 100 { 100 } else { if s < 0 { 0 } else { s } } - } - let importance_100: Int = if str_eq(importance_str, "") { 70 } else { - let v: Int = str_to_int(str_replace(importance_str, ".", "")) - if v > 100 { 100 } else { if v < 0 { 0 } else { v } } - } + // parse_salience_100 handles "0.7" → 70, "0.85" → 85, "1.0" → 100, "1" → 100 + let salience_100: Int = parse_salience_100(salience_str) + let importance_100: Int = parse_salience_100(importance_str) - // Recency: decay from 100 (today) to 10 (30+ days). created_at is Unix seconds. + // Recency: use max(created_at, updated_at, last_activated). + // last_activated is updated by engram_strengthen() every chat turn — nodes + // actively referenced score fresh regardless of original write time. let now_ts: Int = time_now() - let recency_100: Int = if str_eq(created_str, "") { 50 } else { - let created_ts: Int = str_to_int(created_str) - let age_secs: Int = now_ts - created_ts - let age_days: Int = age_secs / 86400 - let decay: Int = if age_days >= 30 { 10 } else { 100 - (age_days * 3) } - if decay < 10 { 10 } else { decay } + let created_ts: Int = if str_eq(created_str, "") { 0 } else { str_to_int(created_str) } + let updated_ts: Int = if str_eq(updated_str, "") { 0 } else { str_to_int(updated_str) } + let activated_ts: Int = if str_eq(activated_str, "") { 0 } else { str_to_int(activated_str) } + let best_ts_ab: Int = if updated_ts > created_ts { updated_ts } else { created_ts } + let best_ts: Int = if activated_ts > best_ts_ab { activated_ts } else { best_ts_ab } + let recency_100: Int = if best_ts == 0 { 50 } else { + let age_secs: Int = now_ts - best_ts + // Guard against clock skew (future timestamps): treat as brand new. + let age_days: Int = if age_secs < 0 { 0 } else { age_secs / 86400 } + // Tier-aware decay, softer floor (30 not 10): + // Canonical: 365-day window — foundational identity/persona nodes. + // Episodic: 90-day window — conversation context fades moderately. + // Working/untiered: 35-day window — transient task state. + let is_canonical: Bool = str_eq(tier_str, "Canonical") + let is_episodic: Bool = str_eq(tier_str, "Episodic") + let decay: Int = if is_canonical { + let drop: Int = if age_days >= 365 { 70 } else { age_days * 70 / 365 } + 100 - drop + } else { + if is_episodic { + if age_days >= 90 { 30 } else { 100 - (age_days * 70 / 90) } + } else { + if age_days >= 35 { 30 } else { 100 - (age_days * 2) } + } + } + if decay < 30 { 30 } else { decay } } - // Combined score 0-1000000 (no floats): salience * importance * recency / 10000 - return salience_100 * importance_100 * recency_100 / 10000 + // Compressed recency weight (50 + recency/2): range 65-100 (1.54x dynamic range). + // Old formula had 10x recency range which drowned out relevance for old-but-important + // nodes. New: relevance (0-100) × recency_weight (65-100) / 100 → score 0-100. + let relevance: Int = salience_100 * importance_100 / 10000 + let recency_weight: Int = 50 + recency_100 / 2 + return relevance * recency_weight / 100 } // engram_compile_ranked — build a context string from a JSON array of node objects, -// ordered best-first by score. Only nodes above a minimum score (25 = salience 0.5 * -// importance 0.5 * recency 1.0) are included; the rest are noise. Returns at most -// max_nodes entries concatenated as JSON array text. Because el has no sort primitive, -// we do a single selection pass picking the top N by linear scan (N=10 cap). +// ordered best-first by score. Only nodes above threshold=15 are included. +// With corrected parsing: sal=0.5 * imp=0.5 at max recency scores 25; threshold 15 +// gives headroom for moderately-relevant older nodes while filtering near-zero noise. +// Returns at most max_nodes entries. max_nodes must not exceed 20 (sentinel limit). fn engram_compile_ranked(nodes_json: String, max_nodes: Int) -> String { if str_eq(nodes_json, "") { return "" } if str_eq(nodes_json, "[]") { return "" } @@ -73,8 +133,9 @@ fn engram_compile_ranked(nodes_json: String, max_nodes: Int) -> String { while ci < total { let node: String = json_array_get(nodes_json, ci) let score: Int = engram_score_node(node) - // Only include reasonably relevant nodes (threshold=25) - let above_thresh: Bool = score >= 25 + // Threshold=15: allows moderately-relevant older nodes while filtering noise. + // Example: a 3-week-old node with sal=0.6, imp=0.6 scores ~14 — passes at 15. + let above_thresh: Bool = score >= 15 // Check this index wasn't already selected (sentinel: look for idx marker) let idx_marker: String = "\"_sel_" + int_to_str(ci) + "\"" let already_picked: Bool = str_contains(selected, idx_marker) @@ -101,7 +162,7 @@ fn engram_compile_ranked(nodes_json: String, max_nodes: Int) -> String { // Strip the _sel_N sentinel fields that were used for duplicate-detection bookkeeping. // The sentinels have the form "\"_sel_N\":1," (trailing comma, space before next key). // We injected them as the first field in each object, so the pattern is predictable. - // Because el has no regex, remove up to 10 possible sentinel variants by literal replace. + // Because el has no regex, remove up to 20 possible sentinel variants by literal replace. let clean: String = "[" + selected + "]" let c0: String = str_replace(clean, "\"_sel_0\":1,", "") let c1: String = str_replace(c0, "\"_sel_1\":1,", "") @@ -113,7 +174,17 @@ fn engram_compile_ranked(nodes_json: String, max_nodes: Int) -> String { let c7: String = str_replace(c6, "\"_sel_7\":1,", "") let c8: String = str_replace(c7, "\"_sel_8\":1,", "") let c9: String = str_replace(c8, "\"_sel_9\":1,", "") - return c9 + let c10: String = str_replace(c9, "\"_sel_10\":1,", "") + let c11: String = str_replace(c10, "\"_sel_11\":1,", "") + let c12: String = str_replace(c11, "\"_sel_12\":1,", "") + let c13: String = str_replace(c12, "\"_sel_13\":1,", "") + let c14: String = str_replace(c13, "\"_sel_14\":1,", "") + let c15: String = str_replace(c14, "\"_sel_15\":1,", "") + let c16: String = str_replace(c15, "\"_sel_16\":1,", "") + let c17: String = str_replace(c16, "\"_sel_17\":1,", "") + let c18: String = str_replace(c17, "\"_sel_18\":1,", "") + let c19: String = str_replace(c18, "\"_sel_19\":1,", "") + return c19 } fn engram_compile(intent: String) -> String { @@ -124,8 +195,11 @@ fn engram_compile(intent: String) -> String { let act_ok: Bool = !str_eq(activate_json, "") && !str_eq(activate_json, "[]") let srch_ok: Bool = !str_eq(search_json, "") && !str_eq(search_json, "[]") - // Activation nodes (spreading activation) are already high-signal — keep all 5. - let act_part: String = if act_ok { activate_json } else { "" } + // Activation nodes (spreading activation) are high-signal but apply scoring via + // engram_compile_ranked with threshold=5 to exclude genuinely zero-quality stale + // nodes that happen to be graph-connected. The threshold of 5 is well below the + // search path threshold of 15 to preserve the activation path's higher recall. + let act_part: String = if act_ok { engram_compile_ranked(activate_json, 5) } else { "" } // Rank search results and keep only the top 8 (was: flat 15 unranked). // This cuts context noise roughly in half while preserving the best-scoring nodes.