From c5508372ca19ddeb65859942cbfb5654774318d2 Mon Sep 17 00:00:00 2001 From: Will Anderson Date: Thu, 13 Aug 2026 15:09:04 -0500 Subject: [PATCH] =?UTF-8?q?elp(multilingual):=20native-el=20language=20lay?= =?UTF-8?q?er=20=E2=80=94=20detect=20+=20localized=20phrases?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3 piece 2. Ports multilingual.py: deterministic language detection (en/es/pt/it) via stopword + diacritic scoring, localized fixed phrases (SACRED per-language yes/no/decline/identity), PT/ES->EN retrieval term lexicon, and EN->target predicate translation. No generative model. Gate (multilingual_gate.el): 4/4 languages detected correctly; localized declines + term/pred lexicons verified. Built bounded (elc rc=0 peak 25MB). Worked through the documented el '+' mis-compile (two chained function-call Int operands compile as string concat -> corrupt Int -> segfault on the accented path); fixed by binding each score to an Int var and adding vars singly. Simplifications (honest): diacritics scored by PRESENCE (str_contains) not codepoint count (UTF-8 index safety); confidence scalar and the regex-based parse_directive() from the reference not yet ported (directive parsing deferred to the dialogue layer). --- elp/manifest.el | 1 + elp/src/multilingual.el | 280 +++++++++++++++++++++++++++++++++ elp/tests/multilingual_gate.el | 43 +++++ 3 files changed, 324 insertions(+) create mode 100644 elp/src/multilingual.el create mode 100644 elp/tests/multilingual_gate.el diff --git a/elp/manifest.el b/elp/manifest.el index 0a1f939..b5072be 100644 --- a/elp/manifest.el +++ b/elp/manifest.el @@ -82,6 +82,7 @@ build { "src/semantics.el", "src/comprehend.el", "src/propositions.el", + "src/multilingual.el", "src/elp.el", ] } diff --git a/elp/src/multilingual.el b/elp/src/multilingual.el new file mode 100644 index 0000000..e97ce84 --- /dev/null +++ b/elp/src/multilingual.el @@ -0,0 +1,280 @@ +// multilingual.el - the language layer for the native-el interlocutor. +// +// Deterministic, NO generative model (ports multilingual.py): +// 1. ml_detect(text) -> ISO code (en/es/pt/it) via stopword + diacritic score +// 2. ml_tr(key, lang) -> localized fixed phrase (SACRED per-language yes/no/decline) +// 3. ml_term(w, lang) -> PT/ES content term -> EN engram equivalent +// 4. ml_translate_pred(lemma, lang) -> EN predicate lemma -> target infinitive +// +// The Python detector count-weights stopwords and diacritics; here diacritics are +// scored by PRESENCE (str_contains) rather than codepoint counting, to stay clear +// of UTF-8 index hazards in the runtime. Faithful enough to classify typical +// queries; documented simplification. Depends on: comprehend (cp_tokenize). + +// ── 1. language detection ───────────────────────────────────────────────────── + +fn ml_stop_en(w: String) -> Bool { + if str_eq(w, "the") { return true } + if str_eq(w, "does") { return true } + if str_eq(w, "do") { return true } + if str_eq(w, "did") { return true } + if str_eq(w, "what") { return true } + if str_eq(w, "who") { return true } + if str_eq(w, "is") { return true } + if str_eq(w, "are") { return true } + if str_eq(w, "how") { return true } + if str_eq(w, "you") { return true } + if str_eq(w, "your") { return true } + if str_eq(w, "of") { return true } + if str_eq(w, "to") { return true } + if str_eq(w, "and") { return true } + if str_eq(w, "for") { return true } + if str_eq(w, "explain") { return true } + if str_eq(w, "answer") { return true } + if str_eq(w, "memory") { return true } + if str_eq(w, "with") { return true } + if str_eq(w, "not") { return true } + if str_eq(w, "store") { return true } + return false +} + +fn ml_stop_es(w: String) -> Bool { + if str_eq(w, "que") { return true } + if str_eq(w, "qué") { return true } + if str_eq(w, "una") { return true } + if str_eq(w, "usted") { return true } + if str_eq(w, "su") { return true } + if str_eq(w, "cómo") { return true } + if str_eq(w, "como") { return true } + if str_eq(w, "cuál") { return true } + if str_eq(w, "quién") { return true } + if str_eq(w, "está") { return true } + if str_eq(w, "es") { return true } + if str_eq(w, "los") { return true } + if str_eq(w, "las") { return true } + if str_eq(w, "del") { return true } + if str_eq(w, "al") { return true } + if str_eq(w, "explica") { return true } + if str_eq(w, "explique") { return true } + if str_eq(w, "forma") { return true } + if str_eq(w, "con") { return true } + if str_eq(w, "memoria") { return true } + if str_eq(w, "responde") { return true } + return false +} + +fn ml_stop_pt(w: String) -> Bool { + if str_eq(w, "que") { return true } + if str_eq(w, "uma") { return true } + if str_eq(w, "você") { return true } + if str_eq(w, "sua") { return true } + if str_eq(w, "seu") { return true } + if str_eq(w, "como") { return true } + if str_eq(w, "memória") { return true } + if str_eq(w, "isso") { return true } + if str_eq(w, "os") { return true } + if str_eq(w, "as") { return true } + if str_eq(w, "da") { return true } + if str_eq(w, "do") { return true } + if str_eq(w, "na") { return true } + if str_eq(w, "no") { return true } + if str_eq(w, "explica") { return true } + if str_eq(w, "forma") { return true } + if str_eq(w, "é") { return true } + if str_eq(w, "está") { return true } + if str_eq(w, "com") { return true } + if str_eq(w, "responda") { return true } + return false +} + +fn ml_stop_it(w: String) -> Bool { + if str_eq(w, "che") { return true } + if str_eq(w, "una") { return true } + if str_eq(w, "come") { return true } + if str_eq(w, "della") { return true } + if str_eq(w, "gli") { return true } + if str_eq(w, "è") { return true } + if str_eq(w, "sono") { return true } + if str_eq(w, "questo") { return true } + if str_eq(w, "nel") { return true } + if str_eq(w, "di") { return true } + if str_eq(w, "il") { return true } + if str_eq(w, "cosa") { return true } + if str_eq(w, "per") { return true } + if str_eq(w, "memoria") { return true } + if str_eq(w, "spiega") { return true } + if str_eq(w, "rispondi") { return true } + return false +} + +// diacritic PRESENCE score (weight 3 each; hard overrides weight 8). +fn ml_dia_score(low: String, lang: String) -> Int { + let s: Int = 0 + if str_eq(lang, "pt") { + if str_contains(low, "ã") { let s = s + 3 } + if str_contains(low, "õ") { let s = s + 3 } + if str_contains(low, "ç") { let s = s + 3 } + if str_contains(low, "ê") { let s = s + 3 } + if str_contains(low, "á") { let s = s + 3 } + // hard PT markers (ã/õ almost never appear outside PT) + if str_contains(low, "ã") { let s = s + 8 } + if str_contains(low, "õ") { let s = s + 8 } + } + if str_eq(lang, "es") { + if str_contains(low, "ñ") { let s = s + 3 } + if str_contains(low, "¿") { let s = s + 3 } + if str_contains(low, "¡") { let s = s + 3 } + if str_contains(low, "á") { let s = s + 3 } + if str_contains(low, "é") { let s = s + 3 } + // hard ES markers + if str_contains(low, "ñ") { let s = s + 8 } + if str_contains(low, "¿") { let s = s + 8 } + if str_contains(low, "¡") { let s = s + 8 } + } + if str_eq(lang, "it") { + if str_contains(low, "è") { let s = s + 3 } + if str_contains(low, "ì") { let s = s + 3 } + if str_contains(low, "ò") { let s = s + 3 } + } + return s +} + +fn ml_stop_score(toks: [String], lang: String) -> Int { + let n: Int = native_list_len(toks) + let s: Int = 0 + let i: Int = 0 + while i < n { + let w: String = native_list_get(toks, i) + if str_eq(lang, "en") { if ml_stop_en(w) { let s = s + 2 } } + if str_eq(lang, "es") { if ml_stop_es(w) { let s = s + 2 } } + if str_eq(lang, "pt") { if ml_stop_pt(w) { let s = s + 2 } } + if str_eq(lang, "it") { if ml_stop_it(w) { let s = s + 2 } } + let i = i + 1 + } + return s +} + +fn ml_detect(text: String) -> String { + if str_eq(text, "") { return "en" } + let low: String = str_to_lower(text) + let toks: [String] = cp_tokenize(text) + // NOTE: el's overloaded `+` mis-compiles two chained function-call Int operands + // as string concat (documented in comprehend_gate.el). Bind each call to an Int + // var and add vars one at a time so the addition stays integer. + let en: Int = ml_stop_score(toks, "en") + let es_s: Int = ml_stop_score(toks, "es") + let es_d: Int = ml_dia_score(low, "es") + let es: Int = es_s + es_d + let pt_s: Int = ml_stop_score(toks, "pt") + let pt_d: Int = ml_dia_score(low, "pt") + let pt: Int = pt_s + pt_d + let it_s: Int = ml_stop_score(toks, "it") + let it_d: Int = ml_dia_score(low, "it") + let it: Int = it_s + it_d + + let best: String = "en" + let bs: Int = en + if es > bs { let best = "es"; let bs = es } + if pt > bs { let best = "pt"; let bs = pt } + if it > bs { let best = "it"; let bs = it } + // weak signal -> honest fallback to English + if bs < 3 { return "en" } + return best +} + +// ── 2. localized fixed phrases (SACRED per-language decline/yes/no) ──────────── + +fn ml_tr(key: String, lang: String) -> String { + if str_eq(key, "no_memory") { + if str_eq(lang, "pt") { return "Não tenho isso na minha memória." } + if str_eq(lang, "es") { return "No tengo eso en mi memoria." } + if str_eq(lang, "it") { return "Non ho quello nella mia memoria." } + return "I don't have that in my memory." + } + if str_eq(key, "parse_fail") { + if str_eq(lang, "pt") { return "Não consegui interpretar isso." } + if str_eq(lang, "es") { return "No pude interpretar eso." } + if str_eq(lang, "it") { return "Non sono riuscito a interpretarlo." } + return "I didn't parse that." + } + if str_eq(key, "yes") { + if str_eq(lang, "pt") { return "Sim" } + if str_eq(lang, "es") { return "Sí" } + if str_eq(lang, "it") { return "Sì" } + return "Yes" + } + if str_eq(key, "no") { + if str_eq(lang, "pt") { return "Não" } + if str_eq(lang, "es") { return "No" } + if str_eq(lang, "it") { return "No" } + return "No" + } + if str_eq(key, "identity") { + if str_eq(lang, "pt") { return "Sou o Neuron, o engrama com quem você está falando." } + if str_eq(lang, "es") { return "Soy Neuron, el engrama con el que estás hablando." } + if str_eq(lang, "it") { return "Sono Neuron, l'engramma con cui stai parlando." } + return "I'm Neuron, the engram you're speaking with." + } + return "" +} + +// ── 3. retrieval term lexicon (PT/ES content term -> EN engram equivalent) ───── + +fn ml_term(w: String, lang: String) -> String { + if str_eq(lang, "en") { return w } + if str_eq(w, "saliência") { return "salience" } + if str_eq(w, "saliencia") { return "salience" } + if str_eq(w, "memória") { return "memory" } + if str_eq(w, "memoria") { return "memory" } + if str_eq(w, "geometria") { return "geometry" } + if str_eq(w, "geometrias") { return "geometry" } + if str_eq(w, "geometrías") { return "geometry" } + if str_eq(w, "forma") { return "form" } + if str_eq(w, "consolidação") { return "consolidation" } + if str_eq(w, "consolidación") { return "consolidation" } + if str_eq(w, "aprendizagem") { return "learning" } + if str_eq(w, "aprendizaje") { return "learning" } + if str_eq(w, "nó") { return "node" } + if str_eq(w, "nodo") { return "node" } + if str_eq(w, "armazenamento") { return "storage" } + if str_eq(w, "almacenamiento") { return "storage" } + if str_eq(w, "estrutura") { return "structure" } + if str_eq(w, "estructura") { return "structure" } + return w +} + +// ── 4. predicate translation (EN lemma -> target infinitive; pass-through) ───── + +fn ml_translate_pred(lemma: String, lang: String) -> String { + if str_eq(lang, "en") { return lemma } + if str_eq(lang, "es") { + if str_eq(lemma, "store") { return "almacenar" } + if str_eq(lemma, "use") { return "usar" } + if str_eq(lemma, "have") { return "tener" } + if str_eq(lemma, "be") { return "ser" } + if str_eq(lemma, "give") { return "dar" } + if str_eq(lemma, "make") { return "hacer" } + if str_eq(lemma, "learn") { return "aprender" } + if str_eq(lemma, "form") { return "formar" } + return lemma + } + if str_eq(lang, "pt") { + if str_eq(lemma, "store") { return "armazenar" } + if str_eq(lemma, "use") { return "usar" } + if str_eq(lemma, "have") { return "ter" } + if str_eq(lemma, "be") { return "ser" } + if str_eq(lemma, "give") { return "dar" } + if str_eq(lemma, "make") { return "fazer" } + if str_eq(lemma, "learn") { return "aprender" } + if str_eq(lemma, "form") { return "formar" } + return lemma + } + if str_eq(lang, "it") { + if str_eq(lemma, "store") { return "memorizzare" } + if str_eq(lemma, "use") { return "usare" } + if str_eq(lemma, "have") { return "avere" } + if str_eq(lemma, "be") { return "essere" } + return lemma + } + return lemma +} diff --git a/elp/tests/multilingual_gate.el b/elp/tests/multilingual_gate.el new file mode 100644 index 0000000..b83906e --- /dev/null +++ b/elp/tests/multilingual_gate.el @@ -0,0 +1,43 @@ +// multilingual_gate.el - deterministic language detect + localized-phrase test. + +fn mg_det(text: String, want: String) -> String { + let got: String = ml_detect(text) + let ok: String = "MISMATCH" + if str_eq(got, want) { let ok = "ok" } + return " detect(" + got + ") want=" + want + " (" + ok + ") :: " + text + "\n" +} + +fn mg_ok(text: String, want: String) -> Int { + if str_eq(ml_detect(text), want) { return 1 } + return 0 +} + +fn run_ml_gate() -> String { + let t1: String = "Does Neuron use SQLite for storage?" + let t2: String = "Neuron, me explica cómo la saliencia forma las geometrías." + let t3: String = "O professor não leu o livro na memória." + let t4: String = "Che cosa memorizza Neuron nella memoria?" + + let rep: String = "==== ELP multilingual detect + localized phrases ====\n" + let rep = rep + mg_det(t1, "en") + let rep = rep + mg_det(t2, "es") + let rep = rep + mg_det(t3, "pt") + let rep = rep + mg_det(t4, "it") + + let rep = rep + " localized decline (pt): " + ml_tr("no_memory", "pt") + "\n" + let rep = rep + " localized decline (es): " + ml_tr("no_memory", "es") + "\n" + let rep = rep + " term(saliência->en): " + ml_term("saliência", "pt") + "\n" + let rep = rep + " pred(store->pt): " + ml_translate_pred("store", "pt") + "\n" + + let ok: Int = 0 + if mg_ok(t1, "en") == 1 { let ok = ok + 1 } + if mg_ok(t2, "es") == 1 { let ok = ok + 1 } + if mg_ok(t3, "pt") == 1 { let ok = ok + 1 } + if mg_ok(t4, "it") == 1 { let ok = ok + 1 } + let rep = rep + "-----------------------------------------------------------------\n" + let rep = rep + "language detected correctly: " + int_to_str(ok) + "/4\n" + if ok == 4 { let rep = rep + "ML GATE: PASS\n" } else { let rep = rep + "ML GATE: FAIL\n" } + return rep +} + +println(run_ml_gate())