"""morphology_fr_full.py — production-grade French morphological generator. Same architecture as morphology_it_full.py (shared Romance engine); French-specific data and rules swapped in. Backed by three real, Wiktionary-lineage sources: VERBS UniMorph French (github.com/unimorph/fra, CC-BY-SA 3.0) 7,535 verb lemmas × full paradigm, CLEAN orthography: indicatif présent / imparfait (PST;IPFV) / passé simple (PST;PFV) / futur, conditionnel (COND), subjonctif présent (SBJV;PRS) / subjonctif imparfait (SBJV;PST), impératif (POS;IMP), infinitif (NFIN), participe présent (V.CVB/V.PTCP;PRS), participe passé (V.PTCP;PST, m.sg). fr_irreg_verbs.json — high-frequency verbs UniMorph MISSES or mis-slots, above all ÊTRE (absent from UniMorph fra), plus avoir/aller/faire/… — the auxiliaries the passé-composé + être-agreement system depends on. Extracted from kaikki.org French (build_fr_irreg.py), reflexive/multiword forms dropped. This layer takes PRIORITY. NOUNS + ADJECTIVES — kaikki.org French (Wiktionary extract, CC-BY-SA 3.0) noun lemmas WITH inherent gender (head-template arg) + real plural (cheval->chevaux, œil->yeux, invariable -s/-x/-z), resolved PER LEMMA. adjective lemmas with real feminine + plural (petit->petite/petits/petites, beau->belle/beaux/belles, heureux->heureuse, rouge invariant-gender). Fallbacks (degrade, never crash, on OOV input): verbs : rule generator for -er / -ir(-iss-) / -re (with -cer/-ger spelling, future/conditional stems, imparfait/subjonctif endings) nouns : gender heuristic (endings) + rule pluralization (-al->-aux, -eau->-eaux) adjs : fem/plural agreement rules (-er->-ère, -eux->-euse, -f->-ve, +e default) Confidence flag on every form: "lexicon" | "rule" | "fallback". Public API (used by realizer_fr.py): identical signature to morphology_it_full. """ import json import os import pickle _HERE = os.path.dirname(os.path.abspath(__file__)) _UNIMORPH = os.path.join(_HERE, "data", "fra.unimorph") _IRREG = os.path.join(_HERE, "data", "fr_irreg_verbs.json") _KAIKKI = os.path.join(_HERE, "data", "kaikki_fr.jsonl") _CACHE = os.path.join(_HERE, "data", "fr_morph_cache.pkl") # ── (mood, tense) -> UniMorph feature set that must ALL be present ──────────────── _VERB_KEYMAP = { ("ind", "present"): {"IND", "PRS"}, ("ind", "imperfect"): {"IND", "PST", "IPFV"}, # imparfait ("ind", "passe_simple"): {"IND", "PST", "PFV"}, # passé simple ("ind", "future"): {"IND", "FUT"}, ("ind", "conditional"): {"COND"}, # French: V;COND;1;SG ("sbjv", "present"): {"SBJV", "PRS"}, ("sbjv", "imperfect"): {"SBJV", "PST"}, ("imp", "affirmative"): {"POS", "IMP"}, } _PERSON = {"first": "1", "second": "2", "third": "3"} _NUMBER = {"singular": "SG", "plural": "PL"} def _feat_set(tag): return set(tag.split(";")) # ── build verb lexicon from UniMorph ───────────────────────────────────────────── def _build_verbs(): verbs = {} part = {} ger = {} with open(_UNIMORPH, encoding="utf-8") as fh: for line in fh: line = line.rstrip("\n") if not line or "\t" not in line: continue parts = line.split("\t") if len(parts) != 3: continue lemma, form, tag = parts f = _feat_set(tag) head = tag.split(";")[0] if head == "V.PTCP": if "PST" in f: part.setdefault(lemma, form) elif "PRS" in f: ger.setdefault(lemma, form) continue if head == "V.CVB": if "PRS" in f: ger.setdefault(lemma, form) continue if head != "V": continue person = next((p for p in ("1", "2", "3") if p in f), None) number = "SG" if "SG" in f else ("PL" if "PL" in f else None) if person is None or number is None: continue for (mood, tense), req in _VERB_KEYMAP.items(): if not req <= f: continue if tense == "imperfect" and "PFV" in f: continue if tense == "passe_simple" and "IPFV" in f: continue verbs.setdefault((lemma, f"{mood}|{tense}|{person}|{number}"), form) break return verbs, part, ger # ── kaikki nouns + adjectives ──────────────────────────────────────────────────── _EXCL_FORM_TAGS = {"alternative", "archaic", "obsolete", "dialectal", "regional", "diminutive", "augmentative", "pejorative", "comparative", "superlative", "misspelling", "rare", "informal", "literary", "poetic", "error-unrecognized-form", "construed", "collective", "nonstandard", "dated", "Louisiana", "Switzerland", "Belgium"} def _kaikki_gender(arg): if not arg: return None a = str(arg).lower() if a.startswith("f"): return "f" if a.startswith("m"): return "m" return None def _build_nouns_adjs(): nouns = {} adjs = {} with open(_KAIKKI, encoding="utf-8") as fh: for line in fh: try: d = json.loads(line) except Exception: continue pos = d.get("pos") word = d.get("word", "") if not word or " " in word: continue forms = d.get("forms", []) or [] if pos == "noun": ht = d.get("head_templates") or [] g = None if ht: g = _kaikki_gender((ht[0].get("args") or {}).get("1")) if g is None: tags = d.get("tags") or [] if "feminine" in tags: g = "f" elif "masculine" in tags: g = "m" pl = None for x in forms: t = set(x.get("tags") or []) if "plural" in t and not (t & _EXCL_FORM_TAGS): fm = x.get("form") if fm and " " not in fm and fm not in ("#", "-", "—"): pl = fm break if word not in nouns: nouns[word] = {"g": g, "SG": word, "PL": pl} else: cur = nouns[word] if cur.get("g") is None and g: cur["g"] = g if not cur.get("PL") and pl: cur["PL"] = pl elif pos == "adj": d0 = adjs.setdefault(word, {}) d0.setdefault(("m", "SG"), word) for x in forms: t = set(x.get("tags") or []) fm = x.get("form") if not fm or " " in fm or (t & _EXCL_FORM_TAGS): continue if "feminine" in t and "plural" in t: d0[("f", "PL")] = d0.get(("f", "PL")) or fm elif "masculine" in t and "plural" in t: d0[("m", "PL")] = d0.get(("m", "PL")) or fm elif "feminine" in t: d0[("f", "SG")] = d0.get(("f", "SG")) or fm elif "plural" in t: d0[("m", "PL")] = d0.get(("m", "PL")) or fm return nouns, adjs def _build_cache(): verbs, part, ger = _build_verbs() nouns, adjs = _build_nouns_adjs() with open(_IRREG, encoding="utf-8") as fh: irreg = json.load(fh) data = {"verbs": verbs, "part": part, "ger": ger, "nouns": nouns, "adjs": adjs, "irreg": irreg} try: with open(_CACHE, "wb") as fh: pickle.dump(data, fh, protocol=pickle.HIGHEST_PROTOCOL) except OSError: pass return data def _load(): if os.path.exists(_CACHE): srcs = [_UNIMORPH, _KAIKKI, _IRREG] newest = max(os.path.getmtime(s) for s in srcs if os.path.exists(s)) if os.path.getmtime(_CACHE) >= newest: try: with open(_CACHE, "rb") as fh: return pickle.load(fh) except Exception: pass return _build_cache() _LEX = _load() _VERBS, _PART, _GER, _NOUNS, _ADJS, _IRREGV = ( _LEX["verbs"], _LEX["part"], _LEX["ger"], _LEX["nouns"], _LEX["adjs"], _LEX["irreg"]) # ── regular-ending rule fallback ───────────────────────────────────────────────── def _vclass(lemma): if lemma.endswith("er"): return "er" if lemma.endswith("ir"): return "ir" if lemma.endswith("re"): return "re" if lemma.endswith("oir"): return "oir" return None # present-tense endings [1sg,2sg,3sg,1pl,2pl,3pl] _REG_PRES = { "er": ["e", "es", "e", "ons", "ez", "ent"], "ir": ["is", "is", "it", "issons", "issez", "issent"], # -iss- class (finir) "re": ["s", "s", "", "ons", "ez", "ent"], # vendre: vends/vend } _REG_IMPF = ["ais", "ais", "ait", "ions", "iez", "aient"] # attaches to pres-1pl stem _REG_SUBJ = ["e", "es", "e", "ions", "iez", "ent"] # attaches to 3pl stem _REG_PS = { # passé simple "er": ["ai", "as", "a", "âmes", "âtes", "èrent"], "ir": ["is", "is", "it", "îmes", "îtes", "irent"], "re": ["is", "is", "it", "îmes", "îtes", "irent"], } _FUT = ["ai", "as", "a", "ons", "ez", "ont"] _COND = ["ais", "ais", "ait", "ions", "iez", "aient"] def _slot_idx(person, number): base = {"first": 0, "second": 1, "third": 2}[person] return base + (0 if number == "singular" else 3) def _fut_stem(lemma, vc): """Future/conditional stem = infinitive (drop final -e of -re).""" if vc == "re": return lemma[:-1] # vendre -> vendr- return lemma # parler-, finir- def _pres_1pl_stem(lemma, vc): """Imparfait stem = present 1pl minus -ons (parlons->parl-, finissons->finiss-).""" if vc == "er": stem = lemma[:-2] if stem.endswith("g"): return stem + "e" # mangeons -> mange- (imparfait mangeais) if stem.endswith("c"): return stem[:-1] + "ç" # commençons -> commenç- return stem if vc == "ir": return lemma[:-1] + "iss" # finir -> finiss- if vc == "re": return lemma[:-2] # vendre -> vend- return lemma[:-2] def _apply_er_spelling(stem, ending): """-cer/-ger softening before a/o (commençons, mangeons).""" if ending and ending[0] in ("a", "o"): if stem.endswith("c"): return stem[:-1] + "ç" + ending if stem.endswith("g"): return stem + "e" + ending return stem + ending def _rule_conjugate(lemma, mood, tense, person, number): vc = _vclass(lemma) if vc is None: return None i = _slot_idx(person, number) if mood == "ind" and tense in ("future", "conditional"): stem = _fut_stem(lemma, vc) end = (_FUT if tense == "future" else _COND)[i] return stem + end if mood == "ind" and tense == "present": table = _REG_PRES.get("ir" if vc == "ir" else vc) if not table: return None body = lemma[:-2] if vc in ("er", "re") else lemma[:-1] if vc == "ir" else lemma[:-2] if vc == "ir": body = lemma[:-2] # fin- ; endings carry -iss- end = table[i] return body + end end = table[i] if vc == "er": return _apply_er_spelling(body, end) return body + end if mood == "ind" and tense == "imperfect": stem = _pres_1pl_stem(lemma, vc) return stem + _REG_IMPF[i] if mood == "ind" and tense == "passe_simple": table = _REG_PS.get("ir" if vc == "ir" else vc) if not table: return None body = lemma[:-2] if vc in ("er", "re") else lemma[:-2] end = table[i] if vc == "er": return _apply_er_spelling(body, end) return body + end if mood == "sbjv" and tense == "present": # subjonctif: present-3pl stem + e/es/e/ions/iez/ent stem3 = _pres_1pl_stem(lemma, vc) if vc == "ir" else ( lemma[:-2] if vc in ("er", "re") else lemma[:-2]) if vc == "ir": stem3 = lemma[:-2] + "iss" end = _REG_SUBJ[i] if vc == "er": return _apply_er_spelling(stem3, end) return stem3 + end if mood == "imp" and tense == "affirmative": # impératif ~ present indicative (tu drops -s for -er verbs) pres = _rule_conjugate(lemma, "ind", "present", person, number) if pres and vc == "er" and person == "second" and number == "singular": return pres[:-1] if pres.endswith("es") else pres return pres return None # ── PUBLIC: verb conjugation ───────────────────────────────────────────────────── def conjugate(lemma, mood, tense, person, number): """Return (surface, confidence).""" lemma = lemma.strip().lower() key = f"{mood}|{tense}|{_PERSON.get(person,'?')}|{number}" ir = _IRREGV.get(lemma) if ir and key in ir: return ir[key], "lexicon" p, n = _PERSON.get(person), _NUMBER.get(number) if p and n: form = _VERBS.get((lemma, f"{mood}|{tense}|{p}|{n}")) if form: return form, "lexicon" r = _rule_conjugate(lemma, mood, tense, person, number) if r: return r, "rule" return lemma, "fallback" # ── PUBLIC: participle + gerund/participe présent ──────────────────────────────── def _participle_msg(lemma): ir = _IRREGV.get(lemma) if ir and "part" in ir: return ir["part"], "lexicon" if lemma in _PART: return _PART[lemma], "lexicon" return None, None # irregular participle fem/plural quirks (drop circonflexe: dû->due, dus) _PART_FIX = {"dû": {"f|SG": "due", "m|PL": "dus", "f|PL": "dues"}} def participle(lemma, gender="m", number="singular"): """Past participle with French gender/number agreement. m.sg = base; f.sg = base+e; m.pl = base+s (invariable if base ends s/x); f.pl = f.sg+s.""" lemma = lemma.strip().lower() g = "f" if gender == "f" else "m" num = "SG" if number == "singular" else "PL" msg, src = _participle_msg(lemma) conf = "lexicon" if msg is None: vc = _vclass(lemma) if vc == "er": msg = lemma[:-2] + "é" elif vc == "ir": msg = lemma[:-1] # finir -> fini, partir -> parti elif vc == "re": msg = lemma[:-2] + "u" # vendre -> vendu elif vc == "oir": msg = lemma[:-3] + "u" # (rough) recevoir handled by irreg else: return lemma, "fallback" conf = "rule" fix = _PART_FIX.get(msg) if fix and f"{g}|{num}" in fix: return fix[f"{g}|{num}"], conf if g == "m" and num == "SG": return msg, conf fem = msg + "e" if not msg.endswith("e") else msg if g == "f" and num == "SG": return fem, conf if g == "m" and num == "PL": return msg if msg.endswith(("s", "x")) else msg + "s", conf # f|PL return fem + "s", conf def gerund(lemma): """Participe présent (base for gérondif 'en -ant').""" lemma = lemma.strip().lower() ir = _IRREGV.get(lemma) if ir and "ger" in ir: return ir["ger"], "lexicon" if lemma in _GER: return _GER[lemma], "lexicon" vc = _vclass(lemma) if vc == "er": stem = lemma[:-2] if stem.endswith("g"): return stem + "eant", "rule" if stem.endswith("c"): return stem[:-1] + "çant", "rule" return stem + "ant", "rule" if vc == "ir": return lemma[:-2] + "issant", "rule" if vc == "re": return lemma[:-2] + "ant", "rule" return lemma, "fallback" # ── PUBLIC: noun gender + number ───────────────────────────────────────────────── _FEM_SUF = ("tion", "sion", "aison", "ance", "ence", "ette", "elle", "esse", "ude", "ade", "ée", "té", "tié", "ie", "ise", "ure", "eur") _MASC_SUF = ("ment", "age", "eau", "isme", "oir", "ier", "eur", "in", "on") def _gender_heuristic(noun): for suf in _FEM_SUF: if noun.endswith(suf): return "f" for suf in _MASC_SUF: if noun.endswith(suf): return "m" if noun.endswith("e"): return "f" return "m" def noun_gender(lemma): lemma = lemma.strip().lower() d = _NOUNS.get(lemma) if d and d.get("g") in ("m", "f"): return d["g"] return _gender_heuristic(lemma) # closed sets for French plural irregularities _OU_X = {"bijou", "caillou", "chou", "genou", "hibou", "joujou", "pou"} _AIL_AUX = {"travail", "vitrail", "corail", "émail", "bail", "soupirail", "vantail"} _AL_S = {"bal", "carnaval", "festival", "récital", "chacal", "régal", "cal", "aval"} def _rule_plural(noun, gender): """Deterministic French pluralization. (form, ok); ok=False FLAGS ambiguity.""" if not noun: return noun, True if noun[-1:] in ("s", "x", "z"): return noun, True # invariable if noun in _OU_X: return noun + "x", True if noun.endswith(("eau", "au", "eu")): if noun in ("pneu", "bleu", "landau", "sarrau"): return noun + "s", True return noun + "x", True # bateau->bateaux, jeu->jeux if noun.endswith("al"): if noun in _AL_S: return noun + "s", True return noun[:-2] + "aux", True # cheval->chevaux if noun.endswith("ail"): if noun in _AIL_AUX: return noun[:-3] + "aux", True # travail->travaux return noun + "s", True return noun + "s", True # default def inflect_noun(lemma, number, gender=None): lemma = lemma.strip().lower() d = _NOUNS.get(lemma) if number == "singular": return (d["SG"] if d and d.get("SG") else lemma), ("lexicon" if d else "rule") if d and d.get("PL"): return d["PL"], "lexicon" g = gender or noun_gender(lemma) form, ok = _rule_plural(lemma, g) return form, ("rule" if ok else "fallback") # adjectives whose kaikki entries are unreliable: audited forms _ADJ_FIX = { "beau": {("m", "SG"): "beau", ("f", "SG"): "belle", ("m", "PL"): "beaux", ("f", "PL"): "belles"}, "nouveau": {("m", "SG"): "nouveau", ("f", "SG"): "nouvelle", ("m", "PL"): "nouveaux", ("f", "PL"): "nouvelles"}, "vieux": {("m", "SG"): "vieux", ("f", "SG"): "vieille", ("m", "PL"): "vieux", ("f", "PL"): "vieilles"}, "fou": {("m", "SG"): "fou", ("f", "SG"): "folle", ("m", "PL"): "fous", ("f", "PL"): "folles"}, "blanc": {("m", "SG"): "blanc", ("f", "SG"): "blanche", ("m", "PL"): "blancs", ("f", "PL"): "blanches"}, "long": {("m", "SG"): "long", ("f", "SG"): "longue", ("m", "PL"): "longs", ("f", "PL"): "longues"}, "bon": {("m", "SG"): "bon", ("f", "SG"): "bonne", ("m", "PL"): "bons", ("f", "PL"): "bonnes"}, } def _rule_fem(a): if a.endswith("e"): return a if a.endswith("er"): return a[:-2] + "ère" if a.endswith("eau"): return a[:-3] + "elle" if a.endswith("eux"): return a[:-3] + "euse" if a.endswith("f"): return a[:-1] + "ve" if a.endswith(("on", "en", "el", "eil", "et")): return a + a[-1] + "e" # bon->bonne, ancien->ancienne, muet->muette if a.endswith("c"): return a[:-1] + "che" # blanc->blanche (public->publique via FIX) return a + "e" # grand->grande, petit->petite, vert->verte def inflect_adj(lemma, gender, number): lemma = lemma.strip().lower() g = "f" if gender == "f" else "m" num = "SG" if number == "singular" else "PL" fix = _ADJ_FIX.get(lemma) if fix and (g, num) in fix: return fix[(g, num)], "lexicon" d = _ADJS.get(lemma) if d and d.get((g, num)): return d[(g, num)], "lexicon" # derive msc = (d.get(("m", "SG")) if d else None) or lemma if g == "m" and num == "SG": return msc, "lexicon" if d else "rule" fem = (d.get(("f", "SG")) if d else None) or _rule_fem(msc) if g == "f" and num == "SG": return fem, "lexicon" if (d and d.get(("f", "SG"))) else "rule" if g == "m" and num == "PL": if msc.endswith(("s", "x")): return msc, "rule" if msc.endswith("al"): return msc[:-2] + "aux", "rule" if msc.endswith("eau"): return msc + "x", "rule" return msc + "s", "rule" # f|PL return (fem if fem.endswith("s") else fem + "s"), "rule" def lexicon_stats(): return { "verb_source": "UniMorph French (github.com/unimorph/fra) + kaikki.org " "irregulars (être + high-frequency)", "noun_adj_source": "kaikki.org French (Wiktionary extract)", "license": "CC-BY-SA 3.0 (Wiktionary/UniMorph lineage)", "unimorph_verb_forms": len(_VERBS), "unimorph_verb_lemmas": len({k[0] for k in _VERBS}), "irregular_verb_lemmas": len(_IRREGV), "participle_lemmas": len(_PART), "gerund_lemmas": len(_GER), "noun_lemmas": len(_NOUNS), "adj_lemmas": len(_ADJS), } if __name__ == "__main__": print(json.dumps(lexicon_stats(), indent=2, ensure_ascii=False)) tests = [ ("parler", "ind", "present", "first", "singular", "parle"), ("être", "ind", "present", "third", "singular", "est"), ("avoir", "ind", "present", "first", "singular", "ai"), ("aller", "ind", "present", "third", "plural", "vont"), ("finir", "ind", "present", "first", "singular", "finis"), ("finir", "ind", "present", "first", "plural", "finissons"), ("manger", "ind", "present", "first", "plural", "mangeons"), ("faire", "ind", "future", "first", "singular", "ferai"), ("pouvoir", "sbjv", "present", "third", "singular", "puisse"), ("prendre", "ind", "passe_simple", "third", "singular", "prit"), ("vendre", "ind", "present", "third", "singular", "vend"), ("commencer", "ind", "imperfect", "first", "singular", "commençais"), ] ok = 0 for lemma, mood, tense, per, num, exp in tests: got, conf = conjugate(lemma, mood, tense, per, num) flag = "OK " if got == exp else "XX " ok += got == exp print(f" {flag}{lemma:10} {mood}/{tense:12} {per[:3]}.{num[:2]} -> {got:12} ({conf}) exp={exp}") print(f"verb tests {ok}/{len(tests)}") print(" gender: maison=", noun_gender("maison"), "chat=", noun_gender("chat"), "cheval=", noun_gender("cheval"), "nation=", noun_gender("nation")) print(" plural: cheval->", inflect_noun("cheval", "plural"), "| bateau->", inflect_noun("bateau", "plural"), "| prix->", inflect_noun("prix", "plural"), "| chat->", inflect_noun("chat", "plural")) print(" adj: petit/f/sg->", inflect_adj("petit", "f", "singular"), "| beau/f/sg->", inflect_adj("beau", "f", "singular"), "| heureux/f/sg->", inflect_adj("heureux", "f", "singular"), "| national/m/pl->", inflect_adj("national", "m", "plural")) print(" part: aller/f/sg->", participle("aller", "f", "singular"), "| prendre/f/pl->", participle("prendre", "f", "plural"), "| finir/m/pl->", participle("finir", "m", "plural")) print(" ger: manger->", gerund("manger"), "| finir->", gerund("finir"))