"""morphology_it_full.py — production-grade Italian morphological generator. NOT a toy. Backed by three real, Wiktionary-lineage lexical sources: VERBS UniMorph Italian (github.com/unimorph/ita, CC-BY-SA 3.0) 10,009 verb lemmas × full paradigm, CLEAN orthography (no stress marks): indicative present / imperfetto (PST;IPFV) / passato remoto (PST;PFV) / futuro, condizionale (COND), congiuntivo presente (SBJV;PRS) / imperfetto (SBJV;PST), affirmative imperative, infinitive, gerundio (V.CVB;PRS), past participle (masc-sg; fem/plural derived by vowel rule). it_irreg_verbs.json — 66 high-frequency verbs UniMorph MISSES (essere, avere, potere, uscire, tenere, prendere, piacere, …), extracted from kaikki.org Italian, filtered to standard forms, and DE-STRESSED to real orthography (kaikki marks tonic stress everywhere: pàrlo->parlo, avùto->avuto; final legit accents kept: sarò, è). Built by build_it_irreg.py. This layer takes priority — it supplies the two auxiliaries essere/avere, which the whole passato-prossimo / essere-agreement system depends on. NOUNS + ADJECTIVES — kaikki.org Italian (Wiktionary extract, CC-BY-SA 3.0) noun lemmas WITH inherent gender (head-template arg) + real (often irregular) plural — uomo->uomini, uovo->uova, dito->dita, città invariant — resolved PER LEMMA, never guessed. adjective lemmas with real feminine + masc/fem plural (italiano->italiana/ italiani/italiane, felice->felici invariant). Fallbacks (degrade, never crash, on OOV input): verbs : rule generator for regular -are/-ere/-ire (with -care/-gare h-insertion and -ciare/-giare/-iare i-drop spelling rules) nouns : gender heuristic (endings) + rule pluralization (ambiguous -co/-go FLAGGED) adjs : -o/-a/-e gender rule + rule pluralization Confidence flag on every form: "lexicon" from UniMorph / kaikki-irregular / kaikki noun-adj (trust: high) "rule" deterministic rule (trust: medium) "fallback" could not inflect; returned lemma / ambiguous (trust: low -> FLAG) Public API (used by realizer_it.py): conjugate(lemma, mood, tense, person, number) -> (form, conf) participle(lemma, gender="m", number="singular") -> (form, conf) gerund(lemma) -> (form, conf) noun_gender(lemma) -> "m"|"f" inflect_noun(lemma, number, gender=None) -> (form, conf) inflect_adj(lemma, gender, number) -> (form, conf) lexicon_stats() -> dict """ import json import os import pickle _HERE = os.path.dirname(os.path.abspath(__file__)) _UNIMORPH = os.path.join(_HERE, "data", "ita.unimorph") _IRREG = os.path.join(_HERE, "data", "it_irreg_verbs.json") _KAIKKI = os.path.join(_HERE, "data", "kaikki_it.jsonl") _CACHE = os.path.join(_HERE, "data", "it_morph_cache.pkl") # ── (mood, tense) -> UniMorph feature set that must ALL be present ──────────────── _VERB_KEYMAP = { ("ind", "present"): {"IND", "PRS"}, ("ind", "imperfect"): {"IND", "PST", "IPFV"}, ("ind", "passato_remoto"): {"IND", "PST", "PFV"}, ("ind", "future"): {"IND", "FUT"}, ("ind", "conditional"): {"COND"}, ("sbjv", "present"): {"SBJV", "PRS"}, ("sbjv", "imperfect"): {"SBJV", "PST"}, ("imp", "affirmative"): {"POS", "IMP"}, } _PERSON = {"first": "1", "second": "2", "third": "3"} _NUMBER = {"singular": "SG", "plural": "PL"} def _feat_set(tag): return set(tag.split(";")) # ── build verb lexicon from UniMorph ───────────────────────────────────────────── def _build_verbs(): verbs = {} # (lemma, "mood|tense|person|number") -> form part = {} # lemma -> masc-sg past participle ger = {} # lemma -> gerundio with open(_UNIMORPH, encoding="utf-8") as fh: for line in fh: line = line.rstrip("\n") if not line or "\t" not in line: continue parts = line.split("\t") if len(parts) != 3: continue lemma, form, tag = parts f = _feat_set(tag) head = tag.split(";")[0] if head == "V.PTCP": if "PST" in f: part.setdefault(lemma, form) continue if head == "V.CVB": # gerundio (converb, present) if "PRS" in f: ger.setdefault(lemma, form) continue if head != "V": continue person = next((p for p in ("1", "2", "3") if p in f), None) number = "SG" if "SG" in f else ("PL" if "PL" in f else None) if person is None or number is None: continue for (mood, tense), req in _VERB_KEYMAP.items(): # exact-set discipline: PST;PFV must not match PST;IPFV, etc. if not req <= f: continue # guard IND;PST ambiguity: require the specific aspect feature if tense == "imperfect" and "PFV" in f: continue if tense == "passato_remoto" and "IPFV" in f: continue # COND must not also be a subjunctive/imperative slot verbs.setdefault((lemma, f"{mood}|{tense}|{person}|{number}"), form) break return verbs, part, ger # ── kaikki nouns + adjectives ──────────────────────────────────────────────────── _EXCL_FORM_TAGS = {"alternative", "archaic", "obsolete", "dialectal", "regional", "diminutive", "augmentative", "pejorative", "comparative", "superlative", "misspelling", "rare", "informal", "literary", "poetic", "error-unrecognized-form", "apocopic", "obsolete", "construed", "collective"} def _kaikki_gender(arg): if not arg: return None a = str(arg).lower() if a.startswith("f"): return "f" if a.startswith("m"): return "m" return None def _build_nouns_adjs(): nouns = {} # lemma -> {"g","SG","PL"} adjs = {} # lemma -> {("m","SG"),("f","SG"),("m","PL"),("f","PL")} with open(_KAIKKI, encoding="utf-8") as fh: for line in fh: try: d = json.loads(line) except Exception: continue pos = d.get("pos") word = d.get("word", "") if not word or " " in word: continue forms = d.get("forms", []) or [] if pos == "noun": ht = d.get("head_templates") or [] g = None if ht: g = _kaikki_gender((ht[0].get("args") or {}).get("1")) if g is None: tags = d.get("tags") or [] if "feminine" in tags: g = "f" elif "masculine" in tags: g = "m" pl = None for x in forms: t = set(x.get("tags") or []) if "plural" in t and not (t & _EXCL_FORM_TAGS): fm = x.get("form") if fm and " " not in fm and fm != "#": pl = fm break if word not in nouns: nouns[word] = {"g": g, "SG": word, "PL": pl} else: cur = nouns[word] if cur.get("g") is None and g: cur["g"] = g if not cur.get("PL") and pl: cur["PL"] = pl elif pos == "adj": d0 = adjs.setdefault(word, {}) d0.setdefault(("m", "SG"), word) for x in forms: t = set(x.get("tags") or []) fm = x.get("form") if not fm or " " in fm or (t & _EXCL_FORM_TAGS): continue if "feminine" in t and "plural" in t: d0[("f", "PL")] = d0.get(("f", "PL")) or fm elif "masculine" in t and "plural" in t: d0[("m", "PL")] = d0.get(("m", "PL")) or fm elif "feminine" in t: d0[("f", "SG")] = d0.get(("f", "SG")) or fm elif "plural" in t: # invariant-gender adj (felice -> felici) d0[("m", "PL")] = d0.get(("m", "PL")) or fm d0[("f", "PL")] = d0.get(("f", "PL")) or fm return nouns, adjs def _build_cache(): verbs, part, ger = _build_verbs() nouns, adjs = _build_nouns_adjs() with open(_IRREG, encoding="utf-8") as fh: irreg = json.load(fh) data = {"verbs": verbs, "part": part, "ger": ger, "nouns": nouns, "adjs": adjs, "irreg": irreg} try: with open(_CACHE, "wb") as fh: pickle.dump(data, fh, protocol=pickle.HIGHEST_PROTOCOL) except OSError: pass return data def _load(): if os.path.exists(_CACHE): srcs = [_UNIMORPH, _KAIKKI, _IRREG] newest = max(os.path.getmtime(s) for s in srcs if os.path.exists(s)) if os.path.getmtime(_CACHE) >= newest: try: with open(_CACHE, "rb") as fh: return pickle.load(fh) except Exception: pass return _build_cache() _LEX = _load() _VERBS, _PART, _GER, _NOUNS, _ADJS, _IRREGV = ( _LEX["verbs"], _LEX["part"], _LEX["ger"], _LEX["nouns"], _LEX["adjs"], _LEX["irreg"]) # ── regular-ending rule fallback ───────────────────────────────────────────────── def _vclass(lemma): if lemma.endswith("are"): return "are" if lemma.endswith("ere"): return "ere" if lemma.endswith("ire"): return "ire" return None # endings [1sg,2sg,3sg,1pl,2pl,3pl] _REG = { ("ind", "present", "are"): ["o", "i", "a", "iamo", "ate", "ano"], ("ind", "present", "ere"): ["o", "i", "e", "iamo", "ete", "ono"], ("ind", "present", "ire"): ["o", "i", "e", "iamo", "ite", "ono"], ("ind", "imperfect", "are"): ["avo", "avi", "ava", "avamo", "avate", "avano"], ("ind", "imperfect", "ere"): ["evo", "evi", "eva", "evamo", "evate", "evano"], ("ind", "imperfect", "ire"): ["ivo", "ivi", "iva", "ivamo", "ivate", "ivano"], ("ind", "passato_remoto", "are"): ["ai", "asti", "ò", "ammo", "aste", "arono"], ("ind", "passato_remoto", "ere"): ["ei", "esti", "é", "emmo", "este", "erono"], ("ind", "passato_remoto", "ire"): ["ii", "isti", "ì", "immo", "iste", "irono"], ("sbjv", "present", "are"): ["i", "i", "i", "iamo", "iate", "ino"], ("sbjv", "present", "ere"): ["a", "a", "a", "iamo", "iate", "ano"], ("sbjv", "present", "ire"): ["a", "a", "a", "iamo", "iate", "ano"], ("sbjv", "imperfect", "are"): ["assi", "assi", "asse", "assimo", "aste", "assero"], ("sbjv", "imperfect", "ere"): ["essi", "essi", "esse", "essimo", "este", "essero"], ("sbjv", "imperfect", "ire"): ["issi", "issi", "isse", "issimo", "iste", "issero"], # imperative: 2sg,3sg(Lei),1pl,2pl,3pl (1sg has none) ("imp", "affirmative", "are"): [None, "a", "i", "iamo", "ate", "ino"], ("imp", "affirmative", "ere"): [None, "i", "a", "iamo", "ete", "ano"], ("imp", "affirmative", "ire"): [None, "i", "a", "iamo", "ite", "ano"], } # future / conditional attach to a stem = infinitive minus final -e, with # -are -> -er (parlare->parler-), -ere/-ire keep (credere->creder-, dormir-) _FUT = ["ò", "ai", "à", "emo", "ete", "anno"] _COND = ["ei", "esti", "ebbe", "emmo", "este", "ebbero"] def _slot_idx(person, number): base = {"first": 0, "second": 1, "third": 2}[person] return base + (0 if number == "singular" else 3) def _fut_stem(lemma, vc): body = lemma[:-3] # drop are/ere/ire if vc == "are": return body + "er" return body + vc[0] + "r" # ere->er? no: keep vowel: creder-, dormir- # NOTE corrected below def _apply_are_spelling(stem, ending): """-care/-gare insert h before front endings; -ciare/-giare/-sciare/-iare drop i.""" front = ending[:1] in ("i", "e") if stem.endswith(("c", "g")) and front: return stem + "h" + ending if stem.endswith(("ci", "gi", "sci")) and ending[:1] == "i": return stem[:-1] + ending # mangi+iamo -> mangiamo if stem.endswith("i") and ending[:1] == "i": return stem[:-1] + ending # studi+iamo -> studiamo return stem + ending def _rule_conjugate(lemma, mood, tense, person, number): vc = _vclass(lemma) if vc is None: return None body = lemma[:-3] i = _slot_idx(person, number) if mood == "ind" and tense in ("future", "conditional"): stem = body + "er" if vc == "are" else body + vc[0] + "r" # ere: creder-, ire: dormir- -> body + 'e'/'i' + 'r' if vc == "ere": stem = body + "er" elif vc == "ire": stem = body + "ir" end = (_FUT if tense == "future" else _COND)[i] # spelling: -care/-gare -> cherò/gherò ; -ciare/-giare -> cerò/gerò if vc == "are": if body.endswith(("c", "g")): stem = body + "her" elif body.endswith(("ci", "gi", "sci")): stem = body[:-1] + "er" elif body.endswith("i"): stem = body[:-1] + "er" return stem + end table = _REG.get((mood, tense, vc)) if not table: return None end = table[i] if end is None: return None if vc == "are": return _apply_are_spelling(body, end) # -ere/-ire: guard against double-i (dormi+iamo -> dormiamo) if body.endswith("i") and end[:1] == "i": return body[:-1] + end return body + end # ── PUBLIC: verb conjugation ───────────────────────────────────────────────────── def conjugate(lemma, mood, tense, person, number): """Return (surface, confidence). mood in ind|sbjv|imp; tense per _VERB_KEYMAP.""" lemma = lemma.strip().lower() key = f"{mood}|{tense}|{_PERSON.get(person,'?')}|{number}" ir = _IRREGV.get(lemma) if ir and key in ir: return ir[key], "lexicon" p, n = _PERSON.get(person), _NUMBER.get(number) if p and n: form = _VERBS.get((lemma, f"{mood}|{tense}|{p}|{n}")) if form: return form, "lexicon" r = _rule_conjugate(lemma, mood, tense, person, number) if r: return r, "rule" return lemma, "fallback" # ── PUBLIC: participle + gerund ────────────────────────────────────────────────── def _participle_msg(lemma): """Return (masc-sg participle, source) or (None, None).""" ir = _IRREGV.get(lemma) if ir and "part" in ir: return ir["part"], "lexicon" if lemma in _PART: return _PART[lemma], "lexicon" return None, None def participle(lemma, gender="m", number="singular"): """Past participle with gender/number agreement (for essere-perfect & passives). UniMorph/irregular give masc-sg; fem/plural derived by final-vowel swap (-o -> -a/-i/-e), valid for regular -ato/-uto/-ito AND irregulars (preso->presa/presi/prese, aperto->aperta/aperti/aperte, morto->morta/...).""" lemma = lemma.strip().lower() g = "f" if gender == "f" else "m" num = "SG" if number == "singular" else "PL" msg, src = _participle_msg(lemma) conf = "lexicon" if msg is None: vc = _vclass(lemma) if vc == "are": msg = lemma[:-3] + "ato" elif vc == "ere": msg = lemma[:-3] + "uto" elif vc == "ire": msg = lemma[:-3] + "ito" else: return lemma, "fallback" conf = "rule" # agreement: only -o participles inflect for gender+number if msg.endswith("o"): stem = msg[:-1] suf = {"m|SG": "o", "f|SG": "a", "m|PL": "i", "f|PL": "e"}[f"{g}|{num}"] return stem + suf, conf return msg, conf # non -o participle: leave as-is (rare) def gerund(lemma): lemma = lemma.strip().lower() ir = _IRREGV.get(lemma) if ir and "ger" in ir: return ir["ger"], "lexicon" if lemma in _GER: return _GER[lemma], "lexicon" vc = _vclass(lemma) if vc == "are": return lemma[:-3] + "ando", "rule" if vc in ("ere", "ire"): return lemma[:-3] + "endo", "rule" return lemma, "fallback" # ── PUBLIC: noun gender + number ───────────────────────────────────────────────── _FEM_SUF = ("zione", "sione", "gione", "tà", "tù", "trice", "aggine", "udine", "igine", "ie", "essa", "izia", "ezza") _MASC_SUF = ("ore", "ame", "iere", "ale", "ile") def _gender_heuristic(noun): for suf in _FEM_SUF: if noun.endswith(suf): return "f" for suf in _MASC_SUF: if noun.endswith(suf): return "m" if noun.endswith("o"): return "m" if noun.endswith("a"): return "f" if noun.endswith("à") or noun.endswith("ù"): return "f" return "m" # -e and consonant-final loanwords default masculine def noun_gender(lemma): lemma = lemma.strip().lower() d = _NOUNS.get(lemma) if d and d.get("g") in ("m", "f"): return d["g"] return _gender_heuristic(lemma) def _rule_plural(noun, gender): """Deterministic Italian pluralization. Returns (form, ok); ok=False FLAGS an ambiguous case the lexicon would normally resolve (-co/-go palatalization).""" if not noun: return noun, True # invariant: accented final vowel, consonant-final, monosyllable, -i final if noun[-1:] in ("à", "è", "é", "ì", "í", "ò", "ó", "ù", "ú"): return noun, True if noun[-1:] not in ("a", "e", "o", "i", "u"): return noun, True # consonant-final loanword: invariant if noun.endswith("i"): return noun, True # e.g. crisi, analisi: invariant if noun.endswith("io"): return noun[:-2] + "i", True # figlio->figli (unstressed i) if noun.endswith("cia") or noun.endswith("gia"): # vowel before cia/gia -> -cie/-gie ; consonant -> -ce/-ge (approx) return noun[:-2] + "e", True # arancia->arance (majority) if noun.endswith("ca"): return noun[:-2] + "che", True # amica->amiche if noun.endswith("ga"): return noun[:-2] + "ghe", True if noun.endswith("co"): return noun[:-2] + "chi", False # AMBIGUOUS (amico->amici) -> flag if noun.endswith("go"): return noun[:-2] + "ghi", False # AMBIGUOUS (psicologo->psicologi) if noun.endswith("a"): return noun[:-1] + "e", True # casa->case (m -a: -i, but rare) if noun.endswith("o"): return noun[:-1] + "i", True # libro->libri if noun.endswith("e"): return noun[:-1] + "i", True # cane->cani, chiave->chiavi return noun, True def inflect_noun(lemma, number, gender=None): lemma = lemma.strip().lower() d = _NOUNS.get(lemma) if number == "singular": return (d["SG"] if d and d.get("SG") else lemma), ("lexicon" if d else "rule") if d and d.get("PL"): return d["PL"], "lexicon" g = gender or noun_gender(lemma) form, ok = _rule_plural(lemma, g) return form, ("rule" if ok else "fallback") # adjectives whose kaikki entries are unreliable (messy inflection templates): # supply audited regular agreement forms (prenominal apocope handled in realizer). _ADJ_FIX = { "bello": {("m", "SG"): "bello", ("f", "SG"): "bella", ("m", "PL"): "belli", ("f", "PL"): "belle"}, "quello": {("m", "SG"): "quello", ("f", "SG"): "quella", ("m", "PL"): "quelli", ("f", "PL"): "quelle"}, } # ── PUBLIC: adjective agreement ────────────────────────────────────────────────── def inflect_adj(lemma, gender, number): lemma = lemma.strip().lower() g = "f" if gender == "f" else "m" num = "SG" if number == "singular" else "PL" fix = _ADJ_FIX.get(lemma) if fix and (g, num) in fix: return fix[(g, num)], "lexicon" d = _ADJS.get(lemma) if d: form = d.get((g, num)) if form: return form, "lexicon" sg = d.get((g, "SG")) or d.get(("m", "SG")) or lemma if num == "PL": pl, ok = _rule_plural(sg, g) return pl, ("rule" if ok else "fallback") return sg, "lexicon" # rule fallback a = lemma if a.endswith("o"): # -o/-a/-i/-e class base = a[:-1] suf = {"m|SG": "o", "f|SG": "a", "m|PL": "i", "f|PL": "e"}[f"{g}|{num}"] return base + suf, "rule" if a.endswith("e"): # felice-class: SG invariant, PL -i if num == "PL": return a[:-1] + "i", "rule" return a, "rule" if num == "PL": p, ok = _rule_plural(a, g) return p, ("rule" if ok else "fallback") return a, "rule" def lexicon_stats(): return { "verb_source": "UniMorph Italian (github.com/unimorph/ita) + kaikki.org " "irregulars (de-stressed)", "noun_adj_source": "kaikki.org Italian (Wiktionary extract)", "license": "CC-BY-SA 3.0 (Wiktionary/UniMorph lineage)", "unimorph_verb_forms": len(_VERBS), "unimorph_verb_lemmas": len({k[0] for k in _VERBS}), "irregular_verb_lemmas": len(_IRREGV), "participle_lemmas": len(_PART), "gerund_lemmas": len(_GER), "noun_lemmas": len(_NOUNS), "adj_lemmas": len(_ADJS), } if __name__ == "__main__": print(json.dumps(lexicon_stats(), indent=2, ensure_ascii=False)) tests = [ ("parlare", "ind", "present", "first", "singular", "parlo"), ("essere", "ind", "present", "third", "singular", "è"), ("avere", "ind", "present", "first", "singular", "ho"), ("mangiare", "ind", "present", "second", "singular", "mangi"), ("finire", "ind", "present", "first", "singular", "finisco"), ("andare", "ind", "present", "third", "plural", "vanno"), ("fare", "ind", "future", "first", "singular", "farò"), ("potere", "sbjv", "present", "third", "singular", "possa"), ("prendere", "ind", "passato_remoto", "first", "singular", "presi"), ("cercare", "ind", "present", "second", "singular", "cerchi"), ("dormire", "ind", "present", "third", "plural", "dormono"), ("credere", "ind", "future", "first", "singular", "crederò"), ] ok = 0 for lemma, mood, tense, per, num, exp in tests: got, conf = conjugate(lemma, mood, tense, per, num) flag = "OK " if got == exp else "XX " ok += got == exp print(f" {flag}{lemma:9} {mood}/{tense:14} {per[:3]}.{num[:2]} -> {got:12} ({conf}) exp={exp}") print(f"verb tests {ok}/{len(tests)}") print(" gender: casa=", noun_gender("casa"), "problema=", noun_gender("problema"), "mano=", noun_gender("mano"), "città=", noun_gender("città"), "cane=", noun_gender("cane")) print(" plural: uomo->", inflect_noun("uomo", "plural"), "| uovo->", inflect_noun("uovo", "plural"), "| città->", inflect_noun("città", "plural"), "| amico->", inflect_noun("amico", "plural"), "| casa->", inflect_noun("casa", "plural")) print(" adj: italiano/f/pl->", inflect_adj("italiano", "f", "plural"), "| felice/m/pl->", inflect_adj("felice", "m", "plural"), "| bello/f/sg->", inflect_adj("bello", "f", "singular")) print(" part: aprire/f/sg->", participle("aprire", "f", "singular"), "| prendere/m/pl->", participle("prendere", "m", "plural"), "| andare/f/sg->", participle("andare", "f", "singular")) print(" ger: fare->", gerund("fare"), "| parlare->", gerund("parlare"))