# -*- coding: utf-8 -*- """morphology_ca_full.py — production-grade Catalan morphological generator. Same design as morphology_it_full.py (its Romance sibling); Catalan-specific data. VERBS UniMorph Catalan (github.com/unimorph/cat, CC-BY-SA 3.0) 7,535 verb lemmas × paradigm, CLEAN orthography: present, imperfet (PST;IPFV), pretèrit simple (PST;PFV), futur, condicional (COND), subjuntiu present (SBJV;PRS) / imperfet (SBJV;PST), imperatiu (POS;IMP), infinitiu (NFIN), gerundi (V.CVB;PRS), participi (V.PTCP;PST) — WITH full gender+number agreement forms (cantat/cantada/cantats/cantades) stored directly. ca_irreg_verbs.json — verbs UniMorph MISSES or under-populates (anar, fer, plus core auxiliaries ser/haver/estar/tenir…), extracted from kaikki.org Catalan by build_ca_irreg.py. Priority layer. Supplies anar, whose present (vaig/vas/va/anem/aneu/van) is ALSO the PERIPHRASTIC-PRETERITE auxiliary (vaig cantar = 'I sang') — a hallmark Catalan construction. NOUNS + ADJECTIVES — kaikki.org Catalan (Wiktionary extract, CC-BY-SA 3.0) noun lemmas WITH inherent gender + real plural (resolved PER LEMMA). adjective lemmas with real feminine + plural forms. Fallbacks degrade, never crash: verbs : regular -ar/-er/-re/-ir rule generator (+ -car/-gar/-çar spelling). nouns : gender heuristic + rule pluralization (-a→-es with ç/c/g/j/qu/gu spelling changes; sibilant-final → -os; else -s). Ambiguous → FLAG. adjs : -o? no (Catalan masc often consonant/-e); fem -a rule + plural rule. Confidence flag per form: "lexicon" | "rule" | "fallback" (low → FLAG). Public API (used by realizer_ca.py): conjugate(lemma, mood, tense, person, number) -> (form, conf) peri_pret_aux(person, number) -> form # anar-present, for vaig+INF participle(lemma, gender, number) -> (form, conf) gerund(lemma) -> (form, conf) noun_gender(lemma) -> "m"|"f" inflect_noun(lemma, number, gender=None) -> (form, conf) inflect_adj(lemma, gender, number) -> (form, conf) lexicon_stats() -> dict """ import json import os import pickle _HERE = os.path.dirname(os.path.abspath(__file__)) _UNIMORPH = os.path.join(_HERE, "data", "cat.unimorph") _IRREG = os.path.join(_HERE, "data", "ca_irreg_verbs.json") _KAIKKI = os.path.join(_HERE, "data", "kaikki_ca.jsonl") _CACHE = os.path.join(_HERE, "data", "ca_morph_cache.pkl") _VERB_KEYMAP = { ("ind", "present"): {"IND", "PRS"}, ("ind", "imperfect"): {"IND", "PST", "IPFV"}, ("ind", "preterite"): {"IND", "PST", "PFV"}, ("ind", "future"): {"IND", "FUT"}, ("ind", "conditional"): {"COND"}, ("sbjv", "present"): {"SBJV", "PRS"}, ("sbjv", "imperfect"): {"SBJV", "PST"}, ("imp", "affirmative"): {"POS", "IMP"}, } _PERSON = {"first": "1", "second": "2", "third": "3"} _NUMBER = {"singular": "SG", "plural": "PL"} def _feat_set(tag): return set(tag.split(";")) # ── verbs from UniMorph ────────────────────────────────────────────────────────── def _build_verbs(): verbs = {} part = {} # lemma -> {("m","SG"):form, ("f","SG"):..., ("m","PL"):..., ("f","PL"):...} ger = {} with open(_UNIMORPH, encoding="utf-8") as fh: for line in fh: line = line.rstrip("\n") if not line or "\t" not in line: continue parts = line.split("\t") if len(parts) != 3: continue lemma, form, tag = parts f = _feat_set(tag) head = tag.split(";")[0] if head == "V.PTCP": if "PST" in f: g = "f" if "FEM" in f else "m" n = "PL" if "PL" in f else "SG" part.setdefault(lemma, {})[(g, n)] = form continue if head == "V.CVB": if "PRS" in f: ger.setdefault(lemma, form) continue if head != "V": continue person = next((p for p in ("1", "2", "3") if p in f), None) number = "SG" if "SG" in f else ("PL" if "PL" in f else None) if person is None or number is None: continue for (mood, tense), req in _VERB_KEYMAP.items(): if not req <= f: continue if tense == "imperfect" and "PFV" in f: continue if tense == "preterite" and "IPFV" in f: continue verbs.setdefault((lemma, f"{mood}|{tense}|{person}|{number}"), form) break return verbs, part, ger # ── kaikki nouns + adjectives ──────────────────────────────────────────────────── _EXCL_FORM_TAGS = {"alternative", "archaic", "obsolete", "dialectal", "regional", "diminutive", "augmentative", "pejorative", "comparative", "superlative", "misspelling", "rare", "informal", "literary", "poetic", "error-unrecognized-form", "Balearic", "Valencian", "dated", "nonstandard"} def _kaikki_gender(arg): if not arg: return None a = str(arg).lower() if a.startswith("f"): return "f" if a.startswith("m"): return "m" return None def _build_nouns_adjs(): nouns = {} adjs = {} with open(_KAIKKI, encoding="utf-8") as fh: for line in fh: try: d = json.loads(line) except Exception: continue pos = d.get("pos") word = d.get("word", "") if not word or " " in word: continue forms = d.get("forms", []) or [] if pos == "noun": ht = d.get("head_templates") or [] g = None if ht: g = _kaikki_gender((ht[0].get("args") or {}).get("1")) if g is None: tags = d.get("tags") or [] if "feminine" in tags: g = "f" elif "masculine" in tags: g = "m" pl = None for x in forms: t = set(x.get("tags") or []) if "plural" in t and not (t & _EXCL_FORM_TAGS): fm = x.get("form") if fm and " " not in fm and fm not in ("#", "—", "-"): pl = fm break if word not in nouns: nouns[word] = {"g": g, "SG": word, "PL": pl} else: cur = nouns[word] if cur.get("g") is None and g: cur["g"] = g if not cur.get("PL") and pl: cur["PL"] = pl elif pos == "adj": d0 = adjs.setdefault(word, {}) d0.setdefault(("m", "SG"), word) for x in forms: t = set(x.get("tags") or []) fm = x.get("form") if not fm or " " in fm or (t & _EXCL_FORM_TAGS): continue if "feminine" in t and "plural" in t: d0[("f", "PL")] = d0.get(("f", "PL")) or fm elif "masculine" in t and "plural" in t: d0[("m", "PL")] = d0.get(("m", "PL")) or fm elif "feminine" in t: d0[("f", "SG")] = d0.get(("f", "SG")) or fm elif "plural" in t: d0[("m", "PL")] = d0.get(("m", "PL")) or fm d0[("f", "PL")] = d0.get(("f", "PL")) or fm return nouns, adjs def _build_cache(): verbs, part, ger = _build_verbs() nouns, adjs = _build_nouns_adjs() with open(_IRREG, encoding="utf-8") as fh: irreg = json.load(fh) data = {"verbs": verbs, "part": part, "ger": ger, "nouns": nouns, "adjs": adjs, "irreg": irreg} try: with open(_CACHE, "wb") as fh: pickle.dump(data, fh, protocol=pickle.HIGHEST_PROTOCOL) except OSError: pass return data def _load(): if os.path.exists(_CACHE): srcs = [_UNIMORPH, _KAIKKI, _IRREG] newest = max(os.path.getmtime(s) for s in srcs if os.path.exists(s)) if os.path.getmtime(_CACHE) >= newest: try: with open(_CACHE, "rb") as fh: return pickle.load(fh) except Exception: pass return _build_cache() _LEX = _load() _VERBS, _PART, _GER, _NOUNS, _ADJS, _IRREGV = ( _LEX["verbs"], _LEX["part"], _LEX["ger"], _LEX["nouns"], _LEX["adjs"], _LEX["irreg"]) _PERI = _IRREGV.get("_peri_pret_aux", {}) # ── regular verb rule fallback ─────────────────────────────────────────────────── def _vclass(lemma): if lemma.endswith("ar"): return "ar" if lemma.endswith("re"): return "re" if lemma.endswith("er"): return "er" if lemma.endswith("ir"): return "ir" return None # endings [1sg,2sg,3sg,1pl,2pl,3pl] — central Catalan _REG = { ("ind", "present", "ar"): ["o", "es", "a", "em", "eu", "en"], ("ind", "present", "re"): ["o", "s", "", "em", "eu", "en"], ("ind", "present", "er"): ["o", "s", "", "em", "eu", "en"], ("ind", "present", "ir"): ["o", "es", "", "im", "iu", "en"], # pure -ir (dormir) ("ind", "imperfect", "ar"): ["ava", "aves", "ava", "àvem", "àveu", "aven"], ("ind", "imperfect", "re"): ["ia", "ies", "ia", "íem", "íeu", "ien"], ("ind", "imperfect", "er"): ["ia", "ies", "ia", "íem", "íeu", "ien"], ("ind", "imperfect", "ir"): ["ia", "ies", "ia", "íem", "íeu", "ien"], ("ind", "preterite", "ar"): ["í", "ares", "à", "àrem", "àreu", "aren"], ("ind", "preterite", "re"): ["í", "eres", "é", "érem", "éreu", "eren"], ("ind", "preterite", "er"): ["í", "eres", "é", "érem", "éreu", "eren"], ("ind", "preterite", "ir"): ["í", "ires", "í", "írem", "íreu", "iren"], ("sbjv", "present", "ar"): ["i", "is", "i", "em", "eu", "in"], ("sbjv", "present", "re"): ["i", "is", "i", "em", "eu", "in"], ("sbjv", "present", "er"): ["i", "is", "i", "em", "eu", "in"], ("sbjv", "present", "ir"): ["i", "is", "i", "im", "iu", "in"], ("sbjv", "imperfect", "ar"): ["és", "essis", "és", "éssim", "éssiu", "essin"], ("sbjv", "imperfect", "re"): ["és", "essis", "és", "éssim", "éssiu", "essin"], ("sbjv", "imperfect", "er"): ["és", "essis", "és", "éssim", "éssiu", "essin"], ("sbjv", "imperfect", "ir"): ["ís", "issis", "ís", "íssim", "íssiu", "issin"], ("imp", "affirmative", "ar"): [None, "a", "i", "em", "eu", "in"], ("imp", "affirmative", "re"): [None, "", "i", "em", "eu", "in"], ("imp", "affirmative", "er"): [None, "", "i", "em", "eu", "in"], ("imp", "affirmative", "ir"): [None, "", "i", "im", "iu", "in"], } _FUT = ["é", "às", "à", "em", "eu", "an"] _COND = ["ia", "ies", "ia", "íem", "íeu", "ien"] def _slot_idx(person, number): base = {"first": 0, "second": 1, "third": 2}[person] return base + (0 if number == "singular" else 3) def _apply_ar_spelling(stem, ending): """-car/-gar/-çar/-jar spelling before front (e/i) endings.""" front = ending[:1] in ("e", "i", "é", "í") if not front: # ç before back vowel stays; but -çar stem already ends ç return stem + ending if stem.endswith("c"): return stem[:-1] + "qu" + ending if stem.endswith("g"): return stem[:-1] + "gu" + ending if stem.endswith("ç"): return stem[:-1] + "c" + ending if stem.endswith("j"): return stem[:-1] + "g" + ending if stem.endswith("qu"): return stem + ending return stem + ending def _rule_conjugate(lemma, mood, tense, person, number): vc = _vclass(lemma) if vc is None: return None body = lemma[:-2] i = _slot_idx(person, number) if mood == "ind" and tense in ("future", "conditional"): # future/cond stem = infinitive (for -re verbs drop final -e) stem = lemma[:-1] if vc == "re" else lemma end = (_FUT if tense == "future" else _COND)[i] return stem + end table = _REG.get((mood, tense, vc)) if not table: return None end = table[i] if end is None: return None if vc == "ar": return _apply_ar_spelling(body, end) # -re/-er/-ir: guard double vowel if body and body[-1:] == end[:1] and end[:1] in "ií": return body[:-1] + end return body + end # ── PUBLIC: verb conjugation ───────────────────────────────────────────────────── def conjugate(lemma, mood, tense, person, number): lemma = lemma.strip().lower() key = f"{mood}|{tense}|{_PERSON.get(person,'?')}|{number and number[:2].upper()}" key = f"{mood}|{tense}|{_PERSON.get(person,'?')}|{_NUMBER.get(number,'?')}" # UniMorph (cleanly accented) takes priority; the kaikki irregulars layer is a # FALLBACK for verbs/slots UniMorph lacks (anar, fer, and rarer paradigm cells). p, n = _PERSON.get(person), _NUMBER.get(number) if p and n: form = _VERBS.get((lemma, f"{mood}|{tense}|{p}|{n}")) if form: return form, "lexicon" ir = _IRREGV.get(lemma) if ir and key in ir: return ir[key], "lexicon" r = _rule_conjugate(lemma, mood, tense, person, number) if r is not None: return r, "rule" return lemma, "fallback" def peri_pret_aux(person, number): """anar-present auxiliary for the periphrastic preterite (vaig cantar).""" return _PERI.get(f"{_PERSON.get(person,'3')}|{_NUMBER.get(number,'SG')}", "va") # ── PUBLIC: participle + gerund ────────────────────────────────────────────────── def participle(lemma, gender="m", number="singular"): lemma = lemma.strip().lower() g = "f" if gender == "f" else "m" num = "SG" if number == "singular" else "PL" ir = _IRREGV.get(lemma) base = None if ir and "part" in ir: # prefer explicit irregular agreement form (part_mSG/part_fSG/...) exact = ir.get("part_" + g + num) if exact: return exact, "lexicon" base = ir["part"] elif lemma in _PART: table = _PART[lemma] if (g, num) in table: return table[(g, num)], "lexicon" base = table.get(("m", "SG")) if base is None: vc = _vclass(lemma) if vc == "ar": base = lemma[:-2] + "at" elif vc == "ir": base = lemma[:-2] + "it" elif vc in ("er", "re"): base = lemma[:-2] + "ut" else: return lemma, "fallback" conf = "rule" else: conf = "lexicon" # agreement on -t/-ut/-at/-it participles: m.sg base, f.sg +a (-da? no: -ada), # Catalan: cantat/cantada/cantats/cantades; -t → f -da, pl -ts/-des if base.endswith("t"): stem = base[:-1] forms = {"m|SG": base, "f|SG": stem + "da", "m|PL": base + "s", "f|PL": stem + "des"} return forms[f"{g}|{num}"], conf if base.endswith("s"): # after sibilant participle (rare): pres->presa stem = base forms = {"m|SG": base, "f|SG": base + "a", "m|PL": base + "os", "f|PL": base + "es"} return forms[f"{g}|{num}"], conf return base, conf def gerund(lemma): lemma = lemma.strip().lower() ir = _IRREGV.get(lemma) if ir and "ger" in ir: return ir["ger"], "lexicon" if lemma in _GER: return _GER[lemma], "lexicon" vc = _vclass(lemma) if vc == "ar": return lemma[:-2] + "ant", "rule" if vc in ("er", "re"): return lemma[:-2] + "ent", "rule" if vc == "ir": return lemma[:-2] + "int", "rule" return lemma, "fallback" # ── PUBLIC: noun gender + number ───────────────────────────────────────────────── _FEM_SUF = ("ció", "sió", "tat", "tud", "esa", "esa", "dat", "ança", "ència", "ància", "tud", "ícia", "esa", "or") # note -or is mixed; kaikki wins _MASC_SUF = ("atge", "ment", " isme", "or") def _gender_heuristic(noun): for suf in ("ció", "sió", "tat", "tud", "esa", "ança", "ència", "ància", "ícia", "etat"): if noun.endswith(suf): return "f" if noun.endswith("a") and not noun.endswith("ma"): return "f" return "m" def noun_gender(lemma): lemma = lemma.strip().lower() d = _NOUNS.get(lemma) if d and d.get("g") in ("m", "f"): return d["g"] return _gender_heuristic(lemma) def _rule_plural(noun, gender): """Deterministic Catalan pluralization. (form, ok); ok=False FLAGS ambiguity.""" if not noun: return noun, True # stressed final vowel with accent → +ns (mà→mans is irregular; but capità→capitans) if noun[-1:] in ("à", "é", "í", "ó", "ú"): return noun + "ns", True if noun.endswith("ça"): return noun[:-2] + "ces", True # plaça→places if noun.endswith("ca"): return noun[:-2] + "ques", True # branca→branques if noun.endswith("ga"): return noun[:-2] + "gues", True # amiga→amigues if noun.endswith("ja"): return noun[:-2] + "ges", True # pluja→pluges if noun.endswith("qua"): return noun[:-3] + "qües", True if noun.endswith("gua"): return noun[:-3] + "gües", True if noun.endswith("a"): return noun[:-1] + "es", True # casa→cases # sibilant-final → -os if noun.endswith(("s", "ç", "x", "ig")) or noun.endswith(("ix", "tx", "tj")): if noun.endswith("ç"): return noun[:-1] + "ços", True # braç→braços return noun + "os", True # peix→peixos, gas→gasos if noun[-1:] in ("e", "i", "o", "u"): return noun + "s", True # consonant-final return noun + "s", True def inflect_noun(lemma, number, gender=None): lemma = lemma.strip().lower() d = _NOUNS.get(lemma) if number == "singular": return (d["SG"] if d and d.get("SG") else lemma), ("lexicon" if d else "rule") if d and d.get("PL"): return d["PL"], "lexicon" g = gender or noun_gender(lemma) form, ok = _rule_plural(lemma, g) return form, ("rule" if ok else "fallback") # ── PUBLIC: adjective agreement ────────────────────────────────────────────────── def _fem_of(adj): """Regular Catalan feminine: consonant/-o? Catalan masc usually consonant or -e. default +a with spelling changes; -e→-a for some; but many are invariable.""" a = adj if a.endswith("a"): return a if a.endswith("e"): return a[:-1] + "a" # ample→? actually 'ample' invariable; kaikki wins if a.endswith("u"): return a + "a" if a.endswith("c"): return a[:-1] + "ca" # ric→rica if a.endswith("t"): return a + "a" # alt→alta return a + "a" def inflect_adj(lemma, gender, number): lemma = lemma.strip().lower() g = "f" if gender == "f" else "m" num = "SG" if number == "singular" else "PL" d = _ADJS.get(lemma) if d: form = d.get((g, num)) if form: return form, "lexicon" sg = d.get((g, "SG")) or d.get(("m", "SG")) or lemma if num == "PL": pl, ok = _rule_plural(sg, g) return pl, ("rule" if ok else "fallback") return sg, "lexicon" # rule fallback base = lemma if g == "m" else _fem_of(lemma) if num == "SG": return base, "rule" pl, ok = _rule_plural(base, g) return pl, ("rule" if ok else "fallback") def lexicon_stats(): return { "verb_source": "UniMorph Catalan (github.com/unimorph/cat) + kaikki.org " "irregulars (anar/fer/auxiliaries)", "noun_adj_source": "kaikki.org Catalan (Wiktionary extract)", "license": "CC-BY-SA 3.0 (Wiktionary/UniMorph lineage)", "unimorph_verb_forms": len(_VERBS), "unimorph_verb_lemmas": len({k[0] for k in _VERBS}), "irregular_verb_lemmas": len([k for k in _IRREGV if not k.startswith("_")]), "participle_lemmas": len(_PART), "gerund_lemmas": len(_GER), "noun_lemmas": len(_NOUNS), "adj_lemmas": len(_ADJS), } if __name__ == "__main__": print(json.dumps(lexicon_stats(), indent=2, ensure_ascii=False)) tests = [ ("cantar", "ind", "present", "first", "singular", "canto"), ("cantar", "ind", "present", "third", "plural", "canten"), ("ser", "ind", "present", "third", "singular", "és"), ("haver", "ind", "present", "first", "singular", "he"), ("anar", "ind", "present", "first", "singular", "vaig"), ("fer", "ind", "present", "third", "singular", "fa"), ("perdre", "ind", "present", "first", "singular", "perdo"), ("dormir", "ind", "present", "third", "plural", "dormen"), ("cantar", "ind", "future", "first", "singular", "cantaré"), ("cantar", "ind", "preterite", "third", "singular", "cantà"), ("tenir", "sbjv", "present", "first", "singular", "tingui"), ] ok = 0 for lemma, mood, tense, per, num, exp in tests: got, conf = conjugate(lemma, mood, tense, per, num) flag = "OK " if got == exp else "XX " ok += got == exp print(f" {flag}{lemma:8} {mood}/{tense:11} {per[:3]}.{num[:2]} -> {got:10} ({conf}) exp={exp}") print(f"verb tests {ok}/{len(tests)}") print(" peri-pret anar: 1sg=", peri_pret_aux("first", "singular"), "3pl=", peri_pret_aux("third", "plural")) print(" gender casa=", noun_gender("casa"), "home=", noun_gender("home"), "cavall=", noun_gender("cavall"), "cançó=", noun_gender("cançó")) print(" plural casa->", inflect_noun("casa", "plural"), "| plaça->", inflect_noun("plaça", "plural"), "| peix->", inflect_noun("peix", "plural"), "| braç->", inflect_noun("braç", "plural"), "| home->", inflect_noun("home", "plural")) print(" adj: alt/f/sg->", inflect_adj("alt", "f", "singular"), "| bonic/f/pl->", inflect_adj("bonic", "f", "plural"), "| vermell/f/sg->", inflect_adj("vermell", "f", "singular")) print(" part: cantar/f/sg->", participle("cantar", "f", "singular"), "| veure/f/pl->", participle("veure", "f", "plural"), "| fer/m/sg->", participle("fer", "m", "singular")) print(" ger: fer->", gerund("fer"), "| cantar->", gerund("cantar"))