import pynini from fun_text_processing.text_normalization.en.graph_utils import DAMO_SIGMA, DAMO_SPACE from fun_text_processing.text_normalization.es import LOCALIZATION from fun_text_processing.text_normalization.es.utils import get_abs_path, load_labels from pynini.lib import pynutil digits = pynini.project(pynini.string_file(get_abs_path("data/numbers/digit.tsv")), "input") tens = pynini.project(pynini.string_file(get_abs_path("data/numbers/ties.tsv")), "input") teens = pynini.project(pynini.string_file(get_abs_path("data/numbers/teen.tsv")), "input") twenties = pynini.project(pynini.string_file(get_abs_path("data/numbers/twenties.tsv")), "input") hundreds = pynini.project(pynini.string_file(get_abs_path("data/numbers/hundreds.tsv")), "input") accents = pynini.string_map([("á", "a"), ("é", "e"), ("í", "i"), ("ó", "o"), ("ú", "u")]) if LOCALIZATION == "am": # Setting localization for central and northern america formatting cardinal_separator = pynini.string_map([",", DAMO_SPACE]) decimal_separator = pynini.accep(".") else: cardinal_separator = pynini.string_map([".", DAMO_SPACE]) decimal_separator = pynini.accep(",") ones = pynini.union("un", "ún") fem_ones = pynini.union( pynini.cross("un", "una"), pynini.cross("ún", "una"), pynini.cross("uno", "una") ) one_to_one_hundred = pynini.union( digits, "uno", tens, teens, twenties, tens + pynini.accep(" y ") + digits ) fem_hundreds = hundreds @ pynini.cdrewrite(pynini.cross("ientos", "ientas"), "", "", DAMO_SIGMA) def strip_accent(fst: "pynini.FstLike") -> "pynini.FstLike": """ Converts all accented vowels to non-accented equivalents Args: fst: Any fst. Composes vowel conversion onto fst's output strings """ return fst @ pynini.cdrewrite(accents, "", "", DAMO_SIGMA) def shift_cardinal_gender(fst: "pynini.FstLike") -> "pynini.FstLike": """ Applies gender conversion rules to a cardinal string. These include: rendering all masculine forms of "uno" (including apocopated forms) as "una" and Converting all gendered numbers in the hundreds series (200,300,400...) to feminine equivalent (e.g. "doscientos" -> "doscientas"). Conversion only applies to value place for <1000 and multiple of 1000. (e.g. "doscientos mil doscientos" -> "doscientas mil doscientas".) For place values greater than the thousands, there is no gender shift as the higher powers of ten ("millones", "billones") are masculine nouns and any conversion would be formally ungrammatical. e.g. "doscientos" -> "doscientas" "doscientos mil" -> "doscientas mil" "doscientos millones" -> "doscientos millones" "doscientos mil millones" -> "doscientos mil millones" "doscientos millones doscientos mil doscientos" -> "doscientos millones doscientas mil doscientas" Args: fst: Any fst. Composes conversion onto fst's output strings """ before_mil = ( DAMO_SPACE + (pynini.accep("mil") | pynini.accep("milésimo")) + pynini.closure(DAMO_SPACE + hundreds, 0, 1) + pynini.closure(DAMO_SPACE + one_to_one_hundred, 0, 1) + pynini.union(pynini.accep("[EOS]"), pynini.accep('"'), decimal_separator) ) before_double_digits = pynini.closure(DAMO_SPACE + one_to_one_hundred, 0, 1) + pynini.union( pynini.accep("[EOS]"), pynini.accep('"') ) fem_allign = pynini.cdrewrite( fem_hundreds, "", before_mil, DAMO_SIGMA ) # doscientas mil dosciento fem_allign @= pynini.cdrewrite( fem_hundreds, "", before_double_digits, DAMO_SIGMA ) # doscientas mil doscienta fem_allign @= pynini.cdrewrite( fem_ones, "", pynini.union("[EOS]", '"', decimal_separator), DAMO_SIGMA ) # If before a quote or EOS, we know it's the end of a string return fst @ fem_allign def shift_number_gender(fst: "pynini.FstLike") -> "pynini.FstLike": """ Performs gender conversion on all verbalized numbers in output. All values in the hundreds series (200,300,400) are changed to feminine gender (e.g. "doscientos" -> "doscientas") and all forms of "uno" (including apocopated forms) are converted to "una". This has no boundary restriction and will perform shift across all values in output string. e.g. "doscientos" -> "doscientas" "doscientos millones" -> "doscientas millones" "doscientos millones doscientos" -> "doscientas millones doscientas" Args: fst: Any fst. Composes conversion onto fst's output strings """ fem_allign = pynini.cdrewrite(fem_hundreds, "", "", DAMO_SIGMA) fem_allign @= pynini.cdrewrite( fem_ones, "", pynini.union(DAMO_SPACE, pynini.accep("[EOS]"), pynini.accep('"')), DAMO_SIGMA ) # If before a quote or EOS, we know it's the end of a string return fst @ fem_allign def strip_cardinal_apocope(fst: "pynini.FstLike") -> "pynini.FstLike": """ Reverts apocope on cardinal strings in line with formation rules. e.g. "un" -> "uno". Due to cardinal formation rules, this in effect only affects strings where the final value is a variation of "un". e.g. "un" -> "uno" "veintiún" -> "veintiuno" Args: fst: Any fst. Composes conversion onto fst's output strings """ # Since cardinals use apocope by default for large values (e.g. "millón"), this only needs to act on the last instance of one strip = pynini.cross("un", "uno") | pynini.cross("ún", "uno") strip = pynini.cdrewrite(strip, "", pynini.union("[EOS]", '"'), DAMO_SIGMA) return fst @ strip def add_cardinal_apocope_fem(fst: "pynini.FstLike") -> "pynini.FstLike": """ Adds apocope on cardinal strings in line with stressing rules. e.g. "una" -> "un". This only occurs when "una" precedes a stressed "a" sound in formal speech. This is not predictable with text string, so is included for non-deterministic cases. e.g. "una" -> "un" "veintiuna" -> "veintiun" Args: fst: Any fst. Composes conversion onto fst's output strings """ # Since the stress trigger follows the cardinal string and only affects the preceding sound, this only needs to act on the last instance of one strip = pynini.cross("una", "un") | pynini.cross("veintiuna", "veintiún") strip = pynini.cdrewrite(strip, "", pynini.union("[EOS]", '"'), DAMO_SIGMA) return fst @ strip def roman_to_int(fst: "pynini.FstLike") -> "pynini.FstLike": """ Alters given fst to convert Roman integers (lower and upper cased) into Arabic numerals. Valid for values up to 1000. e.g. "V" -> "5" "i" -> "1" Args: fst: Any fst. Composes fst onto Roman conversion outputs. """ def _load_roman(file: str): roman = load_labels(get_abs_path(file)) roman_numerals = [(x, y) for x, y in roman] + [(x.upper(), y) for x, y in roman] return pynini.string_map(roman_numerals) digit = _load_roman("data/roman/digit.tsv") ties = _load_roman("data/roman/ties.tsv") hundreds = _load_roman("data/roman/hundreds.tsv") graph = ( digit | ties + (digit | pynutil.add_weight(pynutil.insert("0"), 0.01)) | ( hundreds + (ties | pynutil.add_weight(pynutil.insert("0"), 0.01)) + (digit | pynutil.add_weight(pynutil.insert("0"), 0.01)) ) ).optimize() return graph @ fst