96 lines
3.1 KiB
Python
96 lines
3.1 KiB
Python
import pynini
|
|
from fun_text_processing.text_normalization.en.graph_utils import (
|
|
DAMO_NOT_QUOTE,
|
|
DAMO_NOT_SPACE,
|
|
DAMO_SIGMA,
|
|
TO_UPPER,
|
|
GraphFst,
|
|
delete_extra_space,
|
|
delete_space,
|
|
insert_space,
|
|
)
|
|
from fun_text_processing.text_normalization.en.utils import get_abs_path
|
|
from pynini.examples import plurals
|
|
from pynini.lib import pynutil
|
|
|
|
|
|
class ElectronicFst(GraphFst):
|
|
"""
|
|
Finite state transducer for verbalizing electronic
|
|
e.g. tokens { electronic { username: "cdf1" domain: "abc.edu" } } -> c d f one at a b c dot e d u
|
|
|
|
Args:
|
|
deterministic: if True will provide a single transduction option,
|
|
for False multiple transduction are generated (used for audio-based normalization)
|
|
"""
|
|
|
|
def __init__(self, deterministic: bool = True):
|
|
super().__init__(name="electronic", kind="verbalize", deterministic=deterministic)
|
|
graph_digit_no_zero = pynini.invert(
|
|
pynini.string_file(get_abs_path("data/number/digit.tsv"))
|
|
).optimize()
|
|
graph_zero = pynini.cross("0", "zero")
|
|
|
|
if not deterministic:
|
|
graph_zero |= pynini.cross("0", "o") | pynini.cross("0", "oh")
|
|
|
|
graph_digit = graph_digit_no_zero | graph_zero
|
|
graph_symbols = pynini.string_file(get_abs_path("data/electronic/symbol.tsv")).optimize()
|
|
|
|
default_chars_symbols = pynini.cdrewrite(
|
|
pynutil.insert(" ") + (graph_symbols | graph_digit) + pynutil.insert(" "),
|
|
"",
|
|
"",
|
|
DAMO_SIGMA,
|
|
)
|
|
default_chars_symbols = pynini.compose(
|
|
pynini.closure(DAMO_NOT_SPACE), default_chars_symbols.optimize()
|
|
).optimize()
|
|
|
|
user_name = (
|
|
pynutil.delete("username:")
|
|
+ delete_space
|
|
+ pynutil.delete('"')
|
|
+ default_chars_symbols
|
|
+ pynutil.delete('"')
|
|
)
|
|
|
|
domain_common = pynini.string_file(get_abs_path("data/electronic/domain.tsv"))
|
|
|
|
domain = (
|
|
default_chars_symbols
|
|
+ insert_space
|
|
+ plurals._priority_union(
|
|
domain_common,
|
|
pynutil.add_weight(pynini.cross(".", "dot"), weight=0.0001),
|
|
DAMO_SIGMA,
|
|
)
|
|
+ pynini.closure(
|
|
insert_space
|
|
+ (pynini.cdrewrite(TO_UPPER, "", "", DAMO_SIGMA) @ default_chars_symbols),
|
|
0,
|
|
1,
|
|
)
|
|
)
|
|
domain = (
|
|
pynutil.delete("domain:")
|
|
+ delete_space
|
|
+ pynutil.delete('"')
|
|
+ domain
|
|
+ delete_space
|
|
+ pynutil.delete('"')
|
|
).optimize()
|
|
|
|
protocol = (
|
|
pynutil.delete('protocol: "') + pynini.closure(DAMO_NOT_QUOTE, 1) + pynutil.delete('"')
|
|
)
|
|
graph = (
|
|
pynini.closure(protocol + delete_space, 0, 1)
|
|
+ pynini.closure(user_name + delete_space + pynutil.insert(" at ") + delete_space, 0, 1)
|
|
+ domain
|
|
+ delete_space
|
|
).optimize() @ pynini.cdrewrite(delete_extra_space, "", "", DAMO_SIGMA)
|
|
|
|
delete_tokens = self.delete_tokens(graph)
|
|
self.fst = delete_tokens.optimize()
|