diff --git a/Jenkinsfile b/Jenkinsfile index cd32b8592..42534bf67 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,7 +28,7 @@ pipeline { HY_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-0' MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' - HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-14-26-0' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-17-26-0' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/electronic/__init__.py b/nemo_text_processing/inverse_text_normalization/hi/data/electronic/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/nemo_text_processing/inverse_text_normalization/hi/data/electronic/symbols.tsv b/nemo_text_processing/inverse_text_normalization/hi/data/electronic/symbols.tsv new file mode 100644 index 000000000..c2d15e04c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/hi/data/electronic/symbols.tsv @@ -0,0 +1,25 @@ +backslash बैकवर्ड स्लैश \\ +forwardslash फॉरवर्ड स्लैश / +dot डॉट . +dot DOT . +point प्वाइंट . +hyphen हाइफ़न - +hyphen हाइफन - +underscore अंडर स्कोर _ +at एट +x एक्स x +space स्पेस +openbracket ओपन ब्रेकेट ( +closebracket क्लोज़ ब्रेकेट ) +dollar डॉलर $ +and एंड and +www डब्ल्यू डब्ल्यू डब्ल्यू www +v वी v +tilde टिल्ड +tilde ~ +hashtag हैशटैग # +colon कोलन : +https एच टी टी पी एस कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश https:// +http एच टी टी पी कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश http:// +litslash / / +lithyphen - - \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/hi/graph_utils.py b/nemo_text_processing/inverse_text_normalization/hi/graph_utils.py index b002efa52..70364611c 100644 --- a/nemo_text_processing/inverse_text_normalization/hi/graph_utils.py +++ b/nemo_text_processing/inverse_text_normalization/hi/graph_utils.py @@ -34,6 +34,21 @@ NEMO_HI_DIGIT = pynini.union("०", "१", "२", "३", "४", "५", "६", "७", "८", "९").optimize() DEVANAGARI_DIGIT = ["०", "१", "२", "३", "४", "५", "६", "७", "८", "९"] +DIGIT_GLYPH_TO_ASCII = pynini.union( + *[pynini.cross(glyph, str(value)) for value, glyph in enumerate(DEVANAGARI_DIGIT)] +).optimize() +DIGIT_WORD_TO_DEVANAGARI = ( + pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert() + | pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert() +).optimize() + +# Devanagari characters (consonants, vowels, matras, signs) excluding the digits +# (0x0966-0x096F). Shared so classes like serial and electronic can reuse it. +DEVANAGARI_LETTER = pynini.union( + *[chr(c) for c in range(0x0900, 0x0966)], + *[chr(c) for c in range(0x0970, 0x0980)], +).optimize() + NEMO_HEX = pynini.union(*string.hexdigits).optimize() NEMO_NON_BREAKING_SPACE = u"\u00a0" NEMO_ZWNJ = u"\u200c" @@ -83,6 +98,34 @@ def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): logging.info(f'Created {file_name}') +def load_symbols(path): + """ + Builds a dict mapping a symbol name to an FST that deletes a spoken Hindi + phrase and inserts its written form. TSV columns: name, spoken phrase, output + (optional). Rows sharing a name are unioned; "" in the output inserts a + space. + """ + table = {} + with open(path, encoding="utf-8") as f: + for line in f: + line = line.rstrip("\r\n") + if not line or line.startswith("#"): + continue + cols = line.split("\t") + name = cols[0] + words = cols[1].split(" ") + out = cols[2] if len(cols) > 2 else "" + if out == "": + out = " " + fst = pynutil.delete(words[0]) + for word in words[1:]: + fst += delete_space + pynutil.delete(word) + if out: + fst += pynutil.insert(out) + table[name] = (table[name] | fst) if name in table else fst + return table + + def convert_space(fst) -> 'pynini.FstLike': """ Converts space to nonbreaking space. diff --git a/nemo_text_processing/inverse_text_normalization/hi/taggers/telephone.py b/nemo_text_processing/inverse_text_normalization/hi/taggers/telephone.py index ad584b58b..09e36927f 100644 --- a/nemo_text_processing/inverse_text_normalization/hi/taggers/telephone.py +++ b/nemo_text_processing/inverse_text_normalization/hi/taggers/telephone.py @@ -16,10 +16,12 @@ from pynini.lib import pynutil from nemo_text_processing.inverse_text_normalization.hi.graph_utils import ( + DIGIT_GLYPH_TO_ASCII, NEMO_CHAR, NEMO_WHITE_SPACE, GraphFst, delete_space, + load_symbols, ) from nemo_text_processing.inverse_text_normalization.hi.utils import get_abs_path @@ -37,19 +39,7 @@ def get_context(keywords: list): keywords = pynini.union(*keywords) - # Load Hindi digits from TSV files - hindi_digits = ( - pynini.string_file(get_abs_path("data/numbers/digit.tsv")) - | pynini.string_file(get_abs_path("data/numbers/zero.tsv")) - ).project("output") - - # Load English digits from TSV files - english_digits = ( - pynini.string_file(get_abs_path("data/telephone/eng_digit.tsv")) - | pynini.string_file(get_abs_path("data/telephone/eng_zero.tsv")) - ).project("output") - - all_digits = hindi_digits | english_digits + all_digits = pynini.project(digit, "input") non_digit_char = pynini.difference(NEMO_CHAR, pynini.union(all_digits, NEMO_WHITE_SPACE)) word = pynini.closure(non_digit_char, 1) + NEMO_WHITE_SPACE @@ -124,6 +114,8 @@ class TelephoneFst(GraphFst): """ Finite state transducer for classifying telephone numbers, e.g. e.g. प्लस इक्यानवे नौ आठ सात छह पांच चार तीन दो एक शून्य => tokens { name: "+९१ ९८७६५ ४३२१०" } + This class also supports IP addresses, e.g. + e.g. एक नौ दो डॉट एक छह आठ डॉट एक डॉट एक => tokens { telephone { number_part: "192.168.1.1" } } Args: Cardinal: CardinalFst """ @@ -134,26 +126,26 @@ def __init__(self, cardinal: GraphFst): # Load context cues from TSV file context_cues = pynini.string_file(get_abs_path("data/telephone/context_cues.tsv")) - # Extract keywords for each category - mobile_keywords = pynini.compose(pynutil.delete("mobile"), context_cues).project("output").optimize() - - landline_keywords = pynini.compose(pynutil.delete("landline"), context_cues).project("output").optimize() - - pincode_keywords = pynini.compose(pynutil.delete("pincode"), context_cues).project("output").optimize() + def keywords(category): + return pynini.compose(pynutil.delete(category), context_cues).project("output").optimize() - credit_keywords = pynini.compose(pynutil.delete("credit"), context_cues).project("output").optimize() + mobile = generate_mobile([keywords("mobile")]) + landline = generate_telephone([keywords("landline")]) + pincode = generate_pincode([keywords("pincode")]) + credit = generate_credit([keywords("credit")]) - # Convert FSTs to keyword lists for generate_* functions - mobile = generate_mobile([mobile_keywords]) - landline = generate_telephone([landline_keywords]) - pincode = generate_pincode([pincode_keywords]) - credit = generate_credit([credit_keywords]) + sym = load_symbols(get_abs_path("data/electronic/symbols.tsv")) + ip_dot = delete_space + sym["dot"] + delete_space + ip_digit = pynini.compose(digit, DIGIT_GLYPH_TO_ASCII) | DIGIT_GLYPH_TO_ASCII + ip_octet = ip_digit + pynini.closure(delete_space + ip_digit, 0, 2) + ip_graph = pynutil.insert("number_part: \"") + ip_octet + (ip_dot + ip_octet) ** 3 + pynutil.insert("\" ") graph = ( pynutil.add_weight(mobile, 0.7) | pynutil.add_weight(landline, 0.8) | pynutil.add_weight(credit, 0.9) | pynutil.add_weight(pincode, 1) + | pynutil.add_weight(ip_graph, 0.7) ) self.final = graph.optimize() diff --git a/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_telephone.txt b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_telephone.txt index 3b84a333d..3c1163c7d 100644 --- a/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_telephone.txt +++ b/tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_telephone.txt @@ -25,4 +25,6 @@ zero one three four two three two one five four eight~०१३४२३२१ चार चार चार चार~४४४४ सात आठ नौ एक~७८९१ एक शून्य दो शून्य~१०२० -नौ आठ सात छह~९८७६ \ No newline at end of file +नौ आठ सात छह~९८७६ +दो पाँच पाँच डॉट एक छह आठ डॉट चार छह डॉट एक सात पाँच~255.168.46.175 +दो पाँच डॉट दो तीन तीन डॉट एक चार डॉट दो तीन एक~25.233.14.231