Hi itn telephone

Jenkinsfile

@@ -28,7 +28,7 @@ pipeline {
     HY_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-0'
     MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1'
     JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1'
-    HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-14-26-0'
+    HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-17-26-0'
     KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6'
     DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
   }

nemo_text_processing/inverse_text_normalization/hi/data/electronic/symbols.tsv

@@ -0,0 +1,25 @@
+backslash	बैकवर्ड स्लैश	\\
+forwardslash	फॉरवर्ड स्लैश	/
+dot	डॉट	.
+dot	DOT	.
+point	प्वाइंट	.
+hyphen	हाइफ़न	-
+hyphen	हाइफन	-
+underscore	अंडर स्कोर	_
+at	एट
+x	एक्स	x
+space	स्पेस	<space>
+openbracket	ओपन ब्रेकेट	(
+closebracket	क्लोज़ ब्रेकेट	)
+dollar	डॉलर	$
+and	एंड	and
+www	डब्ल्यू डब्ल्यू डब्ल्यू	www
+v	वी	v
+tilde	टिल्ड
+tilde	~
+hashtag	हैशटैग	#
+colon	कोलन	:
+https	एच टी टी पी एस कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश	https://
+http	एच टी टी पी कोलन फॉरवर्ड स्लैश फॉरवर्ड स्लैश	http://
+litslash	/	/
+lithyphen	-	-

nemo_text_processing/inverse_text_normalization/hi/graph_utils.py

@@ -34,6 +34,21 @@
 NEMO_HI_DIGIT = pynini.union("०", "१", "२", "३", "४", "५", "६", "७", "८", "९").optimize()
 DEVANAGARI_DIGIT = ["०", "१", "२", "३", "४", "५", "६", "७", "८", "९"]
 
+DIGIT_GLYPH_TO_ASCII = pynini.union(
+    *[pynini.cross(glyph, str(value)) for value, glyph in enumerate(DEVANAGARI_DIGIT)]
+).optimize()
+DIGIT_WORD_TO_DEVANAGARI = (
+    pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert()
+    | pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert()
+).optimize()
+
+# Devanagari characters (consonants, vowels, matras, signs) excluding the digits
+# (0x0966-0x096F). Shared so classes like serial and electronic can reuse it.
+DEVANAGARI_LETTER = pynini.union(
+    *[chr(c) for c in range(0x0900, 0x0966)],
+    *[chr(c) for c in range(0x0970, 0x0980)],
+).optimize()
+
 NEMO_HEX = pynini.union(*string.hexdigits).optimize()
 NEMO_NON_BREAKING_SPACE = u"\u00a0"
 NEMO_ZWNJ = u"\u200c"
@@ -83,6 +98,34 @@ def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']):
     logging.info(f'Created {file_name}')
 
 
+def load_symbols(path):
+    """
+    Builds a dict mapping a symbol name to an FST that deletes a spoken Hindi
+    phrase and inserts its written form. TSV columns: name, spoken phrase, output
+    (optional). Rows sharing a name are unioned; "<space>" in the output inserts a
+    space.
+    """
+    table = {}
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.rstrip("\r\n")
+            if not line or line.startswith("#"):
+                continue
+            cols = line.split("\t")
+            name = cols[0]
+            words = cols[1].split(" ")
+            out = cols[2] if len(cols) > 2 else ""
+            if out == "<space>":
+                out = " "
+            fst = pynutil.delete(words[0])
+            for word in words[1:]:
+                fst += delete_space + pynutil.delete(word)
+            if out:
+                fst += pynutil.insert(out)
+            table[name] = (table[name] | fst) if name in table else fst
+    return table
+
+
 def convert_space(fst) -> 'pynini.FstLike':
     """
     Converts space to nonbreaking space.

nemo_text_processing/inverse_text_normalization/hi/taggers/telephone.py

@@ -16,10 +16,12 @@
 from pynini.lib import pynutil
 
 from nemo_text_processing.inverse_text_normalization.hi.graph_utils import (
+    DIGIT_GLYPH_TO_ASCII,
     NEMO_CHAR,
     NEMO_WHITE_SPACE,
     GraphFst,
     delete_space,
+    load_symbols,
 )
 from nemo_text_processing.inverse_text_normalization.hi.utils import get_abs_path
 
@@ -37,19 +39,7 @@
 def get_context(keywords: list):
     keywords = pynini.union(*keywords)
 
-    # Load Hindi digits from TSV files
-    hindi_digits = (
-        pynini.string_file(get_abs_path("data/numbers/digit.tsv"))
-        | pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
-    ).project("output")
-
-    # Load English digits from TSV files
-    english_digits = (
-        pynini.string_file(get_abs_path("data/telephone/eng_digit.tsv"))
-        | pynini.string_file(get_abs_path("data/telephone/eng_zero.tsv"))
-    ).project("output")
-
-    all_digits = hindi_digits | english_digits
+    all_digits = pynini.project(digit, "input")
 
     non_digit_char = pynini.difference(NEMO_CHAR, pynini.union(all_digits, NEMO_WHITE_SPACE))
     word = pynini.closure(non_digit_char, 1) + NEMO_WHITE_SPACE
@@ -124,6 +114,8 @@ class TelephoneFst(GraphFst):
     """
     Finite state transducer for classifying telephone numbers, e.g.
     e.g. प्लस इक्यानवे नौ आठ सात छह पांच चार तीन दो एक शून्य => tokens { name: "+९१ ९८७६५ ४३२१०" }
+    This class also supports IP addresses, e.g.
+    e.g. एक नौ दो डॉट एक छह आठ डॉट एक डॉट एक => tokens { telephone { number_part: "192.168.1.1" } }
     Args:
         Cardinal: CardinalFst
     """
@@ -134,26 +126,26 @@ def __init__(self, cardinal: GraphFst):
         # Load context cues from TSV file
         context_cues = pynini.string_file(get_abs_path("data/telephone/context_cues.tsv"))
 
-        # Extract keywords for each category
-        mobile_keywords = pynini.compose(pynutil.delete("mobile"), context_cues).project("output").optimize()
-
-        landline_keywords = pynini.compose(pynutil.delete("landline"), context_cues).project("output").optimize()
-
-        pincode_keywords = pynini.compose(pynutil.delete("pincode"), context_cues).project("output").optimize()
+        def keywords(category):
+            return pynini.compose(pynutil.delete(category), context_cues).project("output").optimize()
 
-        credit_keywords = pynini.compose(pynutil.delete("credit"), context_cues).project("output").optimize()
+        mobile = generate_mobile([keywords("mobile")])
+        landline = generate_telephone([keywords("landline")])
+        pincode = generate_pincode([keywords("pincode")])
+        credit = generate_credit([keywords("credit")])
 
-        # Convert FSTs to keyword lists for generate_* functions
-        mobile = generate_mobile([mobile_keywords])
-        landline = generate_telephone([landline_keywords])
-        pincode = generate_pincode([pincode_keywords])
-        credit = generate_credit([credit_keywords])
+        sym = load_symbols(get_abs_path("data/electronic/symbols.tsv"))
+        ip_dot = delete_space + sym["dot"] + delete_space
+        ip_digit = pynini.compose(digit, DIGIT_GLYPH_TO_ASCII) | DIGIT_GLYPH_TO_ASCII
+        ip_octet = ip_digit + pynini.closure(delete_space + ip_digit, 0, 2)
+        ip_graph = pynutil.insert("number_part: \"") + ip_octet + (ip_dot + ip_octet) ** 3 + pynutil.insert("\" ")
 
         graph = (
             pynutil.add_weight(mobile, 0.7)
             | pynutil.add_weight(landline, 0.8)
             | pynutil.add_weight(credit, 0.9)
             | pynutil.add_weight(pincode, 1)
+            | pynutil.add_weight(ip_graph, 0.7)
         )
 
         self.final = graph.optimize()

tests/nemo_text_processing/hi/data_inverse_text_normalization/test_cases_telephone.txt

@@ -25,4 +25,6 @@ zero one three four two three two one five four eight~०१३४२३२१
 चार चार चार चार~४४४४
 सात आठ नौ एक~७८९१
 एक शून्य दो शून्य~१०२०
-नौ आठ सात छह~९८७६
+नौ आठ सात छह~९८७६
+दो पाँच पाँच डॉट एक छह आठ डॉट चार छह डॉट एक सात पाँच~255.168.46.175
+दो पाँच डॉट दो तीन तीन डॉट एक चार डॉट दो तीन एक~25.233.14.231