Hi TN: implement natural cardinal verbalization for up to 3-digit numbers in address

Jenkinsfile

@@ -28,7 +28,7 @@ pipeline {
     MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1'
     JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1'
     KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/04-23-26-0'
-    HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-10-26-0'
+    HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-22-26-0'
     DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
   }
   stages {

nemo_text_processing/text_normalization/hi/data/address/cities.tsv

@@ -34,3 +34,4 @@
 कारगिल
 कवरत्ती
 पुडुचेरी
+बैंगलोर

nemo_text_processing/text_normalization/hi/data/address/context.tsv

@@ -45,4 +45,8 @@
 सामने
 पीछे
 वीया
-आर डी
+हिल
+पिनकोड
+पार्कहर्स्ट
+स्टे
+रैपिड्स

nemo_text_processing/text_normalization/hi/taggers/measure.py

@@ -30,6 +30,8 @@
     HYPHEN,
     INPUT_LOWER_CASED,
     LOWERCASE_X,
+    MIN_NEG_WEIGHT,
+    NEMO_ALL_DIGIT,
     NEMO_CHAR,
     NEMO_DIGIT,
     NEMO_HI_DIGIT,
@@ -67,23 +69,21 @@ class MeasureFst(GraphFst):
     Args:
         cardinal: CardinalFst
         decimal: DecimalFst
+        ordinal: OrdinalFst
+        input_case: str
         deterministic: if True will provide a single transduction option,
             for False multiple transduction are generated (used for audio-based normalization)
     """
 
-    def get_structured_address_graph(self, ordinal: GraphFst, input_case: str):
+    def get_structured_address_graph(self, states_graph: "pynini.Fst", cities_graph: "pynini.Fst"):
         """
         Minimal address tagger for state/city + pincode patterns only.
-        Highly optimized for performance.
 
         Examples:
             "मुंबई ८८४४०४" -> "मुंबई आठ आठ चार चार शून्य चार"
             "गोवा १२३४५६" -> "गोवा एक दो तीन चार पाँच छह"
         """
-        # State/city keywords
-        states = pynini.string_file(get_abs_path("data/address/states.tsv"))
-        cities = pynini.string_file(get_abs_path("data/address/cities.tsv"))
-        state_city_names = pynini.union(states, cities).optimize()
+        state_city_names = pynini.union(states_graph, cities_graph).optimize()
 
         # Digit mappings
         num_token = (
@@ -98,7 +98,7 @@ def get_structured_address_graph(self, ordinal: GraphFst, input_case: str):
         # Street number (1-4 digits)
         street_num = (num_token + pynini.closure(insert_space + num_token, 0, 3)).optimize()
 
-        # Text: words with trailing separator (comma? + space)
+        # Text: words with trailing separator
         any_digit = pynini.union(NEMO_HI_DIGIT, NEMO_DIGIT).optimize()
         punctuation = pynini.union(COMMA, PERIOD, HI_PERIOD).optimize()
         word_char = pynini.difference(NEMO_NOT_SPACE, pynini.union(any_digit, punctuation)).optimize()
@@ -107,7 +107,9 @@ def get_structured_address_graph(self, ordinal: GraphFst, input_case: str):
         # Separator: optional comma followed by mandatory space
         sep = pynini.closure(pynini.accep(COMMA), 0, 1) + pynini.accep(NEMO_SPACE)
         word_with_sep = word + sep
-        text = pynini.closure(word_with_sep, 0, 5).optimize()
+
+        # Unbounded word closure preceding state/city to optimize FST compilation size.
+        text = pynini.closure(word_with_sep).optimize()
 
         # Pattern: [street_num + sep]? text state/city [space pincode]
         pattern = (
@@ -122,79 +124,128 @@ def get_structured_address_graph(self, ordinal: GraphFst, input_case: str):
             + pattern
             + pynutil.insert('" } preserve_order: true')
         )
-        return pynutil.add_weight(graph, 1.0).optimize()
-
-    def get_address_graph(self, ordinal: GraphFst, input_case: str):
+        # Weight kept above get_address_graph so address graph takes priority when both can match.
+        return pynutil.add_weight(graph, 2.0).optimize()
+
+    def get_address_graph(
+        self,
+        cardinal: GraphFst,
+        ordinal: GraphFst,
+        input_case: str,
+        address_keywords_hi: "pynini.Fst",
+        states_graph: "pynini.Fst",
+        cities_graph: "pynini.Fst",
+        en_to_hi_map: "pynini.Fst",
+    ):
         """
-        Address tagger that converts digits/hyphens/slashes character-by-character
-        when address context keywords are present.
-        English words and ordinals are converted to Hindi transliterations.
+        Address tagger with natural Hindi verbalization for street numbers.
+        1-3 digit numbers are verbalized as cardinals, 4+ digit numbers and
+        PIN codes as digit-by-digit.
 
         Examples:
-            "७०० ओक स्ट्रीट" -> "सात शून्य शून्य ओक स्ट्रीट"
-            "६६-४ पार्क रोड" -> "छह छह हाइफ़न चार पार्क रोड"
+            "७०० ओक स्ट्रीट"  -> "सात सौ ओक स्ट्रीट"
+            "६६-४ पार्क रोड"  -> "छियासठ हाइफ़न चार पार्क रोड"
+            "14/3 मथुरा रोड"  -> "चौदह बटा तीन मथुरा रोड"
         """
-        ordinal_graph = ordinal.graph
-        # Alphanumeric to word mappings (digits, special characters, telephone digits)
-        char_to_word = (
-            digit
-            | pynini.string_file(get_abs_path("data/numbers/zero.tsv"))
-            | pynini.string_file(get_abs_path("data/address/special_characters.tsv"))
-            | pynini.string_file(get_abs_path("data/telephone/number.tsv"))
+        special_char_map = pynini.string_file(get_abs_path("data/address/special_characters.tsv"))
+        single_digit_word = (cardinal.digit | cardinal.zero).optimize()
+        single_digit_input = NEMO_ALL_DIGIT
+
+        digit_run = (single_digit_word + pynini.closure(insert_space + single_digit_word)).optimize()
+
+        num_1digit = pynini.compose(single_digit_input**1, (cardinal.digit | cardinal.zero)).optimize()
+        num_2digit = pynini.compose(single_digit_input**2, cardinal.teens_and_ties).optimize()
+        num_3digit = pynini.compose(single_digit_input**3, cardinal.graph_hundreds).optimize()
+
+        num_as_cardinal = (
+            pynutil.add_weight(num_3digit, -0.2)
+            | pynutil.add_weight(num_2digit, -0.02)
+            | pynutil.add_weight(num_1digit, MIN_NEG_WEIGHT)
         ).optimize()
-        letter_to_word = pynini.string_file(get_abs_path("data/address/letters.tsv"))
-        letter_to_word = capitalized_input_graph(letter_to_word)
-        address_keywords_hi = pynini.string_file(get_abs_path("data/address/context.tsv"))
 
-        # English address keywords with Hindi translation (case-insensitive)
-        en_to_hi_map = pynini.string_file(get_abs_path("data/address/en_to_hi_mapping.tsv"))
+        digit_run_4_plus = pynini.compose(
+            single_digit_input**4 + pynini.closure(single_digit_input), digit_run
+        ).optimize()
+
+        any_street_num = (num_as_cardinal | pynutil.add_weight(digit_run_4_plus, -0.5)).optimize()
+
+        # Only slash joins two numbers into one token; hyphen is handled by standalone_hyphen.
+        slash_separator = pynini.compose(pynini.accep(SLASH), special_char_map)
+        joined_num = (any_street_num + insert_space + slash_separator + insert_space + any_street_num).optimize()
+
+        # Digit run maps to one token by length: 1-3 digits -> cardinal, 4+ -> digit-by-digit (covers PIN codes too).
+        street_num_1_to_3 = num_as_cardinal
+        street_num_4_plus = digit_run_4_plus
+
+        ordinal_processor = pynutil.add_weight(insert_space + ordinal.graph, MIN_NEG_WEIGHT * 5)
+
         if input_case != INPUT_LOWER_CASED:
             en_to_hi_map = capitalized_input_graph(en_to_hi_map)
-        address_keywords_en = pynini.project(en_to_hi_map, "input")
-        address_keywords = pynini.union(address_keywords_hi, address_keywords_en)
+        english_word_processor = pynutil.add_weight(insert_space + en_to_hi_map, MIN_NEG_WEIGHT)
 
-        # Alphanumeric processing: treat digits, letters, and -/ as convertible tokens
-        single_digit = pynini.union(NEMO_DIGIT, NEMO_HI_DIGIT).optimize()
-        special_chars = pynini.union(HYPHEN, SLASH).optimize()
+        letter_to_word = pynini.string_file(get_abs_path("data/address/letters.tsv"))
+        letter_to_word = capitalized_input_graph(letter_to_word)
         single_letter = pynini.project(letter_to_word, "input").optimize()
-        convertible_char = pynini.union(single_digit, special_chars, single_letter)
+        letter_processor = pynutil.add_weight(insert_space + pynini.compose(single_letter, letter_to_word), 0.5)
+
+        special_chars = pynini.union(HYPHEN, SLASH).optimize()
+        convertible_char = pynini.union(single_digit_input, special_chars, single_letter)
         non_space_char = pynini.difference(
             NEMO_CHAR, pynini.union(NEMO_WHITE_SPACE, convertible_char, pynini.accep(COMMA))
         ).optimize()
 
-        # Token processors with weights: prefer ordinals and known English→Hindi words
-        # Delete space before comma to avoid Sparrowhawk "sil" issue
+        # Letter right after a slash (e.g. "12/A").
+        right_side_letter = (
+            pynini.compose(single_letter, letter_to_word) | pynini.closure(non_space_char, 1)
+        ).optimize()
+        slash_word = pynini.compose(pynini.accep(SLASH), special_char_map)
+        slashed_num_letter = (any_street_num + insert_space + slash_word + insert_space + right_side_letter).optimize()
+
         comma_processor = pynutil.add_weight(delete_space + pynini.accep(COMMA), 0.0)
-        ordinal_processor = pynutil.add_weight(insert_space + ordinal_graph, -5.0)
-        english_word_processor = pynutil.add_weight(insert_space + en_to_hi_map, -3.0)
-        letter_processor = pynutil.add_weight(insert_space + pynini.compose(single_letter, letter_to_word), 0.5)
-        digit_char_processor = pynutil.add_weight(insert_space + pynini.compose(convertible_char, char_to_word), 0.0)
+
+        hyphen_word = pynini.compose(pynini.accep(HYPHEN), special_char_map)
+        standalone_hyphen = hyphen_word
+
         other_word_processor = pynutil.add_weight(insert_space + pynini.closure(non_space_char, 1), 0.1)
 
-        token_processor = (
+        # Kept separate so full_string_processor can forbid two in a row.
+        numeric_token = ((insert_space + street_num_1_to_3) | (insert_space + street_num_4_plus)).optimize()
+
+        non_numeric_token = (
             ordinal_processor
             | english_word_processor
+            | (insert_space + joined_num)
+            | (insert_space + slashed_num_letter)
             | letter_processor
-            | digit_char_processor
             | pynini.accep(NEMO_SPACE)
             | comma_processor
+            | (insert_space + standalone_hyphen)
             | other_word_processor
         ).optimize()
-        full_string_processor = pynini.closure(token_processor, 1).optimize()
+
+        # Forbids two numeric tokens back-to-back, so a digit run is always one maximal match.
+        full_string_processor = (
+            pynini.closure(non_numeric_token | (numeric_token + pynini.closure(non_numeric_token, 1)))
+            + pynini.closure(numeric_token, 0, 1)
+        ).optimize()
 
         # Window-based context matching around address keywords for robust detection
+        address_keywords_en = pynini.project(en_to_hi_map, "input")
+        state_city_keywords = pynini.union(
+            pynini.project(states_graph, "input"),
+            pynini.project(cities_graph, "input"),
+        ).optimize()
+        address_keywords = pynini.union(address_keywords_hi, address_keywords_en, state_city_keywords)
+
         word_boundary = pynini.union(
             NEMO_WHITE_SPACE, pynini.accep(COMMA), pynini.accep(HI_PERIOD), pynini.accep(PERIOD)
         ).optimize()
-        non_boundary_char = pynini.difference(NEMO_CHAR, word_boundary)
-        word = pynini.closure(non_boundary_char, 1).optimize()
-        word_with_boundary = word + pynini.closure(word_boundary)
-        window = pynini.closure(word_with_boundary, 0, 5).optimize()
-        boundary = pynini.closure(word_boundary, 1).optimize()
-        input_pattern = pynini.union(
-            address_keywords + boundary + window,
-            window + boundary + address_keywords + pynini.closure(boundary + window, 0, 1),
-        ).optimize()
+
+        # Unbounded sigma closure for efficient context matching around address keywords.
+        sigma_star = pynini.closure(NEMO_CHAR)
+        left_context = pynini.closure(sigma_star + word_boundary, 0, 1)
+        right_context = pynini.closure(word_boundary + sigma_star, 0, 1)
+        input_pattern = (left_context + address_keywords + right_context).optimize()
         address_graph = pynini.compose(input_pattern, full_string_processor).optimize()
         graph = (
             pynutil.insert('units: "address" cardinal { integer: "')
@@ -451,8 +502,15 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, inp
             + pynutil.insert("\"")
         )
 
-        address_graph = self.get_address_graph(ordinal, input_case)
-        structured_address_graph = self.get_structured_address_graph(ordinal, input_case)
+        address_keywords_hi = pynini.string_file(get_abs_path("data/address/context.tsv"))
+        states_graph = pynini.string_file(get_abs_path("data/address/states.tsv"))
+        cities_graph = pynini.string_file(get_abs_path("data/address/cities.tsv"))
+        en_to_hi_map = pynini.string_file(get_abs_path("data/address/en_to_hi_mapping.tsv"))
+
+        address_graph = self.get_address_graph(
+            cardinal, ordinal, input_case, address_keywords_hi, states_graph, cities_graph, en_to_hi_map
+        )
+        structured_address_graph = self.get_structured_address_graph(states_graph, cities_graph)
 
         graph = (
             pynutil.add_weight(graph_decimal, 0.1)