Ko TN: Add serial grammar and review fixes

nemo_text_processing/text_normalization/ko/data/whitelist.tsv

@@ -24,6 +24,7 @@ No.	번호
 )	오른쪽 괄호
 +	더하기
 -	마이너스
+=	은
 Σ	시그마
 η	에타
 κ	카파

nemo_text_processing/text_normalization/ko/taggers/cardinal.py

@@ -16,7 +16,7 @@
 import pynini
 from pynini.lib import pynutil
 
-from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst
+from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst, delete_space
 from nemo_text_processing.text_normalization.ko.utils import get_abs_path
 
 
@@ -274,6 +274,56 @@ def __init__(self, deterministic: bool = True):
             graph_zero,
         ).optimize()
 
+        # ----------------------------
+        # Context-based digit-by-digit reading
+        # e.g., 번호는 0987654321 -> 번호는 영구팔칠육오사삼이일
+        #
+        # Keep this separate from graph_num so regular cardinal zero
+        # and place-value behavior remain unchanged.
+
+        serial_space = pynini.closure(delete_space)
+
+        # Exclude 0 from graph_digit and force 0 -> 영.
+        # This avoids ambiguity if digit.tsv has another mapping for 0.
+        graph_digit_one_to_nine = (pynini.difference(NEMO_DIGIT, "0") @ graph_digit).optimize()
+
+        serial_digit = pynini.union(
+            pynini.cross("0", "영"),
+            graph_digit_one_to_nine,
+        ).optimize()
+
+        # Optional separators between individual digits.
+        serial_separator = pynini.union(
+            pynutil.delete("-"),
+            pynutil.delete("."),
+            pynutil.delete(" "),
+        ).optimize()
+
+        # Require at least three digits.
+        serial_body = (
+            serial_digit
+            + pynini.closure(serial_separator, 0, 1)
+            + serial_digit
+            + pynini.closure(serial_separator, 0, 1)
+            + serial_digit
+            + pynini.closure(pynini.closure(serial_separator, 0, 1) + serial_digit)
+        ).optimize()
+
+        serial_signal = pynini.string_map(
+            [
+                ("번호는", "번호는 "),
+                ("번호가", "번호가 "),
+                ("번호를", "번호를 "),
+                ("연락처는", "연락처는 "),
+                ("연락처가", "연락처가 "),
+                ("연락처를", "연락처를 "),
+            ]
+        ).optimize()
+
+        serial_case = (
+            pynutil.insert('integer: "') + serial_signal + serial_space + serial_body + pynutil.insert('"')
+        ).optimize()
+
         # ----------------------------
         # Native counting + counters
         # e.g., 3개, 2명, 10살
@@ -319,7 +369,7 @@ def __init__(self, deterministic: bool = True):
         signed_integer = (minus_prefix | plus_prefix).ques + integer_token
 
         # Prefer accounting-form first, then signed form
-        final_graph = paren_negative | signed_integer | counter_case
+        final_graph = serial_case | paren_negative | signed_integer | counter_case
 
         # Wrap with class tokens and finalize
         final_graph = self.add_tokens(final_graph)

nemo_text_processing/text_normalization/ko/taggers/date.py

@@ -84,6 +84,8 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True):
         era = pynini.union("기원전", "기원후").optimize()
         signs = pynutil.delete("/") | pynutil.delete(".") | pynutil.delete("-")
 
+        date_sep = signs + pynini.closure(delete_space, 0, 1) + insert_space
+
         # Strict digit ranges for M/D/Y and Y/M/D
         _d = pynini.union(*[pynini.accep(str(i)) for i in range(10)])
         _1to9 = pynini.union(*[pynini.accep(str(i)) for i in range(1, 10)])
@@ -185,14 +187,24 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True):
         graph_basic_date = (
             pynini.closure(era_component + insert_space, 0, 1)
             + year_component_y4_strict
-            + signs
-            + insert_space
+            + date_sep
             + (pynutil.insert("month: \"") + month_cardinal + pynutil.insert("월") + pynutil.insert("\""))
-            + signs
-            + insert_space
+            + date_sep
             + (pynutil.insert("day: \"") + cardinal_lz + pynutil.insert("일") + pynutil.insert("\""))
-            + pynini.closure(pynini.closure(insert_space, 0, 1) + week_component, 0, 1)
+            + pynini.closure(insert_space + week_component, 0, 1)
         )
+        graph_basic_date_with_dot_weekday = (
+            pynini.closure(era_component + insert_space, 0, 1)
+            + year_component_y4_strict
+            + date_sep
+            + (pynutil.insert("month: \"") + month_cardinal + pynutil.insert("월") + pynutil.insert("\""))
+            + date_sep
+            + (pynutil.insert("day: \"") + cardinal_lz + pynutil.insert("일") + pynutil.insert("\""))
+            + pynutil.delete(".")
+            + pynini.closure(delete_space, 0, 1)
+            + insert_space
+            + week_component
+        ).optimize()
 
         # American: MM/DD/YYYY
         graph_american_date = (
@@ -298,7 +310,8 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True):
         ).optimize()
 
         graph_all_date = (
-            graph_basic_date
+            graph_basic_date_with_dot_weekday
+            | graph_basic_date
             | graph_american_date
             | graph_european_date
             | graph_individual_component

nemo_text_processing/text_normalization/ko/taggers/telephone.py

@@ -37,8 +37,13 @@ class TelephoneFst(GraphFst):
 
     def __init__(self, deterministic: bool = True):
         super().__init__(name="telephone", kind="classify", deterministic=deterministic)
-        # Separator between digit blocks (e.g., "-" or ".")
-        delete_sep = pynutil.delete("-") | pynutil.delete(".")
+        # Separator between number blocks.
+        delete_sep = pynini.union(
+            pynutil.delete("-"),
+            pynutil.delete("."),
+            pynutil.delete(" "),
+        ).optimize()
+
         # Optional space inserted between blocks
         insert_block_space = insert_space
 
@@ -47,6 +52,7 @@ def __init__(self, deterministic: bool = True):
         zero_map = pynini.cross("0", "영")
         digit_ko = (digit | zero_map).optimize()
 
+        two_digits = digit_ko**2
         three_digits = digit_ko**3
         four_digits = digit_ko**4
 
@@ -62,25 +68,49 @@ def __init__(self, deterministic: bool = True):
             + delete_space
         )
 
-        # area part: "123-" | "123." | "(123)" [space?] or "(123)-"
-        area_core = three_digits
-        area_part = (
-            (area_core + delete_sep)
-            | (
-                pynutil.delete("(")
-                + area_core
-                + pynutil.delete(")")
-                + pynini.closure(pynutil.delete(" "), 0, 1)
-                + pynini.closure(delete_sep, 0, 1)
+        # First block may contain 2 or 3 digits.
+        # Examples: 02, 031, 043, 010
+        first_block = pynini.union(
+            two_digits,
+            three_digits,
+        ).optimize()
+
+        # Middle block may contain 3 or 4 digits.
+        # Examples: 123, 1234
+        middle_block = pynini.union(
+            three_digits,
+            four_digits,
+        ).optimize()
+
+        # Plain telephone form:
+        #   02-1234-5678
+        plain_first_part = (first_block + delete_sep + insert_block_space).optimize()
+
+        # Parenthesized telephone form:
+        #   (010)1234-5678
+        parenthesized_first_part = (
+            pynutil.delete("(")
+            + first_block
+            + pynutil.delete(")")
+            + pynini.closure(
+                pynutil.delete(" ") | pynutil.delete("-") | pynutil.delete("."),
+                0,
+                1,
             )
-        ) + insert_block_space
+            + insert_block_space
+        ).optimize()
+
+        first_part = pynini.union(
+            plain_first_part,
+            parenthesized_first_part,
+        ).optimize()
 
-        # 2) allow 3 **or 4** digits in the middle block (to support 010-3713-7050)
-        mid = pynini.union(three_digits, four_digits)
-        last4 = four_digits
+        # Standard telephone layout:
+        #   2 or 3 digits
+        #   followed by 3 or 4 digits
+        #   followed by 4 digits
+        number_part_core = (first_part + middle_block + delete_sep + insert_block_space + four_digits).optimize()
 
-        # consume '-' or '.' between middle and last blocks
-        number_part_core = area_part + mid + delete_sep + insert_block_space + last4
         number_part = pynutil.insert('number_part: "') + number_part_core + pynutil.insert('"')
 
         # final graph: with or without country code

nemo_text_processing/text_normalization/ko/verbalizers/fraction.py

@@ -129,43 +129,24 @@ def __init__(self, deterministic: bool = True):
         # Sigma for rewrite context (entire string)
         sigma = pynini.closure(NEMO_NOT_QUOTE | NEMO_SPACE)
 
-        # Fix subject particle agreement (이 → 가 for vowel-ending numerals)
-        # e.g., 사이 → 사가, 구이 → 구가
-        subject_rewrite = pynini.cdrewrite(
+        # Fix particle agreement for vowel-ending numerals.
+        # Subject: 이 -> 가
+        # Topic: 은 -> 는
+        # Object: 을 -> 를
+        particle_rewrite = pynini.cdrewrite(
             pynini.string_map(
                 [
+                    # Subject particle
                     ("이이", "이가"),
                     ("사이", "사가"),
                     ("오이", "오가"),
                     ("구이", "구가"),
-                ]
-            ),
-            "",
-            "",
-            sigma,
-        )
-
-        # Fix topic particle agreement (은 → 는)
-        # e.g., 이은 → 이는, 사은 → 사는
-        topic_rewrite = pynini.cdrewrite(
-            pynini.string_map(
-                [
+                    # Topic particle
                     ("이은", "이는"),
                     ("사은", "사는"),
                     ("오은", "오는"),
                     ("구은", "구는"),
-                ]
-            ),
-            "",
-            "",
-            sigma,
-        )
-
-        # Fix object particle agreement (을 → 를)
-        # e.g., 오을 → 오를, 이을 → 이를
-        object_rewrite = pynini.cdrewrite(
-            pynini.string_map(
-                [
+                    # Object particle
                     ("이을", "이를"),
                     ("사을", "사를"),
                     ("오을", "오를"),
@@ -178,5 +159,5 @@ def __init__(self, deterministic: bool = True):
         )
 
         # Apply all rewrite rules sequentially and final optimized FST
-        final_graph = final_graph @ subject_rewrite @ topic_rewrite @ object_rewrite
+        final_graph = final_graph @ particle_rewrite
         self.fst = final_graph.optimize()

tests/nemo_text_processing/ko/data_text_normalization/test_cases_cardinal.txt

@@ -45,4 +45,24 @@
 -2~마이너스 이
 -93~마이너스 구십삼
 -90325~마이너스 구만삼백이십오
--3234567~마이너스 삼백이십삼만사천오백육십칠
+-3234567~마이너스 삼백이십삼만사천오백육십칠
+휴대폰 번호는 0987654321입니다~휴대폰 번호는 영구팔칠육오사삼이일 입니다
+전화 번호는 01090817263입니다~전화 번호는 영일영구영팔일칠이육삼 입니다
+계좌 번호는 70501938462011입니다~계좌 번호는 칠영오영일구삼팔사육이영일일 입니다
+예약 번호는 907입니다~예약 번호는 구영칠 입니다
+예약 번호는 830-291입니다~예약 번호는 팔삼영이구일 입니다
+주문 번호는 4829.7301입니다~주문 번호는 사팔이구칠삼영일 입니다
+인증 번호는 000739입니다~인증 번호는 영영영칠삼구 입니다
+인증 번호는 204060입니다~인증 번호는 이영사영육영 입니다
+배송 번호는 591837462입니다~배송 번호는 오구일팔삼칠사육이 입니다
+회원 번호는 3000456789입니다~회원 번호는 삼영영영사오육칠팔구 입니다
+연락처는 84502719입니다~연락처는 팔사오영이칠일구 입니다
+연락처가 77008899입니다~연락처가 칠칠영영팔팔구구 입니다
+연락처를 604 812 930으로 저장했습니다~연락처를 육영사팔일이구삼영 으로 저장했습니다
+번호는 12입니다~번호는 십이 입니다
+배송 번호는 591837462입니다~배송 번호는 오구일팔삼칠사육이 입니다
+회원 번호는 3000456789입니다~회원 번호는 삼영영영사오육칠팔구 입니다
+접수 번호는 812709입니다~접수 번호는 팔일이칠영구 입니다
+신청 번호는 40682015입니다~신청 번호는 사영육팔이영일오 입니다
+확인 번호는 970045입니다~확인 번호는 구칠영영사오 입니다
+관리 번호는 25081973입니다~관리 번호는 이오영팔일구칠삼 입니다

tests/nemo_text_processing/ko/data_text_normalization/test_cases_telephone.txt

@@ -30,4 +30,10 @@
 +82-123.456-7890~국가번호 팔이 일이삼 사오육 칠팔구영
 111-222-3333~일일일 이이이 삼삼삼삼
 909-808-7070~구영구 팔영팔 칠영칠영
-(555)555-5555~오오오 오오오 오오오오
+(555)555-5555~오오오 오오오 오오오오
+02-123-4567~영이 일이삼 사오육칠
+02-1234-5678~영이 일이삼사 오육칠팔
+043-123-4567~영사삼 일이삼 사오육칠
+043-1234-5678~영사삼 일이삼사 오육칠팔
+(02) 1234-5678~영이 일이삼사 오육칠팔
+010 1234 5678~영일영 일이삼사 오육칠팔