diff --git a/nemo_text_processing/text_normalization/ko/data/whitelist.tsv b/nemo_text_processing/text_normalization/ko/data/whitelist.tsv index 82dc1220e..d0bdf4caf 100644 --- a/nemo_text_processing/text_normalization/ko/data/whitelist.tsv +++ b/nemo_text_processing/text_normalization/ko/data/whitelist.tsv @@ -24,6 +24,7 @@ No. 번호 ) 오른쪽 괄호 + 더하기 - 마이너스 += 은 Σ 시그마 η 에타 κ 카파 diff --git a/nemo_text_processing/text_normalization/ko/taggers/cardinal.py b/nemo_text_processing/text_normalization/ko/taggers/cardinal.py index e2b0f87e3..901c5723e 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/ko/taggers/cardinal.py @@ -16,7 +16,7 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst +from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst, delete_space from nemo_text_processing.text_normalization.ko.utils import get_abs_path @@ -274,6 +274,56 @@ def __init__(self, deterministic: bool = True): graph_zero, ).optimize() + # ---------------------------- + # Context-based digit-by-digit reading + # e.g., 번호는 0987654321 -> 번호는 영구팔칠육오사삼이일 + # + # Keep this separate from graph_num so regular cardinal zero + # and place-value behavior remain unchanged. + + serial_space = pynini.closure(delete_space) + + # Exclude 0 from graph_digit and force 0 -> 영. + # This avoids ambiguity if digit.tsv has another mapping for 0. + graph_digit_one_to_nine = (pynini.difference(NEMO_DIGIT, "0") @ graph_digit).optimize() + + serial_digit = pynini.union( + pynini.cross("0", "영"), + graph_digit_one_to_nine, + ).optimize() + + # Optional separators between individual digits. + serial_separator = pynini.union( + pynutil.delete("-"), + pynutil.delete("."), + pynutil.delete(" "), + ).optimize() + + # Require at least three digits. + serial_body = ( + serial_digit + + pynini.closure(serial_separator, 0, 1) + + serial_digit + + pynini.closure(serial_separator, 0, 1) + + serial_digit + + pynini.closure(pynini.closure(serial_separator, 0, 1) + serial_digit) + ).optimize() + + serial_signal = pynini.string_map( + [ + ("번호는", "번호는 "), + ("번호가", "번호가 "), + ("번호를", "번호를 "), + ("연락처는", "연락처는 "), + ("연락처가", "연락처가 "), + ("연락처를", "연락처를 "), + ] + ).optimize() + + serial_case = ( + pynutil.insert('integer: "') + serial_signal + serial_space + serial_body + pynutil.insert('"') + ).optimize() + # ---------------------------- # Native counting + counters # e.g., 3개, 2명, 10살 @@ -319,7 +369,7 @@ def __init__(self, deterministic: bool = True): signed_integer = (minus_prefix | plus_prefix).ques + integer_token # Prefer accounting-form first, then signed form - final_graph = paren_negative | signed_integer | counter_case + final_graph = serial_case | paren_negative | signed_integer | counter_case # Wrap with class tokens and finalize final_graph = self.add_tokens(final_graph) diff --git a/nemo_text_processing/text_normalization/ko/taggers/date.py b/nemo_text_processing/text_normalization/ko/taggers/date.py index 9748abc49..8b5d89d91 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/date.py +++ b/nemo_text_processing/text_normalization/ko/taggers/date.py @@ -84,6 +84,8 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): era = pynini.union("기원전", "기원후").optimize() signs = pynutil.delete("/") | pynutil.delete(".") | pynutil.delete("-") + date_sep = signs + pynini.closure(delete_space, 0, 1) + insert_space + # Strict digit ranges for M/D/Y and Y/M/D _d = pynini.union(*[pynini.accep(str(i)) for i in range(10)]) _1to9 = pynini.union(*[pynini.accep(str(i)) for i in range(1, 10)]) @@ -185,14 +187,24 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): graph_basic_date = ( pynini.closure(era_component + insert_space, 0, 1) + year_component_y4_strict - + signs - + insert_space + + date_sep + (pynutil.insert("month: \"") + month_cardinal + pynutil.insert("월") + pynutil.insert("\"")) - + signs - + insert_space + + date_sep + (pynutil.insert("day: \"") + cardinal_lz + pynutil.insert("일") + pynutil.insert("\"")) - + pynini.closure(pynini.closure(insert_space, 0, 1) + week_component, 0, 1) + + pynini.closure(insert_space + week_component, 0, 1) ) + graph_basic_date_with_dot_weekday = ( + pynini.closure(era_component + insert_space, 0, 1) + + year_component_y4_strict + + date_sep + + (pynutil.insert("month: \"") + month_cardinal + pynutil.insert("월") + pynutil.insert("\"")) + + date_sep + + (pynutil.insert("day: \"") + cardinal_lz + pynutil.insert("일") + pynutil.insert("\"")) + + pynutil.delete(".") + + pynini.closure(delete_space, 0, 1) + + insert_space + + week_component + ).optimize() # American: MM/DD/YYYY graph_american_date = ( @@ -298,7 +310,8 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): ).optimize() graph_all_date = ( - graph_basic_date + graph_basic_date_with_dot_weekday + | graph_basic_date | graph_american_date | graph_european_date | graph_individual_component diff --git a/nemo_text_processing/text_normalization/ko/taggers/telephone.py b/nemo_text_processing/text_normalization/ko/taggers/telephone.py index 90f31bb1f..ba4fbcea0 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/telephone.py +++ b/nemo_text_processing/text_normalization/ko/taggers/telephone.py @@ -37,8 +37,13 @@ class TelephoneFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="telephone", kind="classify", deterministic=deterministic) - # Separator between digit blocks (e.g., "-" or ".") - delete_sep = pynutil.delete("-") | pynutil.delete(".") + # Separator between number blocks. + delete_sep = pynini.union( + pynutil.delete("-"), + pynutil.delete("."), + pynutil.delete(" "), + ).optimize() + # Optional space inserted between blocks insert_block_space = insert_space @@ -47,6 +52,7 @@ def __init__(self, deterministic: bool = True): zero_map = pynini.cross("0", "영") digit_ko = (digit | zero_map).optimize() + two_digits = digit_ko**2 three_digits = digit_ko**3 four_digits = digit_ko**4 @@ -62,25 +68,49 @@ def __init__(self, deterministic: bool = True): + delete_space ) - # area part: "123-" | "123." | "(123)" [space?] or "(123)-" - area_core = three_digits - area_part = ( - (area_core + delete_sep) - | ( - pynutil.delete("(") - + area_core - + pynutil.delete(")") - + pynini.closure(pynutil.delete(" "), 0, 1) - + pynini.closure(delete_sep, 0, 1) + # First block may contain 2 or 3 digits. + # Examples: 02, 031, 043, 010 + first_block = pynini.union( + two_digits, + three_digits, + ).optimize() + + # Middle block may contain 3 or 4 digits. + # Examples: 123, 1234 + middle_block = pynini.union( + three_digits, + four_digits, + ).optimize() + + # Plain telephone form: + # 02-1234-5678 + plain_first_part = (first_block + delete_sep + insert_block_space).optimize() + + # Parenthesized telephone form: + # (010)1234-5678 + parenthesized_first_part = ( + pynutil.delete("(") + + first_block + + pynutil.delete(")") + + pynini.closure( + pynutil.delete(" ") | pynutil.delete("-") | pynutil.delete("."), + 0, + 1, ) - ) + insert_block_space + + insert_block_space + ).optimize() + + first_part = pynini.union( + plain_first_part, + parenthesized_first_part, + ).optimize() - # 2) allow 3 **or 4** digits in the middle block (to support 010-3713-7050) - mid = pynini.union(three_digits, four_digits) - last4 = four_digits + # Standard telephone layout: + # 2 or 3 digits + # followed by 3 or 4 digits + # followed by 4 digits + number_part_core = (first_part + middle_block + delete_sep + insert_block_space + four_digits).optimize() - # consume '-' or '.' between middle and last blocks - number_part_core = area_part + mid + delete_sep + insert_block_space + last4 number_part = pynutil.insert('number_part: "') + number_part_core + pynutil.insert('"') # final graph: with or without country code diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py b/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py index 472b8a86d..d523dba00 100644 --- a/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py @@ -129,43 +129,24 @@ def __init__(self, deterministic: bool = True): # Sigma for rewrite context (entire string) sigma = pynini.closure(NEMO_NOT_QUOTE | NEMO_SPACE) - # Fix subject particle agreement (이 → 가 for vowel-ending numerals) - # e.g., 사이 → 사가, 구이 → 구가 - subject_rewrite = pynini.cdrewrite( + # Fix particle agreement for vowel-ending numerals. + # Subject: 이 -> 가 + # Topic: 은 -> 는 + # Object: 을 -> 를 + particle_rewrite = pynini.cdrewrite( pynini.string_map( [ + # Subject particle ("이이", "이가"), ("사이", "사가"), ("오이", "오가"), ("구이", "구가"), - ] - ), - "", - "", - sigma, - ) - - # Fix topic particle agreement (은 → 는) - # e.g., 이은 → 이는, 사은 → 사는 - topic_rewrite = pynini.cdrewrite( - pynini.string_map( - [ + # Topic particle ("이은", "이는"), ("사은", "사는"), ("오은", "오는"), ("구은", "구는"), - ] - ), - "", - "", - sigma, - ) - - # Fix object particle agreement (을 → 를) - # e.g., 오을 → 오를, 이을 → 이를 - object_rewrite = pynini.cdrewrite( - pynini.string_map( - [ + # Object particle ("이을", "이를"), ("사을", "사를"), ("오을", "오를"), @@ -178,5 +159,5 @@ def __init__(self, deterministic: bool = True): ) # Apply all rewrite rules sequentially and final optimized FST - final_graph = final_graph @ subject_rewrite @ topic_rewrite @ object_rewrite + final_graph = final_graph @ particle_rewrite self.fst = final_graph.optimize() diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_cardinal.txt index 40187f74e..dbde76d4e 100644 --- a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_cardinal.txt @@ -45,4 +45,24 @@ -2~마이너스 이 -93~마이너스 구십삼 -90325~마이너스 구만삼백이십오 --3234567~마이너스 삼백이십삼만사천오백육십칠 \ No newline at end of file +-3234567~마이너스 삼백이십삼만사천오백육십칠 +휴대폰 번호는 0987654321입니다~휴대폰 번호는 영구팔칠육오사삼이일 입니다 +전화 번호는 01090817263입니다~전화 번호는 영일영구영팔일칠이육삼 입니다 +계좌 번호는 70501938462011입니다~계좌 번호는 칠영오영일구삼팔사육이영일일 입니다 +예약 번호는 907입니다~예약 번호는 구영칠 입니다 +예약 번호는 830-291입니다~예약 번호는 팔삼영이구일 입니다 +주문 번호는 4829.7301입니다~주문 번호는 사팔이구칠삼영일 입니다 +인증 번호는 000739입니다~인증 번호는 영영영칠삼구 입니다 +인증 번호는 204060입니다~인증 번호는 이영사영육영 입니다 +배송 번호는 591837462입니다~배송 번호는 오구일팔삼칠사육이 입니다 +회원 번호는 3000456789입니다~회원 번호는 삼영영영사오육칠팔구 입니다 +연락처는 84502719입니다~연락처는 팔사오영이칠일구 입니다 +연락처가 77008899입니다~연락처가 칠칠영영팔팔구구 입니다 +연락처를 604 812 930으로 저장했습니다~연락처를 육영사팔일이구삼영 으로 저장했습니다 +번호는 12입니다~번호는 십이 입니다 +배송 번호는 591837462입니다~배송 번호는 오구일팔삼칠사육이 입니다 +회원 번호는 3000456789입니다~회원 번호는 삼영영영사오육칠팔구 입니다 +접수 번호는 812709입니다~접수 번호는 팔일이칠영구 입니다 +신청 번호는 40682015입니다~신청 번호는 사영육팔이영일오 입니다 +확인 번호는 970045입니다~확인 번호는 구칠영영사오 입니다 +관리 번호는 25081973입니다~관리 번호는 이오영팔일구칠삼 입니다 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_telephone.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_telephone.txt index b6e573aec..a871e4f71 100644 --- a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_telephone.txt +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_telephone.txt @@ -30,4 +30,10 @@ +82-123.456-7890~국가번호 팔이 일이삼 사오육 칠팔구영 111-222-3333~일일일 이이이 삼삼삼삼 909-808-7070~구영구 팔영팔 칠영칠영 -(555)555-5555~오오오 오오오 오오오오 \ No newline at end of file +(555)555-5555~오오오 오오오 오오오오 +02-123-4567~영이 일이삼 사오육칠 +02-1234-5678~영이 일이삼사 오육칠팔 +043-123-4567~영사삼 일이삼 사오육칠 +043-1234-5678~영사삼 일이삼사 오육칠팔 +(02) 1234-5678~영이 일이삼사 오육칠팔 +010 1234 5678~영일영 일이삼사 오육칠팔 \ No newline at end of file