From 98d4da250b44bbdd2261d9f13625b24ec891f304 Mon Sep 17 00:00:00 2001 From: Jinwoo Bae Date: Tue, 7 Apr 2026 14:08:50 -0700 Subject: [PATCH 01/13] Add Korean TN post-processing rules for particle agreement and month handling Signed-off-by: Jinwoo Bae --- .../text_normalization/ko/taggers/date.py | 12 ++++ .../ko/verbalizers/post_processing.py | 68 +++++++++++++++++++ .../text_normalization/normalize.py | 6 +- 3 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/post_processing.py diff --git a/nemo_text_processing/text_normalization/ko/taggers/date.py b/nemo_text_processing/text_normalization/ko/taggers/date.py index 4f2da5702..3c89e104b 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/date.py +++ b/nemo_text_processing/text_normalization/ko/taggers/date.py @@ -249,6 +249,17 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): + pynutil.insert("\"") ) + month_josa = pynini.union("에", "은", "는", "에는").optimize() + + individual_month_component_with_josa = ( + pynutil.insert('month: "') + + month_cardinal + + pynutil.delete("월") + + pynutil.insert("월") + + pynini.closure(month_josa, 0, 1) + + pynutil.insert('"') + ).optimize() + individual_day_component = ( pynutil.insert("day: \"") + cardinal_lz @@ -272,6 +283,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): day_and_weekday_component | month_and_weekday_component | individual_year_component + | individual_month_component_with_josa | individual_month_component | individual_day_component | week_component diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/post_processing.py b/nemo_text_processing/text_normalization/ko/verbalizers/post_processing.py new file mode 100644 index 000000000..b363040fd --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/verbalizers/post_processing.py @@ -0,0 +1,68 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_SIGMA, NEMO_SPACE, generator_main +from nemo_text_processing.utils.logging import logger + + +class PostProcessingFst: + def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "ko_tn_post_processing.far") + + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["post_process_graph"] + logger.info(f"Post processing graph was restored from {far_file}.") + else: + self.fst = self.get_postprocess_graph() + if far_file: + generator_main(far_file, {"post_process_graph": self.fst}) + + def get_postprocess_graph(self): + delete_space = pynutil.delete(NEMO_SPACE) + + vowel_final = pynini.union( + "아", "야", "어", "여", "오", "요", "우", "유", "이", "애", "에", + "사", "오", "구" + ) + + rule_i_to_ga = pynini.cdrewrite( + delete_space + pynini.cross("이 ", "가 "), + vowel_final, + "", + NEMO_SIGMA, + ) + + rule_eun_to_neun = pynini.cdrewrite( + delete_space + pynini.cross("은 ", "는 "), + vowel_final, + "", + NEMO_SIGMA, + ) + + rule_eul_to_reul = pynini.cdrewrite( + delete_space + pynini.cross("을 ", "를 "), + vowel_final, + "", + NEMO_SIGMA, + ) + + graph = rule_i_to_ga @ rule_eun_to_neun @ rule_eul_to_reul + return graph.optimize() \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index 5e2f9ebb5..5930f49bd 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -187,7 +187,11 @@ def __init__( self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) elif lang == 'ko': from nemo_text_processing.text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.ko.verbalizers.post_processing import PostProcessingFst from nemo_text_processing.text_normalization.ko.verbalizers.verbalize_final import VerbalizeFinalFst + + if post_process: + self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) else: raise NotImplementedError(f"Language {lang} has not been supported yet.") @@ -388,7 +392,7 @@ def normalize( return text output = SPACE_DUP.sub(' ', output[1:]) - if self.lang in ["en", "hi", "vi"] and hasattr(self, 'post_processor') and self.post_processor is not None: + if self.lang in ["en", "hi", "vi", "ko"] and hasattr(self, 'post_processor') and self.post_processor is not None: output = self.post_process(output) if punct_post_process: From 19619a9b1d6906318985fcbfe0b549a5102cf85a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Apr 2026 21:14:31 +0000 Subject: [PATCH 02/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../text_normalization/ko/taggers/date.py | 2 +- .../text_normalization/ko/verbalizers/post_processing.py | 7 ++----- nemo_text_processing/text_normalization/normalize.py | 8 ++++++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/nemo_text_processing/text_normalization/ko/taggers/date.py b/nemo_text_processing/text_normalization/ko/taggers/date.py index 3c89e104b..45943e1a3 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/date.py +++ b/nemo_text_processing/text_normalization/ko/taggers/date.py @@ -259,7 +259,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): + pynini.closure(month_josa, 0, 1) + pynutil.insert('"') ).optimize() - + individual_day_component = ( pynutil.insert("day: \"") + cardinal_lz diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/post_processing.py b/nemo_text_processing/text_normalization/ko/verbalizers/post_processing.py index b363040fd..45fcb259f 100644 --- a/nemo_text_processing/text_normalization/ko/verbalizers/post_processing.py +++ b/nemo_text_processing/text_normalization/ko/verbalizers/post_processing.py @@ -38,10 +38,7 @@ def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): def get_postprocess_graph(self): delete_space = pynutil.delete(NEMO_SPACE) - vowel_final = pynini.union( - "아", "야", "어", "여", "오", "요", "우", "유", "이", "애", "에", - "사", "오", "구" - ) + vowel_final = pynini.union("아", "야", "어", "여", "오", "요", "우", "유", "이", "애", "에", "사", "오", "구") rule_i_to_ga = pynini.cdrewrite( delete_space + pynini.cross("이 ", "가 "), @@ -65,4 +62,4 @@ def get_postprocess_graph(self): ) graph = rule_i_to_ga @ rule_eun_to_neun @ rule_eul_to_reul - return graph.optimize() \ No newline at end of file + return graph.optimize() diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index 5930f49bd..12661bd3d 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -189,7 +189,7 @@ def __init__( from nemo_text_processing.text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.ko.verbalizers.post_processing import PostProcessingFst from nemo_text_processing.text_normalization.ko.verbalizers.verbalize_final import VerbalizeFinalFst - + if post_process: self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) else: @@ -392,7 +392,11 @@ def normalize( return text output = SPACE_DUP.sub(' ', output[1:]) - if self.lang in ["en", "hi", "vi", "ko"] and hasattr(self, 'post_processor') and self.post_processor is not None: + if ( + self.lang in ["en", "hi", "vi", "ko"] + and hasattr(self, 'post_processor') + and self.post_processor is not None + ): output = self.post_process(output) if punct_post_process: From 420fb96f8b16de776796c965483926bc42e3881a Mon Sep 17 00:00:00 2001 From: Jinwoo Bae Date: Wed, 8 Apr 2026 00:04:09 -0700 Subject: [PATCH 03/13] Add Korean TN fraction test cases for particle agreement Signed-off-by: Jinwoo Bae --- .../test_cases_fraction.txt | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_fraction.txt index a183be59b..fc39fd495 100644 --- a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_fraction.txt +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_fraction.txt @@ -11,4 +11,18 @@ 1과1/3~일과 삼분의 일 1과√1/4~일과 사분의 루트 일 3분의1~삼분의 일 -121분의3221~백이십일분의 삼천이백이십일 \ No newline at end of file +121분의3221~백이십일분의 삼천이백이십일 +이번 경기의 3/5이 중요하다~이번 경기의 오분의 삼 이 중요하다 +전체 구역의 4/7이 통제되었다~전체 구역의 칠분의 사가 통제되었다 +설문 응답자의 9/10 이 찬성했다~설문 응답자의 십분의 구가 찬성했다 +그 중 2/3은 성공했다~그 중 삼분의 이는 성공했다 +참가자의 5/8이 탈락했다~참가자의 팔분의 오가 탈락했다 +참가자의 6/7 이 통과했다~참가자의 칠분의 육 이 통과했다 +전체의 3/4 이 감소했다~전체의 사분의 삼 이 감소했다 +응답자의 2/5이 반대했다~응답자의 오분의 이가 반대했다 +학생의 7/9 이 합격했다~학생의 구분의 칠 이 합격했다 +전체의 1/2 이 남았다~전체의 이분의 일 이 남았다 +그 중 4/5이 성공했다~그 중 오분의 사가 성공했다 +전체의 5/6이 완료되었다~전체의 육분의 오가 완료되었다 +참가자의 3/8이 탈락했다~참가자의 팔분의 삼 이 탈락했다 +응답자의 6/10 이 동의했다~응답자의 십분의 육 이 동의했다 \ No newline at end of file From ecf34ccb8657f0e91852657750fa310d29cd128d Mon Sep 17 00:00:00 2001 From: Jinwoo Bae Date: Fri, 10 Apr 2026 12:18:56 -0700 Subject: [PATCH 04/13] Fix Korean fraction verbalization with particle-aware handling and remove post_processing dependency Signed-off-by: Jinwoo Bae --- .../text_normalization/ko/taggers/fraction.py | 22 +++- .../ko/verbalizers/fraction.py | 108 ++++++++++++++++-- .../ko/verbalizers/post_processing.py | 65 ----------- .../ko/verbalizers/verbalize_final.py | 2 +- .../text_normalization/normalize.py | 6 +- .../test_cases_fraction.txt | 6 +- 6 files changed, 124 insertions(+), 85 deletions(-) delete mode 100644 nemo_text_processing/text_normalization/ko/verbalizers/post_processing.py diff --git a/nemo_text_processing/text_normalization/ko/taggers/fraction.py b/nemo_text_processing/text_normalization/ko/taggers/fraction.py index 2163f5f7f..6181e82d1 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/fraction.py +++ b/nemo_text_processing/text_normalization/ko/taggers/fraction.py @@ -81,6 +81,26 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): + numerator_component ) + # Optional particles following the fraction + particle_subject = ( + pynutil.insert('morphosyntactic_features: "분의_subject"') + + (pynutil.delete("이") | pynutil.delete("가")) + ) + particle_topic = ( + pynutil.insert('morphosyntactic_features: "분의_topic"') + + (pynutil.delete("은") | pynutil.delete("는")) + ) + particle_object = ( + pynutil.insert('morphosyntactic_features: "분의_object"') + + (pynutil.delete("을") | pynutil.delete("를")) + ) + + optional_particle = pynini.closure( + pynutil.insert(NEMO_SPACE) + (particle_subject | particle_topic | particle_object), + 0, + 1, + ) + # Optional minus sign optional_sign = ( pynutil.insert(f'negative: {DOUBLE_QUOTE}') @@ -90,7 +110,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): ) # Combine full graph - graph = pynini.closure(optional_sign, 0, 1) + (graph_fraction_slash | graph_fraction_word) + graph = pynini.closure(optional_sign, 0, 1) + (graph_fraction_slash | graph_fraction_word) + optional_particle self.graph = graph.optimize() final_graph = self.add_tokens(graph) self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py b/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py index bafbf133d..5886d408c 100644 --- a/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py @@ -17,7 +17,6 @@ from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space - class FractionFst(GraphFst): """ Finite state transducer for verbalizing Korean fractions, e.g. @@ -60,7 +59,46 @@ def __init__(self, deterministic: bool = True): + numerator_component ) - # Match and delete integer_part field (e.g., "2" in "2과3분의1") + # Handle subject particle feature (분의_subject) + # Insert default particle "이" (will be corrected later via rewrite rules) + subject_suffix = ( + pynutil.delete(NEMO_SPACE) + + pynutil.delete('morphosyntactic_features:') + + delete_space + + pynutil.delete('"분의_subject"') + + delete_space + + pynutil.insert("이") # 일단 기본값 + ) + + # Handle topic particle feature (분의_topic) + topic_suffix = ( + pynutil.delete(NEMO_SPACE) + + pynutil.delete('morphosyntactic_features:') + + delete_space + + pynutil.delete('"분의_topic"') + + delete_space + + pynutil.insert("은") + ) + + # Handle object particle feature (분의_object) + object_suffix = ( + pynutil.delete(NEMO_SPACE) + + pynutil.delete('morphosyntactic_features:') + + delete_space + + pynutil.delete('"분의_object"') + + delete_space + + pynutil.insert("을") + ) + + # Combine fraction + optional particle suffix + # Particle is always inserted first in default form and later corrected + graph_fraction_all = ( + graph_fraction + + pynini.closure(subject_suffix | topic_suffix | object_suffix, 0, 1) + ) + + # Handle integer + fraction (e.g., "2과 3/4") + # integer_part is removed and replaced with proper spacing graph_integer = ( pynutil.delete('integer_part:') + delete_space @@ -69,9 +107,10 @@ def __init__(self, deterministic: bool = True): + pynutil.delete('"') + pynutil.insert(NEMO_SPACE) ) - graph_integer_fraction = graph_integer + delete_space + graph_fraction - - # Match and delete optional negative field (e.g., "마이너스") + # Combine integer part with fraction + graph_integer_fraction = graph_integer + delete_space + graph_fraction_all + + # Handle optional negative prefix (e.g., "마이너스") optional_sign = ( pynutil.delete('negative:') + delete_space @@ -82,9 +121,58 @@ def __init__(self, deterministic: bool = True): + pynutil.insert(NEMO_SPACE) ) - # Final graph handles optional negative + (integer + fraction | fraction only) - graph = pynini.closure(optional_sign, 0, 1) + (graph_integer_fraction | graph_fraction) - - # Final optimized verbalizer FST + # Final structure: + # [optional negative] + (integer + fraction OR fraction only) + graph = pynini.closure(optional_sign, 0, 1) + (graph_integer_fraction | graph_fraction_all) + + # Remove token wrappers final_graph = self.delete_tokens(graph) - self.fst = final_graph.optimize() + + # Sigma for rewrite context (entire string) + sigma = pynini.closure(NEMO_NOT_QUOTE | NEMO_SPACE) + + # Fix subject particle agreement (이 → 가 for vowel-ending numerals) + # e.g., 사이 → 사가, 구이 → 구가 + subject_rewrite = pynini.cdrewrite( + pynini.union( + pynini.cross("이이", "이가"), + pynini.cross("사이", "사가"), + pynini.cross("오이", "오가"), + pynini.cross("구이", "구가"), + ), + "", + "", + sigma, + ) + + # Fix topic particle agreement (은 → 는) + # e.g., 이은 → 이는, 사은 → 사는 + topic_rewrite = pynini.cdrewrite( + pynini.union( + pynini.cross("이은", "이는"), + pynini.cross("사은", "사는"), + pynini.cross("오은", "오는"), + pynini.cross("구은", "구는"), + ), + "", + "", + sigma, + ) + + # Fix object particle agreement (을 → 를) + # e.g., 오을 → 오를, 이을 → 이를 + object_rewrite = pynini.cdrewrite( + pynini.union( + pynini.cross("이을", "이를"), + pynini.cross("사을", "사를"), + pynini.cross("오을", "오를"), + pynini.cross("구을", "구를"), + ), + "", + "", + sigma, + ) + + # Apply all rewrite rules sequentially and final optimized FST + final_graph = final_graph @ subject_rewrite @ topic_rewrite @ object_rewrite + self.fst = final_graph.optimize() \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/post_processing.py b/nemo_text_processing/text_normalization/ko/verbalizers/post_processing.py deleted file mode 100644 index 45fcb259f..000000000 --- a/nemo_text_processing/text_normalization/ko/verbalizers/post_processing.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pynini -from pynini.lib import pynutil - -from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_SIGMA, NEMO_SPACE, generator_main -from nemo_text_processing.utils.logging import logger - - -class PostProcessingFst: - def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): - far_file = None - if cache_dir is not None and cache_dir != "None": - os.makedirs(cache_dir, exist_ok=True) - far_file = os.path.join(cache_dir, "ko_tn_post_processing.far") - - if not overwrite_cache and far_file and os.path.exists(far_file): - self.fst = pynini.Far(far_file, mode="r")["post_process_graph"] - logger.info(f"Post processing graph was restored from {far_file}.") - else: - self.fst = self.get_postprocess_graph() - if far_file: - generator_main(far_file, {"post_process_graph": self.fst}) - - def get_postprocess_graph(self): - delete_space = pynutil.delete(NEMO_SPACE) - - vowel_final = pynini.union("아", "야", "어", "여", "오", "요", "우", "유", "이", "애", "에", "사", "오", "구") - - rule_i_to_ga = pynini.cdrewrite( - delete_space + pynini.cross("이 ", "가 "), - vowel_final, - "", - NEMO_SIGMA, - ) - - rule_eun_to_neun = pynini.cdrewrite( - delete_space + pynini.cross("은 ", "는 "), - vowel_final, - "", - NEMO_SIGMA, - ) - - rule_eul_to_reul = pynini.cdrewrite( - delete_space + pynini.cross("을 ", "를 "), - vowel_final, - "", - NEMO_SIGMA, - ) - - graph = rule_i_to_ga @ rule_eun_to_neun @ rule_eul_to_reul - return graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/ko/verbalizers/verbalize_final.py index 0271a4b7b..1c0fabd1b 100644 --- a/nemo_text_processing/text_normalization/ko/verbalizers/verbalize_final.py +++ b/nemo_text_processing/text_normalization/ko/verbalizers/verbalize_final.py @@ -69,4 +69,4 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ self.fst = verbalizer.optimize() if far_file: - generator_main(far_file, {"verbalize": self.fst}) + generator_main(far_file, {"verbalize": self.fst}) \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index 12661bd3d..7682ef047 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -187,11 +187,7 @@ def __init__( self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) elif lang == 'ko': from nemo_text_processing.text_normalization.ko.taggers.tokenize_and_classify import ClassifyFst - from nemo_text_processing.text_normalization.ko.verbalizers.post_processing import PostProcessingFst from nemo_text_processing.text_normalization.ko.verbalizers.verbalize_final import VerbalizeFinalFst - - if post_process: - self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) else: raise NotImplementedError(f"Language {lang} has not been supported yet.") @@ -393,7 +389,7 @@ def normalize( output = SPACE_DUP.sub(' ', output[1:]) if ( - self.lang in ["en", "hi", "vi", "ko"] + self.lang in ["en", "hi", "vi"] and hasattr(self, 'post_processor') and self.post_processor is not None ): diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_fraction.txt index fc39fd495..65e5049b8 100644 --- a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_fraction.txt +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_fraction.txt @@ -12,9 +12,9 @@ 1과√1/4~일과 사분의 루트 일 3분의1~삼분의 일 121분의3221~백이십일분의 삼천이백이십일 -이번 경기의 3/5이 중요하다~이번 경기의 오분의 삼 이 중요하다 +이번 경기의 3/5이 중요하다~이번 경기의 오분의 삼이 중요하다 전체 구역의 4/7이 통제되었다~전체 구역의 칠분의 사가 통제되었다 -설문 응답자의 9/10 이 찬성했다~설문 응답자의 십분의 구가 찬성했다 +설문 응답자의 9/10이 찬성했다~설문 응답자의 십분의 구가 찬성했다 그 중 2/3은 성공했다~그 중 삼분의 이는 성공했다 참가자의 5/8이 탈락했다~참가자의 팔분의 오가 탈락했다 참가자의 6/7 이 통과했다~참가자의 칠분의 육 이 통과했다 @@ -24,5 +24,5 @@ 전체의 1/2 이 남았다~전체의 이분의 일 이 남았다 그 중 4/5이 성공했다~그 중 오분의 사가 성공했다 전체의 5/6이 완료되었다~전체의 육분의 오가 완료되었다 -참가자의 3/8이 탈락했다~참가자의 팔분의 삼 이 탈락했다 +참가자의 3/8이 탈락했다~참가자의 팔분의 삼이 탈락했다 응답자의 6/10 이 동의했다~응답자의 십분의 육 이 동의했다 \ No newline at end of file From 7acdb886c04c102814c4a6b7645a4627f0f89fbb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 Apr 2026 20:02:04 +0000 Subject: [PATCH 05/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../text_normalization/ko/taggers/fraction.py | 17 +++++------- .../ko/verbalizers/fraction.py | 26 +++++++++---------- .../ko/verbalizers/verbalize_final.py | 2 +- .../text_normalization/normalize.py | 6 +---- 4 files changed, 21 insertions(+), 30 deletions(-) diff --git a/nemo_text_processing/text_normalization/ko/taggers/fraction.py b/nemo_text_processing/text_normalization/ko/taggers/fraction.py index 6181e82d1..64ea0c56e 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/fraction.py +++ b/nemo_text_processing/text_normalization/ko/taggers/fraction.py @@ -82,17 +82,14 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): ) # Optional particles following the fraction - particle_subject = ( - pynutil.insert('morphosyntactic_features: "분의_subject"') - + (pynutil.delete("이") | pynutil.delete("가")) + particle_subject = pynutil.insert('morphosyntactic_features: "분의_subject"') + ( + pynutil.delete("이") | pynutil.delete("가") ) - particle_topic = ( - pynutil.insert('morphosyntactic_features: "분의_topic"') - + (pynutil.delete("은") | pynutil.delete("는")) + particle_topic = pynutil.insert('morphosyntactic_features: "분의_topic"') + ( + pynutil.delete("은") | pynutil.delete("는") ) - particle_object = ( - pynutil.insert('morphosyntactic_features: "분의_object"') - + (pynutil.delete("을") | pynutil.delete("를")) + particle_object = pynutil.insert('morphosyntactic_features: "분의_object"') + ( + pynutil.delete("을") | pynutil.delete("를") ) optional_particle = pynini.closure( @@ -100,7 +97,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): 0, 1, ) - + # Optional minus sign optional_sign = ( pynutil.insert(f'negative: {DOUBLE_QUOTE}') diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py b/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py index 5886d408c..84e9db160 100644 --- a/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py @@ -17,6 +17,7 @@ from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space + class FractionFst(GraphFst): """ Finite state transducer for verbalizing Korean fractions, e.g. @@ -67,7 +68,7 @@ def __init__(self, deterministic: bool = True): + delete_space + pynutil.delete('"분의_subject"') + delete_space - + pynutil.insert("이") # 일단 기본값 + + pynutil.insert("이") # 일단 기본값 ) # Handle topic particle feature (분의_topic) @@ -89,14 +90,11 @@ def __init__(self, deterministic: bool = True): + delete_space + pynutil.insert("을") ) - + # Combine fraction + optional particle suffix # Particle is always inserted first in default form and later corrected - graph_fraction_all = ( - graph_fraction - + pynini.closure(subject_suffix | topic_suffix | object_suffix, 0, 1) - ) - + graph_fraction_all = graph_fraction + pynini.closure(subject_suffix | topic_suffix | object_suffix, 0, 1) + # Handle integer + fraction (e.g., "2과 3/4") # integer_part is removed and replaced with proper spacing graph_integer = ( @@ -109,7 +107,7 @@ def __init__(self, deterministic: bool = True): ) # Combine integer part with fraction graph_integer_fraction = graph_integer + delete_space + graph_fraction_all - + # Handle optional negative prefix (e.g., "마이너스") optional_sign = ( pynutil.delete('negative:') @@ -124,13 +122,13 @@ def __init__(self, deterministic: bool = True): # Final structure: # [optional negative] + (integer + fraction OR fraction only) graph = pynini.closure(optional_sign, 0, 1) + (graph_integer_fraction | graph_fraction_all) - + # Remove token wrappers final_graph = self.delete_tokens(graph) - + # Sigma for rewrite context (entire string) sigma = pynini.closure(NEMO_NOT_QUOTE | NEMO_SPACE) - + # Fix subject particle agreement (이 → 가 for vowel-ending numerals) # e.g., 사이 → 사가, 구이 → 구가 subject_rewrite = pynini.cdrewrite( @@ -144,7 +142,7 @@ def __init__(self, deterministic: bool = True): "", sigma, ) - + # Fix topic particle agreement (은 → 는) # e.g., 이은 → 이는, 사은 → 사는 topic_rewrite = pynini.cdrewrite( @@ -172,7 +170,7 @@ def __init__(self, deterministic: bool = True): "", sigma, ) - + # Apply all rewrite rules sequentially and final optimized FST final_graph = final_graph @ subject_rewrite @ topic_rewrite @ object_rewrite - self.fst = final_graph.optimize() \ No newline at end of file + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/ko/verbalizers/verbalize_final.py index 1c0fabd1b..0271a4b7b 100644 --- a/nemo_text_processing/text_normalization/ko/verbalizers/verbalize_final.py +++ b/nemo_text_processing/text_normalization/ko/verbalizers/verbalize_final.py @@ -69,4 +69,4 @@ def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_ self.fst = verbalizer.optimize() if far_file: - generator_main(far_file, {"verbalize": self.fst}) \ No newline at end of file + generator_main(far_file, {"verbalize": self.fst}) diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index 7682ef047..5e2f9ebb5 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -388,11 +388,7 @@ def normalize( return text output = SPACE_DUP.sub(' ', output[1:]) - if ( - self.lang in ["en", "hi", "vi"] - and hasattr(self, 'post_processor') - and self.post_processor is not None - ): + if self.lang in ["en", "hi", "vi"] and hasattr(self, 'post_processor') and self.post_processor is not None: output = self.post_process(output) if punct_post_process: From f47b3708f2573d0d251fa9dc90adb959313677d9 Mon Sep 17 00:00:00 2001 From: Jinwoo Bae Date: Thu, 16 Apr 2026 10:19:44 -0700 Subject: [PATCH 06/13] Fix date and fraction normalization issues based on review feedback Signed-off-by: Jinwoo Bae --- .../text_normalization/ko/taggers/date.py | 17 ++++----- .../ko/verbalizers/fraction.py | 36 +++++++++---------- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/nemo_text_processing/text_normalization/ko/taggers/date.py b/nemo_text_processing/text_normalization/ko/taggers/date.py index 45943e1a3..254eebfc0 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/date.py +++ b/nemo_text_processing/text_normalization/ko/taggers/date.py @@ -226,8 +226,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): + insert_space + pynutil.insert("year: \"") + (YEAR_ERA_1TO4 @ graph_cardinal) - + pynutil.delete("년") - + pynutil.insert("년") + + pynini.accep("년") + pynutil.insert("\"") ) | @@ -235,8 +234,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): ( pynutil.insert("year: \"") + (YEAR_NO_ERA_1TO4 @ graph_cardinal) - + pynutil.delete("년") - + pynutil.insert("년") + + pynini.accep("년") + pynutil.insert("\"") ) ).optimize() @@ -244,18 +242,16 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): individual_month_component = ( pynutil.insert("month: \"") + month_cardinal - + pynutil.delete("월") - + pynutil.insert("월") + + pynini.accep("월") + pynutil.insert("\"") ) - month_josa = pynini.union("에", "은", "는", "에는").optimize() + month_josa = pynini.union("에", "은", "는", "에는") individual_month_component_with_josa = ( pynutil.insert('month: "') + month_cardinal - + pynutil.delete("월") - + pynutil.insert("월") + + pynini.accep("월") + pynini.closure(month_josa, 0, 1) + pynutil.insert('"') ).optimize() @@ -263,8 +259,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): individual_day_component = ( pynutil.insert("day: \"") + cardinal_lz - + pynutil.delete("일") - + pynutil.insert("일") + + pynini.accep("일") + pynutil.insert("\"") ) diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py b/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py index 84e9db160..10a5d7781 100644 --- a/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py @@ -132,12 +132,12 @@ def __init__(self, deterministic: bool = True): # Fix subject particle agreement (이 → 가 for vowel-ending numerals) # e.g., 사이 → 사가, 구이 → 구가 subject_rewrite = pynini.cdrewrite( - pynini.union( - pynini.cross("이이", "이가"), - pynini.cross("사이", "사가"), - pynini.cross("오이", "오가"), - pynini.cross("구이", "구가"), - ), + pynini.string_map([ + ("이이", "이가"), + ("사이", "사가"), + ("오이", "오가"), + ("구이", "구가"), + ]), "", "", sigma, @@ -146,12 +146,12 @@ def __init__(self, deterministic: bool = True): # Fix topic particle agreement (은 → 는) # e.g., 이은 → 이는, 사은 → 사는 topic_rewrite = pynini.cdrewrite( - pynini.union( - pynini.cross("이은", "이는"), - pynini.cross("사은", "사는"), - pynini.cross("오은", "오는"), - pynini.cross("구은", "구는"), - ), + pynini.string_map([ + ("이은", "이는"), + ("사은", "사는"), + ("오은", "오는"), + ("구은", "구는"), + ]), "", "", sigma, @@ -160,12 +160,12 @@ def __init__(self, deterministic: bool = True): # Fix object particle agreement (을 → 를) # e.g., 오을 → 오를, 이을 → 이를 object_rewrite = pynini.cdrewrite( - pynini.union( - pynini.cross("이을", "이를"), - pynini.cross("사을", "사를"), - pynini.cross("오을", "오를"), - pynini.cross("구을", "구를"), - ), + pynini.string_map([ + ("이을", "이를"), + ("사을", "사를"), + ("오을", "오를"), + ("구을", "구를"), + ]), "", "", sigma, From be9a45c8f6b54f5536ca1417d35227db9650cd61 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 17:23:52 +0000 Subject: [PATCH 07/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../text_normalization/ko/taggers/date.py | 12 +----- .../ko/verbalizers/fraction.py | 42 +++++++++++-------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/nemo_text_processing/text_normalization/ko/taggers/date.py b/nemo_text_processing/text_normalization/ko/taggers/date.py index 254eebfc0..9748abc49 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/date.py +++ b/nemo_text_processing/text_normalization/ko/taggers/date.py @@ -240,10 +240,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): ).optimize() individual_month_component = ( - pynutil.insert("month: \"") - + month_cardinal - + pynini.accep("월") - + pynutil.insert("\"") + pynutil.insert("month: \"") + month_cardinal + pynini.accep("월") + pynutil.insert("\"") ) month_josa = pynini.union("에", "은", "는", "에는") @@ -256,12 +253,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): + pynutil.insert('"') ).optimize() - individual_day_component = ( - pynutil.insert("day: \"") - + cardinal_lz - + pynini.accep("일") - + pynutil.insert("\"") - ) + individual_day_component = pynutil.insert("day: \"") + cardinal_lz + pynini.accep("일") + pynutil.insert("\"") week_full_word_acceptor = pynini.project(week, "output") week_component_full_word = pynutil.insert("weekday: \"") + week_full_word_acceptor + pynutil.insert("\"") diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py b/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py index 10a5d7781..472b8a86d 100644 --- a/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py @@ -132,12 +132,14 @@ def __init__(self, deterministic: bool = True): # Fix subject particle agreement (이 → 가 for vowel-ending numerals) # e.g., 사이 → 사가, 구이 → 구가 subject_rewrite = pynini.cdrewrite( - pynini.string_map([ - ("이이", "이가"), - ("사이", "사가"), - ("오이", "오가"), - ("구이", "구가"), - ]), + pynini.string_map( + [ + ("이이", "이가"), + ("사이", "사가"), + ("오이", "오가"), + ("구이", "구가"), + ] + ), "", "", sigma, @@ -146,12 +148,14 @@ def __init__(self, deterministic: bool = True): # Fix topic particle agreement (은 → 는) # e.g., 이은 → 이는, 사은 → 사는 topic_rewrite = pynini.cdrewrite( - pynini.string_map([ - ("이은", "이는"), - ("사은", "사는"), - ("오은", "오는"), - ("구은", "구는"), - ]), + pynini.string_map( + [ + ("이은", "이는"), + ("사은", "사는"), + ("오은", "오는"), + ("구은", "구는"), + ] + ), "", "", sigma, @@ -160,12 +164,14 @@ def __init__(self, deterministic: bool = True): # Fix object particle agreement (을 → 를) # e.g., 오을 → 오를, 이을 → 이를 object_rewrite = pynini.cdrewrite( - pynini.string_map([ - ("이을", "이를"), - ("사을", "사를"), - ("오을", "오를"), - ("구을", "구를"), - ]), + pynini.string_map( + [ + ("이을", "이를"), + ("사을", "사를"), + ("오을", "오를"), + ("구을", "구를"), + ] + ), "", "", sigma, From cbf4bd4667719b78c6aebdc34b0af86e60c04474 Mon Sep 17 00:00:00 2001 From: Jinwoo Bae Date: Thu, 9 Jul 2026 09:03:17 -0700 Subject: [PATCH 08/13] Add Korean TN serial and review fixes Signed-off-by: Jinwoo Bae --- .../text_normalization/ko/data/whitelist.tsv | 1 + .../text_normalization/ko/taggers/date.py | 31 ++++-- .../text_normalization/ko/taggers/serial.py | 96 +++++++++++++++++++ .../ko/taggers/telephone.py | 17 +++- .../ko/taggers/tokenize_and_classify.py | 3 + .../test_cases_serial.txt | 20 ++++ tests/nemo_text_processing/ko/test_serial.py | 31 ++++++ .../ko/test_sparrowhawk_normalization.sh | 5 + 8 files changed, 194 insertions(+), 10 deletions(-) create mode 100644 nemo_text_processing/text_normalization/ko/taggers/serial.py create mode 100644 tests/nemo_text_processing/ko/data_text_normalization/test_cases_serial.txt create mode 100644 tests/nemo_text_processing/ko/test_serial.py diff --git a/nemo_text_processing/text_normalization/ko/data/whitelist.tsv b/nemo_text_processing/text_normalization/ko/data/whitelist.tsv index 82dc1220e..d0bdf4caf 100644 --- a/nemo_text_processing/text_normalization/ko/data/whitelist.tsv +++ b/nemo_text_processing/text_normalization/ko/data/whitelist.tsv @@ -24,6 +24,7 @@ No. 번호 ) 오른쪽 괄호 + 더하기 - 마이너스 += 은 Σ 시그마 η 에타 κ 카파 diff --git a/nemo_text_processing/text_normalization/ko/taggers/date.py b/nemo_text_processing/text_normalization/ko/taggers/date.py index 9748abc49..4914b96d9 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/date.py +++ b/nemo_text_processing/text_normalization/ko/taggers/date.py @@ -84,6 +84,8 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): era = pynini.union("기원전", "기원후").optimize() signs = pynutil.delete("/") | pynutil.delete(".") | pynutil.delete("-") + date_sep = signs + pynini.closure(delete_space, 0, 1) + insert_space + # Strict digit ranges for M/D/Y and Y/M/D _d = pynini.union(*[pynini.accep(str(i)) for i in range(10)]) _1to9 = pynini.union(*[pynini.accep(str(i)) for i in range(1, 10)]) @@ -185,15 +187,25 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): graph_basic_date = ( pynini.closure(era_component + insert_space, 0, 1) + year_component_y4_strict - + signs - + insert_space + + date_sep + (pynutil.insert("month: \"") + month_cardinal + pynutil.insert("월") + pynutil.insert("\"")) - + signs - + insert_space + + date_sep + (pynutil.insert("day: \"") + cardinal_lz + pynutil.insert("일") + pynutil.insert("\"")) - + pynini.closure(pynini.closure(insert_space, 0, 1) + week_component, 0, 1) + + pynini.closure(insert_space + week_component, 0, 1) ) - + graph_basic_date_with_dot_weekday = ( + pynini.closure(era_component + insert_space, 0, 1) + + year_component_y4_strict + + date_sep + + (pynutil.insert("month: \"") + month_cardinal + pynutil.insert("월") + pynutil.insert("\"")) + + date_sep + + (pynutil.insert("day: \"") + cardinal_lz + pynutil.insert("일") + pynutil.insert("\"")) + + pynutil.delete(".") + + pynini.closure(delete_space, 0, 1) + + insert_space + + week_component + ).optimize() + # American: MM/DD/YYYY graph_american_date = ( month_component_md @@ -298,13 +310,14 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): ).optimize() graph_all_date = ( - graph_basic_date + graph_basic_date_with_dot_weekday + | graph_basic_date | graph_american_date | graph_european_date | graph_individual_component | graph_individual_component_combined | era_nendai ).optimize() - + final_graph = self.add_tokens(graph_all_date) - self.fst = final_graph.optimize() + self.fst = final_graph.optimize() \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/taggers/serial.py b/nemo_text_processing/text_normalization/ko/taggers/serial.py new file mode 100644 index 000000000..3597ea3b9 --- /dev/null +++ b/nemo_text_processing/text_normalization/ko/taggers/serial.py @@ -0,0 +1,96 @@ +# Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ko.graph_utils import GraphFst, delete_space +from nemo_text_processing.text_normalization.ko.utils import get_abs_path + + +class SerialFst(GraphFst): + """ + Finite state transducer for classifying Korean serial/number-like strings. + + This class is signal-based, similar to MoneyFst. It only reads digits + one by one when the input contains a clear signal such as "번호" or "연락처". + + Example inputs and outputs: + 번호는 0987654321 -> name: "번호는 영구팔칠육오사삼이일" + 휴대폰 번호는 0987654321 -> tokens { name: "휴대폰" } tokens { name: "번호는 영구팔칠육오사삼이일" } + 연락처는 12345678 -> name: "연락처는 일이삼사오육칠팔" + + Args: + deterministic: If True, provide a single transduction; + if False, allow multiple transductions. + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="serial", kind="classify", deterministic=deterministic) + + sp = pynini.closure(delete_space) + + # Digit mapping. Force 0 -> "영" for serial/number readings. + digit = pynini.string_file(get_abs_path("data/number/digit.tsv")).optimize() + zero_map = pynini.cross("0", "영") + digit_ko = (digit | zero_map).optimize() + + # Optional separators inside number-like strings. + # These separators are deleted so that the number is read digit-by-digit. + # Examples: + # 123-456 -> 일이삼사오육 + # 123.456 -> 일이삼사오육 + # 123 456 -> 일이삼사오육 + sep = pynutil.delete("-") | pynutil.delete(".") | pynutil.delete(" ") + + # Require at least 3 digits. + # This covers common serial-like numbers such as reservation numbers, + # verification numbers, account numbers, and context-based phone-number strings. + # Very short numbers such as "번호는 12" are left to the existing cardinal path. + min_three_digits = ( + digit_ko + + pynini.closure(sep, 0, 1) + + digit_ko + + pynini.closure(sep, 0, 1) + + digit_ko + ) + + serial_body = ( + min_three_digits + + pynini.closure(pynini.closure(sep, 0, 1) + digit_ko) + ).optimize() + + # Minimal context signals. + # "번호는" covers 휴대폰 번호는, 전화 번호는, 계좌 번호는, 예약 번호는, etc., + # because the preceding noun can stay outside the serial token. + signal = pynini.string_map([ + ("번호는", "번호는"), + ("번호가", "번호가"), + ("번호를", "번호를"), + + ("연락처는", "연락처는"), + ("연락처가", "연락처가"), + ("연락처를", "연락처를"), + ]).optimize() + + graph = ( + pynutil.insert('name: "') + + signal + + pynutil.insert('" } tokens { name: "') + + sp + + serial_body + + pynutil.insert('"') + ).optimize() + + self.fst = graph.optimize() \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ko/taggers/telephone.py b/nemo_text_processing/text_normalization/ko/taggers/telephone.py index 90f31bb1f..e014b15ce 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/telephone.py +++ b/nemo_text_processing/text_normalization/ko/taggers/telephone.py @@ -47,6 +47,7 @@ def __init__(self, deterministic: bool = True): zero_map = pynini.cross("0", "영") digit_ko = (digit | zero_map).optimize() + two_digits = digit_ko**2 three_digits = digit_ko**3 four_digits = digit_ko**4 @@ -80,7 +81,21 @@ def __init__(self, deterministic: bool = True): last4 = four_digits # consume '-' or '.' between middle and last blocks - number_part_core = area_part + mid + delete_sep + insert_block_space + last4 + intl_mobile_local = ( + two_digits + + pynini.closure(pynutil.delete(" ") | delete_sep, 0, 1) + + insert_space + + four_digits + + delete_sep + + insert_space + + four_digits + ) + + number_part_core = ( + area_part + mid + delete_sep + insert_block_space + last4 + | intl_mobile_local + ).optimize() + number_part = pynutil.insert('number_part: "') + number_part_core + pynutil.insert('"') # final graph: with or without country code diff --git a/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py index e2a3a5890..170a4a485 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py @@ -36,6 +36,7 @@ from nemo_text_processing.text_normalization.ko.taggers.time import TimeFst from nemo_text_processing.text_normalization.ko.taggers.whitelist import WhiteListFst from nemo_text_processing.text_normalization.ko.taggers.word import WordFst +from nemo_text_processing.text_normalization.ko.taggers.serial import SerialFst from nemo_text_processing.utils.logging import logger @@ -85,6 +86,7 @@ def __init__( telephone = TelephoneFst(deterministic=deterministic) measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) electronic = ElectronicFst(cardinal=cardinal, deterministic=deterministic) + serial = SerialFst(deterministic=deterministic) classify = pynini.union( pynutil.add_weight(cardinal.fst, 1.1), @@ -100,6 +102,7 @@ def __init__( pynutil.add_weight(whitelist.fst, 1.1), pynutil.add_weight(telephone.fst, 1.1), pynutil.add_weight(electronic.fst, 1.11), + pynutil.add_weight(serial.fst, 1.05), ) token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_serial.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_serial.txt new file mode 100644 index 000000000..526ee33a3 --- /dev/null +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_serial.txt @@ -0,0 +1,20 @@ +휴대폰 번호는 0987654321입니다~휴대폰 번호는 영구팔칠육오사삼이일 입니다 +전화 번호는 01090817263입니다~전화 번호는 영일영구영팔일칠이육삼 입니다 +계좌 번호는 70501938462011입니다~계좌 번호는 칠영오영일구삼팔사육이영일일 입니다 +예약 번호는 907입니다~예약 번호는 구영칠 입니다 +예약 번호는 830-291입니다~예약 번호는 팔삼영이구일 입니다 +주문 번호는 4829.7301입니다~주문 번호는 사팔이구칠삼영일 입니다 +인증 번호는 000739입니다~인증 번호는 영영영칠삼구 입니다 +인증 번호는 204060입니다~인증 번호는 이영사영육영 입니다 +배송 번호는 591837462입니다~배송 번호는 오구일팔삼칠사육이 입니다 +회원 번호는 3000456789입니다~회원 번호는 삼영영영사오육칠팔구 입니다 +연락처는 84502719입니다~연락처는 팔사오영이칠일구 입니다 +연락처가 77008899입니다~연락처가 칠칠영영팔팔구구 입니다 +연락처를 604 812 930으로 저장했습니다~연락처를 육영사팔일이구삼영 으로 저장했습니다 +번호는 12입니다~번호는 십이 입니다 +배송 번호는 591837462입니다~배송 번호는 오구일팔삼칠사육이 입니다 +회원 번호는 3000456789입니다~회원 번호는 삼영영영사오육칠팔구 입니다 +접수 번호는 812709입니다~접수 번호는 팔일이칠영구 입니다 +신청 번호는 40682015입니다~신청 번호는 사영육팔이영일오 입니다 +확인 번호는 970045입니다~확인 번호는 구칠영영사오 입니다 +관리 번호는 25081973입니다~관리 번호는 이오영팔일구칠삼 입니다 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/test_serial.py b/tests/nemo_text_processing/ko/test_serial.py new file mode 100644 index 000000000..2b2414e58 --- /dev/null +++ b/tests/nemo_text_processing/ko/test_serial.py @@ -0,0 +1,31 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestSerial: + normalizer_ko = Normalizer(input_case='cased', lang='ko', cache_dir=CACHE_DIR, overwrite_cache=True) + + @parameterized.expand(parse_test_case_file('ko/data_text_normalization/test_cases_serial.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer_ko.normalize(test_input, verbose=False, punct_post_process=False) + assert pred == expected, f"input: {test_input}" \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/ko/test_sparrowhawk_normalization.sh index 9adbc152b..52091fb11 100644 --- a/tests/nemo_text_processing/ko/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/ko/test_sparrowhawk_normalization.sh @@ -76,6 +76,11 @@ testTNElectronicText() { runtest $input } +testTNSerialText() { + input=$TEST_DIR/ko/data_text_normalization/test_cases_serial.txt + runtest $input +} + # Remove all command-line arguments shift $# From c747dafda4d815149370b9049cea53cabfa622f7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 9 Jul 2026 20:34:17 +0000 Subject: [PATCH 09/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../text_normalization/ko/taggers/date.py | 8 ++--- .../text_normalization/ko/taggers/serial.py | 36 ++++++++----------- .../ko/taggers/telephone.py | 9 ++--- .../ko/taggers/tokenize_and_classify.py | 2 +- tests/nemo_text_processing/ko/test_serial.py | 2 +- 5 files changed, 23 insertions(+), 34 deletions(-) diff --git a/nemo_text_processing/text_normalization/ko/taggers/date.py b/nemo_text_processing/text_normalization/ko/taggers/date.py index 4914b96d9..8b5d89d91 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/date.py +++ b/nemo_text_processing/text_normalization/ko/taggers/date.py @@ -85,7 +85,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): signs = pynutil.delete("/") | pynutil.delete(".") | pynutil.delete("-") date_sep = signs + pynini.closure(delete_space, 0, 1) + insert_space - + # Strict digit ranges for M/D/Y and Y/M/D _d = pynini.union(*[pynini.accep(str(i)) for i in range(10)]) _1to9 = pynini.union(*[pynini.accep(str(i)) for i in range(1, 10)]) @@ -205,7 +205,7 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): + insert_space + week_component ).optimize() - + # American: MM/DD/YYYY graph_american_date = ( month_component_md @@ -318,6 +318,6 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): | graph_individual_component_combined | era_nendai ).optimize() - + final_graph = self.add_tokens(graph_all_date) - self.fst = final_graph.optimize() \ No newline at end of file + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/taggers/serial.py b/nemo_text_processing/text_normalization/ko/taggers/serial.py index 3597ea3b9..3eb4a6d63 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/serial.py +++ b/nemo_text_processing/text_normalization/ko/taggers/serial.py @@ -58,31 +58,23 @@ def __init__(self, deterministic: bool = True): # This covers common serial-like numbers such as reservation numbers, # verification numbers, account numbers, and context-based phone-number strings. # Very short numbers such as "번호는 12" are left to the existing cardinal path. - min_three_digits = ( - digit_ko - + pynini.closure(sep, 0, 1) - + digit_ko - + pynini.closure(sep, 0, 1) - + digit_ko - ) - - serial_body = ( - min_three_digits - + pynini.closure(pynini.closure(sep, 0, 1) + digit_ko) - ).optimize() + min_three_digits = digit_ko + pynini.closure(sep, 0, 1) + digit_ko + pynini.closure(sep, 0, 1) + digit_ko + + serial_body = (min_three_digits + pynini.closure(pynini.closure(sep, 0, 1) + digit_ko)).optimize() # Minimal context signals. # "번호는" covers 휴대폰 번호는, 전화 번호는, 계좌 번호는, 예약 번호는, etc., # because the preceding noun can stay outside the serial token. - signal = pynini.string_map([ - ("번호는", "번호는"), - ("번호가", "번호가"), - ("번호를", "번호를"), - - ("연락처는", "연락처는"), - ("연락처가", "연락처가"), - ("연락처를", "연락처를"), - ]).optimize() + signal = pynini.string_map( + [ + ("번호는", "번호는"), + ("번호가", "번호가"), + ("번호를", "번호를"), + ("연락처는", "연락처는"), + ("연락처가", "연락처가"), + ("연락처를", "연락처를"), + ] + ).optimize() graph = ( pynutil.insert('name: "') @@ -93,4 +85,4 @@ def __init__(self, deterministic: bool = True): + pynutil.insert('"') ).optimize() - self.fst = graph.optimize() \ No newline at end of file + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/taggers/telephone.py b/nemo_text_processing/text_normalization/ko/taggers/telephone.py index e014b15ce..aee610158 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/telephone.py +++ b/nemo_text_processing/text_normalization/ko/taggers/telephone.py @@ -90,12 +90,9 @@ def __init__(self, deterministic: bool = True): + insert_space + four_digits ) - - number_part_core = ( - area_part + mid + delete_sep + insert_block_space + last4 - | intl_mobile_local - ).optimize() - + + number_part_core = (area_part + mid + delete_sep + insert_block_space + last4 | intl_mobile_local).optimize() + number_part = pynutil.insert('number_part: "') + number_part_core + pynutil.insert('"') # final graph: with or without country code diff --git a/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py index 170a4a485..4cef486fc 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py @@ -32,11 +32,11 @@ from nemo_text_processing.text_normalization.ko.taggers.money import MoneyFst from nemo_text_processing.text_normalization.ko.taggers.ordinal import OrdinalFst from nemo_text_processing.text_normalization.ko.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.ko.taggers.serial import SerialFst from nemo_text_processing.text_normalization.ko.taggers.telephone import TelephoneFst from nemo_text_processing.text_normalization.ko.taggers.time import TimeFst from nemo_text_processing.text_normalization.ko.taggers.whitelist import WhiteListFst from nemo_text_processing.text_normalization.ko.taggers.word import WordFst -from nemo_text_processing.text_normalization.ko.taggers.serial import SerialFst from nemo_text_processing.utils.logging import logger diff --git a/tests/nemo_text_processing/ko/test_serial.py b/tests/nemo_text_processing/ko/test_serial.py index 2b2414e58..314a3174e 100644 --- a/tests/nemo_text_processing/ko/test_serial.py +++ b/tests/nemo_text_processing/ko/test_serial.py @@ -28,4 +28,4 @@ class TestSerial: @pytest.mark.unit def test_norm(self, test_input, expected): pred = self.normalizer_ko.normalize(test_input, verbose=False, punct_post_process=False) - assert pred == expected, f"input: {test_input}" \ No newline at end of file + assert pred == expected, f"input: {test_input}" From 34b869805c11ff610fa6b2f0a19773297c467b53 Mon Sep 17 00:00:00 2001 From: Jinwoo Bae Date: Thu, 23 Jul 2026 15:08:22 -0700 Subject: [PATCH 10/13] Address Korean TN review feedback Signed-off-by: Jinwoo Bae --- .../text_normalization/ko/taggers/cardinal.py | 62 ++++++++++++- .../text_normalization/ko/taggers/serial.py | 88 ------------------ .../ko/taggers/telephone.py | 89 +++++++++++++------ .../ko/taggers/tokenize_and_classify.py | 2 - .../ko/verbalizers/fraction.py | 35 ++------ .../test_cases_cardinal.txt | 22 ++++- .../test_cases_serial.txt | 20 ----- .../test_cases_telephone.txt | 8 +- tests/nemo_text_processing/ko/test_serial.py | 31 ------- .../ko/test_sparrowhawk_normalization.sh | 5 -- 10 files changed, 161 insertions(+), 201 deletions(-) delete mode 100644 nemo_text_processing/text_normalization/ko/taggers/serial.py delete mode 100644 tests/nemo_text_processing/ko/data_text_normalization/test_cases_serial.txt delete mode 100644 tests/nemo_text_processing/ko/test_serial.py diff --git a/nemo_text_processing/text_normalization/ko/taggers/cardinal.py b/nemo_text_processing/text_normalization/ko/taggers/cardinal.py index e2b0f87e3..142c4a090 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/ko/taggers/cardinal.py @@ -16,7 +16,7 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst +from nemo_text_processing.text_normalization.ko.graph_utils import NEMO_DIGIT, NEMO_SPACE, GraphFst, delete_space from nemo_text_processing.text_normalization.ko.utils import get_abs_path @@ -273,7 +273,65 @@ def __init__(self, deterministic: bool = True): graph_1_to_99, graph_zero, ).optimize() + + # ---------------------------- + # Context-based digit-by-digit reading + # e.g., 번호는 0987654321 -> 번호는 영구팔칠육오사삼이일 + # + # Keep this separate from graph_num so regular cardinal zero + # and place-value behavior remain unchanged. + + serial_space = pynini.closure(delete_space) + + # Exclude 0 from graph_digit and force 0 -> 영. + # This avoids ambiguity if digit.tsv has another mapping for 0. + graph_digit_one_to_nine = ( + pynini.difference(NEMO_DIGIT, "0") @ graph_digit + ).optimize() + + serial_digit = pynini.union( + pynini.cross("0", "영"), + graph_digit_one_to_nine, + ).optimize() + + # Optional separators between individual digits. + serial_separator = pynini.union( + pynutil.delete("-"), + pynutil.delete("."), + pynutil.delete(" "), + ).optimize() + # Require at least three digits. + serial_body = ( + serial_digit + + pynini.closure(serial_separator, 0, 1) + + serial_digit + + pynini.closure(serial_separator, 0, 1) + + serial_digit + + pynini.closure( + pynini.closure(serial_separator, 0, 1) + serial_digit + ) + ).optimize() + + serial_signal = pynini.string_map( + [ + ("번호는", "번호는 "), + ("번호가", "번호가 "), + ("번호를", "번호를 "), + ("연락처는", "연락처는 "), + ("연락처가", "연락처가 "), + ("연락처를", "연락처를 "), + ] + ).optimize() + + serial_case = ( + pynutil.insert('integer: "') + + serial_signal + + serial_space + + serial_body + + pynutil.insert('"') + ).optimize() + # ---------------------------- # Native counting + counters # e.g., 3개, 2명, 10살 @@ -319,7 +377,7 @@ def __init__(self, deterministic: bool = True): signed_integer = (minus_prefix | plus_prefix).ques + integer_token # Prefer accounting-form first, then signed form - final_graph = paren_negative | signed_integer | counter_case + final_graph = serial_case | paren_negative | signed_integer | counter_case # Wrap with class tokens and finalize final_graph = self.add_tokens(final_graph) diff --git a/nemo_text_processing/text_normalization/ko/taggers/serial.py b/nemo_text_processing/text_normalization/ko/taggers/serial.py deleted file mode 100644 index 3eb4a6d63..000000000 --- a/nemo_text_processing/text_normalization/ko/taggers/serial.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) 2025 NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pynini -from pynini.lib import pynutil - -from nemo_text_processing.text_normalization.ko.graph_utils import GraphFst, delete_space -from nemo_text_processing.text_normalization.ko.utils import get_abs_path - - -class SerialFst(GraphFst): - """ - Finite state transducer for classifying Korean serial/number-like strings. - - This class is signal-based, similar to MoneyFst. It only reads digits - one by one when the input contains a clear signal such as "번호" or "연락처". - - Example inputs and outputs: - 번호는 0987654321 -> name: "번호는 영구팔칠육오사삼이일" - 휴대폰 번호는 0987654321 -> tokens { name: "휴대폰" } tokens { name: "번호는 영구팔칠육오사삼이일" } - 연락처는 12345678 -> name: "연락처는 일이삼사오육칠팔" - - Args: - deterministic: If True, provide a single transduction; - if False, allow multiple transductions. - """ - - def __init__(self, deterministic: bool = True): - super().__init__(name="serial", kind="classify", deterministic=deterministic) - - sp = pynini.closure(delete_space) - - # Digit mapping. Force 0 -> "영" for serial/number readings. - digit = pynini.string_file(get_abs_path("data/number/digit.tsv")).optimize() - zero_map = pynini.cross("0", "영") - digit_ko = (digit | zero_map).optimize() - - # Optional separators inside number-like strings. - # These separators are deleted so that the number is read digit-by-digit. - # Examples: - # 123-456 -> 일이삼사오육 - # 123.456 -> 일이삼사오육 - # 123 456 -> 일이삼사오육 - sep = pynutil.delete("-") | pynutil.delete(".") | pynutil.delete(" ") - - # Require at least 3 digits. - # This covers common serial-like numbers such as reservation numbers, - # verification numbers, account numbers, and context-based phone-number strings. - # Very short numbers such as "번호는 12" are left to the existing cardinal path. - min_three_digits = digit_ko + pynini.closure(sep, 0, 1) + digit_ko + pynini.closure(sep, 0, 1) + digit_ko - - serial_body = (min_three_digits + pynini.closure(pynini.closure(sep, 0, 1) + digit_ko)).optimize() - - # Minimal context signals. - # "번호는" covers 휴대폰 번호는, 전화 번호는, 계좌 번호는, 예약 번호는, etc., - # because the preceding noun can stay outside the serial token. - signal = pynini.string_map( - [ - ("번호는", "번호는"), - ("번호가", "번호가"), - ("번호를", "번호를"), - ("연락처는", "연락처는"), - ("연락처가", "연락처가"), - ("연락처를", "연락처를"), - ] - ).optimize() - - graph = ( - pynutil.insert('name: "') - + signal - + pynutil.insert('" } tokens { name: "') - + sp - + serial_body - + pynutil.insert('"') - ).optimize() - - self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ko/taggers/telephone.py b/nemo_text_processing/text_normalization/ko/taggers/telephone.py index aee610158..37d07c59a 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/telephone.py +++ b/nemo_text_processing/text_normalization/ko/taggers/telephone.py @@ -37,8 +37,13 @@ class TelephoneFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="telephone", kind="classify", deterministic=deterministic) - # Separator between digit blocks (e.g., "-" or ".") - delete_sep = pynutil.delete("-") | pynutil.delete(".") + # Separator between number blocks. + delete_sep = pynini.union( + pynutil.delete("-"), + pynutil.delete("."), + pynutil.delete(" "), + ).optimize() + # Optional space inserted between blocks insert_block_space = insert_space @@ -63,36 +68,70 @@ def __init__(self, deterministic: bool = True): + delete_space ) - # area part: "123-" | "123." | "(123)" [space?] or "(123)-" - area_core = three_digits - area_part = ( - (area_core + delete_sep) - | ( - pynutil.delete("(") - + area_core - + pynutil.delete(")") - + pynini.closure(pynutil.delete(" "), 0, 1) - + pynini.closure(delete_sep, 0, 1) - ) - ) + insert_block_space - - # 2) allow 3 **or 4** digits in the middle block (to support 010-3713-7050) - mid = pynini.union(three_digits, four_digits) - last4 = four_digits - - # consume '-' or '.' between middle and last blocks - intl_mobile_local = ( - two_digits - + pynini.closure(pynutil.delete(" ") | delete_sep, 0, 1) - + insert_space - + four_digits + # First block may contain 2 or 3 digits. + # Examples: 02, 031, 043, 010 + first_block = pynini.union( + two_digits, + three_digits, + ).optimize() + + # Middle block may contain 3 or 4 digits. + # Examples: 123, 1234 + middle_block = pynini.union( + three_digits, + four_digits, + ).optimize() + + # Plain telephone form: + # 02-1234-5678 + plain_first_part = ( + first_block + delete_sep +<<<<<<< HEAD + insert_space + four_digits ) number_part_core = (area_part + mid + delete_sep + insert_block_space + last4 | intl_mobile_local).optimize() +======= + + insert_block_space + ).optimize() + + # Parenthesized telephone form: + # (010)1234-5678 + parenthesized_first_part = ( + pynutil.delete("(") + + first_block + + pynutil.delete(")") + + pynini.closure( + pynutil.delete(" ") + | pynutil.delete("-") + | pynutil.delete("."), + 0, + 1, + ) + + insert_block_space + ).optimize() + + first_part = pynini.union( + plain_first_part, + parenthesized_first_part, + ).optimize() + + # Standard telephone layout: + # 2 or 3 digits + # followed by 3 or 4 digits + # followed by 4 digits + number_part_core = ( + first_part + + middle_block + + delete_sep + + insert_block_space + + four_digits + ).optimize() + +>>>>>>> 613cdac9 (Address Korean TN review feedback) number_part = pynutil.insert('number_part: "') + number_part_core + pynutil.insert('"') # final graph: with or without country code diff --git a/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py index 4cef486fc..31732db51 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py @@ -86,7 +86,6 @@ def __init__( telephone = TelephoneFst(deterministic=deterministic) measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) electronic = ElectronicFst(cardinal=cardinal, deterministic=deterministic) - serial = SerialFst(deterministic=deterministic) classify = pynini.union( pynutil.add_weight(cardinal.fst, 1.1), @@ -102,7 +101,6 @@ def __init__( pynutil.add_weight(whitelist.fst, 1.1), pynutil.add_weight(telephone.fst, 1.1), pynutil.add_weight(electronic.fst, 1.11), - pynutil.add_weight(serial.fst, 1.05), ) token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py b/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py index 472b8a86d..37a4a4954 100644 --- a/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py @@ -129,43 +129,26 @@ def __init__(self, deterministic: bool = True): # Sigma for rewrite context (entire string) sigma = pynini.closure(NEMO_NOT_QUOTE | NEMO_SPACE) - # Fix subject particle agreement (이 → 가 for vowel-ending numerals) - # e.g., 사이 → 사가, 구이 → 구가 - subject_rewrite = pynini.cdrewrite( + # Fix particle agreement for vowel-ending numerals. + # Subject: 이 -> 가 + # Topic: 은 -> 는 + # Object: 을 -> 를 + particle_rewrite = pynini.cdrewrite( pynini.string_map( [ + # Subject particle ("이이", "이가"), ("사이", "사가"), ("오이", "오가"), ("구이", "구가"), - ] - ), - "", - "", - sigma, - ) - # Fix topic particle agreement (은 → 는) - # e.g., 이은 → 이는, 사은 → 사는 - topic_rewrite = pynini.cdrewrite( - pynini.string_map( - [ + # Topic particle ("이은", "이는"), ("사은", "사는"), ("오은", "오는"), ("구은", "구는"), - ] - ), - "", - "", - sigma, - ) - # Fix object particle agreement (을 → 를) - # e.g., 오을 → 오를, 이을 → 이를 - object_rewrite = pynini.cdrewrite( - pynini.string_map( - [ + # Object particle ("이을", "이를"), ("사을", "사를"), ("오을", "오를"), @@ -178,5 +161,5 @@ def __init__(self, deterministic: bool = True): ) # Apply all rewrite rules sequentially and final optimized FST - final_graph = final_graph @ subject_rewrite @ topic_rewrite @ object_rewrite + final_graph = final_graph @ particle_rewrite self.fst = final_graph.optimize() diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_cardinal.txt index 40187f74e..dbde76d4e 100644 --- a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_cardinal.txt @@ -45,4 +45,24 @@ -2~마이너스 이 -93~마이너스 구십삼 -90325~마이너스 구만삼백이십오 --3234567~마이너스 삼백이십삼만사천오백육십칠 \ No newline at end of file +-3234567~마이너스 삼백이십삼만사천오백육십칠 +휴대폰 번호는 0987654321입니다~휴대폰 번호는 영구팔칠육오사삼이일 입니다 +전화 번호는 01090817263입니다~전화 번호는 영일영구영팔일칠이육삼 입니다 +계좌 번호는 70501938462011입니다~계좌 번호는 칠영오영일구삼팔사육이영일일 입니다 +예약 번호는 907입니다~예약 번호는 구영칠 입니다 +예약 번호는 830-291입니다~예약 번호는 팔삼영이구일 입니다 +주문 번호는 4829.7301입니다~주문 번호는 사팔이구칠삼영일 입니다 +인증 번호는 000739입니다~인증 번호는 영영영칠삼구 입니다 +인증 번호는 204060입니다~인증 번호는 이영사영육영 입니다 +배송 번호는 591837462입니다~배송 번호는 오구일팔삼칠사육이 입니다 +회원 번호는 3000456789입니다~회원 번호는 삼영영영사오육칠팔구 입니다 +연락처는 84502719입니다~연락처는 팔사오영이칠일구 입니다 +연락처가 77008899입니다~연락처가 칠칠영영팔팔구구 입니다 +연락처를 604 812 930으로 저장했습니다~연락처를 육영사팔일이구삼영 으로 저장했습니다 +번호는 12입니다~번호는 십이 입니다 +배송 번호는 591837462입니다~배송 번호는 오구일팔삼칠사육이 입니다 +회원 번호는 3000456789입니다~회원 번호는 삼영영영사오육칠팔구 입니다 +접수 번호는 812709입니다~접수 번호는 팔일이칠영구 입니다 +신청 번호는 40682015입니다~신청 번호는 사영육팔이영일오 입니다 +확인 번호는 970045입니다~확인 번호는 구칠영영사오 입니다 +관리 번호는 25081973입니다~관리 번호는 이오영팔일구칠삼 입니다 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_serial.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_serial.txt deleted file mode 100644 index 526ee33a3..000000000 --- a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_serial.txt +++ /dev/null @@ -1,20 +0,0 @@ -휴대폰 번호는 0987654321입니다~휴대폰 번호는 영구팔칠육오사삼이일 입니다 -전화 번호는 01090817263입니다~전화 번호는 영일영구영팔일칠이육삼 입니다 -계좌 번호는 70501938462011입니다~계좌 번호는 칠영오영일구삼팔사육이영일일 입니다 -예약 번호는 907입니다~예약 번호는 구영칠 입니다 -예약 번호는 830-291입니다~예약 번호는 팔삼영이구일 입니다 -주문 번호는 4829.7301입니다~주문 번호는 사팔이구칠삼영일 입니다 -인증 번호는 000739입니다~인증 번호는 영영영칠삼구 입니다 -인증 번호는 204060입니다~인증 번호는 이영사영육영 입니다 -배송 번호는 591837462입니다~배송 번호는 오구일팔삼칠사육이 입니다 -회원 번호는 3000456789입니다~회원 번호는 삼영영영사오육칠팔구 입니다 -연락처는 84502719입니다~연락처는 팔사오영이칠일구 입니다 -연락처가 77008899입니다~연락처가 칠칠영영팔팔구구 입니다 -연락처를 604 812 930으로 저장했습니다~연락처를 육영사팔일이구삼영 으로 저장했습니다 -번호는 12입니다~번호는 십이 입니다 -배송 번호는 591837462입니다~배송 번호는 오구일팔삼칠사육이 입니다 -회원 번호는 3000456789입니다~회원 번호는 삼영영영사오육칠팔구 입니다 -접수 번호는 812709입니다~접수 번호는 팔일이칠영구 입니다 -신청 번호는 40682015입니다~신청 번호는 사영육팔이영일오 입니다 -확인 번호는 970045입니다~확인 번호는 구칠영영사오 입니다 -관리 번호는 25081973입니다~관리 번호는 이오영팔일구칠삼 입니다 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_telephone.txt b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_telephone.txt index b6e573aec..a871e4f71 100644 --- a/tests/nemo_text_processing/ko/data_text_normalization/test_cases_telephone.txt +++ b/tests/nemo_text_processing/ko/data_text_normalization/test_cases_telephone.txt @@ -30,4 +30,10 @@ +82-123.456-7890~국가번호 팔이 일이삼 사오육 칠팔구영 111-222-3333~일일일 이이이 삼삼삼삼 909-808-7070~구영구 팔영팔 칠영칠영 -(555)555-5555~오오오 오오오 오오오오 \ No newline at end of file +(555)555-5555~오오오 오오오 오오오오 +02-123-4567~영이 일이삼 사오육칠 +02-1234-5678~영이 일이삼사 오육칠팔 +043-123-4567~영사삼 일이삼 사오육칠 +043-1234-5678~영사삼 일이삼사 오육칠팔 +(02) 1234-5678~영이 일이삼사 오육칠팔 +010 1234 5678~영일영 일이삼사 오육칠팔 \ No newline at end of file diff --git a/tests/nemo_text_processing/ko/test_serial.py b/tests/nemo_text_processing/ko/test_serial.py deleted file mode 100644 index 314a3174e..000000000 --- a/tests/nemo_text_processing/ko/test_serial.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -from parameterized import parameterized - -from nemo_text_processing.text_normalization.normalize import Normalizer - -from ..utils import CACHE_DIR, parse_test_case_file - - -class TestSerial: - normalizer_ko = Normalizer(input_case='cased', lang='ko', cache_dir=CACHE_DIR, overwrite_cache=True) - - @parameterized.expand(parse_test_case_file('ko/data_text_normalization/test_cases_serial.txt')) - @pytest.mark.run_only_on('CPU') - @pytest.mark.unit - def test_norm(self, test_input, expected): - pred = self.normalizer_ko.normalize(test_input, verbose=False, punct_post_process=False) - assert pred == expected, f"input: {test_input}" diff --git a/tests/nemo_text_processing/ko/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/ko/test_sparrowhawk_normalization.sh index 52091fb11..9adbc152b 100644 --- a/tests/nemo_text_processing/ko/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/ko/test_sparrowhawk_normalization.sh @@ -76,11 +76,6 @@ testTNElectronicText() { runtest $input } -testTNSerialText() { - input=$TEST_DIR/ko/data_text_normalization/test_cases_serial.txt - runtest $input -} - # Remove all command-line arguments shift $# From 33b6c0a8f580e13d3d1269e2f80e23c35a875c06 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 22:19:45 +0000 Subject: [PATCH 11/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../text_normalization/ko/taggers/cardinal.py | 18 +++++------------- .../ko/verbalizers/fraction.py | 2 -- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/nemo_text_processing/text_normalization/ko/taggers/cardinal.py b/nemo_text_processing/text_normalization/ko/taggers/cardinal.py index 142c4a090..901c5723e 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/ko/taggers/cardinal.py @@ -273,7 +273,7 @@ def __init__(self, deterministic: bool = True): graph_1_to_99, graph_zero, ).optimize() - + # ---------------------------- # Context-based digit-by-digit reading # e.g., 번호는 0987654321 -> 번호는 영구팔칠육오사삼이일 @@ -285,9 +285,7 @@ def __init__(self, deterministic: bool = True): # Exclude 0 from graph_digit and force 0 -> 영. # This avoids ambiguity if digit.tsv has another mapping for 0. - graph_digit_one_to_nine = ( - pynini.difference(NEMO_DIGIT, "0") @ graph_digit - ).optimize() + graph_digit_one_to_nine = (pynini.difference(NEMO_DIGIT, "0") @ graph_digit).optimize() serial_digit = pynini.union( pynini.cross("0", "영"), @@ -308,9 +306,7 @@ def __init__(self, deterministic: bool = True): + serial_digit + pynini.closure(serial_separator, 0, 1) + serial_digit - + pynini.closure( - pynini.closure(serial_separator, 0, 1) + serial_digit - ) + + pynini.closure(pynini.closure(serial_separator, 0, 1) + serial_digit) ).optimize() serial_signal = pynini.string_map( @@ -325,13 +321,9 @@ def __init__(self, deterministic: bool = True): ).optimize() serial_case = ( - pynutil.insert('integer: "') - + serial_signal - + serial_space - + serial_body - + pynutil.insert('"') + pynutil.insert('integer: "') + serial_signal + serial_space + serial_body + pynutil.insert('"') ).optimize() - + # ---------------------------- # Native counting + counters # e.g., 3개, 2명, 10살 diff --git a/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py b/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py index 37a4a4954..d523dba00 100644 --- a/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/ko/verbalizers/fraction.py @@ -141,13 +141,11 @@ def __init__(self, deterministic: bool = True): ("사이", "사가"), ("오이", "오가"), ("구이", "구가"), - # Topic particle ("이은", "이는"), ("사은", "사는"), ("오은", "오는"), ("구은", "구는"), - # Object particle ("이을", "이를"), ("사을", "사를"), From 5586dcd2d840bf7050264686d78002fc3a28e752 Mon Sep 17 00:00:00 2001 From: Jinwoo Bae Date: Thu, 23 Jul 2026 17:10:32 -0700 Subject: [PATCH 12/13] Resolve Korean telephone conflict and remove serial references Signed-off-by: Jinwoo Bae --- .../text_normalization/ko/taggers/telephone.py | 9 --------- .../ko/taggers/tokenize_and_classify.py | 1 - 2 files changed, 10 deletions(-) diff --git a/nemo_text_processing/text_normalization/ko/taggers/telephone.py b/nemo_text_processing/text_normalization/ko/taggers/telephone.py index 37d07c59a..ae065a5ed 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/telephone.py +++ b/nemo_text_processing/text_normalization/ko/taggers/telephone.py @@ -87,14 +87,6 @@ def __init__(self, deterministic: bool = True): plain_first_part = ( first_block + delete_sep -<<<<<<< HEAD - + insert_space - + four_digits - ) - - number_part_core = (area_part + mid + delete_sep + insert_block_space + last4 | intl_mobile_local).optimize() - -======= + insert_block_space ).optimize() @@ -131,7 +123,6 @@ def __init__(self, deterministic: bool = True): + four_digits ).optimize() ->>>>>>> 613cdac9 (Address Korean TN review feedback) number_part = pynutil.insert('number_part: "') + number_part_core + pynutil.insert('"') # final graph: with or without country code diff --git a/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py index 31732db51..e2a3a5890 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/ko/taggers/tokenize_and_classify.py @@ -32,7 +32,6 @@ from nemo_text_processing.text_normalization.ko.taggers.money import MoneyFst from nemo_text_processing.text_normalization.ko.taggers.ordinal import OrdinalFst from nemo_text_processing.text_normalization.ko.taggers.punctuation import PunctuationFst -from nemo_text_processing.text_normalization.ko.taggers.serial import SerialFst from nemo_text_processing.text_normalization.ko.taggers.telephone import TelephoneFst from nemo_text_processing.text_normalization.ko.taggers.time import TimeFst from nemo_text_processing.text_normalization.ko.taggers.whitelist import WhiteListFst From 7b5ae47036f092a39ed05273cb9c11bdb70c6b3d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:11:52 +0000 Subject: [PATCH 13/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../ko/taggers/telephone.py | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/nemo_text_processing/text_normalization/ko/taggers/telephone.py b/nemo_text_processing/text_normalization/ko/taggers/telephone.py index ae065a5ed..ba4fbcea0 100644 --- a/nemo_text_processing/text_normalization/ko/taggers/telephone.py +++ b/nemo_text_processing/text_normalization/ko/taggers/telephone.py @@ -43,7 +43,7 @@ def __init__(self, deterministic: bool = True): pynutil.delete("."), pynutil.delete(" "), ).optimize() - + # Optional space inserted between blocks insert_block_space = insert_space @@ -84,11 +84,7 @@ def __init__(self, deterministic: bool = True): # Plain telephone form: # 02-1234-5678 - plain_first_part = ( - first_block - + delete_sep - + insert_block_space - ).optimize() + plain_first_part = (first_block + delete_sep + insert_block_space).optimize() # Parenthesized telephone form: # (010)1234-5678 @@ -97,9 +93,7 @@ def __init__(self, deterministic: bool = True): + first_block + pynutil.delete(")") + pynini.closure( - pynutil.delete(" ") - | pynutil.delete("-") - | pynutil.delete("."), + pynutil.delete(" ") | pynutil.delete("-") | pynutil.delete("."), 0, 1, ) @@ -115,14 +109,8 @@ def __init__(self, deterministic: bool = True): # 2 or 3 digits # followed by 3 or 4 digits # followed by 4 digits - number_part_core = ( - first_part - + middle_block - + delete_sep - + insert_block_space - + four_digits - ).optimize() - + number_part_core = (first_part + middle_block + delete_sep + insert_block_space + four_digits).optimize() + number_part = pynutil.insert('number_part: "') + number_part_core + pynutil.insert('"') # final graph: with or without country code