diff --git a/Jenkinsfile b/Jenkinsfile index 38e05bb00..4f5ee5c23 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,6 +27,7 @@ pipeline { HE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-24-25-0' HY_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-0' MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' + TE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-22-26-0' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-26-5' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' @@ -297,6 +298,11 @@ pipeline { sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=mr --text="शून्य " --cache_dir ${MR_TN_CACHE}' } } + stage('L0: TE ITN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=te --text="సున్నా " --cache_dir ${TE_TN_CACHE}' + } + } stage('L0: HY TN grammars') { steps { sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=hy --text="6" --cache_dir ${HY_TN_CACHE}' @@ -463,6 +469,11 @@ pipeline { sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/mr/ -m "not pleasefixme" --cpu --tn_cache_dir ${MR_TN_CACHE}' } } + stage('L1: Run all TE ITN tests (restore grammars from cache)') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/te/ -m "not pleasefixme" --cpu --tn_cache_dir ${TE_TN_CACHE}' + } + } stage('L1: Run all HY TN/ITN tests (restore grammars from cache)') { steps { sh 'CUDA_VISIBLE_DEVICES="" pytest tests/nemo_text_processing/hy/ -m "not pleasefixme" --cpu --tn_cache_dir ${HY_TN_CACHE}' diff --git a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py index 9a6fcc64c..4ce15346c 100644 --- a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py +++ b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py @@ -121,6 +121,11 @@ def __init__( from nemo_text_processing.inverse_text_normalization.hi.verbalizers.verbalize_final import ( VerbalizeFinalFst, ) + elif lang == 'te': # Telugu + from nemo_text_processing.inverse_text_normalization.te.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize_final import ( + VerbalizeFinalFst, + ) elif lang == 'hi_en': # Hindi-English code-switch from nemo_text_processing.inverse_text_normalization.hi_en.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.inverse_text_normalization.hi_en.verbalizers.verbalize_final import ( @@ -207,6 +212,7 @@ def parse_args(): 'zh', 'he', 'hi', + 'te', 'hy', 'mr', 'ja', diff --git a/nemo_text_processing/inverse_text_normalization/run_evaluate.py b/nemo_text_processing/inverse_text_normalization/run_evaluate.py index cf9b29fce..3a989f96c 100644 --- a/nemo_text_processing/inverse_text_normalization/run_evaluate.py +++ b/nemo_text_processing/inverse_text_normalization/run_evaluate.py @@ -44,6 +44,7 @@ def parse_args(): "fr", "hi", "hi_en", + "te", "hy", "ko", "mr", diff --git a/nemo_text_processing/inverse_text_normalization/te/__init__.py b/nemo_text_processing/inverse_text_normalization/te/__init__.py new file mode 100644 index 000000000..d3f64b4cb --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from nemo_text_processing.inverse_text_normalization.te.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize_final import VerbalizeFinalFst diff --git a/nemo_text_processing/inverse_text_normalization/te/data/__init__.py b/nemo_text_processing/inverse_text_normalization/te/data/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/__init__.py b/nemo_text_processing/inverse_text_normalization/te/data/numbers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/case_suffix.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/case_suffix.tsv new file mode 100644 index 000000000..98724c4f5 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/case_suffix.tsv @@ -0,0 +1,5 @@ +కి +కు +లో +తో +కోసం \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit.tsv new file mode 100644 index 000000000..6606ff8b3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +౧ ఒకటి +౨ రెండు +౩ మూడు +౪ నాలుగు +౫ ఐదు +౬ ఆరు +౭ ఏడు +౮ ఎనిమిది +౯ తొమ్మిది \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit_people.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit_people.tsv new file mode 100644 index 000000000..e6de5b8b0 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/digit_people.tsv @@ -0,0 +1,9 @@ +ఇద్దరు ౨ +ముగ్గురు ౩ +నలుగురు ౪ +ఐదుగురు ౫ +ఆరుగురు ౬ +ఏడుగురు ౭ +ఎనిమిదుగురు ౮ +తొమ్మిదుగురు ౯ +పది మంది ౧౦ \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred.tsv new file mode 100644 index 000000000..39f6a81f6 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/hundred.tsv @@ -0,0 +1,3 @@ +వంద +నూట +నూరు diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/special_numbers.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/special_numbers.tsv new file mode 100644 index 000000000..49da8c121 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/special_numbers.tsv @@ -0,0 +1,2 @@ +పంతొమ్మిది ౧౯ +ఒక ౧ diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/teens_and_ties.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/teens_and_ties.tsv new file mode 100644 index 000000000..5c0ca90ed --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/teens_and_ties.tsv @@ -0,0 +1,18 @@ +౧౦ పది +౧౧ పదకొండు +౧౨ పన్నెండు +౧౩ పదమూడు +౧౪ పద్నాలుగు +౧౫ పదిహేను +౧౬ పదహారు +౧౭ పదిహేడు +౧౮ పద్దెనిమిది +౧౯ పందొమ్మిది +౨౦ ఇరవై +౩౦ ముప్పై +౪౦ నలభై +౫౦ యాభై +౬౦ అరవై +౭౦ డెబ్బై +౮౦ ఎనభై +౯౦ తొంభై \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/thousands.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/thousands.tsv new file mode 100644 index 000000000..ebbebeb46 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/thousands.tsv @@ -0,0 +1,3 @@ +వెయ్యి +లక్ష +కోటి diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/ties_prefix.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/ties_prefix.tsv new file mode 100644 index 000000000..7ace1c04e --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/ties_prefix.tsv @@ -0,0 +1,8 @@ +ఇరవై ౨ +ముప్పై ౩ +నలభై ౪ +యాభై ౫ +అరవై ౬ +డెబ్బై ౭ +ఎనభై ౮ +తొంభై ౯ \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/data/numbers/zero.tsv b/nemo_text_processing/inverse_text_normalization/te/data/numbers/zero.tsv new file mode 100644 index 000000000..5f95b48d6 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/data/numbers/zero.tsv @@ -0,0 +1 @@ +౦ సున్నా \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/te/graph_utils.py b/nemo_text_processing/inverse_text_normalization/te/graph_utils.py new file mode 100644 index 000000000..225527dc9 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/graph_utils.py @@ -0,0 +1,199 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.examples import plurals +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from nemo_text_processing.inverse_text_normalization.te.utils import get_abs_path, load_labels + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + +NEMO_TE_DIGIT = pynini.union("౦", "౧", "౨", "౩", "౪", "౫", "౬", "౭", "౮", "౯").optimize() + +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = u"\u00a0" +NEMO_ZWNJ = u"\u200c" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", u"\u00a0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_CHAR, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(" field_order: \"") + NEMO_NOT_QUOTE + pynutil.delete("\"")) +) + + +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("ఋణాత్మక", "మైనస్").optimize() + + +def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logging.info(f'Created {file_name}') + + +def convert_space(fst) -> 'pynini.FstLike': + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [written_capitalized, spoken.capitalize()], + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], + ] + ) + + spoken_no_space = spoken.replace(" ", "") + + if len(spoken) == (2 * len(spoken_no_space) - 1): + logging.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [[written, spoken_no_space], [written_capitalized, spoken_no_space.upper()]] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path(os.path.dirname(__file__) + '/grammars/' + kind + '/' + name + '.far') + if self.far_exist(): + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> 'pynini.FstLike': + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> 'pynini.FstLike': + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> 'pynini.FstLike': + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/te/taggers/__init__.py b/nemo_text_processing/inverse_text_normalization/te/taggers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/taggers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/te/taggers/cardinal.py new file mode 100644 index 000000000..fd99441d7 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/taggers/cardinal.py @@ -0,0 +1,181 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import ( + INPUT_LOWER_CASED, + MINUS, + NEMO_CHAR, + NEMO_NOT_SPACE, + NEMO_SPACE, + NEMO_TE_DIGIT, + GraphFst, + delete_space, +) +from nemo_text_processing.inverse_text_normalization.te.utils import get_abs_path + + +class CardinalFst(GraphFst): + """ + Finite state transducer for classifying Telugu cardinals, e.g. + ఇరవై మూడు -> cardinal { integer: "౨౩" } + మైనస్ ఇరవై మూడు -> cardinal { integer: "౨౩" negative: "-" } + + The grammar follows the Indian numbering system (hundred / thousand / lakh / + crore) using the same uniform, place-value approach as the Hindi/Marathi ITN + grammars: every place-value group emits a *fixed* number of digits (its real + value or a zero-pad), groups are concatenated, and a single final rewrite + strips the leading zeros. This keeps the number of alternative paths small, + so the FST stays compact and builds quickly. + + Telugu spoken numbers are rendered with Telugu digits ("నూట ఇరవై మూడు" -> + "౧౨౩"). As in the Hindi ITN grammar, only the number words in a phrase are + converted and any surrounding non-number words are left untouched + ("ఆరు మంది ఆటగాళ్లు బయటకు" -> "౬ మంది ఆటగాళ్లు బయటకు"). Whatever spacing the + input uses is reproduced in the output: a glued case suffix stays glued + ("ఇరవైకి" -> "౨౦కి") while a spaced one keeps its space ("ఇరవై కి" -> "౨౦ కి"). + + Args: + input_case: accepting either "lower_cased" or "cased" input. + """ + + def __init__(self, input_case: str = INPUT_LOWER_CASED): + super().__init__(name="cardinal", kind="classify") + self.input_case = input_case + + # ---- atoms (spoken word -> digit) ------------------------------------------------- + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")).invert() # సున్నా -> ౦ + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")).invert() # ఒకటి -> ౧ ... + graph_teens_and_ties = pynini.string_file(get_abs_path("data/numbers/teens_and_ties.tsv")).invert() + graph_ties_prefix = pynini.string_file(get_abs_path("data/numbers/ties_prefix.tsv")) # ఇరవై -> ౨ ... + + # These two files are already stored as "worddigit", so no .invert() is needed. + # Route each entry by its digit width so the single-/two-digit invariants below hold: + # single-digit entries (ఇద్దరు -> ౨, ఒక -> ౧) join the digits, the two-digit entry + # (పది మంది -> ౧౦, పంతొమ్మిది -> ౧౯) joins the two-digit numbers. + graph_people = pynini.string_file(get_abs_path("data/numbers/digit_people.tsv")) + graph_special = pynini.string_file(get_abs_path("data/numbers/special_numbers.tsv")) + two_te_digits = NEMO_TE_DIGIT + NEMO_TE_DIGIT + graph_digit |= (graph_people | graph_special) @ NEMO_TE_DIGIT + + self.graph_zero = graph_zero + self.graph_digit = graph_digit + + # ---- two-digit block: always exactly 2 output digits ------------------------------ + graph_two_digit = ( + graph_teens_and_ties + | (graph_ties_prefix + delete_space + graph_digit) + | (pynutil.insert("౦") + graph_digit) + | ((graph_people | graph_special) @ two_te_digits) + ) + self.graph_two_digit = graph_two_digit + two_digit_or_zeros = graph_two_digit | pynutil.insert("౦౦") + + # A grammatical case suffix (కి / కు / లో / తో / కోసం) glued directly onto a number + # (e.g. "ఇరవైకి") has to be absorbed into the cardinal token, because the tokenizer + # only splits tokens on whitespace: without this rule "ఇరవైకి" could not be + # recognised as a number at all. It is preserved in the output ("ఇరవైకి" -> "౨౦కి"). + # A *space-separated* suffix ("ఇరవై కి") is left untouched here: the number graph is + # forbidden from consuming a trailing space (see ``no_trailing_space`` below), so the + # suffix becomes its own word token and the single separating space is preserved + # ("ఇరవై కి" -> "౨౦ కి"). + graph_case_suffix = pynini.string_file(get_abs_path("data/numbers/case_suffix.tsv")) + optional_case = pynini.closure(graph_case_suffix, 0, 1) + + # An explicit "ఒక" (one) may precede a bare magnitude, e.g. "ఒక కోటి" == "కోటి". + optional_one = pynini.closure(pynutil.delete("ఒక") + delete_space, 0, 1) + + # Magnitude words (hundred / thousand / lakh / crore) are a small, fixed, closed + # class, so - exactly like the Hindi and Marathi ITN grammars - they are written + # inline rather than loaded from data files. Each magnitude has "plural" forms that + # follow a multiplier (e.g. రెండు వందల -> 2 hundreds) and a "singular" form that + # stands alone and implies one (e.g. వంద -> 100). + + # ---- hundreds block: always exactly 3 output digits ------------------------------- + delete_hundred = pynutil.delete(pynini.union("వందల", "వందలు")) + one_hundred = optional_one + pynini.cross(pynini.union("నూట", "వంద", "నూరు"), "౧") + hundred_prefix = ( + (graph_digit + delete_space + delete_hundred) # N hundred -> N + | one_hundred # (one) hundred -> ౧ + | pynutil.insert("౦") # no hundred -> ౦ + ) + graph_hundreds = hundred_prefix + delete_space + two_digit_or_zeros + self.graph_hundreds = graph_hundreds + + # ---- thousand / lakh blocks: always exactly 2 output digits ----------------------- + delete_thousand = pynutil.delete(pynini.union("వేల", "వేలు")) + thousand_block = ( + (graph_two_digit + delete_space + delete_thousand) # N thousand -> N + | (optional_one + pynini.cross("వెయ్యి", "౦౧")) # (one) thousand -> ౦౧ + | pynutil.insert("౦౦") # no thousand -> ౦౦ + ) + + delete_lakh = pynutil.delete(pynini.union("లక్షల", "లక్షలు")) + lakh_block = ( + (graph_two_digit + delete_space + delete_lakh) # N lakh -> N + | (optional_one + pynini.cross(pynini.union("లక్ష", "లక్షా"), "౦౧")) # (one) lakh -> ౦౧ + | pynutil.insert("౦౦") # no lakh -> ౦౦ + ) + + # ---- everything below a crore: always exactly 7 output digits --------------------- + graph_below_crore = lakh_block + delete_space + thousand_block + delete_space + graph_hundreds + + at_least_one_non_zero = pynini.closure(NEMO_TE_DIGIT) + (NEMO_TE_DIGIT - "౦") + pynini.closure(NEMO_TE_DIGIT) + + # ---- crore block: always exactly 7 output digits --------------------------------- + # The crore multiplier is itself any (non-zero) below-crore number, which lets the + # grammar reach numbers such as "వెయ్యి నూట రెండు కోట్ల..." (1102 crore) without + # enumerating a separate rule per magnitude. + delete_crore = pynutil.delete(pynini.union("కోట్ల", "కోట్లు")) + crore_block = ( + ((graph_below_crore @ at_least_one_non_zero) + delete_space + delete_crore) # X crore -> X + | (optional_one + pynini.cross("కోటి", "౦౦౦౦౦౦౧")) # (one) crore + | pynutil.insert("౦౦౦౦౦౦౦") # no crore + ) + + # ---- assemble, then strip the zero padding --------------------------------------- + graph_full = crore_block + delete_space + graph_below_crore + strip_leading_zeros = ( + pynutil.delete(pynini.closure("౦")) + (NEMO_TE_DIGIT - "౦") + pynini.closure(NEMO_TE_DIGIT) + ) + # The number may contain spaces *between* its words ("రెండు వేల మూడు వందల") but must + # never consume a *trailing* space. This stops the internal ``delete_space`` of an + # empty (zero-pad) block from swallowing the space before a following word/suffix, so + # spacing in the input is always reproduced in the output ("వంద కి" -> "౧౦౦ కి"). + no_trailing_space = pynini.closure(NEMO_CHAR) + NEMO_NOT_SPACE + graph_number = (no_trailing_space @ graph_full) @ strip_leading_zeros + + # "సున్నా ఐదు" -> ౦౫, "సున్నా సున్నా" -> ౦౦, ... (explicit leading-zero readouts) + graph_leading_zeros = graph_zero + pynini.closure(delete_space + (graph_zero | graph_digit), 1) + + graph = (graph_number | graph_zero) + optional_case + graph |= graph_leading_zeros + + self.graph_no_exception = graph + + self.graph = graph.optimize() + + optional_minus_graph = pynini.closure( + pynutil.insert("negative: ") + pynini.cross(MINUS, "\"-\"") + NEMO_SPACE, + 0, + 1, + ) + + final_graph = optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + + final_graph = self.add_tokens(final_graph) + self.fst = final_graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/te/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/te/taggers/tokenize_and_classify.py new file mode 100644 index 000000000..b55338325 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/taggers/tokenize_and_classify.py @@ -0,0 +1,75 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.inverse_text_normalization.te.taggers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.te.taggers.word import WordFst + + +class ClassifyFst(GraphFst): + """ + Final classification grammar for Telugu ITN cardinal processing. + """ + + def __init__( + self, + cache_dir: str = None, + overwrite_cache: bool = False, + whitelist: str = None, + input_case: str = None, + ): + super().__init__(name="tokenize_and_classify", kind="classify") + + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, "te_itn.far") + + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] + logging.info(f"ClassifyFst.fst was restored from {far_file}.") + else: + logging.info("Creating Telugu ClassifyFst grammar.") + + cardinal = CardinalFst() + cardinal_graph = cardinal.fst + word_graph = WordFst().fst + + classify = pynutil.add_weight(cardinal_graph, 1.1) | pynutil.add_weight(word_graph, 100) + + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + + graph = token + pynini.closure(pynutil.add_weight(delete_extra_space + token, 1000.0)) + graph = delete_space + graph + delete_space + + self.fst = graph.optimize() + + if far_file: + generator_main( + far_file, + {"tokenize_and_classify": self.fst}, + ) + logging.info(f"ClassifyFst grammars are saved to {far_file}.") diff --git a/nemo_text_processing/inverse_text_normalization/te/taggers/word.py b/nemo_text_processing/inverse_text_normalization/te/taggers/word.py new file mode 100644 index 000000000..95045582a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/taggers/word.py @@ -0,0 +1,30 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying plain Telugu tokens. + """ + + def __init__(self): + super().__init__(name="word", kind="classify") + word = pynutil.insert('name: "') + pynini.closure(NEMO_NOT_SPACE, 1) + pynutil.insert('"') + self.fst = word.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/te/utils.py b/nemo_text_processing/inverse_text_normalization/te/utils.py new file mode 100644 index 000000000..5830b50d0 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/utils.py @@ -0,0 +1,63 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import csv +import logging +import os +import pynini + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + abs_path = os.path.dirname(os.path.abspath(__file__)) + os.sep + rel_path + + if not os.path.exists(abs_path): + logging.warning(f'{abs_path} does not exist') + return abs_path + + +def load_labels(abs_path): + """ + loads relative path file as dictionary + + Args: + abs_path: absolute path + + Returns dictionary of mappings + """ + label_tsv = open(abs_path, encoding="utf-8") + labels = list(csv.reader(label_tsv, delimiter="\t")) + return labels + + +from pynini.lib import pynutil + + +def apply_fst(text, fst): + """Given a string input, returns the output string + produced by traversing the path with lowest weight. + If no valid path accepts input string, returns an + error. + """ + try: + print(pynini.shortestpath(text @ fst).string()) + except pynini.FstOpError: + print(f"Error: No valid output with given input: '{text}'") diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/__init__.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/cardinal.py new file mode 100644 index 000000000..cf6c3a903 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/cardinal.py @@ -0,0 +1,55 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space + + +class CardinalFst(GraphFst): + """ + Finite state transducer for verbalizing cardinal + e.g. + cardinal { integer: "౨౩" } -> ౨౩ + cardinal { integer: "౨౩" negative: "-" } -> -౨౩ + """ + + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + + optional_sign = pynini.closure( + pynutil.delete("negative:") + + delete_space + + pynutil.delete("\"") + + NEMO_NOT_QUOTE + + pynutil.delete("\"") + + delete_space, + 0, + 1, + ) + + graph = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) + + self.numbers = graph + graph = optional_sign + graph + + delete_tokens = self.delete_tokens(graph) + self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize.py new file mode 100644 index 000000000..af4080432 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize.py @@ -0,0 +1,32 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or impliesd. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import GraphFst +from nemo_text_processing.inverse_text_normalization.te.verbalizers.cardinal import CardinalFst +from nemo_text_processing.inverse_text_normalization.te.verbalizers.word import WordFst + + +class VerbalizeFst(GraphFst): + """ + Composes the Telugu cardinal verbalizer grammar. + """ + + def __init__(self): + super().__init__(name="verbalize", kind="verbalize") + + cardinal_graph = CardinalFst().fst + word_graph = WordFst().fst + self.fst = (cardinal_graph | word_graph).optimize() diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize_final.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize_final.py new file mode 100644 index 000000000..fe839c467 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/verbalize_final.py @@ -0,0 +1,49 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import GraphFst, delete_extra_space, delete_space +from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize import VerbalizeFst + + +class VerbalizeFinalFst(GraphFst): + """ + Verbalizes full Telugu ITN token output. + + Example: + tokens { cardinal { integer: "౨౩" } } -> ౨౩ + tokens { name: "పరుగులు" } -> పరుగులు + """ + + def __init__(self): + super().__init__(name="verbalize_final", kind="verbalize") + + verbalize = VerbalizeFst().fst + + graph = ( + pynutil.delete("tokens") + + delete_space + + pynutil.delete("{") + + delete_space + + verbalize + + delete_space + + pynutil.delete("}") + ) + + graph = delete_space + graph + pynini.closure(delete_extra_space + graph) + delete_space + + self.fst = graph.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/te/verbalizers/word.py b/nemo_text_processing/inverse_text_normalization/te/verbalizers/word.py new file mode 100644 index 000000000..28603646c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/te/verbalizers/word.py @@ -0,0 +1,38 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2026 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.te.graph_utils import ( + NEMO_CHAR, + NEMO_SIGMA, + GraphFst, + delete_space, +) + + +class WordFst(GraphFst): + """ + Finite state transducer for verbalizing plain Telugu tokens. + """ + + def __init__(self): + super().__init__(name="word", kind="verbalize") + chars = pynini.closure(NEMO_CHAR - " ", 1) + char = pynutil.delete("name:") + delete_space + pynutil.delete('"') + chars + pynutil.delete('"') + graph = char @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) + + self.fst = graph.optimize() diff --git a/tests/nemo_text_processing/te/__init__.py b/tests/nemo_text_processing/te/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/tests/nemo_text_processing/te/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/nemo_text_processing/te/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/te/data_inverse_text_normalization/test_cases_cardinal.txt new file mode 100644 index 000000000..a9e447128 --- /dev/null +++ b/tests/nemo_text_processing/te/data_inverse_text_normalization/test_cases_cardinal.txt @@ -0,0 +1,51 @@ +నాలుగు నాలుగులు~౪ నాలుగులు +ఆరు మంది ఆటగాళ్లు బయటకు~౬ మంది ఆటగాళ్లు బయటకు +నాలుగు ఓవర్లలో పదిహేడు పరుగులు~౪ ఓవర్లలో ౧౭ పరుగులు +ఐదు చాక్లెట్లు తొమ్మిది టాఫీలు~౫ చాక్లెట్లు ౯ టాఫీలు +పది వేల తొంభై తొమ్మిది~౧౦౦౯౯ +లక్షా ఒకటి~౧౦౦౦౦౧ +ఐదు నాలుగులు~౫ నాలుగులు +నాలుగు మంది మృతి పద్దెనిమిది మందికి గాయాలు~౪ మంది మృతి ౧౮ మందికి గాయాలు +ఐదు వేల నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౫౧౦౨౨౩౪౫౫౬౭ +అధ్యాయం పది శ్లోకం రెండు~అధ్యాయం ౧౦ శ్లోకం ౨ +నూట ఒకటి~౧౦౧ +నూట ఇరవై మూడు~౧౨౩ +వంద~౧౦౦ +నూట ఐదు~౧౦౫ +రెండు వందలు~౨౦౦ +రెండు వందల మూడు~౨౦౩ +ఐదు వందల రెండు~౫౦౨ +నూట పదకొండు~౧౧౧ +నూట పంతొమ్మిది~౧౧౯ +నూట ముప్పై~౧౩౦ +నూట యాభై~౧౫౦ +మూడు వందల పన్నెండు~౩౧౨ +ఐదు వందల ఇరవై ఐదు~౫౨౫ +నాలుగు వందల ఇరవై ఐదు~౪౨౫ +తొమ్మిది వందల పదిహేను~౯౧౫ +వెయ్యి ఒకటి~౧౦౦౧ +ఐదు వేల ఐదు వందల యాభై ఒకటి అరటిపండ్లు~౫౫౫౧ అరటిపండ్లు +ఎనభై ఐదు~౮౫ +తొంభై ఒకటి~౯౧ +వెయ్యి మూడు వందల ఇరవై మూడు~౧౩౨౩ +వెయ్యి మూడు వందల నలభై ఐదు~౧౩౪౫ +పదమూడు వేల నాలుగు వందల యాభై ఆరు~౧౩౪౫౬ +పన్నెండు వేల మూడు వందల నలభై ఆరు~౧౨౩౪౬ +లక్ష ఇరవై మూడు వేల నాలుగు వందల యాభై ఆరు~౧౨౩౪౫౬ +ఎనిమిది లక్షల డెబ్బై రెండు వేల తొమ్మిది వందల ఎనభై ఏడు~౮౭౨౯౮౭ +తొమ్మిది లక్షల ఎనభై ఏడు వేల ఆరు వందల తొమ్మిది~౯౮౭౬౦౯ +తొంభై ఎనిమిది లక్షల డెబ్బై ఆరు వేల ఏడు వందల ఎనభై తొమ్మిది~౯౮౭౬౭౮౯ +ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౨౩౪౫౫౬౭ +కోటి ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౧౨౩౪౫౫౬౭ +కోటి ఇరవై ఒక లక్షల ఇరవై ఒక వేల రెండు వందల పన్నెండు~౧౨౧౨౧౨౧౨ +నూట పన్నెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౧౧౨౨౩౪౫౫౬౭ +నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౧౦౨౨౩౪౫౫౬౭ +వెయ్యి నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౧౧౦౨౨౩౪౫౫౬౭ +ఐదు వేల నూట రెండు కోట్ల ఇరవై మూడు లక్షల నలభై ఐదు వేల ఐదు వందల అరవై ఏడు~౫౧౦౨౨౩౪౫౫౬౭ +సున్నా ఐదు~౦౫ +సున్నా ఒకటి~౦౧ +సున్నా ఏడు మూడు~౦౭౩ +సున్నా సున్నా సున్నా ఒకటి~౦౦౦౧ +సున్నా సున్నా సున్నా~౦౦౦ +మైనస్ ఇరవై మూడు~-౨౩ +ఋణాత్మక ఐదు~-౫ diff --git a/tests/nemo_text_processing/te/test_cardinal.py b/tests/nemo_text_processing/te/test_cardinal.py new file mode 100644 index 000000000..85a10a177 --- /dev/null +++ b/tests/nemo_text_processing/te/test_cardinal.py @@ -0,0 +1,35 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +import pytest +from parameterized import parameterized + +from nemo_text_processing.inverse_text_normalization.te.taggers.tokenize_and_classify import ClassifyFst +from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize_final import VerbalizeFinalFst + +from ..utils import parse_test_case_file + + +class TestCardinal: + tagger = ClassifyFst(overwrite_cache=True).fst + verbalizer = VerbalizeFinalFst().fst + + @parameterized.expand(parse_test_case_file('te/data_inverse_text_normalization/test_cases_cardinal.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_denorm(self, test_input, expected): + tagged = pynini.shortestpath(test_input @ self.tagger).string() + pred = pynini.shortestpath(tagged @ self.verbalizer).string() + assert pred == expected diff --git a/tests/nemo_text_processing/te/test_sparrowhawk_inverse_text_normalization.sh b/tests/nemo_text_processing/te/test_sparrowhawk_inverse_text_normalization.sh new file mode 100644 index 000000000..67a4077c3 --- /dev/null +++ b/tests/nemo_text_processing/te/test_sparrowhawk_inverse_text_normalization.sh @@ -0,0 +1,29 @@ +#! /bin/sh + +PROJECT_DIR=/workspace/tests + +runtest () { + input=$1 + cd /workspace/sparrowhawk/documentation/grammars + + # read test file + while read testcase; do + IFS='~' read spoken written <<< $testcase + denorm_pred=$(echo $spoken | normalizer_main --config=sparrowhawk_configuration.ascii_proto 2>&1 | tail -n 1) + + # trim white space + written="$(echo -e "${written}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + denorm_pred="$(echo -e "${denorm_pred}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + + # input expected actual + assertEquals "$spoken" "$written" "$denorm_pred" + done < "$input" +} + +testITNCardinal() { + input=$PROJECT_DIR/te/data_inverse_text_normalization/test_cases_cardinal.txt + runtest $input +} + +# Load shUnit2 +. $PROJECT_DIR/../shunit2/shunit2 diff --git a/tools/text_processing_deployment/pynini_export.py b/tools/text_processing_deployment/pynini_export.py index 73a4fc138..f183fdf0c 100644 --- a/tools/text_processing_deployment/pynini_export.py +++ b/tools/text_processing_deployment/pynini_export.py @@ -106,6 +106,7 @@ def parse_args(): 'hi_en', 'hy', 'mr', + 'te', 'ja', 'rw', 'ko', @@ -140,7 +141,7 @@ def parse_args(): if __name__ == '__main__': args = parse_args() - if args.language in ['ru', 'es_en', 'hi_en', 'mr'] and args.grammars == 'tn_grammars': + if args.language in ['ru', 'es_en', 'hi_en', 'mr', 'te'] and args.grammars == 'tn_grammars': raise ValueError('Only ITN grammars could be deployed in Sparrowhawk for the selected languages.') TNPostProcessingFst = None ITNPostProcessingFst = None @@ -305,6 +306,13 @@ def parse_args(): from nemo_text_processing.inverse_text_normalization.mr.verbalizers.verbalize import ( VerbalizeFst as ITNVerbalizeFst, ) + elif args.language == 'te': + from nemo_text_processing.inverse_text_normalization.te.taggers.tokenize_and_classify import ( + ClassifyFst as ITNClassifyFst, + ) + from nemo_text_processing.inverse_text_normalization.te.verbalizers.verbalize import ( + VerbalizeFst as ITNVerbalizeFst, + ) elif args.language == 'he': from nemo_text_processing.inverse_text_normalization.he.taggers.tokenize_and_classify import ( ClassifyFst as ITNClassifyFst,