diff --git a/Jenkinsfile b/Jenkinsfile index 38e05bb00..454aac441 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,7 +27,7 @@ pipeline { HE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-24-25-0' HY_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-0' MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' - JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' + JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/07-17-26-0' HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-26-5' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' @@ -332,7 +332,7 @@ pipeline { } } } - stage('L0: Create JA ITN Grammars') { + stage('L0: Create JA TN/ITN Grammars') { when { anyOf { branch 'main' @@ -348,6 +348,11 @@ pipeline { sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=ja --text="100" --cache_dir ${JA_TN_CACHE}' } } + stage('L0: JA TN grammars') { + steps { + sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=ja --text="100" --cache_dir ${JA_TN_CACHE}' + } + } } } stage('L0: Create KO TN/ITN Grammars') { @@ -481,7 +486,7 @@ pipeline { } } - stage('L2: EN Sparrowhawk Tests') { + stage('L2: EN Sparrowhawk Tests') { when { anyOf { branch 'main' diff --git a/nemo_text_processing/inverse_text_normalization/ja/data/time_hours.tsv b/nemo_text_processing/inverse_text_normalization/ja/data/time_hours.tsv index 8a5eced19..1957b95ad 100644 --- a/nemo_text_processing/inverse_text_normalization/ja/data/time_hours.tsv +++ b/nemo_text_processing/inverse_text_normalization/ja/data/time_hours.tsv @@ -22,4 +22,4 @@ 二十二 22 二十三 23 二十四 24 -零 0 \ No newline at end of file +ゼロ 0 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/ja/taggers/fraction.py b/nemo_text_processing/inverse_text_normalization/ja/taggers/fraction.py index 0ced0c679..10882ae99 100644 --- a/nemo_text_processing/inverse_text_normalization/ja/taggers/fraction.py +++ b/nemo_text_processing/inverse_text_normalization/ja/taggers/fraction.py @@ -26,7 +26,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): e.g., 四分の三 -> fraction { denominator: "4" numerator: "3" } 一と四分の三 -> fraction { integer: "1" denominator: "4" numerator: "3" } - 一荷四分の三 -> fraction { integer: "1" denominator: "4" numerator: "3" } + 一と四分の三 -> fraction { integer: "1" denominator: "4" numerator: "3" } ルート三分の一 -> fraction { denominator: "√3" numerator: "1" } 一点六五分の五十 -> fraction { denominator: "1.65" numerator: "50" } 二ルート六分の三 -> -> fraction { denominator: "2√6 " numerator: "3" } @@ -40,7 +40,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): pynutil.delete("分の") | pynutil.delete(" 分 の　") | pynutil.delete("分 の　") | pynutil.delete("分 の") ) - integer_word = pynutil.delete("と") | pynutil.delete("荷") + integer_word = pynutil.delete("と") root_word = pynini.accep("√") | pynini.cross("ルート", "√") graph_sign = ( diff --git a/nemo_text_processing/text_normalization/ja/data/address/__init__.py b/nemo_text_processing/text_normalization/ja/data/address/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/address/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/address/postal.tsv b/nemo_text_processing/text_normalization/ja/data/address/postal.tsv new file mode 100644 index 000000000..2ed52b275 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/address/postal.tsv @@ -0,0 +1 @@ +〒 郵便番号 diff --git a/nemo_text_processing/text_normalization/ja/data/address/room_suffix.tsv b/nemo_text_processing/text_normalization/ja/data/address/room_suffix.tsv new file mode 100644 index 000000000..6d259df0d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/address/room_suffix.tsv @@ -0,0 +1 @@ +号室 diff --git a/nemo_text_processing/text_normalization/ja/data/address/separator.tsv b/nemo_text_processing/text_normalization/ja/data/address/separator.tsv new file mode 100644 index 000000000..028b8020d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/address/separator.tsv @@ -0,0 +1,3 @@ +- の +－ の +ー の diff --git a/nemo_text_processing/text_normalization/ja/data/date/day.tsv b/nemo_text_processing/text_normalization/ja/data/date/day.tsv index 09258cb4c..cd6dbe49a 100644 --- a/nemo_text_processing/text_normalization/ja/data/date/day.tsv +++ b/nemo_text_processing/text_normalization/ja/data/date/day.tsv @@ -28,4 +28,4 @@ 28 二十八 29 二十九 30 三十 -31 三十一 \ No newline at end of file +31 三十一 diff --git a/nemo_text_processing/text_normalization/ja/data/date/era.tsv b/nemo_text_processing/text_normalization/ja/data/date/era.tsv index b932b7fb3..85e56bce0 100644 --- a/nemo_text_processing/text_normalization/ja/data/date/era.tsv +++ b/nemo_text_processing/text_normalization/ja/data/date/era.tsv @@ -9,4 +9,4 @@ グレゴリオ暦 紀元前 紀元 -紀元後 \ No newline at end of file +紀元後 diff --git a/nemo_text_processing/text_normalization/ja/data/date/era_abbrev.tsv b/nemo_text_processing/text_normalization/ja/data/date/era_abbrev.tsv index 1c95675c2..996934a15 100644 --- a/nemo_text_processing/text_normalization/ja/data/date/era_abbrev.tsv +++ b/nemo_text_processing/text_normalization/ja/data/date/era_abbrev.tsv @@ -2,4 +2,4 @@ R. 令和 H. 平成 S. 昭和 T. 大正 -M. 明治 \ No newline at end of file +M. 明治 diff --git a/nemo_text_processing/text_normalization/ja/data/date/month.tsv b/nemo_text_processing/text_normalization/ja/data/date/month.tsv index f992b4d28..33896fe82 100644 --- a/nemo_text_processing/text_normalization/ja/data/date/month.tsv +++ b/nemo_text_processing/text_normalization/ja/data/date/month.tsv @@ -9,4 +9,4 @@ 9 九 10 十 11 十一 -12 十二 \ No newline at end of file +12 十二 diff --git a/nemo_text_processing/text_normalization/ja/data/date/suffix.tsv b/nemo_text_processing/text_normalization/ja/data/date/suffix.tsv new file mode 100644 index 000000000..a1064b515 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/date/suffix.tsv @@ -0,0 +1,8 @@ +year 年 +month 月 +day 日 +century 世紀 +decade 年代 +early_ten_days 上旬 +middle_ten_days 中旬 +late_ten_days 下旬 diff --git a/nemo_text_processing/text_normalization/ja/data/date/week.tsv b/nemo_text_processing/text_normalization/ja/data/date/week.tsv index 41556b4db..af2e05c5c 100644 --- a/nemo_text_processing/text_normalization/ja/data/date/week.tsv +++ b/nemo_text_processing/text_normalization/ja/data/date/week.tsv @@ -12,4 +12,4 @@ 木曜日 金曜日 土曜日 -日曜日 \ No newline at end of file +日曜日 diff --git a/nemo_text_processing/text_normalization/ja/data/electronic/__init__.py b/nemo_text_processing/text_normalization/ja/data/electronic/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/electronic/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/electronic/card_cues.tsv b/nemo_text_processing/text_normalization/ja/data/electronic/card_cues.tsv new file mode 100644 index 000000000..769dee97d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/electronic/card_cues.tsv @@ -0,0 +1,2 @@ +カード末尾 +カード番号 diff --git a/nemo_text_processing/text_normalization/ja/data/electronic/card_digit_count_prefix.tsv b/nemo_text_processing/text_normalization/ja/data/electronic/card_digit_count_prefix.tsv new file mode 100644 index 000000000..4a52fb771 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/electronic/card_digit_count_prefix.tsv @@ -0,0 +1 @@ +カード下 diff --git a/nemo_text_processing/text_normalization/ja/data/electronic/card_digit_count_suffix.tsv b/nemo_text_processing/text_normalization/ja/data/electronic/card_digit_count_suffix.tsv new file mode 100644 index 000000000..d4fb9db66 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/electronic/card_digit_count_suffix.tsv @@ -0,0 +1 @@ +桁 diff --git a/nemo_text_processing/text_normalization/ja/data/electronic/file_extensions.tsv b/nemo_text_processing/text_normalization/ja/data/electronic/file_extensions.tsv new file mode 100644 index 000000000..96c957484 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/electronic/file_extensions.tsv @@ -0,0 +1,6 @@ +.jpg ドット jpg +.JPG ドット JPG +.png ドット png +.PNG ドット PNG +.pdf ドット pdf +.PDF ドット PDF diff --git a/nemo_text_processing/text_normalization/ja/data/electronic/protocol.tsv b/nemo_text_processing/text_normalization/ja/data/electronic/protocol.tsv new file mode 100644 index 000000000..d1cfd2593 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/electronic/protocol.tsv @@ -0,0 +1,2 @@ +http +https diff --git a/nemo_text_processing/text_normalization/ja/data/electronic/special_digit_runs.tsv b/nemo_text_processing/text_normalization/ja/data/electronic/special_digit_runs.tsv new file mode 100644 index 000000000..8609135f3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/electronic/special_digit_runs.tsv @@ -0,0 +1 @@ +01 ゼロイチ diff --git a/nemo_text_processing/text_normalization/ja/data/electronic/symbol.tsv b/nemo_text_processing/text_normalization/ja/data/electronic/symbol.tsv new file mode 100644 index 000000000..568588980 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/electronic/symbol.tsv @@ -0,0 +1,5 @@ +@ アット +. ドット +- ハイフン +/ スラッシュ +: コロン diff --git a/nemo_text_processing/text_normalization/ja/data/fraction/__init__.py b/nemo_text_processing/text_normalization/ja/data/fraction/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/fraction/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/fraction/marker.tsv b/nemo_text_processing/text_normalization/ja/data/fraction/marker.tsv new file mode 100644 index 000000000..23fe7022a --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/fraction/marker.tsv @@ -0,0 +1,4 @@ +fraction 分の +root_written √ +root_spoken ルート +mixed と diff --git a/nemo_text_processing/text_normalization/ja/data/latin/__init__.py b/nemo_text_processing/text_normalization/ja/data/latin/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/latin/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/latin/letters.tsv b/nemo_text_processing/text_normalization/ja/data/latin/letters.tsv new file mode 100644 index 000000000..c28af1201 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/latin/letters.tsv @@ -0,0 +1,26 @@ +A エー +B ビー +C シー +D ディー +E イー +F エフ +G ジー +H エイチ +I アイ +J ジェー +K ケー +L エル +M エム +N エヌ +O オー +P ピー +Q キュー +R アール +S エス +T ティー +U ユー +V ブイ +W ダブリュー +X エックス +Y ワイ +Z ゼット diff --git a/nemo_text_processing/text_normalization/ja/data/measure/__init__.py b/nemo_text_processing/text_normalization/ja/data/measure/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/measure/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/measure/per_marker.tsv b/nemo_text_processing/text_normalization/ja/data/measure/per_marker.tsv new file mode 100644 index 000000000..133297769 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/measure/per_marker.tsv @@ -0,0 +1 @@ +毎 diff --git a/nemo_text_processing/text_normalization/ja/data/measure/per_unit.tsv b/nemo_text_processing/text_normalization/ja/data/measure/per_unit.tsv new file mode 100644 index 000000000..ed878a1bf --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/measure/per_unit.tsv @@ -0,0 +1,33 @@ +h 時 +ｈ 時 +hour 時 +hours 時 +s 秒 +ｓ 秒 +sec 秒 +second 秒 +seconds 秒 +min 分 +ｍｉｎ 分 +minute 分 +minutes 分 +kg キロ +ｋｇ キロ +g グラム +ｇ グラム +m メートル +ｍ メートル +km キロ +ｋｍ キロ +cm センチメートル +ｃｍ センチメートル +mm ミリメートル +ｍｍ ミリメートル +L リットル +Ｌ リットル +l リットル +ｌ リットル +ml ミリリットル +mL ミリリットル +ｍｌ ミリリットル +ｍＬ ミリリットル diff --git a/nemo_text_processing/text_normalization/ja/data/measure/rate_numerator.tsv b/nemo_text_processing/text_normalization/ja/data/measure/rate_numerator.tsv new file mode 100644 index 000000000..126b4a3e1 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/measure/rate_numerator.tsv @@ -0,0 +1,4 @@ +km キロ +ｋｍ キロ +m メートル +ｍ メートル diff --git a/nemo_text_processing/text_normalization/ja/data/measure/speed.tsv b/nemo_text_processing/text_normalization/ja/data/measure/speed.tsv new file mode 100644 index 000000000..275b2c14d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/measure/speed.tsv @@ -0,0 +1,2 @@ +キロ 時 時速 +メートル 秒 秒速 diff --git a/nemo_text_processing/text_normalization/ja/data/measure/unit.tsv b/nemo_text_processing/text_normalization/ja/data/measure/unit.tsv new file mode 100644 index 000000000..0573068f8 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/measure/unit.tsv @@ -0,0 +1,79 @@ +kg キロ +ｋｇ キロ +キロ キロ +キログラム キログラム +g グラム +ｇ グラム +グラム グラム +mg ミリグラム +ｍｇ ミリグラム +ミリグラム ミリグラム +μg マイクログラム +µg マイクログラム +マイクログラム マイクログラム +km キロ +ｋｍ キロ +キロメートル キロ +m メートル +ｍ メートル +メートル メートル +cm センチ +ｃｍ センチ +センチ センチ +センチメートル センチメートル +mm ミリメートル +ｍｍ ミリメートル +ミリメートル ミリメートル +m2 平方メートル +m² 平方メートル +ｍ2 平方メートル +ｍ² 平方メートル +平方メートル 平方メートル +cm2 平方センチメートル +cm² 平方センチメートル +平方センチメートル 平方センチメートル +L リットル +l リットル +Ｌ リットル +ｌ リットル +リットル リットル +mL ミリリットル +ml ミリリットル +ｍＬ ミリリットル +ｍｌ ミリリットル +ミリリットル ミリリットル +% パーセント +％ パーセント +パーセント パーセント +° 度 +º 度 +度 度 +°C 度 +℃ 度 +ºC 度 +°F 度エフ +℉ 度エフ +W ワット +w ワット +ワット ワット +kW キロワット +kw キロワット +キロワット キロワット +V ボルト +v ボルト +ボルト ボルト +Hz ヘルツ +hz ヘルツ +ヘルツ ヘルツ +kHz キロヘルツ +khz キロヘルツ +MHz メガヘルツ +mhz メガヘルツ +GHz ギガヘルツ +ghz ギガヘルツ +GB ギガバイト +gb ギガバイト +ギガバイト ギガバイト +MB メガバイト +mb メガバイト +メガバイト メガバイト diff --git a/nemo_text_processing/text_normalization/ja/data/money/__init__.py b/nemo_text_processing/text_normalization/ja/data/money/__init__.py new file mode 100644 index 000000000..4fc25d0d3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/money/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/money/currency_major.tsv b/nemo_text_processing/text_normalization/ja/data/money/currency_major.tsv new file mode 100644 index 000000000..c5f2efa84 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/money/currency_major.tsv @@ -0,0 +1,27 @@ +¥ 円 +￥ 円 +JPY 円 +jpy 円 +円 円 +$ ドル +US$ ドル +USD ドル +usd ドル +ドル ドル +米ドル ドル +€ ユーロ +EUR ユーロ +eur ユーロ +ユーロ ユーロ +£ ポンド +GBP ポンド +gbp ポンド +ポンド ポンド +₩ ウォン +KRW ウォン +krw ウォン +ウォン ウォン +CNY 元 +cny 元 +元 元 +人民元 人民元 diff --git a/nemo_text_processing/text_normalization/ja/data/money/currency_minor.tsv b/nemo_text_processing/text_normalization/ja/data/money/currency_minor.tsv new file mode 100644 index 000000000..3709c9572 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/money/currency_minor.tsv @@ -0,0 +1,2 @@ +銭 銭 +セント セント diff --git a/nemo_text_processing/text_normalization/ja/data/money/currency_minor_by_major.tsv b/nemo_text_processing/text_normalization/ja/data/money/currency_minor_by_major.tsv new file mode 100644 index 000000000..bdb9c4d07 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/money/currency_minor_by_major.tsv @@ -0,0 +1,15 @@ +¥ 銭 +￥ 銭 +JPY 銭 +jpy 銭 +円 銭 +$ セント +US$ セント +USD セント +usd セント +ドル セント +米ドル セント +€ セント +EUR セント +eur セント +ユーロ セント diff --git a/nemo_text_processing/text_normalization/ja/data/money/currency_prefix.tsv b/nemo_text_processing/text_normalization/ja/data/money/currency_prefix.tsv new file mode 100644 index 000000000..c959bf0ca --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/money/currency_prefix.tsv @@ -0,0 +1,19 @@ +¥ 円 +￥ 円 +JPY 円 +jpy 円 +$ ドル +US$ ドル +USD ドル +usd ドル +€ ユーロ +EUR ユーロ +eur ユーロ +£ ポンド +GBP ポンド +gbp ポンド +₩ ウォン +KRW ウォン +krw ウォン +CNY 元 +cny 元 diff --git a/nemo_text_processing/text_normalization/ja/data/money/quantity.tsv b/nemo_text_processing/text_normalization/ja/data/money/quantity.tsv new file mode 100644 index 000000000..bfbb13743 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/money/quantity.tsv @@ -0,0 +1,3 @@ +万 +億 +兆 diff --git a/nemo_text_processing/text_normalization/ja/data/numbers/decimal_point.tsv b/nemo_text_processing/text_normalization/ja/data/numbers/decimal_point.tsv new file mode 100644 index 000000000..ae902786c --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/numbers/decimal_point.tsv @@ -0,0 +1 @@ +. 点 diff --git a/nemo_text_processing/text_normalization/ja/data/numbers/digit.tsv b/nemo_text_processing/text_normalization/ja/data/numbers/digit.tsv index ede6b97b7..3a578d358 100644 --- a/nemo_text_processing/text_normalization/ja/data/numbers/digit.tsv +++ b/nemo_text_processing/text_normalization/ja/data/numbers/digit.tsv @@ -6,4 +6,4 @@ 6 六 7 七 8 八 -9 九 \ No newline at end of file +9 九 diff --git a/nemo_text_processing/text_normalization/ja/data/numbers/magnitude.tsv b/nemo_text_processing/text_normalization/ja/data/numbers/magnitude.tsv new file mode 100644 index 000000000..e5ecf396b --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/numbers/magnitude.tsv @@ -0,0 +1,4 @@ +hundred 百 +thousand 千 +ten_thousand 万 +hundred_million 億 diff --git a/nemo_text_processing/text_normalization/ja/data/numbers/sign.tsv b/nemo_text_processing/text_normalization/ja/data/numbers/sign.tsv new file mode 100644 index 000000000..76f8308b7 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/numbers/sign.tsv @@ -0,0 +1,2 @@ +- マイナス +マイナス マイナス diff --git a/nemo_text_processing/text_normalization/ja/data/numbers/teen.tsv b/nemo_text_processing/text_normalization/ja/data/numbers/teen.tsv index 1585fe836..52dc01917 100644 --- a/nemo_text_processing/text_normalization/ja/data/numbers/teen.tsv +++ b/nemo_text_processing/text_normalization/ja/data/numbers/teen.tsv @@ -7,4 +7,4 @@ 16 十六 17 十七 18 十八 -19 十九 \ No newline at end of file +19 十九 diff --git a/nemo_text_processing/text_normalization/ja/data/numbers/ties.tsv b/nemo_text_processing/text_normalization/ja/data/numbers/ties.tsv index 562e63265..2a73c0399 100644 --- a/nemo_text_processing/text_normalization/ja/data/numbers/ties.tsv +++ b/nemo_text_processing/text_normalization/ja/data/numbers/ties.tsv @@ -5,4 +5,4 @@ 6 六十 7 七十 8 八十 -9 九十 \ No newline at end of file +9 九十 diff --git a/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv b/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv index 7fab21718..ac94cc719 100644 --- a/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv +++ b/nemo_text_processing/text_normalization/ja/data/numbers/zero.tsv @@ -1 +1 @@ -0 零 \ No newline at end of file +0 ゼロ diff --git a/nemo_text_processing/text_normalization/ja/data/numbers/zero_decimal.tsv b/nemo_text_processing/text_normalization/ja/data/numbers/zero_decimal.tsv new file mode 100644 index 000000000..d6b9cece5 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/numbers/zero_decimal.tsv @@ -0,0 +1 @@ +0 零 diff --git a/nemo_text_processing/text_normalization/ja/data/numbers/zero_maru.tsv b/nemo_text_processing/text_normalization/ja/data/numbers/zero_maru.tsv new file mode 100644 index 000000000..67a5bfdbb --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/numbers/zero_maru.tsv @@ -0,0 +1 @@ +0 〇 diff --git a/nemo_text_processing/text_normalization/ja/data/ordinal/__init__.py b/nemo_text_processing/text_normalization/ja/data/ordinal/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/ordinal/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/ordinal/marker.tsv b/nemo_text_processing/text_normalization/ja/data/ordinal/marker.tsv new file mode 100644 index 000000000..fed01aeb8 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/ordinal/marker.tsv @@ -0,0 +1,2 @@ +prefix 第 +suffix 番目 diff --git a/nemo_text_processing/text_normalization/ja/data/post_processing/__init__.py b/nemo_text_processing/text_normalization/ja/data/post_processing/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/post_processing/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/post_processing/sentence_suffix.tsv b/nemo_text_processing/text_normalization/ja/data/post_processing/sentence_suffix.tsv new file mode 100644 index 000000000..ac7eb4c33 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/post_processing/sentence_suffix.tsv @@ -0,0 +1 @@ +です diff --git a/nemo_text_processing/text_normalization/ja/data/punctuation/__init__.py b/nemo_text_processing/text_normalization/ja/data/punctuation/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/punctuation/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/punctuation/extra.tsv b/nemo_text_processing/text_normalization/ja/data/punctuation/extra.tsv new file mode 100644 index 000000000..3cbc272d1 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/punctuation/extra.tsv @@ -0,0 +1,14 @@ +< +> +` +々 +ゝ +ゞ +ヽ +ヾ +〱 +〲 +〳 +〴 +〵 +〆 diff --git a/nemo_text_processing/text_normalization/ja/data/punctuation/range.tsv b/nemo_text_processing/text_normalization/ja/data/punctuation/range.tsv new file mode 100644 index 000000000..f83d02ca2 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/punctuation/range.tsv @@ -0,0 +1 @@ +〜 から diff --git a/nemo_text_processing/text_normalization/ja/data/range/__init__.py b/nemo_text_processing/text_normalization/ja/data/range/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/range/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/range/operator.tsv b/nemo_text_processing/text_normalization/ja/data/range/operator.tsv new file mode 100644 index 000000000..d412189b7 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/range/operator.tsv @@ -0,0 +1,2 @@ +x かける +× かける diff --git a/nemo_text_processing/text_normalization/ja/data/range/separator.tsv b/nemo_text_processing/text_normalization/ja/data/range/separator.tsv new file mode 100644 index 000000000..e774bb177 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/range/separator.tsv @@ -0,0 +1,2 @@ +- から +〜 から diff --git a/nemo_text_processing/text_normalization/ja/data/range/suffix.tsv b/nemo_text_processing/text_normalization/ja/data/range/suffix.tsv new file mode 100644 index 000000000..fd2a91bc5 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/range/suffix.tsv @@ -0,0 +1,2 @@ +人 +歳 diff --git a/nemo_text_processing/text_normalization/ja/data/range/year_suffix.tsv b/nemo_text_processing/text_normalization/ja/data/range/year_suffix.tsv new file mode 100644 index 000000000..76d279961 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/range/year_suffix.tsv @@ -0,0 +1 @@ +s 年代 diff --git a/nemo_text_processing/text_normalization/ja/data/roman/__init__.py b/nemo_text_processing/text_normalization/ja/data/roman/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/roman/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/roman/japanese_prefix.tsv b/nemo_text_processing/text_normalization/ja/data/roman/japanese_prefix.tsv new file mode 100644 index 000000000..4c34f1c09 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/roman/japanese_prefix.tsv @@ -0,0 +1 @@ +第 diff --git a/nemo_text_processing/text_normalization/ja/data/roman/japanese_suffix.tsv b/nemo_text_processing/text_normalization/ja/data/roman/japanese_suffix.tsv new file mode 100644 index 000000000..fa74f5ad8 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/roman/japanese_suffix.tsv @@ -0,0 +1,4 @@ +章 +条 +巻 +回 diff --git a/nemo_text_processing/text_normalization/ja/data/roman/key_cardinal.tsv b/nemo_text_processing/text_normalization/ja/data/roman/key_cardinal.tsv new file mode 100644 index 000000000..6b8f72d6d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/roman/key_cardinal.tsv @@ -0,0 +1,14 @@ +Chapter +chapter +Part +part +Century +century +Class +class +Article +article +Section +section +Paragraph +paragraph diff --git a/nemo_text_processing/text_normalization/ja/data/roman/roman_numerals.tsv b/nemo_text_processing/text_normalization/ja/data/roman/roman_numerals.tsv new file mode 100644 index 000000000..f443fd3a9 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/roman/roman_numerals.tsv @@ -0,0 +1,13 @@ +I 1 +V 5 +X 10 +L 50 +C 100 +D 500 +M 1000 +IV 4 +IX 9 +XL 40 +XC 90 +CD 400 +CM 900 diff --git a/nemo_text_processing/text_normalization/ja/data/serial/__init__.py b/nemo_text_processing/text_normalization/ja/data/serial/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/serial/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/serial/delimiter.tsv b/nemo_text_processing/text_normalization/ja/data/serial/delimiter.tsv new file mode 100644 index 000000000..4a6fc82ef --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/serial/delimiter.tsv @@ -0,0 +1,2 @@ +- ハイフン +/ スラッシュ diff --git a/nemo_text_processing/text_normalization/ja/data/serial/model_cues.tsv b/nemo_text_processing/text_normalization/ja/data/serial/model_cues.tsv new file mode 100644 index 000000000..c3d4f5e18 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/serial/model_cues.tsv @@ -0,0 +1 @@ +型番 diff --git a/nemo_text_processing/text_normalization/ja/data/serial/words.tsv b/nemo_text_processing/text_normalization/ja/data/serial/words.tsv new file mode 100644 index 000000000..87725d259 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/serial/words.tsv @@ -0,0 +1,3 @@ +covid コビッド +COVID コビッド +Covid コビッド diff --git a/nemo_text_processing/text_normalization/ja/data/symbol.tsv b/nemo_text_processing/text_normalization/ja/data/symbol.tsv index 67ad785f0..c1dcfe15e 100644 --- a/nemo_text_processing/text_normalization/ja/data/symbol.tsv +++ b/nemo_text_processing/text_normalization/ja/data/symbol.tsv @@ -1,23 +1,23 @@ -& アンド -# ハッシュタグ -@ アット -§ セクション -™ トレードマーク -® 登録商標マーク -© 著作権 -_ アンダースコア -% パーセント -* 星印 -+ プラス -/ スラッシュ -= エコール -^ 曲折アクセント記号 -| 縦棒 -~ ティルダ -$ ドール -£ ポンド -€ ユーロ -₩ ウォン -¥ 円 -° 度 -º 度 \ No newline at end of file +& アンド +# ハッシュタグ +@ アット +§ セクション +™ トレードマーク +® 登録商標マーク +© 著作権 +_ アンダースコア +% パーセント +* 星印 ++ プラス +/ スラッシュ += エコール +^ 曲折アクセント記号 +| 縦棒 +~ ティルダ +$ ドール +£ ポンド +€ ユーロ +₩ ウォン +¥ 円 +° 度 +º 度 diff --git a/nemo_text_processing/text_normalization/ja/data/telephone/__init__.py b/nemo_text_processing/text_normalization/ja/data/telephone/__init__.py new file mode 100644 index 000000000..9e3fb699d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/telephone/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nemo_text_processing/text_normalization/ja/data/telephone/country_code_prefix.tsv b/nemo_text_processing/text_normalization/ja/data/telephone/country_code_prefix.tsv new file mode 100644 index 000000000..209f307d4 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/telephone/country_code_prefix.tsv @@ -0,0 +1 @@ +プラス diff --git a/nemo_text_processing/text_normalization/ja/data/telephone/extension.tsv b/nemo_text_processing/text_normalization/ja/data/telephone/extension.tsv new file mode 100644 index 000000000..6f19d8b99 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/telephone/extension.tsv @@ -0,0 +1,2 @@ +内線番号 内線 +内線 内線 diff --git a/nemo_text_processing/text_normalization/ja/data/telephone/group_separator.tsv b/nemo_text_processing/text_normalization/ja/data/telephone/group_separator.tsv new file mode 100644 index 000000000..f43a2f5e6 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/telephone/group_separator.tsv @@ -0,0 +1 @@ +、 diff --git a/nemo_text_processing/text_normalization/ja/data/time/division.tsv b/nemo_text_processing/text_normalization/ja/data/time/division.tsv index f6beedce8..59b40ce7c 100644 --- a/nemo_text_processing/text_normalization/ja/data/time/division.tsv +++ b/nemo_text_processing/text_normalization/ja/data/time/division.tsv @@ -20,4 +20,4 @@ 翌日 未明 正午 -真夜中の \ No newline at end of file +真夜中の diff --git a/nemo_text_processing/text_normalization/ja/data/time/hour.tsv b/nemo_text_processing/text_normalization/ja/data/time/hour.tsv index 1d5c08425..14ef0cf3c 100644 --- a/nemo_text_processing/text_normalization/ja/data/time/hour.tsv +++ b/nemo_text_processing/text_normalization/ja/data/time/hour.tsv @@ -21,4 +21,4 @@ 21 二十一 22 二十二 23 二十三 -24 二十四 \ No newline at end of file +24 二十四 diff --git a/nemo_text_processing/text_normalization/ja/data/time/minute.tsv b/nemo_text_processing/text_normalization/ja/data/time/minute_second.tsv similarity index 98% rename from nemo_text_processing/text_normalization/ja/data/time/minute.tsv rename to nemo_text_processing/text_normalization/ja/data/time/minute_second.tsv index 5e8276a00..79ac794c4 100644 --- a/nemo_text_processing/text_normalization/ja/data/time/minute.tsv +++ b/nemo_text_processing/text_normalization/ja/data/time/minute_second.tsv @@ -57,4 +57,4 @@ 57 五十七 58 五十八 59 五十九 -60 六十 \ No newline at end of file +60 六十 diff --git a/nemo_text_processing/text_normalization/ja/data/time/second.tsv b/nemo_text_processing/text_normalization/ja/data/time/second.tsv deleted file mode 100644 index 5e8276a00..000000000 --- a/nemo_text_processing/text_normalization/ja/data/time/second.tsv +++ /dev/null @@ -1,60 +0,0 @@ -1 一 -2 二 -3 三 -4 四 -5 五 -6 六 -7 七 -8 八 -9 九 -10 十 -11 十一 -12 十二 -13 十三 -14 十四 -15 十五 -16 十六 -17 十七 -18 十八 -19 十九 -20 二十 -21 二十一 -22 二十二 -23 二十三 -24 二十四 -25 二十五 -26 二十六 -27 二十七 -28 二十八 -29 二十九 -30 三十 -31 三十一 -32 三十二 -33 三十三 -34 三十四 -35 三十五 -36 三十六 -37 三十七 -38 三十八 -39 三十九 -40 四十 -41 四十一 -42 四十二 -43 四十三 -44 四十四 -45 四十五 -46 四十六 -47 四十七 -48 四十八 -49 四十九 -50 五十 -51 五十一 -52 五十二 -53 五十三 -54 五十四 -55 五十五 -56 五十六 -57 五十七 -58 五十八 -59 五十九 -60 六十 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/ja/data/time/suffix.tsv b/nemo_text_processing/text_normalization/ja/data/time/suffix.tsv new file mode 100644 index 000000000..71bc03f55 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/time/suffix.tsv @@ -0,0 +1,8 @@ +hour 時 +duration_hour 時間 +approximate_hour 時頃 +minute 分 +past 過ぎ +approximate 頃 +half 半 +second 秒 diff --git a/nemo_text_processing/text_normalization/ja/data/whitelist.tsv b/nemo_text_processing/text_normalization/ja/data/whitelist.tsv index d0d7bef70..203f6396d 100644 --- a/nemo_text_processing/text_normalization/ja/data/whitelist.tsv +++ b/nemo_text_processing/text_normalization/ja/data/whitelist.tsv @@ -1,17 +1,7 @@ -Dr. ドクター -dr. ドクター -Mr. ミスター -mr. ミスター -Ms. ミス -ms. ミス -Mrs. ミシーズ -mrs. ミシーず -st. ストリード -St. ストリード +st. ストリート +St. ストリート mt. マウント Mt. マウント -Prof. プロフェッサー -prof. プロフェッサー sr. シニア Sr. シニア jr. ジュニア diff --git a/nemo_text_processing/text_normalization/ja/data/whitelist_title.tsv b/nemo_text_processing/text_normalization/ja/data/whitelist_title.tsv new file mode 100644 index 000000000..649cabe0d --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/data/whitelist_title.tsv @@ -0,0 +1,10 @@ +Dr. ドクター +dr. ドクター +Mr. ミスター +mr. ミスター +Mrs. ミセス +mrs. ミセス +Ms. ミス +ms. ミス +Prof. プロフェッサー +prof. プロフェッサー diff --git a/nemo_text_processing/text_normalization/ja/taggers/address.py b/nemo_text_processing/text_normalization/ja/taggers/address.py new file mode 100644 index 000000000..32a18dcd1 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/address.py @@ -0,0 +1,64 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_DIGIT, NEMO_NOT_SPACE, GraphFst +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class AddressFst(GraphFst): + """ + Finite state transducer for classifying Japanese address-like expressions. + + Examples: + 東京都千代田区丸の内1-1-1 -> name: "東京都千代田区丸の内一の一の一" + 503号室 -> name: "五〇三号室" + 〒100-0001 -> name: "郵便番号一ゼロゼロのゼロゼロゼロ一" + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="address", kind="classify", deterministic=deterministic) + + digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + zero_maru = pynini.string_file(get_abs_path("data/numbers/zero_maru.tsv")) + + address_number = cardinal.just_cardinals + digit_for_room = digit | zero_maru + digit_for_postal = digit | zero + separator = pynini.string_file(get_abs_path("data/address/separator.tsv")) + separator_input = pynini.project(separator, "input") + postal = pynini.string_file(get_abs_path("data/address/postal.tsv")) + room_suffix = pynini.string_file(get_abs_path("data/address/room_suffix.tsv")) + + hyphen_to_no = ( + pynini.closure(pynutil.delete(" "), 0, 1) + separator + pynini.closure(pynutil.delete(" "), 0, 1) + ) + + address_chain = address_number + hyphen_to_no + address_number + hyphen_to_no + address_number + + address_prefix_char = pynini.difference( + NEMO_NOT_SPACE, + NEMO_DIGIT | separator_input, + ) + address_with_prefix = pynini.closure(address_prefix_char, 1) + address_chain + + postal_code = postal + digit_for_postal**3 + hyphen_to_no + digit_for_postal**4 + + room = (NEMO_DIGIT**3 @ (digit_for_room**3)) + room_suffix + + graph = address_with_prefix | postal_code | room + self.fst = (pynutil.insert('name: "') + graph + pynutil.insert('"')).optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/cardinal.py b/nemo_text_processing/text_normalization/ja/taggers/cardinal.py index ff80f6a3b..86e7a22ff 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/cardinal.py +++ b/nemo_text_processing/text_normalization/ja/taggers/cardinal.py @@ -17,7 +17,7 @@ from pynini.lib import pynutil from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_DIGIT, GraphFst -from nemo_text_processing.text_normalization.ja.utils import get_abs_path +from nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels class CardinalFst(GraphFst): @@ -38,35 +38,45 @@ def __init__(self, deterministic: bool = True): graph_digit_alt = no_zero_and_one @ graph_digit graph_ties = pynini.string_file(get_abs_path("data/numbers/ties.tsv")) graph_teen = pynini.string_file(get_abs_path("data/numbers/teen.tsv")) + magnitudes = dict(load_labels(get_abs_path("data/numbers/magnitude.tsv"))) + hundred = magnitudes["hundred"] + thousand = magnitudes["thousand"] + ten_thousand = magnitudes["ten_thousand"] + hundred_million = magnitudes["hundred_million"] - graph_all = (graph_ties + (graph_digit | pynutil.delete('0'))) | graph_teen | graph_digit + graph_all = (graph_ties + (graph_digit | pynutil.delete("0"))) | graph_teen | graph_digit hundreds = NEMO_DIGIT**3 - graph_hundred_component = (pynini.cross('1', '百') | (graph_digit_alt + pynutil.insert('百'))) + pynini.union( - pynini.closure(pynutil.delete('0')), (pynini.closure(pynutil.delete('0')) + graph_all) + graph_hundred_component = ( + pynini.cross("1", hundred) | (graph_digit_alt + pynutil.insert(hundred)) + ) + pynini.union( + pynini.closure(pynutil.delete("0")), + (pynini.closure(pynutil.delete("0")) + graph_all), ) graph_hundred = hundreds @ graph_hundred_component thousands = NEMO_DIGIT**4 - graph_thousand_component = (pynini.cross('1', '千') | (graph_digit_alt + pynutil.insert('千'))) + pynini.union( - pynini.closure(pynutil.delete('0')), + graph_thousand_component = ( + pynini.cross("1", thousand) | (graph_digit_alt + pynutil.insert(thousand)) + ) + pynini.union( + pynini.closure(pynutil.delete("0")), graph_hundred_component, - (pynini.closure(pynutil.delete('0')) + graph_all), + (pynini.closure(pynutil.delete("0")) + graph_all), ) - graph_thousand_component_alt = (graph_digit + pynutil.insert('千')) + pynini.union( - pynini.closure(pynutil.delete('0')), + graph_thousand_component_alt = (graph_digit + pynutil.insert(thousand)) + pynini.union( + pynini.closure(pynutil.delete("0")), graph_hundred_component, - (pynini.closure(pynutil.delete('0')) + graph_all), + (pynini.closure(pynutil.delete("0")) + graph_all), ) # this grammar is for larger number in later gramamr graph_thousand = thousands @ graph_thousand_component ten_thousands = NEMO_DIGIT**5 - graph_ten_thousand_component = (graph_digit + pynutil.insert('万')) + pynini.union( - pynini.closure(pynutil.delete('0')), + graph_ten_thousand_component = (graph_digit + pynutil.insert(ten_thousand)) + pynini.union( + pynini.closure(pynutil.delete("0")), graph_thousand_component, - (pynutil.delete('0') + graph_hundred_component), - (pynini.closure(pynutil.delete('0')) + graph_all), + (pynutil.delete("0") + graph_hundred_component), + (pynini.closure(pynutil.delete("0")) + graph_all), ) graph_ten_thousand = ten_thousands @ graph_ten_thousand_component self.man = graph_ten_thousand.optimize() @@ -74,91 +84,95 @@ def __init__(self, deterministic: bool = True): hundred_thousands = NEMO_DIGIT**6 hundred_thousands_position = NEMO_DIGIT**2 hundred_thousands_position = hundred_thousands_position @ graph_all - graph_hundred_thousand_component = (hundred_thousands_position + pynutil.insert('万')) + pynini.union( - pynini.closure(pynutil.delete('0')), + graph_hundred_thousand_component = (hundred_thousands_position + pynutil.insert(ten_thousand)) + pynini.union( + pynini.closure(pynutil.delete("0")), graph_thousand_component, - (pynutil.delete('0') + graph_hundred_component), - (pynini.closure(pynutil.delete('0')) + graph_all), + (pynutil.delete("0") + graph_hundred_component), + (pynini.closure(pynutil.delete("0")) + graph_all), ) graph_hundred_thousand = hundred_thousands @ graph_hundred_thousand_component millions = NEMO_DIGIT**7 million_position = NEMO_DIGIT**3 million_position = million_position @ graph_hundred_component - graph_million_component = (million_position + pynutil.insert('万')) + pynini.union( - pynini.closure(pynutil.delete('0')), + graph_million_component = (million_position + pynutil.insert(ten_thousand)) + pynini.union( + pynini.closure(pynutil.delete("0")), graph_thousand_component, - (pynutil.delete('0') + graph_hundred_component), - (pynini.closure(pynutil.delete('0')) + graph_all), + (pynutil.delete("0") + graph_hundred_component), + (pynini.closure(pynutil.delete("0")) + graph_all), ) graph_million = millions @ graph_million_component ten_millions = NEMO_DIGIT**8 ten_million_position = NEMO_DIGIT**4 ten_million_position = ten_million_position @ graph_thousand_component_alt - graph_ten_million_component = (ten_million_position + pynutil.insert('万')) + pynini.union( - pynini.closure(pynutil.delete('0')), + graph_ten_million_component = (ten_million_position + pynutil.insert(ten_thousand)) + pynini.union( + pynini.closure(pynutil.delete("0")), graph_thousand_component, - (pynutil.delete('0') + graph_hundred_component), - (pynini.closure(pynutil.delete('0')) + graph_all), + (pynutil.delete("0") + graph_hundred_component), + (pynini.closure(pynutil.delete("0")) + graph_all), ) graph_ten_million = ten_millions @ graph_ten_million_component hundred_millions = NEMO_DIGIT**9 - graph_hundred_million_component = (graph_digit + pynutil.insert('億')) + pynini.union( - pynini.closure(pynutil.delete('0')), + graph_hundred_million_component = (graph_digit + pynutil.insert(hundred_million)) + pynini.union( + pynini.closure(pynutil.delete("0")), graph_ten_million_component, - (pynutil.delete('0') + graph_million_component), - (pynutil.delete('00') + graph_hundred_thousand_component), - (pynutil.delete('000') + graph_ten_thousand_component), - (pynutil.delete('0000') + graph_thousand_component), - ((pynutil.delete('00000') + graph_hundred_component)), - (pynini.closure(pynutil.delete('0')) + graph_all), + (pynutil.delete("0") + graph_million_component), + (pynutil.delete("00") + graph_hundred_thousand_component), + (pynutil.delete("000") + graph_ten_thousand_component), + (pynutil.delete("0000") + graph_thousand_component), + ((pynutil.delete("00000") + graph_hundred_component)), + (pynini.closure(pynutil.delete("0")) + graph_all), ) graph_hundred_million = hundred_millions @ graph_hundred_million_component thousand_millions = NEMO_DIGIT**10 thousand_millions_position = NEMO_DIGIT**2 thousand_millions_position = thousand_millions_position @ graph_all - graph_thousand_million_component = (thousand_millions_position + pynutil.insert('億')) + pynini.union( - pynini.closure(pynutil.delete('0')), + graph_thousand_million_component = ( + thousand_millions_position + pynutil.insert(hundred_million) + ) + pynini.union( + pynini.closure(pynutil.delete("0")), graph_ten_million_component, - (pynutil.delete('0') + graph_million_component), - (pynutil.delete('00') + graph_hundred_thousand_component), - (pynutil.delete('000') + graph_ten_thousand_component), - (pynutil.delete('0000') + graph_thousand_component), - ((pynutil.delete('00000') + graph_hundred_component)), - (pynini.closure(pynutil.delete('0')) + graph_all), + (pynutil.delete("0") + graph_million_component), + (pynutil.delete("00") + graph_hundred_thousand_component), + (pynutil.delete("000") + graph_ten_thousand_component), + (pynutil.delete("0000") + graph_thousand_component), + ((pynutil.delete("00000") + graph_hundred_component)), + (pynini.closure(pynutil.delete("0")) + graph_all), ) graph_thousand_million = thousand_millions @ graph_thousand_million_component ten_billions = NEMO_DIGIT**11 ten_billions_position = NEMO_DIGIT**3 ten_billions_position = ten_billions_position @ graph_hundred_component - graph_ten_billions_component = (ten_billions_position + pynutil.insert('億')) + pynini.union( - pynini.closure(pynutil.delete('0')), + graph_ten_billions_component = (ten_billions_position + pynutil.insert(hundred_million)) + pynini.union( + pynini.closure(pynutil.delete("0")), graph_ten_million_component, - (pynutil.delete('0') + graph_million_component), - (pynutil.delete('00') + graph_hundred_thousand_component), - (pynutil.delete('000') + graph_ten_thousand_component), - (pynutil.delete('0000') + graph_thousand_component), - ((pynutil.delete('00000') + graph_hundred_component)), - (pynini.closure(pynutil.delete('0')) + graph_all), + (pynutil.delete("0") + graph_million_component), + (pynutil.delete("00") + graph_hundred_thousand_component), + (pynutil.delete("000") + graph_ten_thousand_component), + (pynutil.delete("0000") + graph_thousand_component), + ((pynutil.delete("00000") + graph_hundred_component)), + (pynini.closure(pynutil.delete("0")) + graph_all), ) graph_ten_billions = ten_billions @ graph_ten_billions_component hundred_billions = NEMO_DIGIT**12 hundred_billions_position = NEMO_DIGIT**4 hundred_billions_position = hundred_billions_position @ graph_thousand_component_alt - graph_hundred_billions_component = (hundred_billions_position + pynutil.insert('億')) + pynini.union( - pynini.closure(pynutil.delete('0')), + graph_hundred_billions_component = ( + hundred_billions_position + pynutil.insert(hundred_million) + ) + pynini.union( + pynini.closure(pynutil.delete("0")), graph_ten_million_component, - (pynutil.delete('0') + graph_million_component), - (pynutil.delete('00') + graph_hundred_thousand_component), - (pynutil.delete('000') + graph_ten_thousand_component), - (pynutil.delete('0000') + graph_thousand_component), - ((pynutil.delete('00000') + graph_hundred_component)), - (pynini.closure(pynutil.delete('0')) + graph_all), + (pynutil.delete("0") + graph_million_component), + (pynutil.delete("00") + graph_hundred_thousand_component), + (pynutil.delete("000") + graph_ten_thousand_component), + (pynutil.delete("0000") + graph_thousand_component), + ((pynutil.delete("00000") + graph_hundred_component)), + (pynini.closure(pynutil.delete("0")) + graph_all), ) graph_hundred_billions = hundred_billions @ graph_hundred_billions_component @@ -179,12 +193,14 @@ def __init__(self, deterministic: bool = True): self.just_cardinals = graph.optimize() optional_sign = ( - pynutil.insert("negative: \"") + (pynini.accep("-") | pynini.cross("マイナス", "-")) + pynutil.insert("\"") + pynutil.insert('negative: "') + + pynini.string_file(get_abs_path("data/numbers/sign.tsv")) + + pynutil.insert('"') ) final_graph = ( - optional_sign + pynutil.insert(" ") + pynutil.insert("integer: \"") + graph + pynutil.insert("\"") - ) | (pynutil.insert("integer: \"") + graph + pynutil.insert("\"")) + optional_sign + pynutil.insert(" ") + pynutil.insert('integer: "') + graph + pynutil.insert('"') + ) | (pynutil.insert('integer: "') + graph + pynutil.insert('"')) final_graph = self.add_tokens(final_graph) diff --git a/nemo_text_processing/text_normalization/ja/taggers/date.py b/nemo_text_processing/text_normalization/ja/taggers/date.py index a8a469252..26c5f742c 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/date.py +++ b/nemo_text_processing/text_normalization/ja/taggers/date.py @@ -21,7 +21,7 @@ NEMO_NON_BREAKING_SPACE, GraphFst, ) -from nemo_text_processing.text_normalization.ja.utils import get_abs_path +from nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels class DateFst(GraphFst): @@ -70,17 +70,29 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): week = pynini.string_file(get_abs_path("data/date/week.tsv")) era = pynini.string_file(get_abs_path("data/date/era.tsv")) era_abbrev = pynini.string_file(get_abs_path("data/date/era_abbrev.tsv")) + suffixes = dict(load_labels(get_abs_path("data/date/suffix.tsv"))) + year_suffix = suffixes["year"] + month_suffix = suffixes["month"] + day_suffix = suffixes["day"] + century_suffix = suffixes["century"] + decade_suffix = suffixes["decade"] + ten_day_period = pynini.union( + suffixes["early_ten_days"], + suffixes["middle_ten_days"], + suffixes["late_ten_days"], + ) + range_separator = pynini.string_file(get_abs_path("data/punctuation/range.tsv")) signs = pynutil.delete("/") | pynutil.delete(".") | pynutil.delete("-") delete_spaces = pynini.closure( pynutil.delete(" ") | pynutil.delete(NEMO_NARROW_NON_BREAK_SPACE) | pynutil.delete(NEMO_NON_BREAKING_SPACE) ) - era_component = pynutil.insert("era: \"") + era + pynutil.insert("\"") - era_abbrev_component = pynutil.insert("era: \"") + era_abbrev + pynutil.insert("\"") - year_component = pynutil.insert("year: \"") + graph_cardinal + pynutil.insert("年") + pynutil.insert("\"") - month_component = pynutil.insert("month: \"") + month + pynutil.insert("月") + pynutil.insert("\"") - day_component = pynutil.insert("day: \"") + day + pynutil.insert("日") + pynutil.insert("\"") + era_component = pynutil.insert('era: "') + era + pynutil.insert('"') + era_abbrev_component = pynutil.insert('era: "') + era_abbrev + pynutil.insert('"') + year_component = pynutil.insert('year: "') + graph_cardinal + pynutil.insert(year_suffix) + pynutil.insert('"') + month_component = pynutil.insert('month: "') + month + pynutil.insert(month_suffix) + pynutil.insert('"') + day_component = pynutil.insert('day: "') + day + pynutil.insert(day_suffix) + pynutil.insert('"') front_bracket = ( ( @@ -119,24 +131,24 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): # this graph optionally accepts () around weekday to accomodate to inputs like (月〜金), thus being longer week_component = ( - (front_bracket + pynutil.insert("weekday: \"") + week + preceding_bracket + pynutil.insert("\"")) + (front_bracket + pynutil.insert('weekday: "') + week + preceding_bracket + pynutil.insert('"')) | ( front_bracket - + pynutil.insert("weekday: \"") + + pynutil.insert('weekday: "') + week - + pynini.cross("〜", "から") + + range_separator + week + preceding_bracket - + pynutil.insert("\"") + + pynutil.insert('"') ) | ( front_bracket - + pynutil.insert("weekday: \"") + + pynutil.insert('weekday: "') + week + pynutil.delete("・") + week + preceding_bracket - + pynutil.insert("\"") + + pynutil.insert('"') ) ) @@ -154,37 +166,33 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): # 2024年, 9月, 28日 individual_year_component = ( pynini.closure(era_component + pynutil.insert(" "), 0, 1) - + pynutil.insert("year: \"") + + pynutil.insert('year: "') + graph_cardinal - + pynini.accep("年") - + pynutil.insert("\"") + + pynini.accep(year_suffix) + + pynutil.insert('"') ) # this extra individual year component is to accomodate inputs R. 2024 with out "年" # the inputs may or maynot include "年", thus below: individual_year_component_2 = ( pynini.closure(era_component + pynutil.insert(" "), 0, 1) - + pynutil.insert("year: \"") + + pynutil.insert('year: "') + graph_cardinal - + (pynini.accep("世紀") | pynini.accep("")) - + pynutil.insert("\"") + + (pynini.accep(century_suffix) | pynini.accep("")) + + pynutil.insert('"') ) | ( era_abbrev_component + pynutil.insert(" ") - + pynutil.insert("year: \"") + + pynutil.insert('year: "') + graph_cardinal - + pynutil.insert("年") - + pynutil.insert("\"") + + pynutil.insert(year_suffix) + + pynutil.insert('"') ) individual_month_component = ( - pynutil.insert("month: \"") + month + pynini.accep("月") + pynutil.insert("\"") - ) | ( - pynutil.insert("month: \"") - + (pynini.accep("中旬") | pynini.accep("下旬") | pynini.accep("上旬")) - + pynutil.insert("\"") - ) + pynutil.insert('month: "') + month + pynini.accep(month_suffix) + pynutil.insert('"') + ) | (pynutil.insert('month: "') + ten_day_period + pynutil.insert('"')) individual_day_component = ( - pynutil.insert("day: \"") + graph_cardinal + pynini.accep("日") + pynutil.insert("\"") + pynutil.insert('day: "') + graph_cardinal + pynini.accep(day_suffix) + pynutil.insert('"') ) graph_individual_component = ( @@ -208,13 +216,13 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): ) ) + pynini.closure(pynutil.insert(" ") + week_component, 0, 1) - nendai = pynini.accep("年代") + nendai = pynini.accep(decade_suffix) era_nendai = ( pynini.closure(era_component + pynutil.insert(" "), 0, 1) - + pynutil.insert("year: \"") + + pynutil.insert('year: "') + graph_cardinal + nendai - + pynutil.insert("\"") + + pynutil.insert('"') ) graph_all_date = ( diff --git a/nemo_text_processing/text_normalization/ja/taggers/decimal.py b/nemo_text_processing/text_normalization/ja/taggers/decimal.py index 8fdea4c87..228e59e1c 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/decimal.py +++ b/nemo_text_processing/text_normalization/ja/taggers/decimal.py @@ -16,7 +16,7 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_DIGIT, GraphFst from nemo_text_processing.text_normalization.ja.utils import get_abs_path @@ -24,6 +24,7 @@ class DecimalFst(GraphFst): """ Finite state transducer for classifying decimal, e.g. 0.5 -> decimal { integer_part: "零" fractional_part: "五" } + 0.05 -> decimal { integer_part: "零" fractional_part: "零五" } -0.5万 -> decimal { negative: "マイナス" integer_part: "零" fractional_part: "五" quantity: "万"} Args: @@ -35,22 +36,31 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): cardinal_before_decimal = cardinal.just_cardinals cardinal_after_decimal = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) - zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + zero_decimal = pynini.string_file(get_abs_path("data/numbers/zero_decimal.tsv")) + decimal_point = pynini.string_file(get_abs_path("data/numbers/decimal_point.tsv")) + decimal_separator = pynini.project(decimal_point, "input") + sign = pynini.string_file(get_abs_path("data/numbers/sign.tsv")) - graph_integer = pynutil.insert('integer_part: \"') + cardinal_before_decimal + pynutil.insert("\"") + graph_integer = ( + pynutil.insert('integer_part: "') + + ( + zero_decimal + | (pynini.difference(NEMO_DIGIT, "0") @ cardinal_before_decimal) + | (pynini.closure(NEMO_DIGIT, 2) @ cardinal_before_decimal) + ) + + pynutil.insert('"') + ) graph_fraction = ( - pynutil.insert("fractional_part: \"") - + pynini.closure((cardinal_after_decimal | zero), 1) - + pynutil.insert("\"") + pynutil.insert('fractional_part: "') + + pynini.closure((cardinal_after_decimal | zero_decimal), 1) + + pynutil.insert('"') ) - graph_decimal_no_sign = graph_integer + pynutil.delete('.') + pynutil.insert(" ") + graph_fraction - - graph_optional_sign = ( - pynutil.insert("negative: \"") - + (pynini.cross("-", "マイナス") | pynini.accep("マイナス")) - + pynutil.insert("\"") + graph_decimal_no_sign = ( + graph_integer + pynutil.delete(decimal_separator) + pynutil.insert(" ") + graph_fraction ) + graph_optional_sign = pynutil.insert('negative: "') + sign + pynutil.insert('"') + graph_decimal = graph_decimal_no_sign | (graph_optional_sign + pynutil.insert(" ") + graph_decimal_no_sign) self.just_decimal = graph_decimal_no_sign.optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/electronic.py b/nemo_text_processing/text_normalization/ja/taggers/electronic.py new file mode 100644 index 000000000..2c61ad6e3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/electronic.py @@ -0,0 +1,129 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_ALPHA, NEMO_DIGIT, NEMO_NOT_SPACE, GraphFst +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class ElectronicFst(GraphFst): + """ + Finite state transducer for classifying Japanese electronic expressions. + + Examples: + abc@abc.com -> electronic { username: "abc" domain: "abc.com" preserve_order: true } + https://www.nvidia.com + -> electronic { protocol: "https" domain: "www.nvidia.com" preserve_order: true } + 1234-5678-9012-3456 + -> electronic { domain: "1234 5678 9012 3456" preserve_order: true } + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="electronic", kind="classify", deterministic=deterministic) + + alnum = NEMO_ALPHA | NEMO_DIGIT + hyphen = pynini.accep("-") + dot = pynini.accep(".") + slash = pynini.accep("/") + at = pynini.accep("@") + + label = pynini.closure(alnum | hyphen, 1) + tld = pynini.closure(NEMO_ALPHA, 2) + domain_core = label + pynini.closure(dot + label) + dot + tld + domain_field = pynutil.insert('domain: "') + domain_core + pynutil.insert('"') + + username_symbol = dot | hyphen + username_core = alnum + pynini.closure(alnum | username_symbol) + username_field = ( + pynutil.insert('username: "') + + username_core + + pynutil.insert('"') + + pynutil.delete("@") + + pynutil.insert(" ") + ) + email = username_field + domain_field + + protocol = pynini.string_file(get_abs_path("data/electronic/protocol.tsv")) + protocol_field = pynutil.insert('protocol: "') + protocol + pynutil.insert('"') + path_segment = pynini.closure(alnum | hyphen, 1) + path_core = slash + path_segment + pynini.closure(slash + path_segment) + path_field = pynutil.insert(' path: "') + path_core + pynutil.insert('"') + url = ( + protocol_field + + pynutil.delete("://") + + pynutil.insert(" ") + + domain_field + + pynini.closure(path_field, 0, 1) + ) + + four_digits = NEMO_DIGIT**4 + card_separator = pynutil.delete("-") | pynutil.delete(" ") + grouped_card_number = ( + four_digits + + card_separator + + pynutil.insert(" ") + + four_digits + + card_separator + + pynutil.insert(" ") + + four_digits + + card_separator + + pynutil.insert(" ") + + four_digits + ) + card_number_field = pynutil.insert('domain: "') + grouped_card_number + pynutil.insert('"') + short_card_number_field = pynutil.insert('domain: "') + four_digits + pynutil.insert('"') + + card_cue = pynini.string_file(get_abs_path("data/electronic/card_cues.tsv")) + card_cue_field = pynutil.insert('protocol: "') + card_cue + pynutil.insert('" ') + credit_card = card_number_field + card_with_cue = card_cue_field + card_number_field + card_tail_with_cue = card_cue_field + short_card_number_field + + digit_count_prefix = pynini.string_file(get_abs_path("data/electronic/card_digit_count_prefix.tsv")) + digit_count_suffix = pynini.string_file(get_abs_path("data/electronic/card_digit_count_suffix.tsv")) + digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + card_tail = ( + pynutil.insert('protocol: "') + + digit_count_prefix + + digit + + digit_count_suffix + + pynutil.insert('" ') + + short_card_number_field + ) + + extension = pynini.project( + pynini.string_file(get_abs_path("data/electronic/file_extensions.tsv")), + "input", + ) + filename_stem = pynini.closure( + pynini.difference(NEMO_NOT_SPACE, pynini.union(dot, slash, at)), + 1, + ) + filename = pynutil.insert('domain: "') + filename_stem + extension + pynutil.insert('"') + + graph = ( + pynutil.add_weight(credit_card, -0.1) + | pynutil.add_weight(card_with_cue, -0.1) + | pynutil.add_weight(card_tail_with_cue, -0.1) + | card_tail + | email + | url + | domain_field + | filename + ) + graph += pynutil.insert(" preserve_order: true") + + self.fst = self.add_tokens(graph.optimize()).optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/fraction.py b/nemo_text_processing/text_normalization/ja/taggers/fraction.py index 94fb4af68..a0433f97e 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/fraction.py +++ b/nemo_text_processing/text_normalization/ja/taggers/fraction.py @@ -17,7 +17,7 @@ from pynini.lib import pynutil from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_SPACE, GraphFst -from nemo_text_processing.text_normalization.ja.utils import get_abs_path +from nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels class FractionFst(GraphFst): @@ -25,7 +25,7 @@ class FractionFst(GraphFst): Finite state transducer for classifying fractions, e.g. 1/2 -> tokens { fraction { denominator: "二" numerator: "一"} } 1と3/4 -> fraction { integer: "一" denominator: "四" numerator: "三" } - 一荷四分の三 -> fraction { integer: "1" denominator: "4" numerator: "3" } + 一と四分の三 -> fraction { integer: "1" denominator: "4" numerator: "3" } ルート三分の一 -> fraction { denominator: "√3" numerator: "1" } 一点六五分の五十 -> fraction { denominator: "1.65" numerator: "50" } マイナス1/2 -> tokens { fraction { denominator: "二" numerator: "一"} } @@ -40,40 +40,39 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): cardinal = cardinal.just_cardinals graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + markers = dict(load_labels(get_abs_path("data/fraction/marker.tsv"))) + decimal_point = pynini.string_file(get_abs_path("data/numbers/decimal_point.tsv")) + sign = pynini.string_file(get_abs_path("data/numbers/sign.tsv")) - slash = pynutil.delete('/') - morphemes = pynini.accep('分の') - root = pynini.accep('√') + slash = pynutil.delete("/") + morphemes = pynini.accep(markers["fraction"]) + root = pynini.accep(markers["root_written"]) + mixed = pynini.accep(markers["mixed"]) decimal_number = ( - cardinal - + pynini.cross(".", "点") - + pynini.closure(pynini.closure(graph_digit) | pynini.closure(graph_zero)) + cardinal + decimal_point + pynini.closure(pynini.closure(graph_digit) | pynini.closure(graph_zero)) ) integer_component = ( - pynutil.insert('integer_part: \"') + pynutil.insert('integer_part: "') + (cardinal | (root + cardinal) | decimal_number | (root + decimal_number)) - + pynutil.insert("\"") + + pynutil.insert('"') ) integer_component_with_char = ( - pynutil.insert('integer_part: \"') - + ( - (cardinal | (root + cardinal) | decimal_number | (root + decimal_number)) - + (pynini.accep("と") | pynini.accep("荷")) - ) - + pynutil.insert("\"") + pynutil.insert('integer_part: "') + + ((cardinal | (root + cardinal) | decimal_number | (root + decimal_number)) + mixed) + + pynutil.insert('"') + pynutil.insert(NEMO_SPACE) ) denominator_component = ( - pynutil.insert("denominator: \"") + pynutil.insert('denominator: "') + (cardinal | (root + cardinal) | decimal_number | (root + decimal_number)) - + pynutil.insert("\"") + + pynutil.insert('"') ) numerator_component = ( - pynutil.insert("numerator: \"") + pynutil.insert('numerator: "') + (cardinal | (root + cardinal) | decimal_number | (root + decimal_number)) - + pynutil.insert("\"") + + pynutil.insert('"') ) # 3/4, 1 3/4, 1と3/4, -3/4, -1 3/4, 1と3/4, √1と3/4 and any combination of root number, cardinal number and decimal number @@ -102,22 +101,17 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): ) + denominator_component + pynutil.insert(NEMO_SPACE) - + pynutil.insert("morphosyntactic_features: \"") + + pynutil.insert('morphosyntactic_features: "') + morphemes - + pynutil.insert("\"") + + pynutil.insert('"') + pynutil.insert(NEMO_SPACE) + numerator_component ) - optional_sign = ( - pynutil.insert("negative: \"") - + (pynini.accep("マイナス") | pynini.cross("-", "マイナス")) - + pynutil.insert("\"") - ) + optional_sign = pynutil.insert('negative: "') + sign + pynutil.insert('"') - graph_fraction_slash_sigh = pynini.closure(optional_sign + pynutil.insert(NEMO_SPACE), 0, 1) + ( - graph_fraction_slash | graph_fraction_word - ) + self.graph = (graph_fraction_slash | graph_fraction_word).optimize() + graph_fraction_slash_sigh = pynini.closure(optional_sign + pynutil.insert(NEMO_SPACE), 0, 1) + self.graph graph = graph_fraction_slash_sigh # | diff --git a/nemo_text_processing/text_normalization/ja/taggers/measure.py b/nemo_text_processing/text_normalization/ja/taggers/measure.py new file mode 100644 index 000000000..5fd778d5a --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/measure.py @@ -0,0 +1,133 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, delete_space +from nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels + + +class MeasureFst(GraphFst): + """ + Finite state transducer for classifying Japanese measure expressions. + + Examples: + 5kg -> measure { cardinal { integer: "五" } units: "キロ" preserve_order: true } + 0kg -> measure { cardinal { integer: "ゼロ" } units: "キロ" preserve_order: true } + 0.05m + -> measure { decimal { integer_part: "零" fractional_part: "零五" } units: "メートル" preserve_order: true } + 60km/h -> measure { cardinal { integer: "時速六十" } units: "キロ" preserve_order: true } + 50m/s -> measure { cardinal { integer: "秒速五十" } units: "メートル" preserve_order: true } + + Args: + cardinal: CardinalFst + decimal: DecimalFst + fraction: FractionFst + deterministic: if True provides a single transduction option + """ + + def __init__( + self, + cardinal: GraphFst, + decimal: GraphFst, + fraction: GraphFst, + deterministic: bool = True, + ): + super().__init__(name="measure", kind="classify", deterministic=deterministic) + + unit_path = get_abs_path("data/measure/unit.tsv") + per_unit_path = get_abs_path("data/measure/per_unit.tsv") + rate_numerator_path = get_abs_path("data/measure/rate_numerator.tsv") + unit = pynini.string_file(unit_path) + rate_numerator_labels = load_labels(rate_numerator_path) + per_unit_labels = load_labels(per_unit_path) + speed_configs = { + (unit_spoken, per_spoken): prefix + for unit_spoken, per_spoken, prefix in load_labels(get_abs_path("data/measure/speed.tsv")) + } + per_marker = load_labels(get_abs_path("data/measure/per_marker.tsv"))[0][0] + + slash = pynutil.delete("/") | pynutil.delete("／") + + general_per_unit = pynini.Fst() + for unit_written, unit_spoken in load_labels(unit_path): + for per_written, per_spoken in per_unit_labels: + general_per_unit |= ( + pynini.cross(unit_written, unit_spoken) + + delete_space + + slash + + delete_space + + pynutil.insert(per_marker) + + pynini.cross(per_written, per_spoken) + ) + unit_graph = unit | general_per_unit + + unit_component = delete_space + pynutil.insert(' units: "') + unit_graph + pynutil.insert('"') + + optional_sign = ( + pynutil.insert('negative: "') + + pynini.string_file(get_abs_path("data/numbers/sign.tsv")) + + pynutil.insert('" ') + + delete_space + ) + + cardinal_graph = ( + pynutil.insert("cardinal { ") + + pynini.closure(optional_sign, 0, 1) + + pynutil.insert('integer: "') + + cardinal.just_cardinals + + pynutil.insert('" }') + ) + decimal_graph = ( + pynutil.insert("decimal { ") + + pynini.closure(optional_sign, 0, 1) + + decimal.just_decimal + + pynutil.insert(" }") + ) + fraction_graph = ( + pynutil.insert("fraction { ") + pynini.closure(optional_sign, 0, 1) + fraction.graph + pynutil.insert(" }") + ) + + number = cardinal_graph | decimal_graph | fraction_graph + + speed_graph = pynini.Fst() + for unit_written, unit_spoken in rate_numerator_labels: + for per_written, per_spoken in per_unit_labels: + prefix = speed_configs.get((unit_spoken, per_spoken)) + if prefix is None: + continue + speed_number = ( + pynutil.insert("cardinal { ") + + pynini.closure(optional_sign, 0, 1) + + pynutil.insert(f'integer: "{prefix}') + + cardinal.just_cardinals + + pynutil.insert('" }') + ) + speed_unit = ( + delete_space + + pynutil.delete(unit_written) + + delete_space + + slash + + delete_space + + pynutil.delete(per_written) + + pynutil.insert(f' units: "{unit_spoken}"') + ) + speed_graph |= speed_number + speed_unit + pynutil.insert(" preserve_order: true") + + general_graph = number + unit_component + pynutil.insert(" preserve_order: true") + + graph = pynutil.add_weight(speed_graph, -0.1) | general_graph + + self.fst = self.add_tokens(graph.optimize()).optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/money.py b/nemo_text_processing/text_normalization/ja/taggers/money.py new file mode 100644 index 000000000..52f93cb41 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/money.py @@ -0,0 +1,151 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_DIGIT, GraphFst, delete_space +from nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels + + +class MoneyFst(GraphFst): + """ + Finite state transducer for classifying Japanese money expressions. + + Examples: + 100円 -> money { integer_part: "百" currency_maj: "円" preserve_order: true } + ¥3万 -> money { integer_part: "三" quantity: "万" currency_maj: "円" preserve_order: true } + 1.5万円 -> money { integer_part: "一点五" quantity: "万" currency_maj: "円" preserve_order: true } + 5ドル25セント + -> money { integer_part: "五" currency_maj: "ドル" fractional_part: "二十五" + currency_min: "セント" preserve_order: true } + $12.50 + -> money { integer_part: "十二" currency_maj: "ドル" fractional_part: "五十" + currency_min: "セント" preserve_order: true } + + Args: + cardinal: CardinalFst + deterministic: if True will provide a single transduction option + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="classify", deterministic=deterministic) + + graph_cardinal = cardinal.just_cardinals + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + + integer_input = (NEMO_DIGIT + pynini.closure(NEMO_DIGIT | pynutil.delete(","))) @ graph_cardinal + fractional_digits = pynini.closure(graph_digit | graph_zero, 1) + decimal_input = ( + integer_input + pynini.string_file(get_abs_path("data/numbers/decimal_point.tsv")) + fractional_digits + ) + sign_input = pynini.string_file(get_abs_path("data/numbers/sign.tsv")) + delete_space + + integer_component = pynutil.insert('integer_part: "') + integer_input + pynutil.insert('"') + signed_integer_component = ( + pynutil.insert('integer_part: "') + pynini.closure(sign_input, 0, 1) + integer_input + pynutil.insert('"') + ) + signed_decimal_component = ( + pynutil.insert('integer_part: "') + pynini.closure(sign_input, 0, 1) + decimal_input + pynutil.insert('"') + ) + + number_component = signed_decimal_component | signed_integer_component + + quantity = pynini.string_file(get_abs_path("data/money/quantity.tsv")) + quantity_component = delete_space + pynutil.insert(' quantity: "') + quantity + pynutil.insert('"') + + currency_major_labels = load_labels(get_abs_path("data/money/currency_major.tsv")) + currency_major = pynini.string_file(get_abs_path("data/money/currency_major.tsv")) + currency_major_component = ( + delete_space + pynutil.insert(' currency_maj: "') + currency_major + pynutil.insert('"') + ) + + currency_prefix_labels = load_labels(get_abs_path("data/money/currency_prefix.tsv")) + currency_minor_by_major = dict(load_labels(get_abs_path("data/money/currency_minor_by_major.tsv"))) + currency_minor = pynini.string_file(get_abs_path("data/money/currency_minor.tsv")) + non_zero_digit = pynini.difference(NEMO_DIGIT, "0") + minor_decimal_input = (NEMO_DIGIT**2 @ graph_cardinal) | (pynutil.delete("0") + (non_zero_digit @ graph_digit)) + + suffix_graph = number_component + pynini.closure(quantity_component, 0, 1) + currency_major_component + for written, spoken in currency_major_labels: + minor_spoken = currency_minor_by_major.get(written) + if not minor_spoken: + continue + + currency_major_suffix = ( + delete_space + pynutil.delete(written) + pynutil.insert(f' currency_maj: "{spoken}"') + ) + minor_suffix = ( + delete_space + + pynutil.insert(' fractional_part: "') + + integer_input + + pynutil.insert('"') + + delete_space + + (currency_minor @ pynini.cross(minor_spoken, "")) + + pynutil.insert(f' currency_min: "{minor_spoken}"') + ) + suffix_graph |= signed_integer_component + currency_major_suffix + minor_suffix + + prefix_graph = pynini.Fst() + for written, spoken in currency_prefix_labels: + currency_prefix = pynutil.delete(written) + delete_space + currency_field = pynutil.insert(f' currency_maj: "{spoken}"') + minor_spoken = currency_minor_by_major.get(written) + + prefix_graph |= ( + currency_prefix + number_component + pynini.closure(quantity_component, 0, 1) + currency_field + ) + prefix_graph |= ( + pynutil.insert('integer_part: "') + + sign_input + + currency_prefix + + (decimal_input | integer_input) + + pynutil.insert('"') + + pynini.closure(quantity_component, 0, 1) + + currency_field + ) + if minor_spoken: + decimal_minor_component = ( + integer_component + + pynutil.delete(".") + + currency_field + + pynutil.insert(' fractional_part: "') + + minor_decimal_input + + pynutil.insert(f'" currency_min: "{minor_spoken}"') + ) + signed_decimal_minor_component = ( + pynutil.insert('integer_part: "') + + sign_input + + currency_prefix + + integer_input + + pynutil.delete(".") + + pynutil.insert('"') + + currency_field + + pynutil.insert(' fractional_part: "') + + minor_decimal_input + + pynutil.insert(f'" currency_min: "{minor_spoken}"') + ) + prefix_graph |= pynutil.add_weight( + currency_prefix + decimal_minor_component, + -0.1, + ) + prefix_graph |= pynutil.add_weight( + signed_decimal_minor_component, + -0.1, + ) + + graph = (suffix_graph | prefix_graph) + pynutil.insert(" preserve_order: true") + + self.fst = self.add_tokens(graph.optimize()).optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/ordinal.py b/nemo_text_processing/text_normalization/ja/taggers/ordinal.py index d88608e01..e269615fb 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/ordinal.py +++ b/nemo_text_processing/text_normalization/ja/taggers/ordinal.py @@ -17,6 +17,7 @@ from pynini.lib import pynutil from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst +from nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels class OrdinalFst(GraphFst): @@ -32,11 +33,12 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): super().__init__(name="ordinal", kind="classify", deterministic=deterministic) graph_cardinal = cardinal.just_cardinals - morpheme_pre = pynini.accep('第') - morpheme_post = pynini.accep('番目') + markers = dict(load_labels(get_abs_path("data/ordinal/marker.tsv"))) + morpheme_pre = pynini.accep(markers["prefix"]) + morpheme_post = pynini.accep(markers["suffix"]) graph_ordinal = pynini.union(morpheme_pre + graph_cardinal, graph_cardinal + morpheme_post) - final_graph = pynutil.insert("integer: \"") + graph_ordinal + pynutil.insert("\"") + final_graph = pynutil.insert('integer: "') + graph_ordinal + pynutil.insert('"') graph_ordinal_final = self.add_tokens(final_graph) self.fst = graph_ordinal_final.optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/punctuation.py b/nemo_text_processing/text_normalization/ja/taggers/punctuation.py index c5df8388c..0fb45ea6b 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/punctuation.py +++ b/nemo_text_processing/text_normalization/ja/taggers/punctuation.py @@ -35,20 +35,20 @@ class PunctuationFst(GraphFst): """ - def __init__(self, deterministic: bool = True): - super().__init__(name="punctuation", kind="classify", deterministic=deterministic) - s = "!#$%&'()*+,-./:;<=>?@^_`{|}。，；：《》“”·~【】！？、‘’.<>-——_、。.「」『』‘`／・；’”“”‷･〔〕々〃ゝゞヽ〲〱〳〴〵ヾ〆，~" - - punct_symbols_to_exclude = ["[", "]"] - punct_unicode = [ + def __init__(self, deterministic: bool = True): + super().__init__(name="punctuation", kind="classify", deterministic=deterministic) + + punct_symbols_to_exclude = ["[", "]"] + punct_unicode = [ chr(i) for i in range(sys.maxunicode) - if category(chr(i)).startswith("P") and chr(i) not in punct_symbols_to_exclude - ] - - whitelist_symbols = load_labels(get_abs_path("data/symbol.tsv")) - whitelist_symbols = [x[0] for x in whitelist_symbols] - self.punct_marks = [p for p in punct_unicode + list(s) if p not in whitelist_symbols] + if category(chr(i)).startswith("P") and chr(i) not in punct_symbols_to_exclude + ] + extra_symbols = [label[0] for label in load_labels(get_abs_path("data/punctuation/extra.tsv"))] + + whitelist_symbols = load_labels(get_abs_path("data/symbol.tsv")) + whitelist_symbols = [x[0] for x in whitelist_symbols] + self.punct_marks = [p for p in punct_unicode + extra_symbols if p not in whitelist_symbols] punct = pynini.union(*self.punct_marks) punct = pynini.closure(punct, 1) @@ -62,9 +62,7 @@ def __init__(self, deterministic: bool = True): + pynini.accep(">") ) punct = plurals._priority_union(emphasis, punct, NEMO_SIGMA) - range_component = pynini.cross("〜", "から") | pynini.accep( - "から" - ) # forcing this conversion for special tilde + range_component = pynini.string_file(get_abs_path("data/punctuation/range.tsv")) - self.graph = punct | pynutil.add_weight(range_component, -1.0) - self.fst = (pynutil.insert("name: \"") + self.graph + pynutil.insert("\"")).optimize() + self.graph = plurals._priority_union(range_component, punct, NEMO_SIGMA) + self.fst = (pynutil.insert('name: "') + self.graph + pynutil.insert('"')).optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/range.py b/nemo_text_processing/text_normalization/ja/taggers/range.py new file mode 100644 index 000000000..163e01cc3 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/range.py @@ -0,0 +1,71 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_DIGIT, GraphFst, delete_space +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class RangeFst(GraphFst): + """ + Finite state transducer for classifying Japanese ranges. + + Examples: + 2-5 -> name: "二から五" + 10:00-11:00 -> name: "十時から十一時" + 3kg-6kg -> name: "三キロから六キロ" + + Args: + cardinal: composed cardinal tagger and verbalizer + date: composed date tagger and verbalizer + time: composed time tagger and verbalizer + money: composed money tagger and verbalizer + measure: composed measure tagger and verbalizer + deterministic: if True will provide a single transduction option, + for False multiple transductions are generated + """ + + def __init__( + self, + cardinal: GraphFst, + date: GraphFst, + time: GraphFst, + money: GraphFst, + measure: GraphFst, + deterministic: bool = True, + ): + super().__init__(name="range", kind="classify", deterministic=deterministic) + + separator = pynini.string_file(get_abs_path("data/range/separator.tsv")) + sep_to_kara = delete_space + separator + delete_space + + endpoint = cardinal | date | time | money | measure + graph = endpoint + sep_to_kara + endpoint + + # Some range suffixes, such as 人 and 歳, do not have a dedicated + # semiotic class. Keep them as range-specific cardinal patterns. + suffix = pynini.string_file(get_abs_path("data/range/suffix.tsv")) + cardinal_pair = cardinal + sep_to_kara + cardinal + cardinal_range = cardinal_pair + pynini.closure(suffix, 0, 1) + graph |= cardinal_range + graph |= cardinal + pynini.string_file(get_abs_path("data/range/operator.tsv")) + cardinal + + # Normalize English-style decade suffixes before reusing the date graph. + year_alias = (NEMO_DIGIT**4 + pynini.string_file(get_abs_path("data/range/year_suffix.tsv"))) @ date + graph |= pynutil.add_weight(year_alias + sep_to_kara + year_alias, -0.5) + + self.graph = graph.optimize() + self.fst = (pynutil.insert('name: "') + self.graph + pynutil.insert('"')).optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/roman.py b/nemo_text_processing/text_normalization/ja/taggers/roman.py new file mode 100644 index 000000000..3ab71d3fa --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/roman.py @@ -0,0 +1,68 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, insert_space +from nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels + + +class RomanFst(GraphFst): + """ + Finite state transducer for classifying Roman numerals in supported contexts. + + Examples: + 第III章 -> name: "第三章" + Chapter IV -> name: "Chapter 四" + Century XXI -> name: "Century 二十一" + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="roman", kind="classify", deterministic=deterministic) + + roman_values = { + roman: int(value) for roman, value in load_labels(get_abs_path("data/roman/roman_numerals.tsv")) + } + valid_roman_pairs = [] + for number in range(1, 4000): + roman = self._int_to_roman(number, roman_values) + valid_roman_pairs.append((roman, str(number))) + valid_roman_pairs.append((roman.lower(), str(number))) + + roman_to_number = pynini.string_map(valid_roman_pairs).optimize() + roman_to_cardinal = roman_to_number @ cardinal.just_cardinals + + japanese_prefix = pynini.string_file(get_abs_path("data/roman/japanese_prefix.tsv")) + japanese_suffix = pynini.string_file(get_abs_path("data/roman/japanese_suffix.tsv")) + japanese_context = japanese_prefix + roman_to_cardinal + japanese_suffix + + key_cardinal = pynini.union( + *[pynini.accep(x[0]) for x in load_labels(get_abs_path("data/roman/key_cardinal.tsv"))] + ) + cardinal_context = key_cardinal + pynutil.delete(" ") + insert_space + roman_to_cardinal + + graph = japanese_context | cardinal_context + self.fst = (pynutil.insert('name: "') + graph.optimize() + pynutil.insert('"')).optimize() + + @staticmethod + def _int_to_roman(number: int, roman_values: dict) -> str: + value_to_roman = sorted(((value, roman) for roman, value in roman_values.items()), reverse=True) + result = [] + remaining = number + for value, roman in value_to_roman: + while remaining >= value: + result.append(roman) + remaining -= value + return "".join(result) diff --git a/nemo_text_processing/text_normalization/ja/taggers/serial.py b/nemo_text_processing/text_normalization/ja/taggers/serial.py new file mode 100644 index 000000000..7942919e5 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/serial.py @@ -0,0 +1,89 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_UPPER, + TO_UPPER, + GraphFst, + insert_space, +) +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class SerialFst(GraphFst): + """ + Finite state transducer for classifying compact serial/model identifiers. + + Examples: + B2A23C -> name: "ビー 二 エー 二三 シー" + MIG-25/235212-asdg + -> name: "エムアイジー ハイフン 二五 スラッシュ 二三五二一二 ハイフン エーエスディージー" + """ + + def __init__(self, cardinal: GraphFst, deterministic: bool = True): + super().__init__(name="serial", kind="classify", deterministic=deterministic) + + uppercase_letters = pynini.string_file(get_abs_path("data/latin/letters.tsv")) + letters = (NEMO_UPPER | TO_UPPER) @ uppercase_letters + letter_input = pynini.project(letters, "input") + digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) | pynini.string_file( + get_abs_path("data/numbers/zero_maru.tsv") + ) + insert_letter_digit_space = pynini.cdrewrite(pynutil.insert(" "), letter_input, NEMO_DIGIT, NEMO_SIGMA) + insert_digit_letter_space = pynini.cdrewrite(pynutil.insert(" "), NEMO_DIGIT, letter_input, NEMO_SIGMA) + alnum_spacing = insert_letter_digit_space @ insert_digit_letter_space + raw_alnum = pynini.closure(letter_input | NEMO_DIGIT, 1) + raw_letter_leading_alnum = ( + letter_input + + pynini.closure(letter_input | NEMO_DIGIT) + + NEMO_DIGIT + + pynini.closure(letter_input | NEMO_DIGIT) + ) + currency_prefix_input = pynini.project( + pynini.string_file(get_abs_path("data/money/currency_prefix.tsv")), "input" + ) + currency_code_number = currency_prefix_input + pynini.closure(NEMO_DIGIT, 1) + raw_letter_leading_alnum = pynini.difference(raw_letter_leading_alnum, currency_code_number) + raw_short_digit_leading_alnum = NEMO_DIGIT + (pynini.accep("x") | pynini.accep("X")) + raw_mixed_alnum = raw_letter_leading_alnum | raw_short_digit_leading_alnum + alnum_reader = pynini.closure(letters | digit | pynini.accep(" "), 1) + alnum = (raw_mixed_alnum @ alnum_spacing @ alnum_reader).optimize() + + delimiter = insert_space + pynini.string_file(get_abs_path("data/serial/delimiter.tsv")) + insert_space + unit_input = pynini.project(pynini.string_file(get_abs_path("data/measure/unit.tsv")), "input") + numeric_measure_segment = pynini.closure(NEMO_DIGIT, 1) + unit_input + raw_alnum_segment = pynini.difference(raw_alnum, numeric_measure_segment) + raw_alnum_with_letter = ( + pynini.closure(letter_input | NEMO_DIGIT) + letter_input + pynini.closure(letter_input | NEMO_DIGIT) + ) + raw_alnum_with_letter = pynini.difference(raw_alnum_with_letter, numeric_measure_segment) + segment = (raw_alnum_segment @ alnum_spacing @ alnum_reader).optimize() + segment_with_letter = (raw_alnum_with_letter @ alnum_spacing @ alnum_reader).optimize() + delimited = segment_with_letter + pynini.closure( + delimiter + segment, 1 + ) | segment + delimiter + segment_with_letter + pynini.closure(delimiter + segment) + + special_word = pynini.string_file(get_abs_path("data/serial/words.tsv")) + covid_style = special_word + pynutil.delete("-") + insert_space + (NEMO_DIGIT**2 @ cardinal.just_cardinals) + + model_cue = pynini.string_file(get_abs_path("data/serial/model_cues.tsv")) + model_number = model_cue + insert_space + delimited + + graph = pynutil.add_weight(covid_style, -0.1) | model_number | delimited | alnum + self.fst = (pynutil.insert('name: "') + graph.optimize() + pynutil.insert('"')).optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/telephone.py b/nemo_text_processing/text_normalization/ja/taggers/telephone.py new file mode 100644 index 000000000..744083686 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/taggers/telephone.py @@ -0,0 +1,125 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_WHITE_SPACE, + GraphFst, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for classifying Japanese telephone numbers. + + Examples: + 090-1234-5678 -> telephone { number_part: "ゼロ九ゼロ 一二三四 五六七八" preserve_order: true } + 03-1234-5678 -> telephone { number_part: "ゼロ三 一二三四 五六七八" preserve_order: true } + +81 90-1234-5678 -> telephone { country_code: "八一" number_part: "九ゼロ 一二三四 五六七八" preserve_order: true } + + Args: + deterministic: if True will provide a single transduction option + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="classify", deterministic=deterministic) + + graph_digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + graph_zero = pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + digit = graph_digit | graph_zero + extension_cue = pynini.project( + pynini.string_file(get_abs_path("data/telephone/extension.tsv")), + "input", + ) + + sep_char = pynini.union("-", "－", "ー", ".", "．") + delete_sep = pynutil.delete(sep_char) + delete_required_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 1)) + block_sep = delete_space + (delete_sep | delete_required_space) + delete_space + insert_space + optional_after_paren_sep = delete_space + pynini.closure(delete_sep + delete_space, 0, 1) + required_country_sep = delete_space + (delete_sep | delete_required_space) + delete_space + + open_paren = pynutil.delete("(") | pynutil.delete("（") + close_paren = pynutil.delete(")") | pynutil.delete("）") + + digits = {count: digit**count for count in range(1, 11)} + paren = { + count: open_paren + digits[count] + close_paren + optional_after_paren_sep + insert_space + for count in range(1, 4) + } + + local_grouped_number = ( + digits[2] + block_sep + digits[4] + block_sep + digits[4] + | digits[3] + block_sep + digits[3] + block_sep + digits[4] + | digits[3] + block_sep + digits[4] + block_sep + digits[4] + | digits[4] + block_sep + digits[2] + block_sep + digits[4] + | digits[4] + block_sep + digits[3] + block_sep + digits[3] + | digits[4] + block_sep + digits[3] + block_sep + digits[4] + | digits[4] + block_sep + digits[4] + block_sep + digits[3] + ) + + local_parenthesized_number = ( + paren[2] + digits[4] + block_sep + digits[4] + | paren[3] + digits[3] + block_sep + digits[4] + | paren[3] + digits[4] + block_sep + digits[4] + ) + + compact_local_number = graph_zero + (digits[9] | digits[10]) + local_number = local_grouped_number | local_parenthesized_number | compact_local_number + + international_number = ( + digits[1] + block_sep + digits[4] + block_sep + digits[4] + | digits[2] + block_sep + digits[3] + block_sep + digits[4] + | digits[2] + block_sep + digits[4] + block_sep + digits[4] + | digits[3] + block_sep + digits[2] + block_sep + digits[4] + | digits[3] + block_sep + digits[3] + block_sep + digits[4] + | paren[1] + digits[4] + block_sep + digits[4] + | paren[2] + digits[3] + block_sep + digits[4] + | paren[2] + digits[4] + block_sep + digits[4] + | paren[3] + digits[2] + block_sep + digits[4] + | paren[3] + digits[3] + block_sep + digits[4] + ) + + country_code = digits[1] | digits[2] + country_code_component = ( + (pynutil.delete("+") | pynutil.delete("＋")) + + pynutil.insert('country_code: "') + + country_code + + pynutil.insert('"') + + required_country_sep + + pynutil.insert(" ") + ) + + extension = ( + delete_space + + pynutil.delete(extension_cue) + + delete_space + + pynutil.insert(' extension: "') + + pynini.closure(digit, 1, 4) + + pynutil.insert('"') + ) + + number_part = pynutil.insert('number_part: "') + local_number + pynutil.insert('"') + international_number_part = pynutil.insert('number_part: "') + international_number + pynutil.insert('"') + + graph = number_part | (country_code_component + international_number_part) + graph = graph + pynini.closure(extension, 0, 1) + graph = graph + pynutil.insert(" preserve_order: true") + + self.fst = self.add_tokens(graph.optimize()).optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/time.py b/nemo_text_processing/text_normalization/ja/taggers/time.py index 7c74bc53e..81c929cde 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/time.py +++ b/nemo_text_processing/text_normalization/ja/taggers/time.py @@ -17,7 +17,7 @@ from pynini.lib import pynutil from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst -from nemo_text_processing.text_normalization.ja.utils import get_abs_path +from nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels class TimeFst(GraphFst): @@ -36,32 +36,39 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): graph_cardinal = cardinal.just_cardinals hour_clock = pynini.string_file(get_abs_path("data/time/hour.tsv")) - minute_clock = pynini.string_file(get_abs_path("data/time/minute.tsv")) - second_clock = pynini.string_file(get_abs_path("data/time/second.tsv")) + minute_second_clock = pynini.string_file(get_abs_path("data/time/minute_second.tsv")) division = pynini.string_file(get_abs_path("data/time/division.tsv")) + zero_decimal = pynini.string_file(get_abs_path("data/numbers/zero_decimal.tsv")) + decimal_point = pynini.string_file(get_abs_path("data/numbers/decimal_point.tsv")) + suffixes = dict(load_labels(get_abs_path("data/time/suffix.tsv"))) + hour_suffix = suffixes["hour"] + hour_variants = pynini.union(hour_suffix, suffixes["duration_hour"], suffixes["approximate_hour"]) + minute_suffix = suffixes["minute"] + minute_modifier = pynini.union(suffixes["past"], suffixes["approximate"]) + half = suffixes["half"] + second_suffix = suffixes["second"] - division_component = pynutil.insert("suffix: \"") + division + pynutil.insert("\"") + division_component = pynutil.insert('suffix: "') + division + pynutil.insert('"') + hour_number = pynutil.add_weight(zero_decimal, -0.1) | graph_cardinal hour_component = ( - pynutil.insert("hours: \"") - + (graph_cardinal | (graph_cardinal + pynini.cross(".", "点") + graph_cardinal)) - + (pynini.accep("時") | pynini.accep("時間") | pynini.accep("時頃")) - + pynutil.insert("\"") + pynutil.insert('hours: "') + + (hour_number | (graph_cardinal + decimal_point + graph_cardinal)) + + hour_variants + + pynutil.insert('"') ) - minute_component = pynutil.insert("minutes: \"") + ( - graph_cardinal | (graph_cardinal + pynini.cross(".", "点") + graph_cardinal) - ) + pynini.accep("分") + pynini.closure((pynini.accep("過ぎ") | pynini.accep("頃")), 0, 1) + pynutil.insert( - "\"" - ) | ( - pynutil.insert("minutes: \"") - + pynini.accep("半") - + pynini.closure((pynini.accep("過ぎ") | pynini.accep("頃")), 0, 1) - + pynutil.insert("\"") + minute_component = pynutil.insert('minutes: "') + ( + graph_cardinal | (graph_cardinal + decimal_point + graph_cardinal) + ) + pynini.accep(minute_suffix) + pynini.closure(minute_modifier, 0, 1) + pynutil.insert('"') | ( + pynutil.insert('minutes: "') + + pynini.accep(half) + + pynini.closure(minute_modifier, 0, 1) + + pynutil.insert('"') ) second_component = ( - pynutil.insert("seconds: \"") - + (graph_cardinal | (graph_cardinal + pynini.cross(".", "点") + graph_cardinal)) - + pynini.accep("秒") - + pynutil.insert("\"") + pynutil.insert('seconds: "') + + (graph_cardinal | (graph_cardinal + decimal_point + graph_cardinal)) + + pynini.accep(second_suffix) + + pynutil.insert('"') ) graph_individual_time = pynini.closure(division_component + pynutil.insert(" "), 0, 1) + ( @@ -75,28 +82,28 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): colon = pynutil.delete(":") hour_clock_component = ( - pynutil.insert("hours: \"") + pynutil.insert('hours: "') + pynutil.delete("0").ques + hour_clock - + pynutil.insert("時") - + pynutil.insert("\"") + + pynutil.insert(hour_suffix) + + pynutil.insert('"') ) minute_clock_component = ( - pynutil.insert("minutes: \"") + pynutil.insert('minutes: "') + pynutil.delete("0").ques - + minute_clock - + pynutil.insert("分") - + pynutil.insert("\"") + + minute_second_clock + + pynutil.insert(minute_suffix) + + pynutil.insert('"') ) second_clock_component = ( - pynutil.insert("seconds: \"") + pynutil.insert('seconds: "') + pynutil.delete("0").ques - + second_clock - + pynutil.insert("秒") - + pynutil.insert("\"") + + minute_second_clock + + pynutil.insert(second_suffix) + + pynutil.insert('"') ) - graph_clock = ( + graph_clock_with_seconds = ( hour_clock_component + pynutil.insert(" ") + colon @@ -104,7 +111,10 @@ def __init__(self, cardinal: GraphFst, deterministic: bool = True): + pynutil.insert(" ") + colon + second_clock_component - ) | (hour_clock_component + pynutil.insert(" ") + colon + minute_clock_component) + ) + graph_clock_with_minutes = hour_clock_component + pynutil.insert(" ") + colon + minute_clock_component + graph_clock_without_minutes = hour_clock_component + colon + pynutil.delete("00") + graph_clock = graph_clock_with_seconds | graph_clock_with_minutes | graph_clock_without_minutes graph = graph_individual_time | graph_clock diff --git a/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py index f992e9b70..52eaeb4a6 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/ja/taggers/tokenize_and_classify.py @@ -18,16 +18,36 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, generator_main +from nemo_text_processing.text_normalization.ja.graph_utils import ( + GraphFst, + delete_extra_space, + delete_space, + generator_main, +) +from nemo_text_processing.text_normalization.ja.taggers.address import AddressFst from nemo_text_processing.text_normalization.ja.taggers.cardinal import CardinalFst from nemo_text_processing.text_normalization.ja.taggers.date import DateFst from nemo_text_processing.text_normalization.ja.taggers.decimal import DecimalFst +from nemo_text_processing.text_normalization.ja.taggers.electronic import ElectronicFst from nemo_text_processing.text_normalization.ja.taggers.fraction import FractionFst +from nemo_text_processing.text_normalization.ja.taggers.measure import MeasureFst +from nemo_text_processing.text_normalization.ja.taggers.money import MoneyFst from nemo_text_processing.text_normalization.ja.taggers.ordinal import OrdinalFst from nemo_text_processing.text_normalization.ja.taggers.punctuation import PunctuationFst +from nemo_text_processing.text_normalization.ja.taggers.range import RangeFst +from nemo_text_processing.text_normalization.ja.taggers.roman import RomanFst +from nemo_text_processing.text_normalization.ja.taggers.serial import SerialFst +from nemo_text_processing.text_normalization.ja.taggers.telephone import TelephoneFst from nemo_text_processing.text_normalization.ja.taggers.time import TimeFst from nemo_text_processing.text_normalization.ja.taggers.whitelist import WhiteListFst from nemo_text_processing.text_normalization.ja.taggers.word import WordFst +from nemo_text_processing.text_normalization.ja.verbalizers.cardinal import CardinalFst as CardinalVerbalizer +from nemo_text_processing.text_normalization.ja.verbalizers.date import DateFst as DateVerbalizer +from nemo_text_processing.text_normalization.ja.verbalizers.decimal import DecimalFst as DecimalVerbalizer +from nemo_text_processing.text_normalization.ja.verbalizers.fraction import FractionFst as FractionVerbalizer +from nemo_text_processing.text_normalization.ja.verbalizers.measure import MeasureFst as MeasureVerbalizer +from nemo_text_processing.text_normalization.ja.verbalizers.money import MoneyFst as MoneyVerbalizer +from nemo_text_processing.text_normalization.ja.verbalizers.time import TimeFst as TimeVerbalizer class ClassifyFst(GraphFst): @@ -59,7 +79,7 @@ def __init__( if cache_dir is not None and cache_dir != "None": os.makedirs(cache_dir, exist_ok=True) whitelist_file = os.path.basename(whitelist) if whitelist else "" - far_file = os.path.join(cache_dir, f"zh_tn_{deterministic}_deterministic_{whitelist_file}_tokenize.far") + far_file = os.path.join(cache_dir, f"ja_tn_{deterministic}_deterministic_{whitelist_file}_tokenize.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["tokenize_and_classify"] else: @@ -68,25 +88,66 @@ def __init__( decimal = DecimalFst(cardinal=cardinal, deterministic=deterministic) time = TimeFst(cardinal=cardinal, deterministic=deterministic) fraction = FractionFst(cardinal=cardinal, deterministic=deterministic) + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + money = MoneyFst(cardinal=cardinal, deterministic=deterministic) + telephone = TelephoneFst(deterministic=deterministic) ordinal = OrdinalFst(cardinal=cardinal, deterministic=deterministic) + address = AddressFst(cardinal=cardinal, deterministic=deterministic) + electronic = ElectronicFst(cardinal=cardinal, deterministic=deterministic) + roman = RomanFst(cardinal=cardinal, deterministic=deterministic) + serial = SerialFst(cardinal=cardinal, deterministic=deterministic) + + cardinal_verbalizer = CardinalVerbalizer(deterministic=deterministic) + decimal_verbalizer = DecimalVerbalizer(deterministic=deterministic) + fraction_verbalizer = FractionVerbalizer(deterministic=deterministic) + date_final = date.fst @ DateVerbalizer(deterministic=deterministic).fst + time_final = time.fst @ TimeVerbalizer(deterministic=deterministic).fst + money_final = money.fst @ MoneyVerbalizer(decimal=decimal_verbalizer, deterministic=deterministic).fst + measure_final = ( + measure.fst + @ MeasureVerbalizer( + cardinal=cardinal_verbalizer, + decimal=decimal_verbalizer, + fraction=fraction_verbalizer, + deterministic=deterministic, + ).fst + ) + range_graph = RangeFst( + cardinal=cardinal.fst @ cardinal_verbalizer.fst, + date=date_final, + time=time_final, + money=money_final, + measure=measure_final, + deterministic=deterministic, + ) whitelist = WhiteListFst(deterministic=deterministic) word = WordFst(deterministic=deterministic) punctuation = PunctuationFst(deterministic=deterministic) classify = pynini.union( + pynutil.add_weight(electronic.fst, 1.1), + pynutil.add_weight(address.fst, 1.1), + pynutil.add_weight(roman.fst, 1.1), + pynutil.add_weight(serial.fst, 1.1), + pynutil.add_weight(range_graph.fst, 1.1), pynutil.add_weight(date.fst, 1.1), - pynutil.add_weight(fraction.fst, 1.0), + pynutil.add_weight(fraction.fst, 1.1), + pynutil.add_weight(money.fst, 1.1), + pynutil.add_weight(measure.fst, 1.1), + pynutil.add_weight(telephone.fst, 1.1), pynutil.add_weight(time.fst, 1.1), pynutil.add_weight(whitelist.fst, 1.1), pynutil.add_weight(cardinal.fst, 1.1), - pynutil.add_weight(decimal.fst, 3.05), + pynutil.add_weight(decimal.fst, 1.1), pynutil.add_weight(ordinal.fst, 1.1), - pynutil.add_weight(punctuation.fst, 1.0), + pynutil.add_weight(punctuation.fst, 1.1), pynutil.add_weight(word.fst, 100), ) - token = pynutil.insert("tokens { ") + classify + pynutil.insert(" } ") - tagger = pynini.closure(token, 1) + token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") + tagger = ( + delete_space + token + pynini.closure((delete_extra_space | pynini.accep("")) + token) + delete_space + ) self.fst = tagger diff --git a/nemo_text_processing/text_normalization/ja/taggers/whitelist.py b/nemo_text_processing/text_normalization/ja/taggers/whitelist.py index 2f9391ddf..b272a8e22 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/whitelist.py +++ b/nemo_text_processing/text_normalization/ja/taggers/whitelist.py @@ -17,7 +17,7 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst +from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, delete_space from nemo_text_processing.text_normalization.ja.utils import get_abs_path @@ -34,7 +34,9 @@ class WhiteListFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="whitelist", kind="classify", deterministic=deterministic) - whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")) - graph = (pynutil.insert('name: "')) + (whitelist) + pynutil.insert('"') - - self.fst = graph.optimize() + whitelist = pynini.string_file(get_abs_path("data/whitelist.tsv")) + title = pynini.string_file(get_abs_path("data/whitelist_title.tsv")) + title_with_space = title + delete_space + pynutil.insert(" ") + graph = (pynutil.insert('name: "')) + (title_with_space | whitelist) + pynutil.insert('"') + + self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ja/taggers/word.py b/nemo_text_processing/text_normalization/ja/taggers/word.py index b1403221b..7caaa0e37 100644 --- a/nemo_text_processing/text_normalization/ja/taggers/word.py +++ b/nemo_text_processing/text_normalization/ja/taggers/word.py @@ -1,30 +1,30 @@ -# Copyright (c) 2024 NVIDIA CORPORATION. All rights reserved. -# Copyright 2015 and onwards Google, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pynini.lib import pynutil - -from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_SPACE, GraphFst - - -class WordFst(GraphFst): - """ - Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class. - e.g. 文字 -> tokens { name: "文字" } - """ - - def __init__(self, deterministic: bool = True): - super().__init__(name="word", kind="classify", deterministic=deterministic) - word = pynutil.insert("name: \"") + NEMO_NOT_SPACE + pynutil.insert("\"") - self.fst = word.optimize() +# Copyright (c) 2024 NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_SPACE, GraphFst + + +class WordFst(GraphFst): + """ + Finite state transducer for classifying plain tokens, that do not belong to any special class. This can be considered as the default class. + e.g. 文字 -> tokens { name: "文字" } + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="word", kind="classify", deterministic=deterministic) + word = pynutil.insert("name: \"") + NEMO_NOT_SPACE + pynutil.insert("\"") + self.fst = word.optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/decimal.py b/nemo_text_processing/text_normalization/ja/verbalizers/decimal.py index f4200001c..dbb28c9af 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/decimal.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/decimal.py @@ -17,6 +17,7 @@ from pynini.lib import pynutil from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, GraphFst, delete_space +from nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels class DecimalFst(GraphFst): @@ -29,20 +30,22 @@ class DecimalFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="decimal", kind="verbalize", deterministic=deterministic) - graph_integer = pynutil.delete("integer_part: \"") + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete("\"") + decimal_point = load_labels(get_abs_path("data/numbers/decimal_point.tsv"))[0][1] + + graph_integer = pynutil.delete('integer_part: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') graph_fraction = ( - pynutil.delete("fractional_part: \"") - + pynutil.insert("点") + pynutil.delete('fractional_part: "') + + pynutil.insert(decimal_point) + pynini.closure(NEMO_NOT_QUOTE, 1) - + pynutil.delete("\"") + + pynutil.delete('"') ) graph_optional_sign = pynini.closure( pynutil.delete("negative:") + delete_space - + pynutil.delete("\"") + + pynutil.delete('"') + pynini.closure(NEMO_NOT_QUOTE, 1) - + pynutil.delete("\"") + + pynutil.delete('"') ) graph_decimal_no_sign = graph_integer + pynutil.delete(" ") + graph_fraction diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/electronic.py b/nemo_text_processing/text_normalization/ja/verbalizers/electronic.py new file mode 100644 index 000000000..1789062c9 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/electronic.py @@ -0,0 +1,147 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NARROW_NON_BREAK_SPACE, + NEMO_NOT_QUOTE, + NEMO_SIGMA, + GraphFst, + delete_preserve_order, + delete_space, + insert_space, +) +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class ElectronicFst(GraphFst): + """Verbalizes structured Japanese electronic tokens.""" + + def __init__(self, deterministic: bool = True): + super().__init__(name="electronic", kind="verbalize", deterministic=deterministic) + + symbol = pynini.string_file(get_abs_path("data/electronic/symbol.tsv")) + + def spaced_symbol(written: str): + return insert_space + (pynini.accep(written) @ symbol) + insert_space + + def insert_spoken_symbol(written: str): + return insert_space + (pynutil.insert(written) @ symbol) + insert_space + + dot = spaced_symbol(".") + hyphen = spaced_symbol("-") + slash = spaced_symbol("/") + insert_at = insert_spoken_symbol("@") + insert_colon = insert_spoken_symbol(":") + insert_slash = insert_spoken_symbol("/") + + digit = pynini.string_file(get_abs_path("data/numbers/digit.tsv")) + digit_zero_maru = digit | pynini.string_file(get_abs_path("data/numbers/zero_maru.tsv")) + digit_zero_user = digit | pynini.string_file(get_abs_path("data/numbers/zero.tsv")) + special_digit_run = pynutil.add_weight( + pynini.string_file(get_abs_path("data/electronic/special_digit_runs.tsv")), + -0.1, + ) + + raw_label = pynini.closure(NEMO_ALPHA | NEMO_DIGIT, 1) + alpha_label = pynini.closure(NEMO_ALPHA, 1) + digit_label = pynini.closure(NEMO_DIGIT, 1) + + insert_alpha_digit_space = pynini.cdrewrite(pynutil.insert(" "), NEMO_ALPHA, NEMO_DIGIT, NEMO_SIGMA) + insert_digit_alpha_space = pynini.cdrewrite(pynutil.insert(" "), NEMO_DIGIT, NEMO_ALPHA, NEMO_SIGMA) + alnum_spacing = insert_alpha_digit_space @ insert_digit_alpha_space + username_reader = pynini.closure( + NEMO_ALPHA | special_digit_run | digit_zero_user | pynini.accep(" "), + 1, + ) + raw_mixed_alnum = ( + pynini.closure(NEMO_ALPHA | NEMO_DIGIT) + + NEMO_ALPHA + + pynini.closure(NEMO_ALPHA | NEMO_DIGIT) + + NEMO_DIGIT + + pynini.closure(NEMO_ALPHA | NEMO_DIGIT) + ) | ( + pynini.closure(NEMO_ALPHA | NEMO_DIGIT) + + NEMO_DIGIT + + pynini.closure(NEMO_ALPHA | NEMO_DIGIT) + + NEMO_ALPHA + + pynini.closure(NEMO_ALPHA | NEMO_DIGIT) + ) + username_alnum = (raw_mixed_alnum @ alnum_spacing @ username_reader).optimize() + username_segment = pynutil.add_weight(username_alnum, -0.1) | alpha_label | digit_label + username_value = username_segment + pynini.closure((dot | hyphen) + username_segment) + + domain_value = raw_label + pynini.closure(dot + raw_label) + dot + alpha_label + path_label = raw_label + pynini.closure(hyphen + raw_label) + path_value = slash + path_label + pynini.closure(slash + path_label) + + username_field = ( + pynutil.delete("username:") + delete_space + pynutil.delete('"') + username_value + pynutil.delete('"') + ) + domain_field = ( + pynutil.delete("domain:") + delete_space + pynutil.delete('"') + domain_value + pynutil.delete('"') + ) + protocol_value = pynini.string_file(get_abs_path("data/electronic/protocol.tsv")) + protocol_field = ( + pynutil.delete("protocol:") + + delete_space + + pynutil.delete('"') + + protocol_value + + pynutil.delete('"') + + insert_colon + + insert_slash + + insert_slash + ) + path_field = pynutil.delete("path:") + delete_space + pynutil.delete('"') + path_value + pynutil.delete('"') + + email = username_field + delete_space + insert_at + domain_field + domain = domain_field + url = protocol_field + delete_space + domain_field + pynini.closure(delete_space + path_field, 0, 1) + + card_digit = pynini.closure(digit_zero_maru, 1) + protected_space = pynutil.insert(NEMO_NARROW_NON_BREAK_SPACE) + card_number_value = card_digit + pynini.closure(pynutil.delete(" ") + protected_space + card_digit) + card_number_field = ( + pynutil.delete("domain:") + delete_space + pynutil.delete('"') + card_number_value + pynutil.delete('"') + ) + card_cue = pynini.string_file(get_abs_path("data/electronic/card_cues.tsv")) | ( + pynini.string_file(get_abs_path("data/electronic/card_digit_count_prefix.tsv")) + + pynini.project(digit, "output") + + pynini.string_file(get_abs_path("data/electronic/card_digit_count_suffix.tsv")) + ) + card_cue_field = ( + pynutil.delete("protocol:") + delete_space + pynutil.delete('"') + card_cue + pynutil.delete('"') + ) + card = pynini.closure(card_cue_field + delete_space, 0, 1) + card_number_field + + filename_stem = pynini.closure( + pynini.difference(NEMO_NOT_QUOTE, pynini.union(".", "/", "@")), + 1, + ) + filename = ( + pynutil.delete("domain:") + + delete_space + + pynutil.delete('"') + + filename_stem + + protected_space + + pynini.string_file(get_abs_path("data/electronic/file_extensions.tsv")) + + pynutil.delete('"') + ) + + graph = (email | url | domain | card | filename) + delete_preserve_order + self.fst = self.delete_tokens(graph.optimize()).optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py b/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py index 4743d3bcd..d14325b30 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/fraction.py @@ -17,6 +17,7 @@ from pynini.lib import pynutil from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, NEMO_SPACE, GraphFst, delete_space +from nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels class FractionFst(GraphFst): @@ -24,7 +25,7 @@ class FractionFst(GraphFst): Finite state transducer for verbalizing fractionss, e.g. tokens { fraction { denominator: "二" numerator: "一"} } -> 1/2 tokens { fraction { integer: "一" denominator: "四" numerator: "三" } } -> 1と3/4 - tokens { fraction { integer: "1" denominator: "4" numerator: "3" } } -> 一荷四分の三 + tokens { fraction { integer: "1" denominator: "4" numerator: "3" } } -> 一と四分の三 tokens { fraction { denominator: "√3" numerator: "1" } } -> ルート三分の一 tokens { fraction { denominator: "1.65" numerator: "50" } } -> 一点六五分の五十 tokens { fraction { denominator: "二" numerator: "一"} } -> マイナス1/2 @@ -33,35 +34,41 @@ class FractionFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="fraction", kind="verbalize", deterministic=deterministic) + markers = dict(load_labels(get_abs_path("data/fraction/marker.tsv"))) + fraction_marker = markers["fraction"] + root_written = markers["root_written"] + root_spoken = markers["root_spoken"] + mixed = markers["mixed"] + denominator_component = ( - pynutil.delete('denominator: \"') + pynini.closure(NEMO_NOT_QUOTE - "√") + pynutil.delete("\"") + pynutil.delete('denominator: "') + pynini.closure(NEMO_NOT_QUOTE - root_written) + pynutil.delete('"') ) numerator_component = ( - pynutil.delete('numerator: \"') + pynini.closure(NEMO_NOT_QUOTE - "√") + pynutil.delete("\"") + pynutil.delete('numerator: "') + pynini.closure(NEMO_NOT_QUOTE - root_written) + pynutil.delete('"') ) # 1/3 graph_regular_fraction = ( - denominator_component + pynutil.delete(NEMO_SPACE) + pynutil.insert("分の") + numerator_component + denominator_component + pynutil.delete(NEMO_SPACE) + pynutil.insert(fraction_marker) + numerator_component ) denominator_component_root = ( - pynutil.delete('denominator: \"') - + pynini.cross("√", "ルート") - + pynini.closure(NEMO_NOT_QUOTE - "√") - + pynutil.delete("\"") + pynutil.delete('denominator: "') + + pynini.cross(root_written, root_spoken) + + pynini.closure(NEMO_NOT_QUOTE - root_written) + + pynutil.delete('"') ) numerator_component_root = ( - pynutil.delete('numerator: \"') - + pynini.cross("√", "ルート") - + pynini.closure(NEMO_NOT_QUOTE - "√") - + pynutil.delete("\"") + pynutil.delete('numerator: "') + + pynini.cross(root_written, root_spoken) + + pynini.closure(NEMO_NOT_QUOTE - root_written) + + pynutil.delete('"') ) # √3/1 graph_regular_fraction_root = ( (denominator_component_root | denominator_component) + pynutil.delete(NEMO_SPACE) - + pynutil.insert("分の") + + pynutil.insert(fraction_marker) + (numerator_component_root | numerator_component) ) @@ -69,9 +76,9 @@ def __init__(self, deterministic: bool = True): graph_regular_fraction_char = ( (denominator_component | denominator_component_root) + pynutil.delete(NEMO_SPACE) - + pynutil.delete("morphosyntactic_features: \"") + + pynutil.delete('morphosyntactic_features: "') + pynini.closure(NEMO_NOT_QUOTE) - + pynutil.delete("\"") + + pynutil.delete('"') + pynutil.delete(NEMO_SPACE) + (numerator_component | numerator_component_root) ) @@ -79,23 +86,21 @@ def __init__(self, deterministic: bool = True): graph_integer = ( pynutil.delete("integer_part:") + delete_space - + pynutil.delete("\"") - + pynini.closure(pynini.cross("√", "ルート"), 0, 1) - + pynini.closure( - NEMO_NOT_QUOTE - pynini.union("荷", "と", "√") - ) # had to remove these 3 items fron nemo_not _quote so the root is properly converted in a deterministic way. - + pynutil.insert("荷") - + pynutil.delete("\"") + + pynutil.delete('"') + + pynini.closure(pynini.cross(root_written, root_spoken), 0, 1) + + pynini.closure(NEMO_NOT_QUOTE - pynini.union(mixed, root_written)) + + pynutil.insert(mixed) + + pynutil.delete('"') ) graph_integer_with_char = ( pynutil.delete("integer_part:") + delete_space - + pynutil.delete("\"") - + pynini.closure(pynini.cross("√", "ルート"), 0, 1) - + pynini.closure(NEMO_NOT_QUOTE - pynini.union("荷", "と", "√")) - + (pynini.accep("と") | pynini.accep("荷")) - + pynutil.delete("\"") + + pynutil.delete('"') + + pynini.closure(pynini.cross(root_written, root_spoken), 0, 1) + + pynini.closure(NEMO_NOT_QUOTE - pynini.union(mixed, root_written)) + + pynini.accep(mixed) + + pynutil.delete('"') ) graph_regular_integer = ( @@ -107,9 +112,9 @@ def __init__(self, deterministic: bool = True): optional_sign = ( pynutil.delete("negative:") + delete_space - + pynutil.delete("\"") + + pynutil.delete('"') + pynini.closure(NEMO_NOT_QUOTE) - + pynutil.delete("\"") + + pynutil.delete('"') + delete_space ) diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/measure.py b/nemo_text_processing/text_normalization/ja/verbalizers/measure.py new file mode 100644 index 000000000..12057afdc --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/measure.py @@ -0,0 +1,50 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, +) + + +class MeasureFst(GraphFst): + """ + Finite state transducer for verbalizing Japanese measure tokens. + + Examples: + measure { cardinal { integer: "五" } units: "キロ" preserve_order: true } -> 五キロ + measure { cardinal { integer: "時速六十" } units: "キロ" preserve_order: true } -> 時速六十キロ + measure { cardinal { integer: "秒速五十" } units: "メートル" preserve_order: true } -> 秒速五十メートル + """ + + def __init__( + self, + cardinal: GraphFst, + decimal: GraphFst, + fraction: GraphFst, + deterministic: bool = True, + ): + super().__init__(name="measure", kind="verbalize", deterministic=deterministic) + + unit = delete_space + pynutil.delete('units: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') + number = cardinal.fst | decimal.fst | fraction.fst + + graph = number + unit + delete_preserve_order + + self.fst = self.delete_tokens(graph.optimize()).optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/money.py b/nemo_text_processing/text_normalization/ja/verbalizers/money.py new file mode 100644 index 000000000..83b8349cc --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/money.py @@ -0,0 +1,64 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, +) + + +class MoneyFst(GraphFst): + """ + Finite state transducer for verbalizing Japanese money. + + Example: + money { integer_part: "百" currency_maj: "円" preserve_order: true } -> 百円 + """ + + def __init__(self, decimal: GraphFst, deterministic: bool = True): + super().__init__(name="money", kind="verbalize", deterministic=deterministic) + + field_value = pynini.closure(NEMO_NOT_QUOTE, 1) + + integer_part = pynutil.delete('integer_part: "') + field_value + pynutil.delete('"') + + quantity = pynini.closure( + delete_space + pynutil.delete('quantity: "') + field_value + pynutil.delete('"'), + 0, + 1, + ) + + currency_major = delete_space + pynutil.delete('currency_maj: "') + field_value + pynutil.delete('"') + + fractional_part = pynini.closure( + delete_space + + pynutil.delete('fractional_part: "') + + field_value + + pynutil.delete('"') + + delete_space + + pynutil.delete('currency_min: "') + + field_value + + pynutil.delete('"'), + 0, + 1, + ) + + graph = integer_part + quantity + currency_major + fractional_part + delete_preserve_order + + self.fst = self.delete_tokens(graph.optimize()).optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py b/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py index 8b196dcaf..e260abcd4 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/post_processing.py @@ -16,20 +16,26 @@ import os import pynini +from pynini.lib import pynutil -from nemo_text_processing.text_normalization.en.graph_utils import ( +from nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_NARROW_NON_BREAK_SPACE, + NEMO_NON_BREAKING_SPACE, NEMO_NOT_SPACE, NEMO_SIGMA, - delete_space, generator_main, ) +from nemo_text_processing.text_normalization.ja.utils import get_abs_path, load_labels from nemo_text_processing.utils.logging import logger class PostProcessingFst: """ Finite state transducer that post-processing an entire sentence after verbalization is complete, e.g. - removes extra spaces around punctuation marks " ( one hundred and twenty three ) " -> "(one hundred and twenty three)" + removes extra spaces around punctuation marks + " ( one hundred and twenty three ) " -> "(one hundred and twenty three)" Args: cache_dir: path to a dir with .far grammar file. Set to None to avoid using cache. @@ -41,73 +47,98 @@ def __init__(self, cache_dir: str = None, overwrite_cache: bool = False): far_file = None if cache_dir is not None and cache_dir != "None": os.makedirs(cache_dir, exist_ok=True) - far_file = os.path.join(cache_dir, "zh_tn_post_processing.far") + far_file = os.path.join(cache_dir, "ja_tn_post_processing.far") if not overwrite_cache and far_file and os.path.exists(far_file): self.fst = pynini.Far(far_file, mode="r")["post_process_graph"] logger.info(f'Post processing graph was restored from {far_file}.') else: - self.set_punct_dict() self.fst = self.get_punct_postprocess_graph() if far_file: generator_main(far_file, {"post_process_graph": self.fst}) - def set_punct_dict(self): - self.punct_marks = { - "'": [ - "'", - '´', - 'ʹ', - 'ʻ', - 'ʼ', - 'ʽ', - 'ʾ', - 'ˈ', - 'ˊ', - 'ˋ', - '˴', - 'ʹ', - '΄', - '՚', - '՝', - 'י', - '׳', - 'ߴ', - 'ߵ', - 'ᑊ', - 'ᛌ', - '᾽', - '᾿', - '`', - '´', - '῾', - '‘', - '’', - '‛', - '′', - '‵', - 'ꞌ', - '＇', - '｀', - '𖽑', - '𖽒', - ], - } - def get_punct_postprocess_graph(self): """ - Returns graph to post process punctuation marks. + Returns graph to post process Japanese TN output. - {``} quotes are converted to {"}. Note, if there are spaces around single quote {'}, they will be kept. - By default, a space is added after a punctuation mark, and spaces are removed before punctuation marks. + Japanese verbalizers need ordinary inter-token spaces removed, but some + classes intentionally use spaces internally. Protect those spaces as NBSP + before deleting remaining technical spaces, then restore them as regular + spaces in the final output. """ - remove_space_around_single_quote = pynini.cdrewrite( - delete_space, NEMO_NOT_SPACE, NEMO_NOT_SPACE, pynini.closure(NEMO_SIGMA) + protect_space = pynini.cross(" ", NEMO_NON_BREAKING_SPACE) + ascii_char = NEMO_ALPHA | NEMO_DIGIT + phone_digit = ( + pynini.project(pynini.string_file(get_abs_path("data/numbers/digit.tsv")), "output") + | pynini.project(pynini.string_file(get_abs_path("data/numbers/zero.tsv")), "output") + | pynini.project(pynini.string_file(get_abs_path("data/numbers/zero_decimal.tsv")), "output") + | pynini.project(pynini.string_file(get_abs_path("data/numbers/zero_maru.tsv")), "output") + ).optimize() + space_sensitive_tokens = ( + pynini.project(pynini.string_file(get_abs_path("data/electronic/symbol.tsv")), "output") + | pynini.project(pynini.string_file(get_abs_path("data/latin/letters.tsv")), "output") + | pynini.project(pynini.string_file(get_abs_path("data/serial/words.tsv")), "output") + ).optimize() + title_tokens = pynini.union( + *{spoken for _, spoken in load_labels(get_abs_path("data/whitelist_title.tsv"))} + ).optimize() + ten = dict(load_labels(get_abs_path("data/numbers/teen.tsv")))["10"] + japanese_number = phone_digit | pynini.accep(ten) + sentence_suffix = pynini.string_file(get_abs_path("data/post_processing/sentence_suffix.tsv")) + collapse_double_space = pynini.cdrewrite(pynini.cross(" ", " "), "", "", pynini.closure(NEMO_SIGMA)) + + protect_whitelist_internal_space = pynini.closure(NEMO_SIGMA) + for spoken in {spoken for _, spoken in load_labels(get_abs_path("data/whitelist.tsv")) if " " in spoken}: + parts = spoken.split() + for left, right in zip(parts, parts[1:]): + protect_whitelist_internal_space @= pynini.cdrewrite( + protect_space, left, right, pynini.closure(NEMO_SIGMA) + ) + delete_ascii_inner_space = pynini.cdrewrite( + pynutil.delete(" "), ascii_char, ascii_char, pynini.closure(NEMO_SIGMA) + ) + protect_ascii_word_space = pynini.cdrewrite( + protect_space, ascii_char**2, ascii_char**2, pynini.closure(NEMO_SIGMA) + ) + protect_ascii_before_desu = pynini.cdrewrite( + protect_space, ascii_char**2, sentence_suffix, pynini.closure(NEMO_SIGMA) + ) + protect_ascii_before_japanese_number = pynini.cdrewrite( + protect_space, ascii_char**2, japanese_number, pynini.closure(NEMO_SIGMA) + ) + protect_title_before_ascii = pynini.cdrewrite( + protect_space, title_tokens, ascii_char**2, pynini.closure(NEMO_SIGMA) + ) + protect_after_space_sensitive_token = pynini.cdrewrite( + protect_space, space_sensitive_tokens, "", pynini.closure(NEMO_SIGMA) + ) + protect_before_space_sensitive_token = pynini.cdrewrite( + protect_space, "", space_sensitive_tokens, pynini.closure(NEMO_SIGMA) + ) + delete_technical_space = pynini.cdrewrite( + pynutil.delete(" "), NEMO_NOT_SPACE, NEMO_NOT_SPACE, pynini.closure(NEMO_SIGMA) + ) + restore_protected_space = pynini.cdrewrite( + pynini.cross(pynini.union(NEMO_NON_BREAKING_SPACE, NEMO_NARROW_NON_BREAK_SPACE), " "), + "", + "", + pynini.closure(NEMO_SIGMA), ) - # this works if spaces in between (good) - # delete space between 2 NEMO_NOT_SPACE（left and right to the space) that are with in a content of NEMO_SIGMA - graph = remove_space_around_single_quote.optimize() + graph = ( + collapse_double_space + @ collapse_double_space + @ protect_whitelist_internal_space + @ protect_ascii_word_space + @ delete_ascii_inner_space + @ protect_ascii_before_desu + @ protect_ascii_before_japanese_number + @ protect_title_before_ascii + @ protect_after_space_sensitive_token + @ protect_before_space_sensitive_token + @ delete_technical_space + @ restore_protected_space + ).optimize() return graph diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py b/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py index 3ff05fa57..cd4b6291a 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/postprocessor.py @@ -14,26 +14,20 @@ import pynini -from pynini.lib import pynutil, utf8 +from pynini.lib import pynutil -from nemo_text_processing.text_normalization.ja.graph_utils import ( - NEMO_ALPHA, - NEMO_DIGIT, - NEMO_PUNCT, - NEMO_SIGMA, - NEMO_WHITE_SPACE, - GraphFst, -) -from nemo_text_processing.text_normalization.ja.utils import get_abs_path +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_SIGMA, TO_LOWER, TO_UPPER, GraphFst +from nemo_text_processing.text_normalization.ja.taggers.punctuation import PunctuationFst class PostProcessor(GraphFst): - ''' - Postprocessing of TN, now contains: - 1. punctuation removal - 2. letter case conversion - 3. oov tagger - ''' + """ + Optional postprocessing for Japanese TN. + + The default graph is an identity rewrite. Optional punctuation removal and + ASCII case conversion are kept generic; OOV tagging needs a Japanese-specific + character inventory and is intentionally not implemented here. + """ def __init__( self, @@ -44,38 +38,20 @@ def __init__( ): super().__init__(name="PostProcessor", kind="processor") - graph = pynini.cdrewrite('', '', '', NEMO_SIGMA) + if to_upper and to_lower: + raise ValueError("to_upper and to_lower cannot both be enabled.") + if tag_oov: + raise ValueError("tag_oov is not supported for Japanese TN without a Japanese charset inventory.") + + graph = pynini.cdrewrite("", "", "", NEMO_SIGMA) if remove_puncts: - remove_puncts_graph = pynutil.delete( - pynini.union(NEMO_PUNCT, pynini.string_file(get_abs_path('data/char/punctuations_zh.tsv'))) - ) + remove_puncts_graph = pynutil.delete(pynini.union(*PunctuationFst().punct_marks)) graph @= pynini.cdrewrite(remove_puncts_graph, "", "", NEMO_SIGMA).optimize() - if to_upper or to_lower: - if to_upper: - conv_cases_graph = pynini.inverse(pynini.string_file(get_abs_path('data/char/upper_to_lower.tsv'))) - else: - conv_cases_graph = pynini.string_file(get_abs_path('data/char/upper_to_lower.tsv')) - + if to_upper: + graph @= pynini.cdrewrite(TO_UPPER, "", "", NEMO_SIGMA).optimize() + elif to_lower: + conv_cases_graph = TO_LOWER graph @= pynini.cdrewrite(conv_cases_graph, "", "", NEMO_SIGMA).optimize() - if tag_oov: - zh_charset_std = pynini.string_file(get_abs_path("data/char/charset_national_standard_2013_8105.tsv")) - zh_charset_ext = pynini.string_file(get_abs_path("data/char/charset_extension.tsv")) - - zh_charset = ( - zh_charset_std | zh_charset_ext | pynini.string_file(get_abs_path("data/char/punctuations_zh.tsv")) - ) - en_charset = NEMO_DIGIT | NEMO_ALPHA | NEMO_PUNCT | NEMO_WHITE_SPACE - charset = zh_charset | en_charset - - with open(get_abs_path("data/char/oov_tags.tsv"), "r") as f: - tags = f.readline().strip().split('\t') - assert len(tags) == 2 - ltag, rtag = tags - - oov_charset = pynini.difference(utf8.VALID_UTF8_CHAR, charset) - tag_oov_graph = pynutil.insert(ltag) + oov_charset + pynutil.insert(rtag) - graph @= pynini.cdrewrite(tag_oov_graph, "", "", NEMO_SIGMA).optimize() - self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/telephone.py b/nemo_text_processing/text_normalization/ja/verbalizers/telephone.py new file mode 100644 index 000000000..f1584a8c5 --- /dev/null +++ b/nemo_text_processing/text_normalization/ja/verbalizers/telephone.py @@ -0,0 +1,76 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import ( + NEMO_NARROW_NON_BREAK_SPACE, + NEMO_NOT_QUOTE, + GraphFst, + delete_preserve_order, + delete_space, +) +from nemo_text_processing.text_normalization.ja.utils import get_abs_path + + +class TelephoneFst(GraphFst): + """ + Finite state transducer for verbalizing Japanese telephone numbers. + + Example: + telephone { number_part: "ゼロ九ゼロ 一二三四 五六七八" preserve_order: true } + -> ゼロ九ゼロ、 一二三四、 五六七八 + """ + + def __init__(self, deterministic: bool = True): + super().__init__(name="telephone", kind="verbalize", deterministic=deterministic) + + country_code_prefix = pynini.string_file(get_abs_path("data/telephone/country_code_prefix.tsv")) + group_separator = pynini.string_file(get_abs_path("data/telephone/group_separator.tsv")) + extension_cue = pynini.project( + pynini.string_file(get_abs_path("data/telephone/extension.tsv")), + "output", + ) + country_code = ( + pynutil.delete('country_code: "') + + pynutil.insert(country_code_prefix) + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + + protected_space = pynutil.insert(NEMO_NARROW_NON_BREAK_SPACE) + spoken_group_separator = pynutil.insert(group_separator) + protected_space + number_group = pynini.closure( + (NEMO_NOT_QUOTE - " ") | (pynutil.delete(" ") + spoken_group_separator), + 1, + ) + number_part = pynutil.delete('number_part: "') + number_group + pynutil.delete('"') + + extension = ( + pynutil.delete('extension: "') + + pynutil.insert(extension_cue) + + protected_space + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete('"') + ) + optional_extension = pynini.closure(delete_space + spoken_group_separator + extension, 0, 1) + + graph = ( + ((country_code + delete_space + spoken_group_separator + number_part) | number_part) + + optional_extension + + delete_preserve_order + ) + + self.fst = self.delete_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/verbalize.py b/nemo_text_processing/text_normalization/ja/verbalizers/verbalize.py index 6a16f96d9..392f9fdde 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/verbalize.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/verbalize.py @@ -19,8 +19,12 @@ from nemo_text_processing.text_normalization.ja.verbalizers.cardinal import CardinalFst from nemo_text_processing.text_normalization.ja.verbalizers.date import DateFst from nemo_text_processing.text_normalization.ja.verbalizers.decimal import DecimalFst +from nemo_text_processing.text_normalization.ja.verbalizers.electronic import ElectronicFst from nemo_text_processing.text_normalization.ja.verbalizers.fraction import FractionFst +from nemo_text_processing.text_normalization.ja.verbalizers.measure import MeasureFst +from nemo_text_processing.text_normalization.ja.verbalizers.money import MoneyFst from nemo_text_processing.text_normalization.ja.verbalizers.ordinal import OrdinalFst +from nemo_text_processing.text_normalization.ja.verbalizers.telephone import TelephoneFst from nemo_text_processing.text_normalization.ja.verbalizers.time import TimeFst from nemo_text_processing.text_normalization.ja.verbalizers.whitelist import WhiteListFst from nemo_text_processing.text_normalization.ja.verbalizers.word import WordFst @@ -45,11 +49,12 @@ def __init__(self, deterministic: bool = True): decimal = DecimalFst(deterministic=deterministic) word = WordFst(deterministic=deterministic) fraction = FractionFst(deterministic=deterministic) - - # money = MoneyFst(decimal=decimal, deterministic=deterministic) - # measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + money = MoneyFst(decimal=decimal, deterministic=deterministic) + measure = MeasureFst(cardinal=cardinal, decimal=decimal, fraction=fraction, deterministic=deterministic) + telephone = TelephoneFst(deterministic=deterministic) time = TimeFst(deterministic=deterministic) whitelist = WhiteListFst(deterministic=deterministic) + electronic = ElectronicFst(deterministic=deterministic) graph = pynini.union( date.fst, @@ -57,9 +62,13 @@ def __init__(self, deterministic: bool = True): ordinal.fst, decimal.fst, fraction.fst, + money.fst, + measure.fst, + telephone.fst, word.fst, time.fst, whitelist.fst, + electronic.fst, ) graph = pynini.closure(delete_space) + graph + pynini.closure(delete_space) diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/verbalize_final.py b/nemo_text_processing/text_normalization/ja/verbalizers/verbalize_final.py index 750598649..82132de01 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/verbalize_final.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/verbalize_final.py @@ -1,54 +1,56 @@ -# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import os - -import pynini -from pynini.lib import pynutil - -from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, delete_space -from nemo_text_processing.text_normalization.ja.verbalizers.postprocessor import PostProcessor -from nemo_text_processing.text_normalization.ja.verbalizers.verbalize import VerbalizeFst - -# from nemo.utils import logging - - -class VerbalizeFinalFst(GraphFst): - """ """ - - def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): - super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) - far_file = None - if cache_dir is not None and cache_dir != "None": - os.makedirs(cache_dir, exist_ok=True) - far_file = os.path.join(cache_dir, f"jp_tn_{deterministic}_deterministic_verbalizer.far") - if not overwrite_cache and far_file and os.path.exists(far_file): - self.fst = pynini.Far(far_file, mode="r")["verbalize"] - else: - token_graph = VerbalizeFst(deterministic=deterministic) - - token_verbalizer = ( - pynutil.delete("tokens {") + delete_space + token_graph.fst + delete_space + pynutil.delete(" }") - ) - verbalizer = pynini.closure(delete_space + token_verbalizer + delete_space) - - postprocessor = PostProcessor( - remove_puncts=False, - to_upper=False, - to_lower=False, - tag_oov=False, - ) - - self.fst = (verbalizer @ postprocessor.fst).optimize() +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +import pynini +from pynini.lib import pynutil + +from nemo_text_processing.text_normalization.ja.graph_utils import GraphFst, delete_space, generator_main +from nemo_text_processing.text_normalization.ja.verbalizers.postprocessor import PostProcessor +from nemo_text_processing.text_normalization.ja.verbalizers.verbalize import VerbalizeFst +from nemo_text_processing.utils.logging import logger + + +class VerbalizeFinalFst(GraphFst): + """ """ + + def __init__(self, deterministic: bool = True, cache_dir: str = None, overwrite_cache: bool = False): + super().__init__(name="verbalize_final", kind="verbalize", deterministic=deterministic) + far_file = None + if cache_dir is not None and cache_dir != "None": + os.makedirs(cache_dir, exist_ok=True) + far_file = os.path.join(cache_dir, f"ja_tn_{deterministic}_deterministic_verbalizer.far") + if not overwrite_cache and far_file and os.path.exists(far_file): + self.fst = pynini.Far(far_file, mode="r")["verbalize"] + logger.info(f"VerbalizeFinalFst graph was restored from {far_file}.") + else: + token_graph = VerbalizeFst(deterministic=deterministic) + + token_verbalizer = ( + pynutil.delete("tokens {") + delete_space + token_graph.fst + delete_space + pynutil.delete(" }") + ) + verbalizer = pynini.closure(delete_space + token_verbalizer + delete_space) + + postprocessor = PostProcessor( + remove_puncts=False, + to_upper=False, + to_lower=False, + tag_oov=False, + ) + + self.fst = (verbalizer @ postprocessor.fst).optimize() + if far_file: + generator_main(far_file, {"verbalize": self.fst}) diff --git a/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py b/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py index 11b0b3ae0..44d694323 100644 --- a/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py +++ b/nemo_text_processing/text_normalization/ja/verbalizers/whitelist.py @@ -16,7 +16,7 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_CHAR, NEMO_SIGMA, GraphFst, delete_space +from nemo_text_processing.text_normalization.ja.graph_utils import NEMO_NOT_QUOTE, NEMO_SIGMA, GraphFst, delete_space class WhiteListFst(GraphFst): @@ -28,11 +28,11 @@ class WhiteListFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="whitelist", kind="verbalize", deterministic=deterministic) graph = ( - pynutil.delete("name:") - + delete_space - + pynutil.delete("\"") - + pynini.closure(NEMO_CHAR - " ", 1) - + pynutil.delete("\"") - ) + pynutil.delete("name:") + + delete_space + + pynutil.delete("\"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + ) graph = graph @ pynini.cdrewrite(pynini.cross(u"\u00a0", " "), "", "", NEMO_SIGMA) self.fst = graph.optimize() diff --git a/nemo_text_processing/text_normalization/normalize.py b/nemo_text_processing/text_normalization/normalize.py index d8ebf2f4d..c6a256eb5 100644 --- a/nemo_text_processing/text_normalization/normalize.py +++ b/nemo_text_processing/text_normalization/normalize.py @@ -177,7 +177,11 @@ def __init__( from nemo_text_processing.text_normalization.rw.verbalizers.verbalize_final import VerbalizeFinalFst elif lang == 'ja': from nemo_text_processing.text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst + from nemo_text_processing.text_normalization.ja.verbalizers.post_processing import PostProcessingFst from nemo_text_processing.text_normalization.ja.verbalizers.verbalize_final import VerbalizeFinalFst + + if post_process: + self.post_processor = PostProcessingFst(cache_dir=cache_dir, overwrite_cache=overwrite_cache) elif lang == 'vi': from nemo_text_processing.text_normalization.vi.taggers.tokenize_and_classify import ClassifyFst from nemo_text_processing.text_normalization.vi.verbalizers.post_processing import PostProcessingFst @@ -391,7 +395,11 @@ def normalize( return text output = SPACE_DUP.sub(' ', output[1:]) - if self.lang in ["en", "hi", "vi"] and hasattr(self, 'post_processor') and self.post_processor is not None: + if ( + self.lang in ["en", "hi", "ja", "vi"] + and hasattr(self, 'post_processor') + and self.post_processor is not None + ): output = self.post_process(output) if punct_post_process: diff --git a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_decimal.txt index d5080acd0..4f91cf83a 100644 --- a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_decimal.txt +++ b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_decimal.txt @@ -1,32 +1,32 @@ -マイナス一点零六~-1.06 -マイナス七点零零六~-7.006 -マイナス三十九点五七四~-39.574 -マイナス三点八六~-3.86 -マイナス九十二点一五七四~-92.1574 -マイナス九点零三八~-9.038 -マイナス二点八七四一~-2.8741 -マイナス二百三十一点四六零九~-231.4609 -マイナス五十二点一八~-52.18 -マイナス五点三~-5.3 -マイナス五百七十九点三零零二~-579.3002 -マイナス八十六点四~-86.4 -マイナス八点四零九~-8.409 -マイナス八百二十一点七九五四~-821.7954 -マイナス八百五十二点七~-852.7 -マイナス六十一点零七~-61.07 -マイナス六点八一四~-6.814 -マイナス六百五十七点三零二四~-657.3024 -マイナス四十二点六零五~-42.605 -マイナス四百八十九点零五二一~-489.0521 -答えはマイナス一点零六~答えは-1.06 -計算の結果はマイナス七点零零六~計算の結果は-7.006 -マイナス二点八七四はかなり悪いスコア~-2.874はかなり悪いスコア -五点三は平均点~5.3は平均点 -テストの点数は八十六点四~テストの点数は86.4 -マイナス三十九点五七四は低すぎる~-39.574は低すぎる -答えはマイナス一点零六~答えは-1.06 -計算の結果はマイナス八十六点四~計算の結果は-86.4 -マイナス五十二点一八はかなり悪いスコア~-52.18はかなり悪いスコア -六点八一四は平均点~6.814は平均点 -テストの点数は九十二点一五七四~テストの点数は92.1574 -マイナス七点零零六は低すぎる~-7.006は低すぎる +マイナス一点ゼロ六~-1.06 +マイナス七点ゼロゼロ六~-7.006 +マイナス三十九点五七四~-39.574 +マイナス三点八六~-3.86 +マイナス九十二点一五七四~-92.1574 +マイナス九点ゼロ三八~-9.038 +マイナス二点八七四一~-2.8741 +マイナス二百三十一点四六ゼロ九~-231.4609 +マイナス五十二点一八~-52.18 +マイナス五点三~-5.3 +マイナス五百七十九点三ゼロゼロ二~-579.3002 +マイナス八十六点四~-86.4 +マイナス八点四ゼロ九~-8.409 +マイナス八百二十一点七九五四~-821.7954 +マイナス八百五十二点七~-852.7 +マイナス六十一点ゼロ七~-61.07 +マイナス六点八一四~-6.814 +マイナス六百五十七点三ゼロ二四~-657.3024 +マイナス四十二点六ゼロ五~-42.605 +マイナス四百八十九点ゼロ五二一~-489.0521 +答えはマイナス一点ゼロ六~答えは-1.06 +計算の結果はマイナス七点ゼロゼロ六~計算の結果は-7.006 +マイナス二点八七四はかなり悪いスコア~-2.874はかなり悪いスコア +五点三は平均点~5.3は平均点 +テストの点数は八十六点四~テストの点数は86.4 +マイナス三十九点五七四は低すぎる~-39.574は低すぎる +答えはマイナス一点ゼロ六~答えは-1.06 +計算の結果はマイナス八十六点四~計算の結果は-86.4 +マイナス五十二点一八はかなり悪いスコア~-52.18はかなり悪いスコア +六点八一四は平均点~6.814は平均点 +テストの点数は九十二点一五七四~テストの点数は92.1574 +マイナス七点ゼロゼロ六は低すぎる~-7.006は低すぎる diff --git a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_fraction.txt index 32f80e812..d2abd62f6 100644 --- a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_fraction.txt +++ b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_fraction.txt @@ -1,34 +1,34 @@ -マイナス一と四分の三~-1 3/4 -一と四分の三~1 3/4 -マイナス一分の九~-9/1 -マイナス一分の六十~-60/1 -マイナス一分の百二十三~-123/1 -マイナス一荷四分の三~-1 3/4 -マイナス七百二十分の一~-1/720 -マイナス三十二分の三十一~-31/32 -マイナス三百九十七分の四~-4/397 -マイナス三百五十分の一~-1/350 -マイナス九十八分の四百七十一~-471/98 -マイナス二と五分の三~-2 3/5 -マイナス二十分の九~-9/20 -マイナス二十分の二十一~-21/20 -マイナス二十四分の一~-1/24 -マイナス二百二十分の一~-1/220 -マイナス二百五十二分の百四十七~-147/252 -マイナス二百五十六分の一~-1/256 -マイナス二荷五分の三~-2 3/5 -マイナス五分の七~-7/5 -マイナス五分の八~-8/5 -マイナス五分の十四~-14/5 -マイナス五分の百三十二~-132/5 -マイナス八分の五~-5/8 -答えはマイナス八分の五~答えは-5/8 -三分の一の人がその場を離れた~1/3の人がその場を離れた -約二分の一を削る~約1/2を削る -十分の三を削って吟醸をつくる~3/10を削って吟醸をつくる -一人三分の一ぐらい取る~1人1/3ぐらい取る -答えは九分の一~答えは1/9 -三分の二の人がその場を離れた~2/3の人がその場を離れた -約十分の一を削る~約1/10を削る -三分の一を削って吟醸をつくる~1/3を削って吟醸をつくる +マイナス一と四分の三~-1 3/4 +一と四分の三~1 3/4 +マイナス一分の九~-9/1 +マイナス一分の六十~-60/1 +マイナス一分の百二十三~-123/1 +マイナス一と四分の三~-1 3/4 +マイナス七百二十分の一~-1/720 +マイナス三十二分の三十一~-31/32 +マイナス三百九十七分の四~-4/397 +マイナス三百五十分の一~-1/350 +マイナス九十八分の四百七十一~-471/98 +マイナス二と五分の三~-2 3/5 +マイナス二十分の九~-9/20 +マイナス二十分の二十一~-21/20 +マイナス二十四分の一~-1/24 +マイナス二百二十分の一~-1/220 +マイナス二百五十二分の百四十七~-147/252 +マイナス二百五十六分の一~-1/256 +マイナス二と五分の三~-2 3/5 +マイナス五分の七~-7/5 +マイナス五分の八~-8/5 +マイナス五分の十四~-14/5 +マイナス五分の百三十二~-132/5 +マイナス八分の五~-5/8 +答えはマイナス八分の五~答えは-5/8 +三分の一の人がその場を離れた~1/3の人がその場を離れた +約二分の一を削る~約1/2を削る +十分の三を削って吟醸をつくる~3/10を削って吟醸をつくる +一人三分の一ぐらい取る~1人1/3ぐらい取る +答えは九分の一~答えは1/9 +三分の二の人がその場を離れた~2/3の人がその場を離れた +約十分の一を削る~約1/10を削る +三分の一を削って吟醸をつくる~1/3を削って吟醸をつくる 一人二分の一とぐらい取る~1人1/2とぐらい取る \ No newline at end of file diff --git a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_time.txt index 6a5082124..14d0ab42e 100644 --- a/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/ja/data_inverse_text_normalization/test_cases_time.txt @@ -1,40 +1,41 @@ -七時一分~7時1分 -七時四分~7時4分 -九時五十八分~9時58分 -九時十分前~9時10分前 -九時四十分~9時40分 -五時二十六分~5時26分 -六時五十五分~6時55分 -三時~3時 -三時~3時 -正午一分前~正午1分前 -正午十分過ぎ~正午10分過ぎ -九時三十分~9時30分 -七時五十分頃~7時50分頃 -一時~1時 -一時十分~1時10分 -三時~3時 -十七時~17時 -二十時~20時 -二十一時~21時 -二時~2時 -十二時三十分~12時30分 -零時~0時 -零時一分前~0時1分前 -二時~2時 -十二時~12時 -二十時~20時 -二十三時~23時 -二十四時~24時 -零時~0時 -四時~4時 -毎日五時に起きる~毎日5時に起きる -九時四十分の予約になります~9時40分の予約になります -現在の時間は十二時三十分~現在の時間は12時30分 -ちょうど零時になった~ちょうど0時になった -四時で店を閉める~4時で店を閉める -毎日六時に起きる~毎日6時に起きる -十時三十分の予約になります~10時30分の予約になります -現在の時間は十時三分~現在の時間は10時3分 -ちょうど一時になった~ちょうど1時になった -七時で店を閉める~7時で店を閉める +七時一分~7時1分 +七時四分~7時4分 +九時五十八分~9時58分 +九時十分前~9時10分前 +九時四十分~9時40分 +五時二十六分~5時26分 +六時五十五分~6時55分 +三時~3時 +三時~3時 +正午一分前~正午1分前 +正午十分過ぎ~正午10分過ぎ +九時三十分~9時30分 +七時五十分頃~7時50分頃 +一時~1時 +一時十分~1時10分 +三時~3時 +十七時~17時 +二十時~20時 +二十一時~21時 +二時~2時 +十二時三十分~12時30分 +ゼロ時~0時 +零時~0時 +ゼロ時一分前~0時1分前 +二時~2時 +十二時~12時 +二十時~20時 +二十三時~23時 +二十四時~24時 +ゼロ時~0時 +四時~4時 +毎日五時に起きる~毎日5時に起きる +九時四十分の予約になります~9時40分の予約になります +現在の時間は十二時三十分~現在の時間は12時30分 +ちょうどゼロ時になった~ちょうど0時になった +四時で店を閉める~4時で店を閉める +毎日六時に起きる~毎日6時に起きる +十時三十分の予約になります~10時30分の予約になります +現在の時間は十時三分~現在の時間は10時3分 +ちょうど一時になった~ちょうど1時になった +七時で店を閉める~7時で店を閉める diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_address.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_address.txt new file mode 100644 index 000000000..4e10ad910 --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_address.txt @@ -0,0 +1,9 @@ +東京都千代田区丸の内1-1-1~東京都千代田区丸の内一の一の一 +大阪府大阪市北区梅田3-1-1~大阪府大阪市北区梅田三の一の一 +神奈川県横浜市西区みなとみらい2-2-1~神奈川県横浜市西区みなとみらい二の二の一 +1丁目2番3号~一丁目二番三号 +3丁目5番1号~三丁目五番一号 +503号室~五〇三号室 +25階~二十五階 +〒100-0001~郵便番号一ゼロゼロのゼロゼロゼロ一 +住所は東京都港区六本木6-10-1です。~住所は東京都港区六本木六の十の一です。 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_cardinal.txt index 1d8a2801a..44a159017 100644 --- a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_cardinal.txt @@ -36,7 +36,7 @@ お年玉50000あげる~お年玉五万あげる 500000000000円分の株式を買った~五千億円分の株式を買った 今年の収益は100000000000になる~今年の収益は一千億になる -隣の会社の年収益は990000000000だそうだ~隣の会社の年収益は九千九百億だそうだ +隣の会社の収益は990000000000だそうだ~隣の会社の収益は九千九百億だそうだ 政府は100000000000の赤字で困っている~政府は一千億の赤字で困っている 兵士500人を派遣する~兵士五百人を派遣する お寺に10000寄付した~お寺に一万寄付した diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_date.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_date.txt index 6625ad4d0..b6eb392bd 100644 --- a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_date.txt +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_date.txt @@ -7,7 +7,6 @@ R.4~令和四年 1日から来年2月末まで~一日から来年二月末まで 1月〜12月~一月から十二月 1月の最終金曜日~一月の最終金曜日 -1月1日(月)〜3日(水)~一月一日月曜日から三日水曜日 1月22日~一月二十二日 70〜80年代~七十から八十年代 70年代~七十年代 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_decimal.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_decimal.txt index 622b61d1a..a1e6f8c13 100644 --- a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_decimal.txt +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_decimal.txt @@ -12,4 +12,4 @@ 今年の冬の平均気温は-4.2度でした。~今年の冬の平均気温はマイナス四点二度でした。 昨日の株価は-3.6ポイント下落しました。~昨日の株価はマイナス三点六ポイント下落しました。 彼の体重は-2.4キログラム減りました。~彼の体重はマイナス二点四キログラム減りました。 -その製品の評価は-1.5ポイントでした。~その製品の評価はマイナス一点五ポイントでした。 \ No newline at end of file +その製品の評価は-1.5ポイントでした。~その製品の評価はマイナス一点五ポイントでした。 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_electronic.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_electronic.txt new file mode 100644 index 000000000..086082b5f --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_electronic.txt @@ -0,0 +1,35 @@ +a@hotmail.de~a アット hotmail ドット de +a@hotmail.fr~a アット hotmail ドット fr +a@hotmail.it~a アット hotmail ドット it +a@aol.it~a アット aol ドット it +a@msn.it~a アット msn ドット it +abc@nvidia.app~abc アット nvidia ドット app +nvidia.co.jp~nvidia ドット co ドット jp +a.bc@gmail.com~a ドット bc アット gmail ドット com +cdf@abc.edu~cdf アット abc ドット edu +abc@gmail.abc~abc アット gmail ドット abc +abc@abc.com~abc アット abc ドット com +asdf123@abc.com~asdf 一二三 アット abc ドット com +ab3.sdd.3@gmail.com~ab 三 ドット sdd ドット 3 アット gmail ドット com +ab3-sdd-3@gmail.com~ab 三 ハイフン sdd ハイフン 3 アット gmail ドット com +www.nvidia.com~www ドット nvidia ドット com +nvidia.com~nvidia ドット com +google.co.jp~google ドット co ドット jp +www.google.co.jp~www ドット google ドット co ドット jp +nvidia.ai~nvidia ドット ai +http://www.nvidia.com~http コロン スラッシュ スラッシュ www ドット nvidia ドット com +https://www.nvidia.com~https コロン スラッシュ スラッシュ www ドット nvidia ドット com +https://developer.nvidia.com/drive-cuda/early-access~https コロン スラッシュ スラッシュ developer ドット nvidia ドット com スラッシュ drive ハイフン cuda スラッシュ early ハイフン access +abc@abc.com.~abc アット abc ドット com. +1234-5678-9012-3456~一二三四 五六七八 九〇一二 三四五六 +2345-2222-3333-4444~二三四五 二二二二 三三三三 四四四四 +9090-1234-5555-9876~九〇九〇 一二三四 五五五五 九八七六 +カード末尾3456~カード末尾三四五六 +カード下4桁7890~カード下四桁七八九〇 +カード番号1234-5678-9012-3456~カード番号一二三四 五六七八 九〇一二 三四五六 +写真.jpg~写真 ドット jpg +写真.JPG~写真 ドット JPG +写真.png~写真 ドット png +写真.PNG~写真 ドット PNG +資料.pdf~資料 ドット pdf +資料.PDF~資料 ドット PDF diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_fraction.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_fraction.txt index e2095fbfa..0d1ca7aa3 100644 --- a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_fraction.txt +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_fraction.txt @@ -1,18 +1,9 @@ 1/2~二分の一 -1/2~マイナス二分の一 -1 1/2~一荷二分の一 +1 1/2~一と二分の一 1と1/2~一と二分の一 -1荷1/2~一荷二分の一 --1荷1/2~マイナス一荷二分の一 -マイナス1荷1/2~マイナス一荷二分の一 -マイナス√1荷1/2~マイナスルート一荷二分の一 --√1荷1/2~マイナスルート一荷二分の一 --1荷√1/2~マイナス一荷二分のルート一 3分の1~三分の一 -3分の1~マイナス三分の一 -√3分の1~マイナスルート三分の一 -1荷√1/2~一荷二分のルート一 -1荷√1/3~一荷三分のルート一 1と1/3~一と三分の一 -1荷√1/4~一荷四分のルート一 1と√1/4~一と四分のルート一 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_measure.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_measure.txt new file mode 100644 index 000000000..6eab137ce --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_measure.txt @@ -0,0 +1,34 @@ +5kg~五キロ +0kg~ゼロキロ +1 kg~一キロ +2ｋｇ~二キロ +100g~百グラム +500mg~五百ミリグラム +-3kg~マイナス三キロ +1.5kg~一点五キロ +-2.5kg~マイナス二点五キロ +0.05m~零点零五メートル +2.5cm~二点五センチ +3.14m2~三点一四平方メートル +3.14m²~三点一四平方メートル +1/2L~二分の一リットル +-1/2L~マイナス二分の一リットル +3分の1m~三分の一メートル +60km/h~時速六十キロ +60 km / h~時速六十キロ +60kg/h~六十キロ毎時 +50m／s~秒速五十メートル +50ｍ／ｓ~秒速五十メートル +100%~百パーセント +100％~百パーセント +30°C~三十度 +5μg~五マイクログラム +250mL~二百五十ミリリットル +440Hz~四百四十ヘルツ +220V~二百二十ボルト +10GB~十ギガバイト +5kg.~五キロ. +重さ 5kgです。~重さ五キロです。 +この箱は5kgです。~この箱は五キロです。 +温度は30°C。~温度は三十度。 +容量は1.5L、重さは500gです。~容量は一点五リットル、重さは五百グラムです。 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_money.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_money.txt new file mode 100644 index 000000000..b930dda27 --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_money.txt @@ -0,0 +1,38 @@ +100円~百円 +1円~一円 +0円~ゼロ円 +10円~十円 +1,000円~千円 +12,345円~一万二千三百四十五円 +-500円~マイナス五百円 +マイナス500円~マイナス五百円 +¥100~百円 +￥2500~二千五百円 +JPY 3000~三千円 +jpy400~四百円 +¥3万~三万円 +1.5万円~一点五万円 +2億円~二億円 +3兆円~三兆円 +12円50銭~十二円五十銭 +¥12.50~十二円五十銭 +￥12.05~十二円五銭 +$5~五ドル +US$ 12~十二ドル +USD99~九十九ドル +usd 1.25~一ドル二十五セント +5ドル25セント~五ドル二十五セント +5米ドル25セント~五ドル二十五セント +$12.50~十二ドル五十セント +$0.99~ゼロドル九十九セント +-US$10.50~マイナス十ドル五十セント +€7~七ユーロ +EUR 8.40~八ユーロ四十セント +eur9~九ユーロ +£15~十五ポンド +GBP 20~二十ポンド +gbp30~三十ポンド +₩1000~千ウォン +KRW 5000~五千ウォン +CNY 10~十元 +100人民元~百人民元 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_range.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_range.txt new file mode 100644 index 000000000..3778b9993 --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_range.txt @@ -0,0 +1,18 @@ +2-5~二から五 +10〜20~十から二十 +10〜20人~十から二十人 +3〜5歳~三から五歳 +10-15分~十から十五分 +1-3日~一から三日 +1000-2000円~千から二千円 +5000〜8000円~五千から八千円 +3kg-6kg~三キロから六キロ +15cm-25cm~十五センチから二十五センチ +10:00-11:00~十時から十一時 +9:00-10:30~九時から十時三十分 +10%〜15%~十パーセントから十五パーセント +2x3~二かける三 +70〜80年代~七十から八十年代 +3〜4月~三から四月 +1960s-1980s~千九百六十年代から千九百八十年代 +平成20〜30年代~平成二十から三十年代 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_roman.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_roman.txt new file mode 100644 index 000000000..ad6617b6c --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_roman.txt @@ -0,0 +1,9 @@ +第III章~第三章 +第IV条~第四条 +第XII巻~第十二巻 +Chapter IV~Chapter 四 +第IX回~第九回 +Part III~Part 三 +第v条~第五条 +chapter ix~chapter 九 +Century XXI~Century 二十一 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_serial.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_serial.txt new file mode 100644 index 000000000..a5e89a2e9 --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_serial.txt @@ -0,0 +1,14 @@ +B2A23C~ビー 二 エー 二三 シー +C24~シー 二四 +W2s~ダブリュー 二 エス +2x~二 エックス +covid-19~コビッド 十九 +133-ABC~一三三 ハイフン エービーシー +MIG-25/235212-asdg~エムアイジー ハイフン 二五 スラッシュ 二三五二一二 ハイフン エーエスディージー +JL123~ジェーエル 一二三 +NH456~エヌエイチ 四五六 +CX-5~シーエックス ハイフン 五 +型番ABC-1234~型番 エービーシー ハイフン 一二三四 +〒100-0001~郵便番号一ゼロゼロのゼロゼロゼロ一 +ISBN978-4-123456-78-9~アイエスビーエヌ 九七八 ハイフン 四 ハイフン 一二三四五六 ハイフン 七八 ハイフン 九 +3号車~三号車 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_telephone.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_telephone.txt new file mode 100644 index 000000000..23e757cfb --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_telephone.txt @@ -0,0 +1,12 @@ +090-1234-5678~ゼロ九ゼロ、 一二三四、 五六七八 +080.1234.5678~ゼロ八ゼロ、 一二三四、 五六七八 +090－1234－5678~ゼロ九ゼロ、 一二三四、 五六七八 +03-1234-5678~ゼロ三、 一二三四、 五六七八 +(03) 1234-5678~ゼロ三、 一二三四、 五六七八 +045-123-4567~ゼロ四五、 一二三、 四五六七 +0120-123-456~ゼロ一二ゼロ、 一二三、 四五六 +050-1234-5678~ゼロ五ゼロ、 一二三四、 五六七八 ++81 90-1234-5678~プラス八一、 九ゼロ、 一二三四、 五六七八 ++81-90-1234-5678~プラス八一、 九ゼロ、 一二三四、 五六七八 +090-1234-5678.~ゼロ九ゼロ、 一二三四、 五六七八. +電話番号は090-1234-5678です。~電話番号はゼロ九ゼロ、 一二三四、 五六七八です。 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_telephone_extended.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_telephone_extended.txt new file mode 100644 index 000000000..b16083126 --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_telephone_extended.txt @@ -0,0 +1,18 @@ +09012345678~ゼロ九ゼロ一二三四五六七八 +090 1234 5678~ゼロ九ゼロ、 一二三四、 五六七八 +(090) 1234-5678~ゼロ九ゼロ、 一二三四、 五六七八 +0570-123-456~ゼロ五七ゼロ、 一二三、 四五六 ++81 (3) 1234-5678~プラス八一、 三、 一二三四、 五六七八 ++1 (415) 555-0123~プラス一、 四一五、 五五五、 ゼロ一二三 +0800-123-4567~ゼロ八ゼロゼロ、 一二三、 四五六七 +0120-1234-567~ゼロ一二ゼロ、 一二三四、 五六七 +082-123-4567~ゼロ八二、 一二三、 四五六七 +0466-12-3456~ゼロ四六六、 一二、 三四五六 ++81 82-123-4567~プラス八一、 八二、 一二三、 四五六七 ++81 (45) 123-4567~プラス八一、 四五、 一二三、 四五六七 ++44 20-7946-0958~プラス四四、 二ゼロ、 七九四六、 ゼロ九五八 ++84 (28) 3822-9999~プラス八四、 二八、 三八二二、 九九九九 +111-222-3333~一一一、 二二二、 三三三三 +03-1234-5678 内線123~ゼロ三、 一二三四、 五六七八、 内線 一二三 +090-1234-5678に電話してください。~ゼロ九ゼロ、 一二三四、 五六七八に電話してください。 +0120-123-456に電話~ゼロ一二ゼロ、 一二三、 四五六に電話 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_time.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_time.txt index 42e7600a7..ca805d214 100644 --- a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_time.txt +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_time.txt @@ -199,4 +199,4 @@ 翌日27時~翌日二十七時 翌日25時~翌日二十五時 翌日26時~翌日二十六時 -0時~零時 \ No newline at end of file +0時~零時 diff --git a/tests/nemo_text_processing/ja/data_text_normalization/test_cases_whitelist.txt b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_whitelist.txt new file mode 100644 index 000000000..1f637ed7e --- /dev/null +++ b/tests/nemo_text_processing/ja/data_text_normalization/test_cases_whitelist.txt @@ -0,0 +1,14 @@ +Dr.田中~ドクター田中 +Mr.山田~ミスター山田 +Mrs.佐藤~ミセス佐藤 +Ms.鈴木~ミス鈴木 +Prof. Smith~プロフェッサー Smith +vs.~バーサス +No.3~ナンバー三 +etc.~エトセトラ +Ph.D.~p h d +St.~ストリート +Mt.~マウント +Ave.~アベニュー +jr.~ジュニア +Sr.~シニア diff --git a/tests/nemo_text_processing/ja/test_address.py b/tests/nemo_text_processing/ja/test_address.py new file mode 100644 index 000000000..535791054 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_address.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestAddress: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_address.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_address(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_date.py b/tests/nemo_text_processing/ja/test_date.py index cd7127698..d6125b842 100644 --- a/tests/nemo_text_processing/ja/test_date.py +++ b/tests/nemo_text_processing/ja/test_date.py @@ -37,6 +37,6 @@ def test_denorm(self, test_input, expected): @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_date.txt')) @pytest.mark.run_only_on('CPU') @pytest.mark.unit - def test_denorm(self, test_input, expected): + def test_norm(self, test_input, expected): pred = self.normalizer.normalize(test_input, verbose=False) assert pred == expected diff --git a/tests/nemo_text_processing/ja/test_electronic.py b/tests/nemo_text_processing/ja/test_electronic.py new file mode 100644 index 000000000..a2665c1ae --- /dev/null +++ b/tests/nemo_text_processing/ja/test_electronic.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestElectronic: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_electronic.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_electronic(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_measure.py b/tests/nemo_text_processing/ja/test_measure.py new file mode 100644 index 000000000..dc2489427 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_measure.py @@ -0,0 +1,32 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMeasure: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_measure.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_measure(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_money.py b/tests/nemo_text_processing/ja/test_money.py new file mode 100644 index 000000000..df4c04d6a --- /dev/null +++ b/tests/nemo_text_processing/ja/test_money.py @@ -0,0 +1,32 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMoney: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_money.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_money(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_range.py b/tests/nemo_text_processing/ja/test_range.py new file mode 100644 index 000000000..5b24746d5 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_range.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestRange: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_range.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_range(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_roman.py b/tests/nemo_text_processing/ja/test_roman.py new file mode 100644 index 000000000..289625c38 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_roman.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestRoman: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_roman.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_roman(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_serial.py b/tests/nemo_text_processing/ja/test_serial.py new file mode 100644 index 000000000..f991251dd --- /dev/null +++ b/tests/nemo_text_processing/ja/test_serial.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestSerial: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_serial.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_serial(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/ja/test_sparrowhawk_normalization.sh index 42db11fd6..e8f676337 100644 --- a/tests/nemo_text_processing/ja/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/ja/test_sparrowhawk_normalization.sh @@ -46,6 +46,42 @@ testTNDateText() { input=$PROJECT_DIR/ja/data_text_normalization/test_cases_date.txt runtest $input } +testTNMoneyText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_money.txt + runtest $input +} +testTNMeasureText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_measure.txt + runtest $input +} +testTNTelephoneText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_telephone.txt + runtest $input +} +testTNElectronicText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_electronic.txt + runtest $input +} +testTNSerialText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_serial.txt + runtest $input +} +testTNRangeText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_range.txt + runtest $input +} +testTNAddressText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_address.txt + runtest $input +} +testTNRomanText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_roman.txt + runtest $input +} +testTNWhitelistText() { + input=$PROJECT_DIR/ja/data_text_normalization/test_cases_whitelist.txt + runtest $input +} # Load shUnit2 diff --git a/tests/nemo_text_processing/ja/test_telephone.py b/tests/nemo_text_processing/ja/test_telephone.py new file mode 100644 index 000000000..4fcd8e116 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_telephone.py @@ -0,0 +1,32 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTelephone: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_telephone.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_telephone(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_telephone_extended.py b/tests/nemo_text_processing/ja/test_telephone_extended.py new file mode 100644 index 000000000..8e950da8f --- /dev/null +++ b/tests/nemo_text_processing/ja/test_telephone_extended.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestTelephoneExtended: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_telephone_extended.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_telephone_extended(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds diff --git a/tests/nemo_text_processing/ja/test_whitelist.py b/tests/nemo_text_processing/ja/test_whitelist.py new file mode 100644 index 000000000..ece7bbe63 --- /dev/null +++ b/tests/nemo_text_processing/ja/test_whitelist.py @@ -0,0 +1,31 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestWhitelist: + normalizer_ja = Normalizer(lang='ja', cache_dir=CACHE_DIR, overwrite_cache=False, input_case='cased') + + @parameterized.expand(parse_test_case_file('ja/data_text_normalization/test_cases_whitelist.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm_whitelist(self, test_input, expected): + preds = self.normalizer_ja.normalize(test_input) + assert expected == preds