NVIDIA · fayejf · Jun 30, 2023 · Feb 8, 2023 · Feb 9, 2023 · Feb 9, 2023
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -22,7 +22,7 @@ pipeline {
     RU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
     VI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
     SV_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
-    ZH_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
+    ZH_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-29-23-0'
     DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
 
   }
@@ -319,11 +319,11 @@ pipeline {
             sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=zh --text="你" --cache_dir ${ZH_TN_CACHE}'
           }
         }
-       // stage('L0: ZH ITN grammars') {
-       //   steps {
-       //     sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=zh --text="二零零二年一月二十八日 " --cache_dir ${ZH_TN_CACHE}'
-       //   }
-       // }
+        stage('L0: ZH ITN grammars') {
+          steps {
+            sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=zh --text="二零零二年一月二十八日 " --cache_dir ${ZH_TN_CACHE}'
+          }
+        }
       }
     }
 

diff --git a/nemo_text_processing/inverse_text_normalization/inverse_normalize.py b/nemo_text_processing/inverse_text_normalization/inverse_normalize.py
@@ -106,6 +106,11 @@ def __init__(
             from nemo_text_processing.inverse_text_normalization.es_en.verbalizers.verbalize_final import (
                 VerbalizeFinalFst,
             )
+        elif lang == 'zh':  # Mandarin
+            from nemo_text_processing.inverse_text_normalization.zh.taggers.tokenize_and_classify import ClassifyFst
+            from nemo_text_processing.inverse_text_normalization.zh.verbalizers.verbalize_final import (
+                VerbalizeFinalFst,
+            )
 
         self.tagger = ClassifyFst(
             cache_dir=cache_dir, whitelist=whitelist, overwrite_cache=overwrite_cache, input_case=input_case
@@ -150,7 +155,7 @@ def parse_args():
     parser.add_argument(
         "--language",
         help="language",
-        choices=['en', 'de', 'es', 'pt', 'ru', 'fr', 'sv', 'vi', 'ar', 'es_en'],
+        choices=['en', 'de', 'es', 'pt', 'ru', 'fr', 'vi', 'ar', 'es_en', 'zh'],
         default="en",
         type=str,
     )

diff --git a/nemo_text_processing/inverse_text_normalization/zh/__init__.py b/nemo_text_processing/inverse_text_normalization/zh/__init__.py
@@ -0,0 +1,17 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nemo_text_processing.inverse_text_normalization.zh.taggers.tokenize_and_classify import ClassifyFst
+from nemo_text_processing.inverse_text_normalization.zh.verbalizers.verbalize import VerbalizeFst
+from nemo_text_processing.inverse_text_normalization.zh.verbalizers.verbalize_final import VerbalizeFinalFst
diff --git a/nemo_text_processing/inverse_text_normalization/zh/data/__init__.py b/nemo_text_processing/inverse_text_normalization/zh/data/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nemo_text_processing/inverse_text_normalization/zh/data/date/__init__.py b/nemo_text_processing/inverse_text_normalization/zh/data/date/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nemo_text_processing/inverse_text_normalization/zh/data/date/day.tsv b/nemo_text_processing/inverse_text_normalization/zh/data/date/day.tsv
@@ -0,0 +1,74 @@
+一	1
+二	2
+三	3
+四	4
+五	5
+六	6
+七	7
+八	8
+九	9
+十	10
+十一	11
+十二	12
+十三	13
+十四	14
+十五	15
+十六	16
+十七	17
+十八	18
+十九	19
+二十	20
+二十一	21
+二十二	22
+二十三	23
+二十四	24
+二十五	25
+二十六	26
+二十七	27
+二十八	28
+二十九	29
+三十	30
+三十一	31
+壹	1
+貳	2
+參	3
+肆	4
+伍	5
+陸	6
+柒	7
+捌	8
+玖	9
+幺	1
+两	2
+兩	2
+拾	10
+拾壹	11
+拾貳	12
+拾叁	13
+拾肆	14
+拾伍	15
+拾陸	16
+拾柒	17
+拾捌	18
+拾玖	19
+貳拾	20
+貳拾壹	21
+貳拾貳	22
+貳拾叁	23
+貳拾肆	24
+貳拾伍	25
+貳拾陸	26
+貳拾柒	27
+貳拾捌	28
+貳拾玖	29
+叁拾	30
+叁拾壹	31
+壹	1
+拾壹	11
+贰拾壹	21
+贰	2
+陆	6
+拾贰	12
+拾陆	16
+贰拾贰  22
+贰拾陆  26
diff --git a/nemo_text_processing/inverse_text_normalization/zh/data/date/months.tsv b/nemo_text_processing/inverse_text_normalization/zh/data/date/months.tsv
@@ -0,0 +1,49 @@
+一	1
+二	2
+三	3
+四	4
+五	5
+六	6
+七	7
+八	8
+九	9
+十	10
+十一	11
+十二	12
+一十	10
+零一	1
+零二	2
+零三	3
+零四	4
+零五	5
+零六	6
+零七	7
+零八	8
+零九	9
+壹	1
+贰	2
+叁	3
+肆	4
+伍	5
+陆	6
+柒	7
+捌	8
+玖	9
+拾	10
+拾壹	11
+拾贰	12
+壹拾	10
+零壹	1
+零贰	2
+零叁	3
+零肆	4
+零伍	5
+零陆	6
+零柒	7
+零捌	8
+零玖	9
+貳	2
+零貳	2
+陸	6
+零陸	6
+拾貳	12
diff --git a/nemo_text_processing/inverse_text_normalization/zh/data/money/__init__.py b/nemo_text_processing/inverse_text_normalization/zh/data/money/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nemo_text_processing/inverse_text_normalization/zh/data/money/currency_major.tsv b/nemo_text_processing/inverse_text_normalization/zh/data/money/currency_major.tsv
@@ -0,0 +1,75 @@
+美元	US$
+欧元	€
+歐元	€
+英镑	£
+英鎊	£
+加拿大元	CAD$
+加拿大币	CAD$
+加拿大幣	CAD$
+加元	CAD$
+加币	CAD$
+加幣	CAD$
+瑞士法郎	Fr
+法郎	₣
+圆	¥
+圓	¥
+瑞典克朗	Kr
+墨西哥比索	NXN$
+新西兰元	NZD$
+新西蘭元	NZD$
+新加坡币	SGD$
+新加坡幣	SGD$
+新加坡元	SGD$
+港元	HKD$
+港币	HKD$
+港幣	HKD$
+挪威克朗	NOKkr
+韩元	₩
+韓元	₩
+韩币	₩
+韓幣	₩
+土耳其里拉	TRY₺
+印度卢布	₹
+印度盧布	₹
+印度卢比	₹
+印度盧比	₹
+俄罗斯卢布	₽
+俄羅斯盧布	₽
+俄罗斯卢比	₽
+俄羅斯盧比	₽
+巴西雷亚尔	BRLR$
+巴西雷亞爾	BRLR$
+南非兰特	R
+南非蘭特	R
+丹麦克朗	DKKkr
+丹麥克朗	DKKkr
+波兰兹罗提	zł
+波蘭兹儸提	zł
+新台币	TWDNT$
+新臺幣	TWDNT$
+泰铢	฿
+泰銖	฿
+马来西亚林吉特	RM
+馬來西亞林吉特	RM
+印尼盾	Rp
+匈牙利福林	Ft
+捷克克朗	Kč
+以色列新谢克尔	₪  
+以色列新謝克爾	₪
+智利披索	CLP$
+菲律宾披索	₱
+菲律賓披索	₱
+阿联酋迪拉姆	د.إ
+阿聯酋迪拉姆	د.إ
+哥伦比亚披索	COL$
+哥倫比亞披索	COL$
+马来西亚令吉	RM
+馬來西亞令吉	RM
+罗马尼亚列伊	L
+羅馬尼亞列伊	L
+日元	JPY¥
+日圆	JPY¥
+日圓	JPY¥
+人民币	¥
+人民幣	¥
+元	¥
diff --git a/nemo_text_processing/inverse_text_normalization/zh/data/money/currency_minor.tsv b/nemo_text_processing/inverse_text_normalization/zh/data/money/currency_minor.tsv
@@ -0,0 +1,9 @@
+美分	US$
+欧分	€
+便士	£
+加拿大分	CAD$
+生丁	₣
+瑞典欧尔	KrOre
+分	NXN$
+新西兰仙	NZD$
+挪威欧尔	NOKOre
diff --git a/nemo_text_processing/inverse_text_normalization/zh/data/money/currency_rmb_minor_cent.tsv b/nemo_text_processing/inverse_text_normalization/zh/data/money/currency_rmb_minor_cent.tsv
@@ -0,0 +1 @@
+分	￥
diff --git a/nemo_text_processing/inverse_text_normalization/zh/data/money/currency_rmb_minor_tencent.tsv b/nemo_text_processing/inverse_text_normalization/zh/data/money/currency_rmb_minor_tencent.tsv
@@ -0,0 +1,2 @@
+毛	￥
+角	￥
diff --git a/nemo_text_processing/inverse_text_normalization/zh/data/numbers/__init__.py b/nemo_text_processing/inverse_text_normalization/zh/data/numbers/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/nemo_text_processing/inverse_text_normalization/zh/data/numbers/digit-nano.tsv b/nemo_text_processing/inverse_text_normalization/zh/data/numbers/digit-nano.tsv
@@ -0,0 +1,22 @@
+一	1
+二	2
+三	3
+四	4
+五	5
+六	6
+七	7
+八	8
+九	9
+壹	1
+贰	2
+叁	3
+肆	4
+伍	5
+陆	6
+柒	7
+捌	8
+玖	9
+貳	2
+陸	6
+两	2
+兩	2
diff --git a/nemo_text_processing/inverse_text_normalization/zh/data/numbers/ties-nano.tsv b/nemo_text_processing/inverse_text_normalization/zh/data/numbers/ties-nano.tsv
@@ -0,0 +1,18 @@
+二十	2
+三十	3
+四十	4
+五十	5
+六十	6
+七十	7
+八十	8
+九十	9
+贰拾	2
+叁拾	3
+肆拾	4
+伍拾	5
+陆拾	6
+柒拾	7
+捌拾	8
+玖拾	9
+貳拾	2
+陸拾	6
diff --git a/nemo_text_processing/inverse_text_normalization/zh/data/numbers/zero.tsv b/nemo_text_processing/inverse_text_normalization/zh/data/numbers/zero.tsv
@@ -0,0 +1 @@
+零	0
diff --git a/nemo_text_processing/inverse_text_normalization/zh/data/time/__init__.py b/nemo_text_processing/inverse_text_normalization/zh/data/time/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
-Original file line number
+Diff line change
@@ -0,0 +1,22 @@
+    一	1
+    二	2
+    三	3
+    四	4
+    五	5
+    六	6
+    七	7
+    八	8
+    九	9
+    壹	1
+    贰	2
+    叁	3
+    肆	4
+    伍	5
+    陆	6
+    柒	7
+    捌	8
+    玖	9
+    貳	2
+    陸	6
+    两	2
+    兩	2