NVIDIA · ekmb · Apr 30, 2024 · May 11, 2023 · May 17, 2023 · May 27, 2023
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -22,7 +22,7 @@ pipeline {
     RU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
     VI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
     SV_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
-    ZH_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/07-27-23-0'
+    ZH_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/04-30-24-0'
     IT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-26-23-0'
     HY_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-0'
     MR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-1'
@@ -189,7 +189,7 @@ pipeline {
       }
     }
 
-    stage('L0: Create RU TN/ITN Grammars & SV & PT & ZH') {
+    stage('L0: Create RU TN/ITN Grammars & SV & PT') {
       when {
         anyOf {
           branch 'main'
@@ -228,16 +228,6 @@ pipeline {
             sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=pt --text="dez " --cache_dir ${PT_TN_CACHE}'
           }
         }
-        stage('L0: ZH TN grammars') {
-         steps {
-            sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=zh --text="你" --cache_dir ${ZH_TN_CACHE}'
-          }
-        }
-        stage('L0: ZH ITN grammars') {
-          steps {
-            sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=zh --text="二零零二年一月二十八日 " --cache_dir ${ZH_TN_CACHE}'
-          }
-        }
       }
     }
 
@@ -267,9 +257,31 @@ pipeline {
         }
       }
     }
+    stage('L0: Create ZH TN/ITN Grammar') {
+      when {
+        anyOf {
+          branch 'main'
+          changeRequest target: 'main'
+        }
+      }
+      failFast true
+      parallel {
+        stage('L0: ZH ITN grammars') {
+          steps {
+            sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=zh --text="你" --cache_dir ${ZH_TN_CACHE}'
+          }
+        }
+        stage('L0: ZH TN grammars') {
+          steps {
+            sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=zh --text="6" --cache_dir ${ZH_TN_CACHE}'
+          }
+        }
+      }
+    }
 
 
 // L1 Tests starts here
+
     stage('L1: TN/ITN Tests CPU') {
       when {
         anyOf {

diff --git a/nemo_text_processing/inverse_text_normalization/zh/graph_utils.py b/nemo_text_processing/inverse_text_normalization/zh/graph_utils.py
@@ -22,6 +22,8 @@
 from pynini.export import export
 from pynini.lib import byte, pynutil, utf8
 
+from nemo_text_processing.inverse_text_normalization.zh.utils import load_labels
+
 NEMO_CHAR = utf8.VALID_UTF8_CHAR
 NEMO_DIGIT = byte.DIGIT
 NEMO_HEX = pynini.union(*string.hexdigits).optimize()

diff --git a/nemo_text_processing/inverse_text_normalization/zh/utils.py b/nemo_text_processing/inverse_text_normalization/zh/utils.py
@@ -60,3 +60,17 @@ def get_various_formats(text: str) -> List[str]:
         result.append(t.upper())
         result.append(t.capitalize())
     return result
+
+
+def load_labels(abs_path):
+    """
+    loads relative path file as dictionary
+
+    Args:
+        abs_path: absolute path
+
+    Returns dictionary of mappings
+    """
+    with open(abs_path, encoding="utf-8") as label_tsv:
+        labels = list(csv.reader(label_tsv, delimiter="\t"))
+    return labels
diff --git a/nemo_text_processing/text_normalization/zh/data/char/punctuations_zh.tsv b/nemo_text_processing/text_normalization/zh/data/char/punctuations_zh.tsv
@@ -70,3 +70,5 @@
 …
 ‧
 ﹏
+<
+>
diff --git a/nemo_text_processing/text_normalization/zh/data/math/symbol.tsv b/nemo_text_processing/text_normalization/zh/data/math/symbol.tsv
@@ -5,3 +5,4 @@
 ×	乘
 ÷	除
 °	度
+-	减
diff --git a/nemo_text_processing/text_normalization/zh/data/measure/units_en.tsv b/nemo_text_processing/text_normalization/zh/data/measure/units_en.tsv
@@ -1,7 +1,5 @@
 amu	原子质量
 bar	巴
-°	度
-º	度
 °c	摄氏度
 °C	摄氏度
 ºc	摄氏度
@@ -40,23 +38,6 @@ kw	千瓦
 kW	千瓦
 lb	磅
 lbs	磅
-m2	平方米
-m²	平方米
-m3	立方米
-m³	立方米
-mbps	兆比特每秒
-mg	毫克
-mhz	兆赫兹
-mi2	平方英里
-mi²	平方英里
-mi	英里
-min	分钟哦
-ml	毫升
-mm2	平方毫米
-mm²	平方毫米
-mol	摩尔
-mpa	兆帕
-mph	英里每小时
 ng	纳克
 nm	纳米
 ns	纳秒
@@ -80,13 +61,7 @@ gb	吉字节
 gpa	吉帕斯卡
 gy	戈瑞
 ha	公顷
-m	米
-mm	毫米
-ms	毫秒
-mv	毫伏
-mw	毫瓦
 pg	皮克
 ps	皮秒
 s	秒
-ms	毫秒
 g	克
diff --git a/nemo_text_processing/text_normalization/zh/data/measure/units_zh.tsv b/nemo_text_processing/text_normalization/zh/data/measure/units_zh.tsv
diff --git a/nemo_text_processing/text_normalization/zh/data/money/currency_major.tsv b/nemo_text_processing/text_normalization/zh/data/money/currency_major.tsv
@@ -168,7 +168,6 @@ Ft	匈牙利福林
 ₪	以色列谢克尔
 J$	牙买加元
 лв	哈萨克斯坦腾格
-₩	朝鲜园
 лв	吉尔吉斯斯坦索姆
 ₭	老挝基普
 ден	马其顿代纳尔

diff --git a/nemo_text_processing/text_normalization/zh/data/number/digit_alt.tsv b/nemo_text_processing/text_normalization/zh/data/number/digit_alt.tsv
@@ -0,0 +1,9 @@
+1	一
+2	两
+3	三
+4	四
+5	五
+6	六
+7	七
+8	八
+9	九
-Original file line number
+Diff line change
@@ Expand Up / @@ -70,3 +70,5 @@ @@
     …
     ‧
     ﹏
+    <
+    >