Jp itn

Jenkinsfile

@@ -22,8 +22,10 @@ pipeline {
     RU_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
     VI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
     SV_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
-    ZH_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/07-27-23-0'
+    ZH_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/02-16-24-0'
     IT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-26-23-0'
+    JP_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/09-11-23-0'
+    JP_ITN_CACHE='home/jenkinsci/TestData/text_norm/ci/grammars/02-20-24-0'
     DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
   }
   stages {
@@ -200,7 +202,28 @@ pipeline {
 
       }
     }
-    stage('L0: Create VI ITN & HU TN & IT TN') {
+    stage('L0: Create HU TN/ITN Grammars') {
+      when {
+        anyOf {
+          branch 'main'
+          changeRequest target: 'main'
+        }
+      }
+      failFast true
+      parallel {
+        stage('L0: HU TN grammars') {
+         steps {
+            sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=hu --text="100" --cache_dir ${HU_TN_CACHE}'
+          }
+        }
+        stage('L0: HU ITN grammars') {
+          steps {
+            sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=hu --text="száz " --cache_dir ${HU_TN_CACHE}'
+          }
+        }
+      }
+    }
+    stage('L0: Create VI TN/ITN Grammars') {
       when {
         anyOf {
           branch 'main'
@@ -291,6 +314,28 @@ pipeline {
       //  }
       }
     }
+    stage('L1: Create JP TN/ITN Grammars') {
+      when {
+        anyOf {
+          branch 'main'
+          changeRequest target: 'main'
+        }
+      }
+      failFast true
+      parallel {
+        stage('L1: JP ITN grammars') {
+         steps {
+            sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/inverse_text_normalization/inverse_normalize.py --lang=jp --text="百" --cache_dir ${JP_TN_CACHE}'
+          }
+        }
+      //  stage('L1: JP TN grammars') {
+      //    steps {
+      //      sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/inverse_normalize.py --lang=jp --text="100" --cache_dir ${JP_TN_CACHE}'
+      //    }
+      //  }
+      }
+    }
+
     stage('L0: Create ZH TN/ITN Grammars') {
       when {
         anyOf {
@@ -313,6 +358,9 @@ pipeline {
       }
     }
 
+
+
+
 // L1 Tests starts here
     stage('L1: TN/ITN Tests CPU') {
       when {

nemo_text_processing/inverse_text_normalization/fr/graph_utils.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import logging
 import os
 import string
 from pathlib import Path
@@ -81,7 +82,11 @@ def generator_main(file_name: str, graphs: Dict[str, pynini.FstLike]):
     for rule, graph in graphs.items():
         exporter[rule] = graph.optimize()
     exporter.close()
+<<<<<<< HEAD
+    logging.info(f'Created {file_name}')
+=======
     logger.info(f'Created {file_name}')
+>>>>>>> 42c0071bbeb3141ba013d3965693bb100c06a8e6
 
 
 def get_plurals(fst):

nemo_text_processing/inverse_text_normalization/inverse_normalize.py

@@ -117,6 +117,12 @@ def __init__(
                 VerbalizeFinalFst,
             )
 
+        elif lang == 'ja':  # Japanese
+            from nemo_text_processing.inverse_text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst
+            from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize_final import (
+                VerbalizeFinalFst,
+            )
+
         self.tagger = ClassifyFst(
             cache_dir=cache_dir, whitelist=whitelist, overwrite_cache=overwrite_cache, input_case=input_case
         )
@@ -160,7 +166,7 @@ def parse_args():
     parser.add_argument(
         "--language",
         help="language",
-        choices=['en', 'de', 'es', 'pt', 'ru', 'fr', 'sv', 'vi', 'ar', 'es_en', 'zh', 'hy'],
+        choices=['en', 'de', 'es', 'pt', 'ru', 'fr', 'sv', 'vi', 'ar', 'es_en', 'zh'],
         default="en",
         type=str,
     )

nemo_text_processing/inverse_text_normalization/ja/__init__.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nemo_text_processing.inverse_text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst
+from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize import VerbalizeFst
+from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize_final import VerbalizeFinalFst

nemo_text_processing/inverse_text_normalization/ja/data/__init__.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nemo_text_processing.inverse_text_normalization.ja.taggers.tokenize_and_classify import ClassifyFst
+from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize import VerbalizeFst
+from nemo_text_processing.inverse_text_normalization.ja.verbalizers.verbalize_final import VerbalizeFinalFst

nemo_text_processing/inverse_text_normalization/ja/data/char/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/inverse_text_normalization/ja/data/char/charset_extension.tsv

@@ -0,0 +1,6 @@
+吶
+诶
+屌
+囧
+飚
+屄