From c45622dfebdc3aeb5e8a4d0e3dc79b849245a019 Mon Sep 17 00:00:00 2001 From: Matvei Novikov Date: Thu, 3 Nov 2022 16:10:01 +0400 Subject: [PATCH 1/3] fixes Signed-off-by: Matvei Novikov --- .../data/get_libritts_data.py | 64 ++- ...ion_and_Capitalization_Lexical_Audio.ipynb | 363 +++++++++++++++++- 2 files changed, 405 insertions(+), 22 deletions(-) diff --git a/examples/nlp/token_classification/data/get_libritts_data.py b/examples/nlp/token_classification/data/get_libritts_data.py index 6b83b1fbaea2..8e08cc5a8665 100644 --- a/examples/nlp/token_classification/data/get_libritts_data.py +++ b/examples/nlp/token_classification/data/get_libritts_data.py @@ -21,10 +21,10 @@ import os import re import shutil +import string import subprocess import tarfile -from examples.nlp.token_classification.data.get_tatoeba_data import create_text_and_labels from tqdm import tqdm from nemo.utils import logging @@ -40,6 +40,68 @@ } +def remove_punctuation(word: str): + """ + Removes all punctuation marks from a word except for ' + that is often a part of word: don't, it's, and so on + """ + all_punct_marks = string.punctuation.replace("'", '') + return re.sub('[' + all_punct_marks + ']', '', word) + + +def create_text_and_labels(output_dir: str, file_path: str, punct_marks: str = ',.?'): + """ + Create datasets for training and evaluation. + + Args: + output_dir: path to the output data directory + file_path: path to file name + punct_marks: supported punctuation marks + + The data will be split into 2 files: text.txt and labels.txt. \ + Each line of the text.txt file contains text sequences, where words\ + are separated with spaces. The labels.txt file contains \ + corresponding labels for each word in text.txt, the labels are \ + separated with spaces. Each line of the files should follow the \ + format: \ + [WORD] [SPACE] [WORD] [SPACE] [WORD] (for text.txt) and \ + [LABEL] [SPACE] [LABEL] [SPACE] [LABEL] (for labels.txt).' + """ + if not os.path.exists(file_path): + raise ValueError(f'{file_path} not found') + + os.makedirs(output_dir, exist_ok=True) + + base_name = os.path.basename(file_path) + labels_file = os.path.join(output_dir, 'labels_' + base_name) + text_file = os.path.join(output_dir, 'text_' + base_name) + + with open(file_path, 'r') as f: + with open(text_file, 'w') as text_f: + with open(labels_file, 'w') as labels_f: + for line in f: + line = line.split() + text = '' + labels = '' + for word in line: + label = word[-1] if word[-1] in punct_marks else 'O' + word = remove_punctuation(word) + if len(word) > 0: + if word[0].isupper(): + label += 'U' + else: + label += 'O' + + word = word.lower() + text += word + ' ' + labels += label + ' ' + + text_f.write(text.strip() + '\n') + labels_f.write(labels.strip() + '\n') + + print(f'{text_file} and {labels_file} created from {file_path}.') + + def __extract_file(filepath, data_dir): try: tar = tarfile.open(filepath) diff --git a/tutorials/nlp/Punctuation_and_Capitalization_Lexical_Audio.ipynb b/tutorials/nlp/Punctuation_and_Capitalization_Lexical_Audio.ipynb index fb544c24e0a2..784f7efe94ae 100644 --- a/tutorials/nlp/Punctuation_and_Capitalization_Lexical_Audio.ipynb +++ b/tutorials/nlp/Punctuation_and_Capitalization_Lexical_Audio.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": { "pycharm": { "name": "#%%\n" @@ -15,13 +15,303 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "pycharm": { "name": "#%%\n" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting nemo_toolkit[all]\n", + " Cloning https://github.com/NVIDIA/NeMo.git (to revision r1.13.0) to /tmp/pip-install-m40oh75q/nemo-toolkit_dc4ac2f47f55476a85af4d6a94a9e037\n", + " Running command git clone --filter=blob:none --quiet https://github.com/NVIDIA/NeMo.git /tmp/pip-install-m40oh75q/nemo-toolkit_dc4ac2f47f55476a85af4d6a94a9e037\n", + " Running command git checkout -b r1.13.0 --track origin/r1.13.0\n", + " Switched to a new branch 'r1.13.0'\n", + " Branch 'r1.13.0' set up to track remote branch 'r1.13.0' from 'origin'.\n", + " Resolved https://github.com/NVIDIA/NeMo.git to commit 26e3e1d1931a9f34986f73f071f82f446d90c1c5\n", + " Preparing metadata (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25hRequirement already satisfied: setuptools==59.5.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (59.5.0)\n", + "Requirement already satisfied: numpy>=1.21 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.23.4)\n", + "Requirement already satisfied: onnx>=1.7.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.12.0)\n", + "Requirement already satisfied: python-dateutil in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.8.2)\n", + "Requirement already satisfied: torch in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.13.0)\n", + "Requirement already satisfied: wrapt in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.14.1)\n", + "Requirement already satisfied: ruamel.yaml in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.17.21)\n", + "Requirement already satisfied: scikit-learn in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.1.3)\n", + "Requirement already satisfied: tqdm>=4.41.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.64.1)\n", + "Requirement already satisfied: numba in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.56.3)\n", + "Requirement already satisfied: wget in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.2)\n", + "Requirement already satisfied: text-unidecode in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.3)\n", + "Requirement already satisfied: huggingface_hub in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.10.1)\n", + "Requirement already satisfied: click==8.0.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (8.0.2)\n", + "Requirement already satisfied: black==19.10b0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (19.10b0)\n", + "Requirement already satisfied: isort[requirements]<5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.3.21)\n", + "Requirement already satisfied: parameterized in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.8.1)\n", + "Requirement already satisfied: pytest in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (7.2.0)\n", + "Requirement already satisfied: pytest-runner in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (6.0.0)\n", + "Requirement already satisfied: sphinx in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (5.3.0)\n", + "Requirement already satisfied: sphinxcontrib-bibtex in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.5.0)\n", + "Requirement already satisfied: wandb in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.13.4)\n", + "Requirement already satisfied: inflect in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (6.0.2)\n", + "Requirement already satisfied: regex in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2022.10.31)\n", + "Requirement already satisfied: pytorch-lightning<=1.7.7,>=1.7.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.7.7)\n", + "Requirement already satisfied: torchmetrics>=0.4.1rc0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.10.2)\n", + "Requirement already satisfied: transformers<=4.21.2,>=4.0.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.21.2)\n", + "Requirement already satisfied: webdataset<=0.1.62,>=0.1.48 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.1.62)\n", + "Requirement already satisfied: omegaconf<2.2,>=2.1.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.1.2)\n", + "Requirement already satisfied: hydra-core<1.2,>=1.1.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.1.2)\n", + "Requirement already satisfied: pyyaml<6 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (5.4.1)\n", + "Requirement already satisfied: sentencepiece<1.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.1.97)\n", + "Requirement already satisfied: youtokentome>=1.0.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.0.6)\n", + "Requirement already satisfied: pandas in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.5.1)\n", + "Requirement already satisfied: sacremoses>=0.0.43 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.0.53)\n", + "Requirement already satisfied: braceexpand in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.1.7)\n", + "Requirement already satisfied: editdistance in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.6.1)\n", + "Requirement already satisfied: librosa>=0.9.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.9.2)\n", + "Requirement already satisfied: marshmallow in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.18.0)\n", + "Requirement already satisfied: packaging in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (21.3)\n", + "Requirement already satisfied: soundfile in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.11.0)\n", + "Requirement already satisfied: sox in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.4.1)\n", + "Requirement already satisfied: kaldi-python-io in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.2.2)\n", + "Requirement already satisfied: kaldiio in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.17.2)\n", + "Requirement already satisfied: scipy>=0.14 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.9.3)\n", + "Requirement already satisfied: g2p_en in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.1.0)\n", + "Requirement already satisfied: pydub in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.25.1)\n", + "Requirement already satisfied: pyannote.core in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.5)\n", + "Requirement already satisfied: pyannote.metrics in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.2.1)\n", + "Requirement already satisfied: ipywidgets in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (7.6.5)\n", + "Requirement already satisfied: matplotlib in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.6.2)\n", + "Requirement already satisfied: texterrors in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.4.4)\n", + "Requirement already satisfied: pillow in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (9.3.0)\n", + "Requirement already satisfied: torchvision in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.14.0)\n", + "Requirement already satisfied: boto3 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.26.1)\n", + "Requirement already satisfied: h5py in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.7.0)\n", + "Requirement already satisfied: rapidfuzz in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.13.0)\n", + "Requirement already satisfied: gdown in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.5.3)\n", + "Requirement already satisfied: sacrebleu[ja] in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.3.1)\n", + "Requirement already satisfied: nltk>=3.6.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.7)\n", + "Requirement already satisfied: fasttext in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.9.2)\n", + "Requirement already satisfied: opencc in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.1.4)\n", + "Requirement already satisfied: pangu in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.0.6.1)\n", + "Requirement already satisfied: jieba in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.42.1)\n", + "Requirement already satisfied: ftfy in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (6.1.1)\n", + "Requirement already satisfied: flask_restful in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.3.9)\n", + "Requirement already satisfied: einops in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.5.0)\n", + "Requirement already satisfied: ijson in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.1.4)\n", + "Requirement already satisfied: faiss-cpu in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.7.2)\n", + "Requirement already satisfied: sentence_transformers in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.2.2)\n", + "Requirement already satisfied: attrdict in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.0.1)\n", + "Requirement already satisfied: pesq in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.0.4)\n", + "Requirement already satisfied: pypinyin in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.47.1)\n", + "Requirement already satisfied: pystoi in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.3.3)\n", + "Requirement already satisfied: progress>=1.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.6)\n", + "Requirement already satisfied: tabulate>=0.8.7 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.9.0)\n", + "Requirement already satisfied: textdistance>=4.1.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.5.0)\n", + "Requirement already satisfied: jiwer>=2.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.5.1)\n", + "Requirement already satisfied: typed-ast>=1.4.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from black==19.10b0->nemo_toolkit[all]) (1.5.4)\n", + "Requirement already satisfied: toml>=0.9.4 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from black==19.10b0->nemo_toolkit[all]) (0.10.2)\n", + "Requirement already satisfied: pathspec<1,>=0.6 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from black==19.10b0->nemo_toolkit[all]) (0.10.1)\n", + "Requirement already satisfied: attrs>=18.1.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from black==19.10b0->nemo_toolkit[all]) (21.4.0)\n", + "Requirement already satisfied: appdirs in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from black==19.10b0->nemo_toolkit[all]) (1.4.4)\n", + "Requirement already satisfied: antlr4-python3-runtime==4.8 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from hydra-core<1.2,>=1.1.0->nemo_toolkit[all]) (4.8)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: pipreqs in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from isort[requirements]<5->nemo_toolkit[all]) (0.4.11)\n", + "Requirement already satisfied: pip-api in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from isort[requirements]<5->nemo_toolkit[all]) (0.0.30)\n", + "Requirement already satisfied: levenshtein==0.20.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from jiwer>=2.0.0->nemo_toolkit[all]) (0.20.2)\n", + "Requirement already satisfied: joblib>=0.14 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from librosa>=0.9.0->nemo_toolkit[all]) (1.2.0)\n", + "Requirement already satisfied: audioread>=2.1.9 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from librosa>=0.9.0->nemo_toolkit[all]) (3.0.0)\n", + "Requirement already satisfied: pooch>=1.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from librosa>=0.9.0->nemo_toolkit[all]) (1.6.0)\n", + "Requirement already satisfied: decorator>=4.0.10 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from librosa>=0.9.0->nemo_toolkit[all]) (5.1.1)\n", + "Requirement already satisfied: resampy>=0.2.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from librosa>=0.9.0->nemo_toolkit[all]) (0.4.2)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from matplotlib->nemo_toolkit[all]) (1.4.4)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from matplotlib->nemo_toolkit[all]) (1.0.6)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from matplotlib->nemo_toolkit[all]) (4.38.0)\n", + "Requirement already satisfied: pyparsing>=2.2.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from matplotlib->nemo_toolkit[all]) (3.0.9)\n", + "Requirement already satisfied: cycler>=0.10 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from matplotlib->nemo_toolkit[all]) (0.11.0)\n", + "Requirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from numba->nemo_toolkit[all]) (0.39.1)\n", + "Requirement already satisfied: typing-extensions>=3.6.2.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from onnx>=1.7.0->nemo_toolkit[all]) (4.3.0)\n", + "Requirement already satisfied: protobuf<=3.20.1,>=3.12.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from onnx>=1.7.0->nemo_toolkit[all]) (3.19.6)\n", + "Requirement already satisfied: six>=1.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from python-dateutil->nemo_toolkit[all]) (1.16.0)\n", + "Requirement already satisfied: pyDeprecate>=0.3.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (0.3.2)\n", + "Requirement already satisfied: fsspec[http]!=2021.06.0,>=2021.05.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (2022.10.0)\n", + "Requirement already satisfied: tensorboard>=2.9.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (2.10.1)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from scikit-learn->nemo_toolkit[all]) (3.1.0)\n", + "Requirement already satisfied: cffi>=1.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from soundfile->nemo_toolkit[all]) (1.15.1)\n", + "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from torch->nemo_toolkit[all]) (11.10.3.66)\n", + "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from torch->nemo_toolkit[all]) (8.5.0.96)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from torch->nemo_toolkit[all]) (11.7.99)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from torch->nemo_toolkit[all]) (11.7.99)\n", + "Requirement already satisfied: wheel in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch->nemo_toolkit[all]) (0.37.1)\n", + "Requirement already satisfied: requests in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from transformers<=4.21.2,>=4.0.1->nemo_toolkit[all]) (2.28.1)\n", + "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from transformers<=4.21.2,>=4.0.1->nemo_toolkit[all]) (0.12.1)\n", + "Requirement already satisfied: filelock in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from transformers<=4.21.2,>=4.0.1->nemo_toolkit[all]) (3.8.0)\n", + "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from boto3->nemo_toolkit[all]) (0.6.0)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from boto3->nemo_toolkit[all]) (1.0.1)\n", + "Requirement already satisfied: botocore<1.30.0,>=1.29.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from boto3->nemo_toolkit[all]) (1.29.1)\n", + "Requirement already satisfied: pybind11>=2.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from fasttext->nemo_toolkit[all]) (2.10.1)\n", + "Requirement already satisfied: aniso8601>=0.82 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from flask_restful->nemo_toolkit[all]) (9.0.1)\n", + "Requirement already satisfied: Flask>=0.8 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from flask_restful->nemo_toolkit[all]) (2.2.2)\n", + "Requirement already satisfied: pytz in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from flask_restful->nemo_toolkit[all]) (2022.1)\n", + "Requirement already satisfied: wcwidth>=0.2.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ftfy->nemo_toolkit[all]) (0.2.5)\n", + "Requirement already satisfied: distance>=0.1.3 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from g2p_en->nemo_toolkit[all]) (0.1.3)\n", + "Requirement already satisfied: pydantic>=1.9.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from inflect->nemo_toolkit[all]) (1.10.2)\n", + "Requirement already satisfied: beautifulsoup4 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from gdown->nemo_toolkit[all]) (4.11.1)\n", + "Requirement already satisfied: ipython-genutils~=0.2.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (0.2.0)\n", + "Requirement already satisfied: traitlets>=4.3.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (5.1.1)\n", + "Requirement already satisfied: nbformat>=4.2.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (5.5.0)\n", + "Requirement already satisfied: ipython>=4.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (8.4.0)\n", + "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (1.0.0)\n", + "Requirement already satisfied: ipykernel>=4.5.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (6.15.2)\n", + "Requirement already satisfied: widgetsnbextension~=3.5.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (3.5.2)\n", + "Requirement already satisfied: sortedcontainers>=2.0.4 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pyannote.core->nemo_toolkit[all]) (2.4.0)\n", + "Requirement already satisfied: simplejson>=3.8.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pyannote.core->nemo_toolkit[all]) (3.17.6)\n", + "Requirement already satisfied: pyannote.database>=4.0.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pyannote.metrics->nemo_toolkit[all]) (4.1.3)\n", + "Requirement already satisfied: sympy>=1.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pyannote.metrics->nemo_toolkit[all]) (1.11.1)\n", + "Requirement already satisfied: docopt>=0.6.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pyannote.metrics->nemo_toolkit[all]) (0.6.2)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: iniconfig in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pytest->nemo_toolkit[all]) (1.1.1)\n", + "Requirement already satisfied: pluggy<2.0,>=0.12 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pytest->nemo_toolkit[all]) (1.0.0)\n", + "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pytest->nemo_toolkit[all]) (1.0.0)\n", + "Requirement already satisfied: tomli>=1.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pytest->nemo_toolkit[all]) (2.0.1)\n", + "Requirement already satisfied: ruamel.yaml.clib>=0.2.6 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ruamel.yaml->nemo_toolkit[all]) (0.2.7)\n", + "Requirement already satisfied: lxml in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sacrebleu[ja]->nemo_toolkit[all]) (4.9.1)\n", + "Requirement already satisfied: portalocker in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sacrebleu[ja]->nemo_toolkit[all]) (2.6.0)\n", + "Requirement already satisfied: colorama in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sacrebleu[ja]->nemo_toolkit[all]) (0.4.6)\n", + "Requirement already satisfied: mecab-python3==1.0.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sacrebleu[ja]->nemo_toolkit[all]) (1.0.5)\n", + "Requirement already satisfied: ipadic<2.0,>=1.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sacrebleu[ja]->nemo_toolkit[all]) (1.0.0)\n", + "Requirement already satisfied: sphinxcontrib-qthelp in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.0.3)\n", + "Requirement already satisfied: Jinja2>=3.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (3.1.2)\n", + "Requirement already satisfied: alabaster<0.8,>=0.7 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (0.7.12)\n", + "Requirement already satisfied: Pygments>=2.12 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (2.13.0)\n", + "Requirement already satisfied: importlib-metadata>=4.8 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (4.11.3)\n", + "Requirement already satisfied: sphinxcontrib-devhelp in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.0.2)\n", + "Requirement already satisfied: sphinxcontrib-serializinghtml>=1.1.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.1.5)\n", + "Requirement already satisfied: docutils<0.20,>=0.14 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (0.19)\n", + "Requirement already satisfied: snowballstemmer>=2.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (2.2.0)\n", + "Requirement already satisfied: sphinxcontrib-htmlhelp>=2.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (2.0.0)\n", + "Requirement already satisfied: sphinxcontrib-applehelp in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.0.2)\n", + "Requirement already satisfied: sphinxcontrib-jsmath in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.0.1)\n", + "Requirement already satisfied: babel>=2.9 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (2.9.1)\n", + "Requirement already satisfied: imagesize>=1.3 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.4.1)\n", + "Requirement already satisfied: pybtex-docutils>=1.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinxcontrib-bibtex->nemo_toolkit[all]) (1.0.2)\n", + "Requirement already satisfied: pybtex>=0.24 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinxcontrib-bibtex->nemo_toolkit[all]) (0.24.0)\n", + "Requirement already satisfied: termcolor in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from texterrors->nemo_toolkit[all]) (2.1.0)\n", + "Requirement already satisfied: plac in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from texterrors->nemo_toolkit[all]) (1.3.5)\n", + "Requirement already satisfied: loguru in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from texterrors->nemo_toolkit[all]) (0.6.0)\n", + "Requirement already satisfied: promise<3,>=2.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (2.3)\n", + "Requirement already satisfied: sentry-sdk>=1.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (1.10.1)\n", + "Requirement already satisfied: pathtools in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (0.1.2)\n", + "Requirement already satisfied: docker-pycreds>=0.4.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (0.4.0)\n", + "Requirement already satisfied: shortuuid>=0.5.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (1.0.9)\n", + "Requirement already satisfied: setproctitle in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (1.3.2)\n", + "Requirement already satisfied: psutil>=5.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (5.9.0)\n", + "Requirement already satisfied: GitPython>=1.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (3.1.29)\n", + "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from botocore<1.30.0,>=1.29.1->boto3->nemo_toolkit[all]) (1.26.12)\n", + "Requirement already satisfied: pycparser in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from cffi>=1.0->soundfile->nemo_toolkit[all]) (2.21)\n", + "Requirement already satisfied: itsdangerous>=2.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from Flask>=0.8->flask_restful->nemo_toolkit[all]) (2.1.2)\n", + "Requirement already satisfied: Werkzeug>=2.2.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from Flask>=0.8->flask_restful->nemo_toolkit[all]) (2.2.2)\n", + "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (3.8.3)\n", + "Requirement already satisfied: gitdb<5,>=4.0.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from GitPython>=1.0.0->wandb->nemo_toolkit[all]) (4.0.9)\n", + "Requirement already satisfied: zipp>=0.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from importlib-metadata>=4.8->sphinx->nemo_toolkit[all]) (3.8.0)\n", + "Requirement already satisfied: matplotlib-inline>=0.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (0.1.6)\n", + "Requirement already satisfied: jupyter-client>=6.1.12 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (7.3.5)\n", + "Requirement already satisfied: pyzmq>=17 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (23.2.0)\n", + "Requirement already satisfied: tornado>=6.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (6.2)\n", + "Requirement already satisfied: debugpy>=1.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (1.5.1)\n", + "Requirement already satisfied: nest-asyncio in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (1.5.5)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: stack-data in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.2.0)\n", + "Requirement already satisfied: jedi>=0.16 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.18.1)\n", + "Requirement already satisfied: pickleshare in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.7.5)\n", + "Requirement already satisfied: pexpect>4.3 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (4.8.0)\n", + "Requirement already satisfied: backcall in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.2.0)\n", + "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (3.0.20)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from Jinja2>=3.0->sphinx->nemo_toolkit[all]) (2.1.1)\n", + "Requirement already satisfied: fastjsonschema in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbformat>=4.2.0->ipywidgets->nemo_toolkit[all]) (2.16.2)\n", + "Requirement already satisfied: jsonschema>=2.6 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbformat>=4.2.0->ipywidgets->nemo_toolkit[all]) (4.16.0)\n", + "Requirement already satisfied: jupyter_core in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbformat>=4.2.0->ipywidgets->nemo_toolkit[all]) (4.11.1)\n", + "Requirement already satisfied: typer[all]>=0.2.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pyannote.database>=4.0.1->pyannote.metrics->nemo_toolkit[all]) (0.6.1)\n", + "Requirement already satisfied: latexcodec>=1.0.4 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pybtex>=0.24->sphinxcontrib-bibtex->nemo_toolkit[all]) (2.0.1)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from requests->transformers<=4.21.2,>=4.0.1->nemo_toolkit[all]) (2022.9.24)\n", + "Requirement already satisfied: charset-normalizer<3,>=2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from requests->transformers<=4.21.2,>=4.0.1->nemo_toolkit[all]) (2.0.4)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from requests->transformers<=4.21.2,>=4.0.1->nemo_toolkit[all]) (3.4)\n", + "Requirement already satisfied: mpmath>=0.19 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sympy>=1.1->pyannote.metrics->nemo_toolkit[all]) (1.2.1)\n", + "Requirement already satisfied: absl-py>=0.4 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (1.3.0)\n", + "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (0.6.1)\n", + "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (0.4.6)\n", + "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (1.8.1)\n", + "Requirement already satisfied: grpcio>=1.24.3 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (1.50.0)\n", + "Requirement already satisfied: markdown>=2.6.8 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (3.4.1)\n", + "Requirement already satisfied: google-auth<3,>=1.6.3 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (2.14.0)\n", + "Requirement already satisfied: notebook>=4.4.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (6.4.12)\n", + "Requirement already satisfied: soupsieve>1.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from beautifulsoup4->gdown->nemo_toolkit[all]) (2.3.2.post1)\n", + "Requirement already satisfied: pip in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pip-api->isort[requirements]<5->nemo_toolkit[all]) (22.2.2)\n", + "Requirement already satisfied: yarg in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pipreqs->isort[requirements]<5->nemo_toolkit[all]) (0.1.9)\n", + "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from requests->transformers<=4.21.2,>=4.0.1->nemo_toolkit[all]) (1.7.1)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (6.0.2)\n", + "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (4.0.2)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (1.2.0)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (1.3.1)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (1.8.1)\n", + "Requirement already satisfied: smmap<6,>=3.0.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from gitdb<5,>=4.0.1->GitPython>=1.0.0->wandb->nemo_toolkit[all]) (5.0.0)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (0.2.8)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (5.2.0)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (4.9)\n", + "Requirement already satisfied: requests-oauthlib>=0.7.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (1.3.1)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from jedi>=0.16->ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.8.3)\n", + "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from jsonschema>=2.6->nbformat>=4.2.0->ipywidgets->nemo_toolkit[all]) (0.18.0)\n", + "Requirement already satisfied: entrypoints in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from jupyter-client>=6.1.12->ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (0.4)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: argon2-cffi in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (21.3.0)\n", + "Requirement already satisfied: prometheus-client in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.14.1)\n", + "Requirement already satisfied: nbconvert>=5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (6.4.4)\n", + "Requirement already satisfied: terminado>=0.8.3 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.13.1)\n", + "Requirement already satisfied: Send2Trash>=1.8.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (1.8.0)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pexpect>4.3->ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.7.0)\n", + "Requirement already satisfied: shellingham<2.0.0,>=1.3.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from typer[all]>=0.2.1->pyannote.database>=4.0.1->pyannote.metrics->nemo_toolkit[all]) (1.5.0)\n", + "Requirement already satisfied: rich<13.0.0,>=10.11.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from typer[all]>=0.2.1->pyannote.database>=4.0.1->pyannote.metrics->nemo_toolkit[all]) (12.6.0)\n", + "Requirement already satisfied: asttokens in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from stack-data->ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (2.0.5)\n", + "Requirement already satisfied: pure-eval in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from stack-data->ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.2.2)\n", + "Requirement already satisfied: executing in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from stack-data->ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (1.2.0)\n", + "Requirement already satisfied: pandocfilters>=1.4.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (1.5.0)\n", + "Requirement already satisfied: nbclient<0.6.0,>=0.5.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.5.13)\n", + "Requirement already satisfied: testpath in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.6.0)\n", + "Requirement already satisfied: bleach in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (4.1.0)\n", + "Requirement already satisfied: mistune<2,>=0.8.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.8.4)\n", + "Requirement already satisfied: defusedxml in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.7.1)\n", + "Requirement already satisfied: jupyterlab-pygments in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.1.2)\n", + "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (0.4.8)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (3.2.2)\n", + "Requirement already satisfied: commonmark<0.10.0,>=0.9.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from rich<13.0.0,>=10.11.0->typer[all]>=0.2.1->pyannote.database>=4.0.1->pyannote.metrics->nemo_toolkit[all]) (0.9.1)\n", + "Requirement already satisfied: argon2-cffi-bindings in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (21.2.0)\n", + "Requirement already satisfied: webencodings in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from bleach->nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.5.1)\n" + ] + } + ], "source": [ "\"\"\"\n", "You can run either this notebook locally (if you have all the dependencies and a GPU) or on Google Colab.\n", @@ -59,13 +349,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "pycharm": { "name": "#%%\n" } }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[NeMo W 2022-11-03 05:00:48 optimizers:55] Apex was not found. Using the lamb or fused_adam optimizer will error out.\n", + "[NeMo W 2022-11-03 05:00:48 experimental:27] Module is experimental, not ready for production and is not fully supported. Use at your own risk.\n", + "[NeMo W 2022-11-03 05:00:49 experimental:27] Module is experimental, not ready for production and is not fully supported. Use at your own risk.\n" + ] + } + ], "source": [ "from nemo.utils.exp_manager import exp_manager\n", "from nemo.collections import nlp as nemo_nlp\n", @@ -260,7 +560,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { "pycharm": { "name": "#%%\n" @@ -268,8 +568,8 @@ }, "outputs": [], "source": [ - "DATA_DIR = 'PATH_TO_A_DIRECTORY_WHERE_DATA_FROM_THIS_TUTORIAL_IS_STORED'\n", - "WORK_DIR = 'PATH_TO_A_DIRECTORY_WHERE_SCRIPTS_FOR_THIS_TUTORIAL_ARE_SAVED'\n", + "DATA_DIR = './data'\n", + "WORK_DIR = './scripts'\n", "MODEL_CONFIG = \"punctuation_capitalization_lexical_audio_config.yaml\"\n", "\n", "# model parameters\n", @@ -279,32 +579,53 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 5, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading get_libritts_data.py...\n", + "100% [................................................................................] 4804 / 4804" + ] + } + ], "source": [ - "## download get_libritts_data.py script to download and preprocess the LibriTTS data\n", + "# download get_libritts_data.py script to download and preprocess the LibriTTS data\n", "os.makedirs(WORK_DIR, exist_ok=True)\n", "if not os.path.exists(WORK_DIR + '/get_libritts_data.py'):\n", " print('Downloading get_libritts_data.py...')\n", " wget.download(f'https://raw.githubusercontent.com/NVIDIA/NeMo/{BRANCH}/examples/nlp/token_classification/data/get_libritts_data.py', WORK_DIR)\n", "else:\n", " print ('get_libritts_data.py already exists')" - ], - "metadata": { - "collapsed": false, - "pycharm": { - "name": "#%% md\n" - } - } + ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": { "pycharm": { "name": "#%%\n" } }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Traceback (most recent call last):\r\n", + " File \"/home/mnovikov/NeMo/tutorials/nlp/./scripts/get_libritts_data.py\", line 29, in \r\n", + " from get_tatoeba_data import create_text_and_labels\r\n", + "ModuleNotFoundError: No module named 'get_tatoeba_data'\r\n" + ] + } + ], "source": [ "# download and preprocess the data\n", "# we will use dev_clean and dev_other subsets\n", @@ -996,9 +1317,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.0" + "version": "3.9.13" } }, "nbformat": 4, "nbformat_minor": 1 -} \ No newline at end of file +} From dadb9b3daa957c0dc7cd7ad4fea340561e5d25c0 Mon Sep 17 00:00:00 2001 From: Matvei Novikov Date: Thu, 3 Nov 2022 16:12:55 +0400 Subject: [PATCH 2/3] fixes Signed-off-by: Matvei Novikov --- ...ion_and_Capitalization_Lexical_Audio.ipynb | 344 +----------------- 1 file changed, 12 insertions(+), 332 deletions(-) diff --git a/tutorials/nlp/Punctuation_and_Capitalization_Lexical_Audio.ipynb b/tutorials/nlp/Punctuation_and_Capitalization_Lexical_Audio.ipynb index 784f7efe94ae..4c20cae8af19 100644 --- a/tutorials/nlp/Punctuation_and_Capitalization_Lexical_Audio.ipynb +++ b/tutorials/nlp/Punctuation_and_Capitalization_Lexical_Audio.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "pycharm": { "name": "#%%\n" @@ -15,303 +15,13 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "pycharm": { "name": "#%%\n" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting nemo_toolkit[all]\n", - " Cloning https://github.com/NVIDIA/NeMo.git (to revision r1.13.0) to /tmp/pip-install-m40oh75q/nemo-toolkit_dc4ac2f47f55476a85af4d6a94a9e037\n", - " Running command git clone --filter=blob:none --quiet https://github.com/NVIDIA/NeMo.git /tmp/pip-install-m40oh75q/nemo-toolkit_dc4ac2f47f55476a85af4d6a94a9e037\n", - " Running command git checkout -b r1.13.0 --track origin/r1.13.0\n", - " Switched to a new branch 'r1.13.0'\n", - " Branch 'r1.13.0' set up to track remote branch 'r1.13.0' from 'origin'.\n", - " Resolved https://github.com/NVIDIA/NeMo.git to commit 26e3e1d1931a9f34986f73f071f82f446d90c1c5\n", - " Preparing metadata (setup.py) ... \u001b[?25ldone\n", - "\u001b[?25hRequirement already satisfied: setuptools==59.5.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (59.5.0)\n", - "Requirement already satisfied: numpy>=1.21 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.23.4)\n", - "Requirement already satisfied: onnx>=1.7.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.12.0)\n", - "Requirement already satisfied: python-dateutil in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.8.2)\n", - "Requirement already satisfied: torch in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.13.0)\n", - "Requirement already satisfied: wrapt in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.14.1)\n", - "Requirement already satisfied: ruamel.yaml in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.17.21)\n", - "Requirement already satisfied: scikit-learn in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.1.3)\n", - "Requirement already satisfied: tqdm>=4.41.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.64.1)\n", - "Requirement already satisfied: numba in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.56.3)\n", - "Requirement already satisfied: wget in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.2)\n", - "Requirement already satisfied: text-unidecode in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.3)\n", - "Requirement already satisfied: huggingface_hub in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.10.1)\n", - "Requirement already satisfied: click==8.0.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (8.0.2)\n", - "Requirement already satisfied: black==19.10b0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (19.10b0)\n", - "Requirement already satisfied: isort[requirements]<5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.3.21)\n", - "Requirement already satisfied: parameterized in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.8.1)\n", - "Requirement already satisfied: pytest in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (7.2.0)\n", - "Requirement already satisfied: pytest-runner in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (6.0.0)\n", - "Requirement already satisfied: sphinx in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (5.3.0)\n", - "Requirement already satisfied: sphinxcontrib-bibtex in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.5.0)\n", - "Requirement already satisfied: wandb in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.13.4)\n", - "Requirement already satisfied: inflect in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (6.0.2)\n", - "Requirement already satisfied: regex in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2022.10.31)\n", - "Requirement already satisfied: pytorch-lightning<=1.7.7,>=1.7.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.7.7)\n", - "Requirement already satisfied: torchmetrics>=0.4.1rc0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.10.2)\n", - "Requirement already satisfied: transformers<=4.21.2,>=4.0.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.21.2)\n", - "Requirement already satisfied: webdataset<=0.1.62,>=0.1.48 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.1.62)\n", - "Requirement already satisfied: omegaconf<2.2,>=2.1.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.1.2)\n", - "Requirement already satisfied: hydra-core<1.2,>=1.1.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.1.2)\n", - "Requirement already satisfied: pyyaml<6 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (5.4.1)\n", - "Requirement already satisfied: sentencepiece<1.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.1.97)\n", - "Requirement already satisfied: youtokentome>=1.0.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.0.6)\n", - "Requirement already satisfied: pandas in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.5.1)\n", - "Requirement already satisfied: sacremoses>=0.0.43 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.0.53)\n", - "Requirement already satisfied: braceexpand in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.1.7)\n", - "Requirement already satisfied: editdistance in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.6.1)\n", - "Requirement already satisfied: librosa>=0.9.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.9.2)\n", - "Requirement already satisfied: marshmallow in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.18.0)\n", - "Requirement already satisfied: packaging in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (21.3)\n", - "Requirement already satisfied: soundfile in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.11.0)\n", - "Requirement already satisfied: sox in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.4.1)\n", - "Requirement already satisfied: kaldi-python-io in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.2.2)\n", - "Requirement already satisfied: kaldiio in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.17.2)\n", - "Requirement already satisfied: scipy>=0.14 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.9.3)\n", - "Requirement already satisfied: g2p_en in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.1.0)\n", - "Requirement already satisfied: pydub in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.25.1)\n", - "Requirement already satisfied: pyannote.core in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.5)\n", - "Requirement already satisfied: pyannote.metrics in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.2.1)\n", - "Requirement already satisfied: ipywidgets in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (7.6.5)\n", - "Requirement already satisfied: matplotlib in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.6.2)\n", - "Requirement already satisfied: texterrors in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.4.4)\n", - "Requirement already satisfied: pillow in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (9.3.0)\n", - "Requirement already satisfied: torchvision in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.14.0)\n", - "Requirement already satisfied: boto3 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.26.1)\n", - "Requirement already satisfied: h5py in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.7.0)\n", - "Requirement already satisfied: rapidfuzz in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.13.0)\n", - "Requirement already satisfied: gdown in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.5.3)\n", - "Requirement already satisfied: sacrebleu[ja] in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.3.1)\n", - "Requirement already satisfied: nltk>=3.6.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.7)\n", - "Requirement already satisfied: fasttext in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.9.2)\n", - "Requirement already satisfied: opencc in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.1.4)\n", - "Requirement already satisfied: pangu in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.0.6.1)\n", - "Requirement already satisfied: jieba in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.42.1)\n", - "Requirement already satisfied: ftfy in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (6.1.1)\n", - "Requirement already satisfied: flask_restful in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.3.9)\n", - "Requirement already satisfied: einops in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.5.0)\n", - "Requirement already satisfied: ijson in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (3.1.4)\n", - "Requirement already satisfied: faiss-cpu in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.7.2)\n", - "Requirement already satisfied: sentence_transformers in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.2.2)\n", - "Requirement already satisfied: attrdict in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.0.1)\n", - "Requirement already satisfied: pesq in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.0.4)\n", - "Requirement already satisfied: pypinyin in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.47.1)\n", - "Requirement already satisfied: pystoi in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.3.3)\n", - "Requirement already satisfied: progress>=1.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (1.6)\n", - "Requirement already satisfied: tabulate>=0.8.7 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (0.9.0)\n", - "Requirement already satisfied: textdistance>=4.1.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (4.5.0)\n", - "Requirement already satisfied: jiwer>=2.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nemo_toolkit[all]) (2.5.1)\n", - "Requirement already satisfied: typed-ast>=1.4.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from black==19.10b0->nemo_toolkit[all]) (1.5.4)\n", - "Requirement already satisfied: toml>=0.9.4 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from black==19.10b0->nemo_toolkit[all]) (0.10.2)\n", - "Requirement already satisfied: pathspec<1,>=0.6 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from black==19.10b0->nemo_toolkit[all]) (0.10.1)\n", - "Requirement already satisfied: attrs>=18.1.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from black==19.10b0->nemo_toolkit[all]) (21.4.0)\n", - "Requirement already satisfied: appdirs in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from black==19.10b0->nemo_toolkit[all]) (1.4.4)\n", - "Requirement already satisfied: antlr4-python3-runtime==4.8 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from hydra-core<1.2,>=1.1.0->nemo_toolkit[all]) (4.8)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: pipreqs in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from isort[requirements]<5->nemo_toolkit[all]) (0.4.11)\n", - "Requirement already satisfied: pip-api in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from isort[requirements]<5->nemo_toolkit[all]) (0.0.30)\n", - "Requirement already satisfied: levenshtein==0.20.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from jiwer>=2.0.0->nemo_toolkit[all]) (0.20.2)\n", - "Requirement already satisfied: joblib>=0.14 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from librosa>=0.9.0->nemo_toolkit[all]) (1.2.0)\n", - "Requirement already satisfied: audioread>=2.1.9 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from librosa>=0.9.0->nemo_toolkit[all]) (3.0.0)\n", - "Requirement already satisfied: pooch>=1.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from librosa>=0.9.0->nemo_toolkit[all]) (1.6.0)\n", - "Requirement already satisfied: decorator>=4.0.10 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from librosa>=0.9.0->nemo_toolkit[all]) (5.1.1)\n", - "Requirement already satisfied: resampy>=0.2.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from librosa>=0.9.0->nemo_toolkit[all]) (0.4.2)\n", - "Requirement already satisfied: kiwisolver>=1.0.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from matplotlib->nemo_toolkit[all]) (1.4.4)\n", - "Requirement already satisfied: contourpy>=1.0.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from matplotlib->nemo_toolkit[all]) (1.0.6)\n", - "Requirement already satisfied: fonttools>=4.22.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from matplotlib->nemo_toolkit[all]) (4.38.0)\n", - "Requirement already satisfied: pyparsing>=2.2.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from matplotlib->nemo_toolkit[all]) (3.0.9)\n", - "Requirement already satisfied: cycler>=0.10 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from matplotlib->nemo_toolkit[all]) (0.11.0)\n", - "Requirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from numba->nemo_toolkit[all]) (0.39.1)\n", - "Requirement already satisfied: typing-extensions>=3.6.2.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from onnx>=1.7.0->nemo_toolkit[all]) (4.3.0)\n", - "Requirement already satisfied: protobuf<=3.20.1,>=3.12.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from onnx>=1.7.0->nemo_toolkit[all]) (3.19.6)\n", - "Requirement already satisfied: six>=1.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from python-dateutil->nemo_toolkit[all]) (1.16.0)\n", - "Requirement already satisfied: pyDeprecate>=0.3.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (0.3.2)\n", - "Requirement already satisfied: fsspec[http]!=2021.06.0,>=2021.05.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (2022.10.0)\n", - "Requirement already satisfied: tensorboard>=2.9.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (2.10.1)\n", - "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from scikit-learn->nemo_toolkit[all]) (3.1.0)\n", - "Requirement already satisfied: cffi>=1.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from soundfile->nemo_toolkit[all]) (1.15.1)\n", - "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from torch->nemo_toolkit[all]) (11.10.3.66)\n", - "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from torch->nemo_toolkit[all]) (8.5.0.96)\n", - "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from torch->nemo_toolkit[all]) (11.7.99)\n", - "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from torch->nemo_toolkit[all]) (11.7.99)\n", - "Requirement already satisfied: wheel in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch->nemo_toolkit[all]) (0.37.1)\n", - "Requirement already satisfied: requests in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from transformers<=4.21.2,>=4.0.1->nemo_toolkit[all]) (2.28.1)\n", - "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from transformers<=4.21.2,>=4.0.1->nemo_toolkit[all]) (0.12.1)\n", - "Requirement already satisfied: filelock in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from transformers<=4.21.2,>=4.0.1->nemo_toolkit[all]) (3.8.0)\n", - "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from boto3->nemo_toolkit[all]) (0.6.0)\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from boto3->nemo_toolkit[all]) (1.0.1)\n", - "Requirement already satisfied: botocore<1.30.0,>=1.29.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from boto3->nemo_toolkit[all]) (1.29.1)\n", - "Requirement already satisfied: pybind11>=2.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from fasttext->nemo_toolkit[all]) (2.10.1)\n", - "Requirement already satisfied: aniso8601>=0.82 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from flask_restful->nemo_toolkit[all]) (9.0.1)\n", - "Requirement already satisfied: Flask>=0.8 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from flask_restful->nemo_toolkit[all]) (2.2.2)\n", - "Requirement already satisfied: pytz in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from flask_restful->nemo_toolkit[all]) (2022.1)\n", - "Requirement already satisfied: wcwidth>=0.2.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ftfy->nemo_toolkit[all]) (0.2.5)\n", - "Requirement already satisfied: distance>=0.1.3 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from g2p_en->nemo_toolkit[all]) (0.1.3)\n", - "Requirement already satisfied: pydantic>=1.9.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from inflect->nemo_toolkit[all]) (1.10.2)\n", - "Requirement already satisfied: beautifulsoup4 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from gdown->nemo_toolkit[all]) (4.11.1)\n", - "Requirement already satisfied: ipython-genutils~=0.2.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (0.2.0)\n", - "Requirement already satisfied: traitlets>=4.3.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (5.1.1)\n", - "Requirement already satisfied: nbformat>=4.2.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (5.5.0)\n", - "Requirement already satisfied: ipython>=4.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (8.4.0)\n", - "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (1.0.0)\n", - "Requirement already satisfied: ipykernel>=4.5.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (6.15.2)\n", - "Requirement already satisfied: widgetsnbextension~=3.5.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipywidgets->nemo_toolkit[all]) (3.5.2)\n", - "Requirement already satisfied: sortedcontainers>=2.0.4 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pyannote.core->nemo_toolkit[all]) (2.4.0)\n", - "Requirement already satisfied: simplejson>=3.8.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pyannote.core->nemo_toolkit[all]) (3.17.6)\n", - "Requirement already satisfied: pyannote.database>=4.0.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pyannote.metrics->nemo_toolkit[all]) (4.1.3)\n", - "Requirement already satisfied: sympy>=1.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pyannote.metrics->nemo_toolkit[all]) (1.11.1)\n", - "Requirement already satisfied: docopt>=0.6.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pyannote.metrics->nemo_toolkit[all]) (0.6.2)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: iniconfig in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pytest->nemo_toolkit[all]) (1.1.1)\n", - "Requirement already satisfied: pluggy<2.0,>=0.12 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pytest->nemo_toolkit[all]) (1.0.0)\n", - "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pytest->nemo_toolkit[all]) (1.0.0)\n", - "Requirement already satisfied: tomli>=1.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pytest->nemo_toolkit[all]) (2.0.1)\n", - "Requirement already satisfied: ruamel.yaml.clib>=0.2.6 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ruamel.yaml->nemo_toolkit[all]) (0.2.7)\n", - "Requirement already satisfied: lxml in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sacrebleu[ja]->nemo_toolkit[all]) (4.9.1)\n", - "Requirement already satisfied: portalocker in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sacrebleu[ja]->nemo_toolkit[all]) (2.6.0)\n", - "Requirement already satisfied: colorama in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sacrebleu[ja]->nemo_toolkit[all]) (0.4.6)\n", - "Requirement already satisfied: mecab-python3==1.0.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sacrebleu[ja]->nemo_toolkit[all]) (1.0.5)\n", - "Requirement already satisfied: ipadic<2.0,>=1.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sacrebleu[ja]->nemo_toolkit[all]) (1.0.0)\n", - "Requirement already satisfied: sphinxcontrib-qthelp in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.0.3)\n", - "Requirement already satisfied: Jinja2>=3.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (3.1.2)\n", - "Requirement already satisfied: alabaster<0.8,>=0.7 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (0.7.12)\n", - "Requirement already satisfied: Pygments>=2.12 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (2.13.0)\n", - "Requirement already satisfied: importlib-metadata>=4.8 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (4.11.3)\n", - "Requirement already satisfied: sphinxcontrib-devhelp in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.0.2)\n", - "Requirement already satisfied: sphinxcontrib-serializinghtml>=1.1.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.1.5)\n", - "Requirement already satisfied: docutils<0.20,>=0.14 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (0.19)\n", - "Requirement already satisfied: snowballstemmer>=2.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (2.2.0)\n", - "Requirement already satisfied: sphinxcontrib-htmlhelp>=2.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (2.0.0)\n", - "Requirement already satisfied: sphinxcontrib-applehelp in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.0.2)\n", - "Requirement already satisfied: sphinxcontrib-jsmath in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.0.1)\n", - "Requirement already satisfied: babel>=2.9 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (2.9.1)\n", - "Requirement already satisfied: imagesize>=1.3 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinx->nemo_toolkit[all]) (1.4.1)\n", - "Requirement already satisfied: pybtex-docutils>=1.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinxcontrib-bibtex->nemo_toolkit[all]) (1.0.2)\n", - "Requirement already satisfied: pybtex>=0.24 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sphinxcontrib-bibtex->nemo_toolkit[all]) (0.24.0)\n", - "Requirement already satisfied: termcolor in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from texterrors->nemo_toolkit[all]) (2.1.0)\n", - "Requirement already satisfied: plac in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from texterrors->nemo_toolkit[all]) (1.3.5)\n", - "Requirement already satisfied: loguru in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from texterrors->nemo_toolkit[all]) (0.6.0)\n", - "Requirement already satisfied: promise<3,>=2.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (2.3)\n", - "Requirement already satisfied: sentry-sdk>=1.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (1.10.1)\n", - "Requirement already satisfied: pathtools in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (0.1.2)\n", - "Requirement already satisfied: docker-pycreds>=0.4.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (0.4.0)\n", - "Requirement already satisfied: shortuuid>=0.5.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (1.0.9)\n", - "Requirement already satisfied: setproctitle in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (1.3.2)\n", - "Requirement already satisfied: psutil>=5.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (5.9.0)\n", - "Requirement already satisfied: GitPython>=1.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from wandb->nemo_toolkit[all]) (3.1.29)\n", - "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from botocore<1.30.0,>=1.29.1->boto3->nemo_toolkit[all]) (1.26.12)\n", - "Requirement already satisfied: pycparser in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from cffi>=1.0->soundfile->nemo_toolkit[all]) (2.21)\n", - "Requirement already satisfied: itsdangerous>=2.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from Flask>=0.8->flask_restful->nemo_toolkit[all]) (2.1.2)\n", - "Requirement already satisfied: Werkzeug>=2.2.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from Flask>=0.8->flask_restful->nemo_toolkit[all]) (2.2.2)\n", - "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (3.8.3)\n", - "Requirement already satisfied: gitdb<5,>=4.0.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from GitPython>=1.0.0->wandb->nemo_toolkit[all]) (4.0.9)\n", - "Requirement already satisfied: zipp>=0.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from importlib-metadata>=4.8->sphinx->nemo_toolkit[all]) (3.8.0)\n", - "Requirement already satisfied: matplotlib-inline>=0.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (0.1.6)\n", - "Requirement already satisfied: jupyter-client>=6.1.12 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (7.3.5)\n", - "Requirement already satisfied: pyzmq>=17 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (23.2.0)\n", - "Requirement already satisfied: tornado>=6.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (6.2)\n", - "Requirement already satisfied: debugpy>=1.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (1.5.1)\n", - "Requirement already satisfied: nest-asyncio in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (1.5.5)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: stack-data in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.2.0)\n", - "Requirement already satisfied: jedi>=0.16 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.18.1)\n", - "Requirement already satisfied: pickleshare in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.7.5)\n", - "Requirement already satisfied: pexpect>4.3 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (4.8.0)\n", - "Requirement already satisfied: backcall in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.2.0)\n", - "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (3.0.20)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from Jinja2>=3.0->sphinx->nemo_toolkit[all]) (2.1.1)\n", - "Requirement already satisfied: fastjsonschema in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbformat>=4.2.0->ipywidgets->nemo_toolkit[all]) (2.16.2)\n", - "Requirement already satisfied: jsonschema>=2.6 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbformat>=4.2.0->ipywidgets->nemo_toolkit[all]) (4.16.0)\n", - "Requirement already satisfied: jupyter_core in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbformat>=4.2.0->ipywidgets->nemo_toolkit[all]) (4.11.1)\n", - "Requirement already satisfied: typer[all]>=0.2.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pyannote.database>=4.0.1->pyannote.metrics->nemo_toolkit[all]) (0.6.1)\n", - "Requirement already satisfied: latexcodec>=1.0.4 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pybtex>=0.24->sphinxcontrib-bibtex->nemo_toolkit[all]) (2.0.1)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from requests->transformers<=4.21.2,>=4.0.1->nemo_toolkit[all]) (2022.9.24)\n", - "Requirement already satisfied: charset-normalizer<3,>=2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from requests->transformers<=4.21.2,>=4.0.1->nemo_toolkit[all]) (2.0.4)\n", - "Requirement already satisfied: idna<4,>=2.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from requests->transformers<=4.21.2,>=4.0.1->nemo_toolkit[all]) (3.4)\n", - "Requirement already satisfied: mpmath>=0.19 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from sympy>=1.1->pyannote.metrics->nemo_toolkit[all]) (1.2.1)\n", - "Requirement already satisfied: absl-py>=0.4 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (1.3.0)\n", - "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (0.6.1)\n", - "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (0.4.6)\n", - "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (1.8.1)\n", - "Requirement already satisfied: grpcio>=1.24.3 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (1.50.0)\n", - "Requirement already satisfied: markdown>=2.6.8 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (3.4.1)\n", - "Requirement already satisfied: google-auth<3,>=1.6.3 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (2.14.0)\n", - "Requirement already satisfied: notebook>=4.4.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (6.4.12)\n", - "Requirement already satisfied: soupsieve>1.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from beautifulsoup4->gdown->nemo_toolkit[all]) (2.3.2.post1)\n", - "Requirement already satisfied: pip in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pip-api->isort[requirements]<5->nemo_toolkit[all]) (22.2.2)\n", - "Requirement already satisfied: yarg in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pipreqs->isort[requirements]<5->nemo_toolkit[all]) (0.1.9)\n", - "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from requests->transformers<=4.21.2,>=4.0.1->nemo_toolkit[all]) (1.7.1)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (6.0.2)\n", - "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (4.0.2)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (1.2.0)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (1.3.1)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (1.8.1)\n", - "Requirement already satisfied: smmap<6,>=3.0.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from gitdb<5,>=4.0.1->GitPython>=1.0.0->wandb->nemo_toolkit[all]) (5.0.0)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (0.2.8)\n", - "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (5.2.0)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (4.9)\n", - "Requirement already satisfied: requests-oauthlib>=0.7.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (1.3.1)\n", - "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from jedi>=0.16->ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.8.3)\n", - "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from jsonschema>=2.6->nbformat>=4.2.0->ipywidgets->nemo_toolkit[all]) (0.18.0)\n", - "Requirement already satisfied: entrypoints in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from jupyter-client>=6.1.12->ipykernel>=4.5.1->ipywidgets->nemo_toolkit[all]) (0.4)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: argon2-cffi in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (21.3.0)\n", - "Requirement already satisfied: prometheus-client in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.14.1)\n", - "Requirement already satisfied: nbconvert>=5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (6.4.4)\n", - "Requirement already satisfied: terminado>=0.8.3 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.13.1)\n", - "Requirement already satisfied: Send2Trash>=1.8.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (1.8.0)\n", - "Requirement already satisfied: ptyprocess>=0.5 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pexpect>4.3->ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.7.0)\n", - "Requirement already satisfied: shellingham<2.0.0,>=1.3.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from typer[all]>=0.2.1->pyannote.database>=4.0.1->pyannote.metrics->nemo_toolkit[all]) (1.5.0)\n", - "Requirement already satisfied: rich<13.0.0,>=10.11.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from typer[all]>=0.2.1->pyannote.database>=4.0.1->pyannote.metrics->nemo_toolkit[all]) (12.6.0)\n", - "Requirement already satisfied: asttokens in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from stack-data->ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (2.0.5)\n", - "Requirement already satisfied: pure-eval in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from stack-data->ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (0.2.2)\n", - "Requirement already satisfied: executing in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from stack-data->ipython>=4.0.0->ipywidgets->nemo_toolkit[all]) (1.2.0)\n", - "Requirement already satisfied: pandocfilters>=1.4.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (1.5.0)\n", - "Requirement already satisfied: nbclient<0.6.0,>=0.5.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.5.13)\n", - "Requirement already satisfied: testpath in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.6.0)\n", - "Requirement already satisfied: bleach in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (4.1.0)\n", - "Requirement already satisfied: mistune<2,>=0.8.1 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.8.4)\n", - "Requirement already satisfied: defusedxml in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.7.1)\n", - "Requirement already satisfied: jupyterlab-pygments in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.1.2)\n", - "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (0.4.8)\n", - "Requirement already satisfied: oauthlib>=3.0.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.9.1->pytorch-lightning<=1.7.7,>=1.7.0->nemo_toolkit[all]) (3.2.2)\n", - "Requirement already satisfied: commonmark<0.10.0,>=0.9.0 in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from rich<13.0.0,>=10.11.0->typer[all]>=0.2.1->pyannote.database>=4.0.1->pyannote.metrics->nemo_toolkit[all]) (0.9.1)\n", - "Requirement already satisfied: argon2-cffi-bindings in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from argon2-cffi->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (21.2.0)\n", - "Requirement already satisfied: webencodings in /home/mnovikov/miniconda3/envs/NeMo_main/lib/python3.9/site-packages (from bleach->nbconvert>=5->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets->nemo_toolkit[all]) (0.5.1)\n" - ] - } - ], + "outputs": [], "source": [ "\"\"\"\n", "You can run either this notebook locally (if you have all the dependencies and a GPU) or on Google Colab.\n", @@ -349,23 +59,13 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "pycharm": { "name": "#%%\n" } }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[NeMo W 2022-11-03 05:00:48 optimizers:55] Apex was not found. Using the lamb or fused_adam optimizer will error out.\n", - "[NeMo W 2022-11-03 05:00:48 experimental:27] Module is experimental, not ready for production and is not fully supported. Use at your own risk.\n", - "[NeMo W 2022-11-03 05:00:49 experimental:27] Module is experimental, not ready for production and is not fully supported. Use at your own risk.\n" - ] - } - ], + "outputs": [], "source": [ "from nemo.utils.exp_manager import exp_manager\n", "from nemo.collections import nlp as nemo_nlp\n", @@ -560,7 +260,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "pycharm": { "name": "#%%\n" @@ -568,8 +268,8 @@ }, "outputs": [], "source": [ - "DATA_DIR = './data'\n", - "WORK_DIR = './scripts'\n", + "DATA_DIR = 'PATH_TO_A_DIRECTORY_WHERE_DATA_FROM_THIS_TUTORIAL_IS_STORED'\n", + "WORK_DIR = 'PATH_TO_A_DIRECTORY_WHERE_SCRIPTS_FOR_THIS_TUTORIAL_ARE_SAVED'\n", "MODEL_CONFIG = \"punctuation_capitalization_lexical_audio_config.yaml\"\n", "\n", "# model parameters\n", @@ -580,22 +280,13 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { "pycharm": { "name": "#%% md\n" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloading get_libritts_data.py...\n", - "100% [................................................................................] 4804 / 4804" - ] - } - ], + "outputs": [], "source": [ "# download get_libritts_data.py script to download and preprocess the LibriTTS data\n", "os.makedirs(WORK_DIR, exist_ok=True)\n", @@ -608,24 +299,13 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": { "pycharm": { "name": "#%%\n" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Traceback (most recent call last):\r\n", - " File \"/home/mnovikov/NeMo/tutorials/nlp/./scripts/get_libritts_data.py\", line 29, in \r\n", - " from get_tatoeba_data import create_text_and_labels\r\n", - "ModuleNotFoundError: No module named 'get_tatoeba_data'\r\n" - ] - } - ], + "outputs": [], "source": [ "# download and preprocess the data\n", "# we will use dev_clean and dev_other subsets\n", From aa2880a5d144bf5470ff5b6af4640ef102e0b8b2 Mon Sep 17 00:00:00 2001 From: Matvei Novikov Date: Mon, 7 Nov 2022 20:32:17 +0400 Subject: [PATCH 3/3] moved `create_text_and_labels` to token_classification_utils.py Signed-off-by: Matvei Novikov --- .../data/get_libritts_data.py | 64 +----------------- .../data/get_tatoeba_data.py | 64 +----------------- .../token_classification_utils.py | 66 ++++++++++++++++++- 3 files changed, 67 insertions(+), 127 deletions(-) diff --git a/examples/nlp/token_classification/data/get_libritts_data.py b/examples/nlp/token_classification/data/get_libritts_data.py index 8e08cc5a8665..86a5d01eb9dc 100644 --- a/examples/nlp/token_classification/data/get_libritts_data.py +++ b/examples/nlp/token_classification/data/get_libritts_data.py @@ -21,12 +21,12 @@ import os import re import shutil -import string import subprocess import tarfile from tqdm import tqdm +from nemo.collections.nlp.data.token_classification.token_classification_utils import create_text_and_labels from nemo.utils import logging URL = { @@ -40,68 +40,6 @@ } -def remove_punctuation(word: str): - """ - Removes all punctuation marks from a word except for ' - that is often a part of word: don't, it's, and so on - """ - all_punct_marks = string.punctuation.replace("'", '') - return re.sub('[' + all_punct_marks + ']', '', word) - - -def create_text_and_labels(output_dir: str, file_path: str, punct_marks: str = ',.?'): - """ - Create datasets for training and evaluation. - - Args: - output_dir: path to the output data directory - file_path: path to file name - punct_marks: supported punctuation marks - - The data will be split into 2 files: text.txt and labels.txt. \ - Each line of the text.txt file contains text sequences, where words\ - are separated with spaces. The labels.txt file contains \ - corresponding labels for each word in text.txt, the labels are \ - separated with spaces. Each line of the files should follow the \ - format: \ - [WORD] [SPACE] [WORD] [SPACE] [WORD] (for text.txt) and \ - [LABEL] [SPACE] [LABEL] [SPACE] [LABEL] (for labels.txt).' - """ - if not os.path.exists(file_path): - raise ValueError(f'{file_path} not found') - - os.makedirs(output_dir, exist_ok=True) - - base_name = os.path.basename(file_path) - labels_file = os.path.join(output_dir, 'labels_' + base_name) - text_file = os.path.join(output_dir, 'text_' + base_name) - - with open(file_path, 'r') as f: - with open(text_file, 'w') as text_f: - with open(labels_file, 'w') as labels_f: - for line in f: - line = line.split() - text = '' - labels = '' - for word in line: - label = word[-1] if word[-1] in punct_marks else 'O' - word = remove_punctuation(word) - if len(word) > 0: - if word[0].isupper(): - label += 'U' - else: - label += 'O' - - word = word.lower() - text += word + ' ' - labels += label + ' ' - - text_f.write(text.strip() + '\n') - labels_f.write(labels.strip() + '\n') - - print(f'{text_file} and {labels_file} created from {file_path}.') - - def __extract_file(filepath, data_dir): try: tar = tarfile.open(filepath) diff --git a/examples/nlp/token_classification/data/get_tatoeba_data.py b/examples/nlp/token_classification/data/get_tatoeba_data.py index 727848b550ab..6a4cd23b249d 100644 --- a/examples/nlp/token_classification/data/get_tatoeba_data.py +++ b/examples/nlp/token_classification/data/get_tatoeba_data.py @@ -17,9 +17,9 @@ import os import random import re -import string import subprocess +from nemo.collections.nlp.data.token_classification.token_classification_utils import create_text_and_labels from nemo.utils import logging URL = {'tatoeba': 'https://downloads.tatoeba.org/exports/sentences.csv'} @@ -120,68 +120,6 @@ def __split_into_train_dev(in_file: str, train_file: str, dev_file: str, percent dev_file.write(' '.join(lines[-dev_size:])) -def remove_punctuation(word: str): - """ - Removes all punctuation marks from a word except for ' - that is often a part of word: don't, it's, and so on - """ - all_punct_marks = string.punctuation.replace("'", '') - return re.sub('[' + all_punct_marks + ']', '', word) - - -def create_text_and_labels(output_dir: str, file_path: str, punct_marks: str = ',.?'): - """ - Create datasets for training and evaluation. - - Args: - output_dir: path to the output data directory - file_path: path to file name - punct_marks: supported punctuation marks - - The data will be split into 2 files: text.txt and labels.txt. \ - Each line of the text.txt file contains text sequences, where words\ - are separated with spaces. The labels.txt file contains \ - corresponding labels for each word in text.txt, the labels are \ - separated with spaces. Each line of the files should follow the \ - format: \ - [WORD] [SPACE] [WORD] [SPACE] [WORD] (for text.txt) and \ - [LABEL] [SPACE] [LABEL] [SPACE] [LABEL] (for labels.txt).' - """ - if not os.path.exists(file_path): - raise ValueError(f'{file_path} not found') - - os.makedirs(output_dir, exist_ok=True) - - base_name = os.path.basename(file_path) - labels_file = os.path.join(output_dir, 'labels_' + base_name) - text_file = os.path.join(output_dir, 'text_' + base_name) - - with open(file_path, 'r') as f: - with open(text_file, 'w') as text_f: - with open(labels_file, 'w') as labels_f: - for line in f: - line = line.split() - text = '' - labels = '' - for word in line: - label = word[-1] if word[-1] in punct_marks else 'O' - word = remove_punctuation(word) - if len(word) > 0: - if word[0].isupper(): - label += 'U' - else: - label += 'O' - - word = word.lower() - text += word + ' ' - labels += label + ' ' - - text_f.write(text.strip() + '\n') - labels_f.write(labels.strip() + '\n') - - print(f'{text_file} and {labels_file} created from {file_path}.') - - def __delete_file(file_to_del: str): """ Deletes the file diff --git a/nemo/collections/nlp/data/token_classification/token_classification_utils.py b/nemo/collections/nlp/data/token_classification/token_classification_utils.py index 828ef1180e0b..94acd69d3b11 100644 --- a/nemo/collections/nlp/data/token_classification/token_classification_utils.py +++ b/nemo/collections/nlp/data/token_classification/token_classification_utils.py @@ -14,6 +14,8 @@ import os import pickle +import re +import string from typing import Dict from nemo.collections.nlp.data.data_utils.data_preprocessing import ( @@ -23,7 +25,69 @@ ) from nemo.utils import logging -__all__ = ['get_label_ids'] +__all__ = ['get_label_ids', 'create_text_and_labels'] + + +def remove_punctuation(word: str): + """ + Removes all punctuation marks from a word except for ' + that is often a part of word: don't, it's, and so on + """ + all_punct_marks = string.punctuation.replace("'", '') + return re.sub('[' + all_punct_marks + ']', '', word) + + +def create_text_and_labels(output_dir: str, file_path: str, punct_marks: str = ',.?'): + """ + Create datasets for training and evaluation. + + Args: + output_dir: path to the output data directory + file_path: path to file name + punct_marks: supported punctuation marks + + The data will be split into 2 files: text.txt and labels.txt. \ + Each line of the text.txt file contains text sequences, where words\ + are separated with spaces. The labels.txt file contains \ + corresponding labels for each word in text.txt, the labels are \ + separated with spaces. Each line of the files should follow the \ + format: \ + [WORD] [SPACE] [WORD] [SPACE] [WORD] (for text.txt) and \ + [LABEL] [SPACE] [LABEL] [SPACE] [LABEL] (for labels.txt).' + """ + if not os.path.exists(file_path): + raise ValueError(f'{file_path} not found') + + os.makedirs(output_dir, exist_ok=True) + + base_name = os.path.basename(file_path) + labels_file = os.path.join(output_dir, 'labels_' + base_name) + text_file = os.path.join(output_dir, 'text_' + base_name) + + with open(file_path, 'r') as f: + with open(text_file, 'w') as text_f: + with open(labels_file, 'w') as labels_f: + for line in f: + line = line.split() + text = '' + labels = '' + for word in line: + label = word[-1] if word[-1] in punct_marks else 'O' + word = remove_punctuation(word) + if len(word) > 0: + if word[0].isupper(): + label += 'U' + else: + label += 'O' + + word = word.lower() + text += word + ' ' + labels += label + ' ' + + text_f.write(text.strip() + '\n') + labels_f.write(labels.strip() + '\n') + + print(f'{text_file} and {labels_file} created from {file_path}.') def get_label_ids(