From f5d93c1346b376143c6d499f164a69bb6007d450 Mon Sep 17 00:00:00 2001 From: Virginia Adams Date: Thu, 26 May 2022 14:38:37 +0000 Subject: [PATCH 1/3] Set Save on train end to false Signed-off-by: Virginia Adams --- .../conf/megatron_gpt_prompt_learning_config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/nlp/language_modeling/conf/megatron_gpt_prompt_learning_config.yaml b/examples/nlp/language_modeling/conf/megatron_gpt_prompt_learning_config.yaml index 4d72760a8f25..38c30ab1ae2e 100644 --- a/examples/nlp/language_modeling/conf/megatron_gpt_prompt_learning_config.yaml +++ b/examples/nlp/language_modeling/conf/megatron_gpt_prompt_learning_config.yaml @@ -32,7 +32,7 @@ exp_manager: monitor: val_loss save_top_k: 2 mode: min - save_nemo_on_train_end: True + save_nemo_on_train_end: False filename: 'megatron_gpt_prompt_tune--{val_loss:.3f}-{step}' model_parallel_size: ${model.tensor_model_parallel_size} save_best_model: True @@ -71,8 +71,8 @@ model: - taskname: 'rte' prompt_template: '<|VIRTUAL_PROMPT_0|>{text}{answer}' - total_virtual_tokens: 100 - virtual_token_splits: [100] + total_virtual_tokens: 10 + virtual_token_splits: [10] truncate_field: null answer_only_loss: True answer_field: 'answer' @@ -86,8 +86,8 @@ model: num_layers: 2 data: - train_ds: [data/squad_train.jsonl,] - validation_ds: [data/squad_val.jsonl,] + train_ds: [data/rte_train.jsonl,] + validation_ds: [data/rte_val.jsonl,] add_eos: True shuffle: True num_workers: 1 From b4781bcb6ab70b623cea25420c5cd3693860d655 Mon Sep 17 00:00:00 2001 From: Virginia Adams <78445382+vadam5@users.noreply.github.com> Date: Thu, 26 May 2022 11:05:31 -0400 Subject: [PATCH 2/3] Update prompt_learning.rst --- docs/source/nlp/prompt_learning.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/source/nlp/prompt_learning.rst b/docs/source/nlp/prompt_learning.rst index b2d3b1cc1c21..fc9531afbaf0 100644 --- a/docs/source/nlp/prompt_learning.rst +++ b/docs/source/nlp/prompt_learning.rst @@ -10,7 +10,7 @@ Instead of selecting discrete text prompts in a manual or automated fashion, pro Our continuous learning capability for combined p-tuning and prompt tuning with GPT style models is a NeMo specific extension of the author's original work. -Please also checkout our `prompt learning tutorial notebook. `_ +Please also checkout our `prompt learning tutorial notebook. `_ Terminology @@ -199,16 +199,16 @@ Prompt Learning Specific Config Values - bool - Whether to add an EOS token at the end of each training example (recommended). -An example config file can be found at https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/language_modeling/conf/megatron_gpt_prompt_learning_config.yaml +An example config file can be found at https://github.com/NVIDIA/NeMo/blob/r1.9.0/examples/nlp/language_modeling/conf/megatron_gpt_prompt_learning_config.yaml Setting New Tasks ^^^^^^^^^^^^^^^^^ After you p-tune or prompt-tune your model, you can always go back and p-tune or prompt-tune your model on more tasks without over writing the virtual prompts who've trained already. You can also use a different number of ``total_virtual_tokens`` between each training session as long as tasks ptuned or prompt tuned at the same time have the same number of ``total_virtual_tokens``. For this reason, when you ptune on a new task, you need to tell your model which of your tasks are new and which ones already exist (and thus you don't want to tune them). You do this by setting the ``new_tasks`` and ``existing_tasks`` values in the config file. -Example Multi-Task Prompt Tuning Command +Example Multi-Task Prompt Tuning Config and Command ^^^^^^^^^^ -First define a config called ``multitask-prompt-learning.yaml`` that looks like: +First define a config called ``multitask-prompt-learning.yaml`` demonstrated below. **In the** ``exp_manager`` **portion of the config,** ``save_on_train_end`` **should be set to** ``False`` **to avoid unnecessarily saving the incorrect model weights.** This is already done in the example `megatron_gpt_prompt_learning_config.yaml config `_ that you should use as your starting point. The correct prompt learning model will be saved at the ``model.nemo_path`` that you set. .. code:: @@ -261,7 +261,7 @@ First define a config called ``multitask-prompt-learning.yaml`` that looks like: optim: ... -(See https://github.com/NVIDIA/NeMo/blob/main/examples/nlp/language_modeling/conf/megatron_gpt_prompt_learning_config.yaml for what should go in the ``trainer``, ``exp_manager``, and ``optim`` sections.) +(See https://github.com/NVIDIA/NeMo/blob/r1.9.0/examples/nlp/language_modeling/conf/megatron_gpt_prompt_learning_config.yaml for what should go in the ``trainer``, ``exp_manager``, and ``optim`` sections.) Then run the command @@ -270,7 +270,7 @@ Then run the command python megatron_gpt_prompt_learning.py --config-name=multitask-prompt-learning.yaml -Example Multi-Task P-Tuning Command After Prompt-Tuning +Example Multi-Task P-Tuning Config and Command After Prompt-Tuning ^^^^^^^^^^ Update ``multitask-prompt-learning.yaml`` from the example above with p-tuning parameters for the new task. Be sure to update ``model.existing_tasks`` with the tasknames from previous prompt learning runs and to use the ``.nemo`` file saved at the end of your last prompt learning session. Values different from the config above have stars commented next to them. @@ -387,6 +387,6 @@ And the dataset class will automatically format your input to have the form: Instead of prompt dicts, you can also pass in a list of string paths to .json files on which you want to run inference. Similarly for all other scenarios, just add virtual_prompt_model=True if you're using a p-tuned/prompt-tuned model. -Example prompt learning script: `NeMo/examples/nlp/language_modeling/megatron_gpt_prompt_learning.py.py `__. +Example prompt learning script: `NeMo/examples/nlp/language_modeling/megatron_gpt_prompt_learning.py.py `__. -Example prompt tuned inference script: `NeMo/examples/nlp/language_modeling/megatron_gpt_eval.py `__. +Example prompt tuned inference script: `NeMo/examples/nlp/language_modeling/megatron_gpt_eval.py `__. From a1a6d01d105a9454a8856a6673a5180f93d14014 Mon Sep 17 00:00:00 2001 From: Virginia Adams <78445382+vadam5@users.noreply.github.com> Date: Thu, 26 May 2022 11:14:06 -0400 Subject: [PATCH 3/3] Update prompt_learning.rst --- docs/source/nlp/prompt_learning.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/nlp/prompt_learning.rst b/docs/source/nlp/prompt_learning.rst index fc9531afbaf0..3135783787ea 100644 --- a/docs/source/nlp/prompt_learning.rst +++ b/docs/source/nlp/prompt_learning.rst @@ -208,7 +208,7 @@ After you p-tune or prompt-tune your model, you can always go back and p-tune or Example Multi-Task Prompt Tuning Config and Command ^^^^^^^^^^ -First define a config called ``multitask-prompt-learning.yaml`` demonstrated below. **In the** ``exp_manager`` **portion of the config,** ``save_on_train_end`` **should be set to** ``False`` **to avoid unnecessarily saving the incorrect model weights.** This is already done in the example `megatron_gpt_prompt_learning_config.yaml config `_ that you should use as your starting point. The correct prompt learning model will be saved at the ``model.nemo_path`` that you set. +First define a config called ``multitask-prompt-learning.yaml`` demonstrated below. **In the** ``exp_manager`` **portion of the config,** ``save_on_train_end`` **should be set to** ``False`` **to avoid unnecessarily saving the incorrect model weights.** This is already done in the example `megatron_gpt_prompt_learning_config.yaml config `_ that you should use as your starting point. The correct prompt learning model will be saved at the ``model.nemo_path`` you set. .. code::