From 5c69dc152aa77b0395002574775ec5bc66f5fa10 Mon Sep 17 00:00:00 2001 From: Camille Zhong <44392324+Camille7777@users.noreply.github.com> Date: Wed, 22 Mar 2023 17:18:13 +0800 Subject: [PATCH 01/14] Add RoBERTa for RLHF Stage 2 & 3 (test) RoBERTa for RLHF Stage 2 & 3 (still in testing) --- .../chatgpt/models/roberta/__init__.py | 5 +++ .../chatgpt/models/roberta/roberta_actor.py | 35 +++++++++++++++++ .../chatgpt/models/roberta/roberta_critic.py | 38 ++++++++++++++++++ .../chatgpt/models/roberta/roberta_rm.py | 39 +++++++++++++++++++ 4 files changed, 117 insertions(+) create mode 100644 applications/ChatGPT/chatgpt/models/roberta/__init__.py create mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py create mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py create mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py diff --git a/applications/ChatGPT/chatgpt/models/roberta/__init__.py b/applications/ChatGPT/chatgpt/models/roberta/__init__.py new file mode 100644 index 000000000000..7b38d5bdbfed --- /dev/null +++ b/applications/ChatGPT/chatgpt/models/roberta/__init__.py @@ -0,0 +1,5 @@ +from .roberta_actor import RoBERTaActor +from .roberta_critic import RoBERTaCritic +from .roberta_rm import RoBERTaRM + +__all__ = ['RoBERTaActor', 'RoBERTaCritic', 'RoBERTaRM'] \ No newline at end of file diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py new file mode 100644 index 000000000000..e35fa6eb19a8 --- /dev/null +++ b/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py @@ -0,0 +1,35 @@ +from typing import Optional + +from transformers.models.roberta.configuration_roberta import RobertaConfig +from transformers.models.roberta.modeling_roberta import RobertaForCausalLM + +from ..base import Actor + +class RoBERTaActor(Actor): + """ + RoBERTa Actor model. + + Args: + pretrained (str): Pretrained model name or path. + config (RoBERTaConfig): Model config. + checkpoint (bool): Enable gradient checkpointing. + lora_rank (int): Rank of the low-rank approximation. + lora_train_bias (str): LoRA bias training mode. + """ + + + def __init__(self, + pretrained: Optional[str] = None, + config: Optional[RobertaConfig] = None, + checkpoint: bool = False, + lora_rank: int = 0, + lora_train_bias: str = 'none') -> None: + if pretrained is not None: + model = RobertaForCausalLM.from_pretrained(pretrained) + elif config is not None: + model = RobertaForCausalLM(config) + else: + model = RobertaForCausalLM(RobertaConfig()) + if checkpoint: + model.gradient_checkpointing_enable() + super().__init__(model, lora_rank, lora_train_bias) diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py new file mode 100644 index 000000000000..3215473ab2f8 --- /dev/null +++ b/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py @@ -0,0 +1,38 @@ +from typing import Optional + +import torch.nn as nn +from transformers.models.roberta.configuration_roberta import RobertaConfig +from transformers.models.roberta.modeling_roberta import RobertaModel + +from ..base import Critic + + +class RoBERTaCritic(Critic): + """ + RoBERTa Critic model. + + Args: + pretrained (str): Pretrained model name or path. + config (RoBERTa Config): Model config. + checkpoint (bool): Enable gradient checkpointing. + lora_rank (int): Rank of the low-rank approximation. + lora_train_bias (str): LoRA bias training mode. + """ + + def __init__(self, + pretrained: Optional[str] = None, + config: Optional[RobertaConfig] = None, + checkpoint: bool = False, + lora_rank: int = 0, + lora_train_bias: str = 'none', + **kwargs) -> None: + if pretrained is not None: + model = RobertaModel.from_pretrained(pretrained) + elif config is not None: + model = RobertaModel(config) + else: + model = RobertaModel(RobertaConfig()) + if checkpoint: + model.gradient_checkpointing_enable() + value_head = nn.Linear(model.config.hidden_size, 1) + super().__init__(model, value_head, lora_rank, lora_train_bias, **kwargs) diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py new file mode 100644 index 000000000000..9930b8ef0770 --- /dev/null +++ b/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py @@ -0,0 +1,39 @@ +from typing import Optional + +import torch.nn as nn +from transformers import RobertaConfig, RobertaModel + + +from ..base import RewardModel + + +class RoBERTaRM(RewardModel): + """ + RoBERTa Reward model. + + Args: + pretrained (str): Pretrained model name or path. + config (RoBERTaConfig): Model config. + checkpoint (bool): Enable gradient checkpointing. + lora_rank (int): Rank of the low-rank approximation. + lora_train_bias (str): LoRA bias training mode. + """ + + def __init__(self, + pretrained: Optional[str] = None, + config: Optional[RobertaConfig] = None, + checkpoint: bool = False, + lora_rank: int = 0, + lora_train_bias: str = 'none') -> None: + if pretrained is not None: + model = RobertaModel.from_pretrained(pretrained) + elif config is not None: + model = RobertaModel(config) + else: + model = RobertaModel(RobertaConfig()) + if checkpoint: + model.gradient_checkpointing_enable() + + value_head = nn.Linear(model.config.hidden_size, 1) + value_head.weight.data.normal_(mean=0.0, std=1/(model.config.hidden_size + 1)) + super().__init__(model, value_head, lora_rank, lora_train_bias) From 548e499c50199731e8f20cd55cdde51d8aec79c8 Mon Sep 17 00:00:00 2001 From: Camille Zhong <44392324+Camille7777@users.noreply.github.com> Date: Wed, 22 Mar 2023 17:30:46 +0800 Subject: [PATCH 02/14] Revert "Add RoBERTa for RLHF Stage 2 & 3 (test)" This reverts commit 06741d894dcbe958acd4e10d771f22275e20e368. --- .../chatgpt/models/roberta/__init__.py | 5 --- .../chatgpt/models/roberta/roberta_actor.py | 35 ----------------- .../chatgpt/models/roberta/roberta_critic.py | 38 ------------------ .../chatgpt/models/roberta/roberta_rm.py | 39 ------------------- 4 files changed, 117 deletions(-) delete mode 100644 applications/ChatGPT/chatgpt/models/roberta/__init__.py delete mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py delete mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py delete mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py diff --git a/applications/ChatGPT/chatgpt/models/roberta/__init__.py b/applications/ChatGPT/chatgpt/models/roberta/__init__.py deleted file mode 100644 index 7b38d5bdbfed..000000000000 --- a/applications/ChatGPT/chatgpt/models/roberta/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .roberta_actor import RoBERTaActor -from .roberta_critic import RoBERTaCritic -from .roberta_rm import RoBERTaRM - -__all__ = ['RoBERTaActor', 'RoBERTaCritic', 'RoBERTaRM'] \ No newline at end of file diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py deleted file mode 100644 index e35fa6eb19a8..000000000000 --- a/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py +++ /dev/null @@ -1,35 +0,0 @@ -from typing import Optional - -from transformers.models.roberta.configuration_roberta import RobertaConfig -from transformers.models.roberta.modeling_roberta import RobertaForCausalLM - -from ..base import Actor - -class RoBERTaActor(Actor): - """ - RoBERTa Actor model. - - Args: - pretrained (str): Pretrained model name or path. - config (RoBERTaConfig): Model config. - checkpoint (bool): Enable gradient checkpointing. - lora_rank (int): Rank of the low-rank approximation. - lora_train_bias (str): LoRA bias training mode. - """ - - - def __init__(self, - pretrained: Optional[str] = None, - config: Optional[RobertaConfig] = None, - checkpoint: bool = False, - lora_rank: int = 0, - lora_train_bias: str = 'none') -> None: - if pretrained is not None: - model = RobertaForCausalLM.from_pretrained(pretrained) - elif config is not None: - model = RobertaForCausalLM(config) - else: - model = RobertaForCausalLM(RobertaConfig()) - if checkpoint: - model.gradient_checkpointing_enable() - super().__init__(model, lora_rank, lora_train_bias) diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py deleted file mode 100644 index 3215473ab2f8..000000000000 --- a/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Optional - -import torch.nn as nn -from transformers.models.roberta.configuration_roberta import RobertaConfig -from transformers.models.roberta.modeling_roberta import RobertaModel - -from ..base import Critic - - -class RoBERTaCritic(Critic): - """ - RoBERTa Critic model. - - Args: - pretrained (str): Pretrained model name or path. - config (RoBERTa Config): Model config. - checkpoint (bool): Enable gradient checkpointing. - lora_rank (int): Rank of the low-rank approximation. - lora_train_bias (str): LoRA bias training mode. - """ - - def __init__(self, - pretrained: Optional[str] = None, - config: Optional[RobertaConfig] = None, - checkpoint: bool = False, - lora_rank: int = 0, - lora_train_bias: str = 'none', - **kwargs) -> None: - if pretrained is not None: - model = RobertaModel.from_pretrained(pretrained) - elif config is not None: - model = RobertaModel(config) - else: - model = RobertaModel(RobertaConfig()) - if checkpoint: - model.gradient_checkpointing_enable() - value_head = nn.Linear(model.config.hidden_size, 1) - super().__init__(model, value_head, lora_rank, lora_train_bias, **kwargs) diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py deleted file mode 100644 index 9930b8ef0770..000000000000 --- a/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import Optional - -import torch.nn as nn -from transformers import RobertaConfig, RobertaModel - - -from ..base import RewardModel - - -class RoBERTaRM(RewardModel): - """ - RoBERTa Reward model. - - Args: - pretrained (str): Pretrained model name or path. - config (RoBERTaConfig): Model config. - checkpoint (bool): Enable gradient checkpointing. - lora_rank (int): Rank of the low-rank approximation. - lora_train_bias (str): LoRA bias training mode. - """ - - def __init__(self, - pretrained: Optional[str] = None, - config: Optional[RobertaConfig] = None, - checkpoint: bool = False, - lora_rank: int = 0, - lora_train_bias: str = 'none') -> None: - if pretrained is not None: - model = RobertaModel.from_pretrained(pretrained) - elif config is not None: - model = RobertaModel(config) - else: - model = RobertaModel(RobertaConfig()) - if checkpoint: - model.gradient_checkpointing_enable() - - value_head = nn.Linear(model.config.hidden_size, 1) - value_head.weight.data.normal_(mean=0.0, std=1/(model.config.hidden_size + 1)) - super().__init__(model, value_head, lora_rank, lora_train_bias) From 026c363ff28b1aef6a04fe3329cd310c65b186f0 Mon Sep 17 00:00:00 2001 From: Camille Zhong <44392324+Camille7777@users.noreply.github.com> Date: Thu, 23 Mar 2023 18:44:06 +0800 Subject: [PATCH 03/14] Add RoBERTa for RLHF stage 2 & 3 1. add roberta folder under model folder 2. add roberta option in train_reward_model.py 3. add some test in testci --- .../chatgpt/models/roberta/__init__.py | 5 +++ .../chatgpt/models/roberta/roberta_actor.py | 35 +++++++++++++++++ .../chatgpt/models/roberta/roberta_critic.py | 38 ++++++++++++++++++ .../chatgpt/models/roberta/roberta_rm.py | 39 +++++++++++++++++++ 4 files changed, 117 insertions(+) create mode 100644 applications/ChatGPT/chatgpt/models/roberta/__init__.py create mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py create mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py create mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py diff --git a/applications/ChatGPT/chatgpt/models/roberta/__init__.py b/applications/ChatGPT/chatgpt/models/roberta/__init__.py new file mode 100644 index 000000000000..0f4a8de067b1 --- /dev/null +++ b/applications/ChatGPT/chatgpt/models/roberta/__init__.py @@ -0,0 +1,5 @@ +from .roberta_actor import RoBERTaActor +from .roberta_critic import RoBERTaCritic +from .roberta_rm import RoBERTaRM + +__all__ = ['RoBERTaActor', 'RoBERTaCritic', 'RoBERTaRM'] \ No newline at end of file diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py new file mode 100644 index 000000000000..e35fa6eb19a8 --- /dev/null +++ b/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py @@ -0,0 +1,35 @@ +from typing import Optional + +from transformers.models.roberta.configuration_roberta import RobertaConfig +from transformers.models.roberta.modeling_roberta import RobertaForCausalLM + +from ..base import Actor + +class RoBERTaActor(Actor): + """ + RoBERTa Actor model. + + Args: + pretrained (str): Pretrained model name or path. + config (RoBERTaConfig): Model config. + checkpoint (bool): Enable gradient checkpointing. + lora_rank (int): Rank of the low-rank approximation. + lora_train_bias (str): LoRA bias training mode. + """ + + + def __init__(self, + pretrained: Optional[str] = None, + config: Optional[RobertaConfig] = None, + checkpoint: bool = False, + lora_rank: int = 0, + lora_train_bias: str = 'none') -> None: + if pretrained is not None: + model = RobertaForCausalLM.from_pretrained(pretrained) + elif config is not None: + model = RobertaForCausalLM(config) + else: + model = RobertaForCausalLM(RobertaConfig()) + if checkpoint: + model.gradient_checkpointing_enable() + super().__init__(model, lora_rank, lora_train_bias) diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py new file mode 100644 index 000000000000..47b083435f03 --- /dev/null +++ b/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py @@ -0,0 +1,38 @@ +from typing import Optional + +import torch.nn as nn +from transformers.models.roberta.configuration_roberta import RobertaConfig +from transformers.models.roberta.modeling_roberta import RobertaModel + +from ..base import Critic + + +class RoBERTaCritic(Critic): + """ + RoBERTa Critic model. + + Args: + pretrained (str): Pretrained model name or path. + config (RoBERTa Config): Model config. + checkpoint (bool): Enable gradient checkpointing. + lora_rank (int): Rank of the low-rank approximation. + lora_train_bias (str): LoRA bias training mode. + """ + + def __init__(self, + pretrained: Optional[str] = None, + config: Optional[RobertaConfig] = None, + checkpoint: bool = False, + lora_rank: int = 0, + lora_train_bias: str = 'none', + **kwargs) -> None: + if pretrained is not None: + model = RobertaModel.from_pretrained(pretrained, add_pooling_layer = False) + elif config is not None: + model = RobertaModel(config) + else: + model = RobertaModel(RobertaConfig()) + if checkpoint: + model.gradient_checkpointing_enable() + value_head = nn.Linear(model.config.hidden_size, 1) + super().__init__(model, value_head, lora_rank, lora_train_bias, **kwargs) diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py new file mode 100644 index 000000000000..1fb89deaf2a7 --- /dev/null +++ b/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py @@ -0,0 +1,39 @@ +from typing import Optional + +import torch.nn as nn +from transformers import RobertaConfig, RobertaModel + + +from ..base import RewardModel + + +class RoBERTaRM(RewardModel): + """ + RoBERTa Reward model. + + Args: + pretrained (str): Pretrained model name or path. + config (RoBERTaConfig): Model config. + checkpoint (bool): Enable gradient checkpointing. + lora_rank (int): Rank of the low-rank approximation. + lora_train_bias (str): LoRA bias training mode. + """ + + def __init__(self, + pretrained: Optional[str] = None, + config: Optional[RobertaConfig] = None, + checkpoint: bool = False, + lora_rank: int = 0, + lora_train_bias: str = 'none') -> None: + if pretrained is not None: + model = RobertaModel.from_pretrained(pretrained, add_pooling_layer = False) + elif config is not None: + model = RobertaModel(config) + else: + model = RobertaModel(RobertaConfig()) + if checkpoint: + model.gradient_checkpointing_enable() + + value_head = nn.Linear(model.config.hidden_size, 1) + value_head.weight.data.normal_(mean=0.0, std=1/(model.config.hidden_size + 1)) + super().__init__(model, value_head, lora_rank, lora_train_bias) \ No newline at end of file From a6aa921a36376fe87c4cb21304d41ed36ef0db95 Mon Sep 17 00:00:00 2001 From: Camille Zhong <44392324+Camille7777@users.noreply.github.com> Date: Thu, 23 Mar 2023 20:31:59 +0800 Subject: [PATCH 04/14] Update test_ci.sh --- applications/Chat/examples/test_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/Chat/examples/test_ci.sh b/applications/Chat/examples/test_ci.sh index 64cf68a0a13f..2a5b1b50b989 100755 --- a/applications/Chat/examples/test_ci.sh +++ b/applications/Chat/examples/test_ci.sh @@ -110,7 +110,7 @@ torchrun --standalone --nproc_per_node=2 ${BASE}/train_reward_model.py \ torchrun --standalone --nproc_per_node=2 ${BASE}/train_reward_model.py \ --pretrain 'roberta-base' --model 'roberta' \ - --strategy colossalai_zero2 --loss_fn 'log_exp'\ + --strategy colossalai_zero2 --loss_fn 'log_sig'\ --dataset 'Anthropic/hh-rlhf' --subset 'harmless-base'\ --test True --lora_rank 4 From 96dda63d9caafd767966de3a59ee56595ddeb8f9 Mon Sep 17 00:00:00 2001 From: Camille Zhong <44392324+Camille7777@users.noreply.github.com> Date: Thu, 23 Mar 2023 20:58:59 +0800 Subject: [PATCH 05/14] Revert "Update test_ci.sh" This reverts commit 9c7352b81766f3177d31eeec0ec178a301df966a. --- applications/Chat/examples/test_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/Chat/examples/test_ci.sh b/applications/Chat/examples/test_ci.sh index 2a5b1b50b989..64cf68a0a13f 100755 --- a/applications/Chat/examples/test_ci.sh +++ b/applications/Chat/examples/test_ci.sh @@ -110,7 +110,7 @@ torchrun --standalone --nproc_per_node=2 ${BASE}/train_reward_model.py \ torchrun --standalone --nproc_per_node=2 ${BASE}/train_reward_model.py \ --pretrain 'roberta-base' --model 'roberta' \ - --strategy colossalai_zero2 --loss_fn 'log_sig'\ + --strategy colossalai_zero2 --loss_fn 'log_exp'\ --dataset 'Anthropic/hh-rlhf' --subset 'harmless-base'\ --test True --lora_rank 4 From a4a860f9ba02794657478f6d3b3c8e344af2fb0b Mon Sep 17 00:00:00 2001 From: Camille Zhong <44392324+Camille7777@users.noreply.github.com> Date: Wed, 22 Mar 2023 17:18:13 +0800 Subject: [PATCH 06/14] Add RoBERTa for RLHF Stage 2 & 3 (test) RoBERTa for RLHF Stage 2 & 3 (still in testing) --- applications/ChatGPT/chatgpt/models/roberta/__init__.py | 2 +- applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py | 2 +- applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/applications/ChatGPT/chatgpt/models/roberta/__init__.py b/applications/ChatGPT/chatgpt/models/roberta/__init__.py index 0f4a8de067b1..7b38d5bdbfed 100644 --- a/applications/ChatGPT/chatgpt/models/roberta/__init__.py +++ b/applications/ChatGPT/chatgpt/models/roberta/__init__.py @@ -1,4 +1,4 @@ -from .roberta_actor import RoBERTaActor +from .roberta_actor import RoBERTaActor from .roberta_critic import RoBERTaCritic from .roberta_rm import RoBERTaRM diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py index 47b083435f03..3215473ab2f8 100644 --- a/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py +++ b/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py @@ -27,7 +27,7 @@ def __init__(self, lora_train_bias: str = 'none', **kwargs) -> None: if pretrained is not None: - model = RobertaModel.from_pretrained(pretrained, add_pooling_layer = False) + model = RobertaModel.from_pretrained(pretrained) elif config is not None: model = RobertaModel(config) else: diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py index 1fb89deaf2a7..9930b8ef0770 100644 --- a/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py +++ b/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py @@ -26,7 +26,7 @@ def __init__(self, lora_rank: int = 0, lora_train_bias: str = 'none') -> None: if pretrained is not None: - model = RobertaModel.from_pretrained(pretrained, add_pooling_layer = False) + model = RobertaModel.from_pretrained(pretrained) elif config is not None: model = RobertaModel(config) else: @@ -36,4 +36,4 @@ def __init__(self, value_head = nn.Linear(model.config.hidden_size, 1) value_head.weight.data.normal_(mean=0.0, std=1/(model.config.hidden_size + 1)) - super().__init__(model, value_head, lora_rank, lora_train_bias) \ No newline at end of file + super().__init__(model, value_head, lora_rank, lora_train_bias) From 2825ec41dac9ee29faf0413584806f65aaf40419 Mon Sep 17 00:00:00 2001 From: Camille Zhong <44392324+Camille7777@users.noreply.github.com> Date: Wed, 22 Mar 2023 17:30:46 +0800 Subject: [PATCH 07/14] Revert "Add RoBERTa for RLHF Stage 2 & 3 (test)" This reverts commit 06741d894dcbe958acd4e10d771f22275e20e368. --- .../chatgpt/models/roberta/__init__.py | 5 --- .../chatgpt/models/roberta/roberta_actor.py | 35 ----------------- .../chatgpt/models/roberta/roberta_critic.py | 38 ------------------ .../chatgpt/models/roberta/roberta_rm.py | 39 ------------------- 4 files changed, 117 deletions(-) delete mode 100644 applications/ChatGPT/chatgpt/models/roberta/__init__.py delete mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py delete mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py delete mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py diff --git a/applications/ChatGPT/chatgpt/models/roberta/__init__.py b/applications/ChatGPT/chatgpt/models/roberta/__init__.py deleted file mode 100644 index 7b38d5bdbfed..000000000000 --- a/applications/ChatGPT/chatgpt/models/roberta/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .roberta_actor import RoBERTaActor -from .roberta_critic import RoBERTaCritic -from .roberta_rm import RoBERTaRM - -__all__ = ['RoBERTaActor', 'RoBERTaCritic', 'RoBERTaRM'] \ No newline at end of file diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py deleted file mode 100644 index e35fa6eb19a8..000000000000 --- a/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py +++ /dev/null @@ -1,35 +0,0 @@ -from typing import Optional - -from transformers.models.roberta.configuration_roberta import RobertaConfig -from transformers.models.roberta.modeling_roberta import RobertaForCausalLM - -from ..base import Actor - -class RoBERTaActor(Actor): - """ - RoBERTa Actor model. - - Args: - pretrained (str): Pretrained model name or path. - config (RoBERTaConfig): Model config. - checkpoint (bool): Enable gradient checkpointing. - lora_rank (int): Rank of the low-rank approximation. - lora_train_bias (str): LoRA bias training mode. - """ - - - def __init__(self, - pretrained: Optional[str] = None, - config: Optional[RobertaConfig] = None, - checkpoint: bool = False, - lora_rank: int = 0, - lora_train_bias: str = 'none') -> None: - if pretrained is not None: - model = RobertaForCausalLM.from_pretrained(pretrained) - elif config is not None: - model = RobertaForCausalLM(config) - else: - model = RobertaForCausalLM(RobertaConfig()) - if checkpoint: - model.gradient_checkpointing_enable() - super().__init__(model, lora_rank, lora_train_bias) diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py deleted file mode 100644 index 3215473ab2f8..000000000000 --- a/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Optional - -import torch.nn as nn -from transformers.models.roberta.configuration_roberta import RobertaConfig -from transformers.models.roberta.modeling_roberta import RobertaModel - -from ..base import Critic - - -class RoBERTaCritic(Critic): - """ - RoBERTa Critic model. - - Args: - pretrained (str): Pretrained model name or path. - config (RoBERTa Config): Model config. - checkpoint (bool): Enable gradient checkpointing. - lora_rank (int): Rank of the low-rank approximation. - lora_train_bias (str): LoRA bias training mode. - """ - - def __init__(self, - pretrained: Optional[str] = None, - config: Optional[RobertaConfig] = None, - checkpoint: bool = False, - lora_rank: int = 0, - lora_train_bias: str = 'none', - **kwargs) -> None: - if pretrained is not None: - model = RobertaModel.from_pretrained(pretrained) - elif config is not None: - model = RobertaModel(config) - else: - model = RobertaModel(RobertaConfig()) - if checkpoint: - model.gradient_checkpointing_enable() - value_head = nn.Linear(model.config.hidden_size, 1) - super().__init__(model, value_head, lora_rank, lora_train_bias, **kwargs) diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py deleted file mode 100644 index 9930b8ef0770..000000000000 --- a/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import Optional - -import torch.nn as nn -from transformers import RobertaConfig, RobertaModel - - -from ..base import RewardModel - - -class RoBERTaRM(RewardModel): - """ - RoBERTa Reward model. - - Args: - pretrained (str): Pretrained model name or path. - config (RoBERTaConfig): Model config. - checkpoint (bool): Enable gradient checkpointing. - lora_rank (int): Rank of the low-rank approximation. - lora_train_bias (str): LoRA bias training mode. - """ - - def __init__(self, - pretrained: Optional[str] = None, - config: Optional[RobertaConfig] = None, - checkpoint: bool = False, - lora_rank: int = 0, - lora_train_bias: str = 'none') -> None: - if pretrained is not None: - model = RobertaModel.from_pretrained(pretrained) - elif config is not None: - model = RobertaModel(config) - else: - model = RobertaModel(RobertaConfig()) - if checkpoint: - model.gradient_checkpointing_enable() - - value_head = nn.Linear(model.config.hidden_size, 1) - value_head.weight.data.normal_(mean=0.0, std=1/(model.config.hidden_size + 1)) - super().__init__(model, value_head, lora_rank, lora_train_bias) From b1403ea91eb7325fa435f5c3fd903224426b7285 Mon Sep 17 00:00:00 2001 From: Camille Zhong <44392324+Camille7777@users.noreply.github.com> Date: Thu, 23 Mar 2023 18:44:06 +0800 Subject: [PATCH 08/14] Add RoBERTa for RLHF stage 2 & 3 1. add roberta folder under model folder 2. add roberta option in train_reward_model.py 3. add some test in testci --- .../chatgpt/models/roberta/__init__.py | 5 +++ .../chatgpt/models/roberta/roberta_actor.py | 35 +++++++++++++++++ .../chatgpt/models/roberta/roberta_critic.py | 38 ++++++++++++++++++ .../chatgpt/models/roberta/roberta_rm.py | 39 +++++++++++++++++++ 4 files changed, 117 insertions(+) create mode 100644 applications/ChatGPT/chatgpt/models/roberta/__init__.py create mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py create mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py create mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py diff --git a/applications/ChatGPT/chatgpt/models/roberta/__init__.py b/applications/ChatGPT/chatgpt/models/roberta/__init__.py new file mode 100644 index 000000000000..0f4a8de067b1 --- /dev/null +++ b/applications/ChatGPT/chatgpt/models/roberta/__init__.py @@ -0,0 +1,5 @@ +from .roberta_actor import RoBERTaActor +from .roberta_critic import RoBERTaCritic +from .roberta_rm import RoBERTaRM + +__all__ = ['RoBERTaActor', 'RoBERTaCritic', 'RoBERTaRM'] \ No newline at end of file diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py new file mode 100644 index 000000000000..e35fa6eb19a8 --- /dev/null +++ b/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py @@ -0,0 +1,35 @@ +from typing import Optional + +from transformers.models.roberta.configuration_roberta import RobertaConfig +from transformers.models.roberta.modeling_roberta import RobertaForCausalLM + +from ..base import Actor + +class RoBERTaActor(Actor): + """ + RoBERTa Actor model. + + Args: + pretrained (str): Pretrained model name or path. + config (RoBERTaConfig): Model config. + checkpoint (bool): Enable gradient checkpointing. + lora_rank (int): Rank of the low-rank approximation. + lora_train_bias (str): LoRA bias training mode. + """ + + + def __init__(self, + pretrained: Optional[str] = None, + config: Optional[RobertaConfig] = None, + checkpoint: bool = False, + lora_rank: int = 0, + lora_train_bias: str = 'none') -> None: + if pretrained is not None: + model = RobertaForCausalLM.from_pretrained(pretrained) + elif config is not None: + model = RobertaForCausalLM(config) + else: + model = RobertaForCausalLM(RobertaConfig()) + if checkpoint: + model.gradient_checkpointing_enable() + super().__init__(model, lora_rank, lora_train_bias) diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py new file mode 100644 index 000000000000..47b083435f03 --- /dev/null +++ b/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py @@ -0,0 +1,38 @@ +from typing import Optional + +import torch.nn as nn +from transformers.models.roberta.configuration_roberta import RobertaConfig +from transformers.models.roberta.modeling_roberta import RobertaModel + +from ..base import Critic + + +class RoBERTaCritic(Critic): + """ + RoBERTa Critic model. + + Args: + pretrained (str): Pretrained model name or path. + config (RoBERTa Config): Model config. + checkpoint (bool): Enable gradient checkpointing. + lora_rank (int): Rank of the low-rank approximation. + lora_train_bias (str): LoRA bias training mode. + """ + + def __init__(self, + pretrained: Optional[str] = None, + config: Optional[RobertaConfig] = None, + checkpoint: bool = False, + lora_rank: int = 0, + lora_train_bias: str = 'none', + **kwargs) -> None: + if pretrained is not None: + model = RobertaModel.from_pretrained(pretrained, add_pooling_layer = False) + elif config is not None: + model = RobertaModel(config) + else: + model = RobertaModel(RobertaConfig()) + if checkpoint: + model.gradient_checkpointing_enable() + value_head = nn.Linear(model.config.hidden_size, 1) + super().__init__(model, value_head, lora_rank, lora_train_bias, **kwargs) diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py new file mode 100644 index 000000000000..1fb89deaf2a7 --- /dev/null +++ b/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py @@ -0,0 +1,39 @@ +from typing import Optional + +import torch.nn as nn +from transformers import RobertaConfig, RobertaModel + + +from ..base import RewardModel + + +class RoBERTaRM(RewardModel): + """ + RoBERTa Reward model. + + Args: + pretrained (str): Pretrained model name or path. + config (RoBERTaConfig): Model config. + checkpoint (bool): Enable gradient checkpointing. + lora_rank (int): Rank of the low-rank approximation. + lora_train_bias (str): LoRA bias training mode. + """ + + def __init__(self, + pretrained: Optional[str] = None, + config: Optional[RobertaConfig] = None, + checkpoint: bool = False, + lora_rank: int = 0, + lora_train_bias: str = 'none') -> None: + if pretrained is not None: + model = RobertaModel.from_pretrained(pretrained, add_pooling_layer = False) + elif config is not None: + model = RobertaModel(config) + else: + model = RobertaModel(RobertaConfig()) + if checkpoint: + model.gradient_checkpointing_enable() + + value_head = nn.Linear(model.config.hidden_size, 1) + value_head.weight.data.normal_(mean=0.0, std=1/(model.config.hidden_size + 1)) + super().__init__(model, value_head, lora_rank, lora_train_bias) \ No newline at end of file From ff527701997eba2e7242143af50195fa122c950b Mon Sep 17 00:00:00 2001 From: Camille Zhong <44392324+Camille7777@users.noreply.github.com> Date: Thu, 23 Mar 2023 20:31:59 +0800 Subject: [PATCH 09/14] Update test_ci.sh --- applications/Chat/examples/test_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/Chat/examples/test_ci.sh b/applications/Chat/examples/test_ci.sh index 64cf68a0a13f..2a5b1b50b989 100755 --- a/applications/Chat/examples/test_ci.sh +++ b/applications/Chat/examples/test_ci.sh @@ -110,7 +110,7 @@ torchrun --standalone --nproc_per_node=2 ${BASE}/train_reward_model.py \ torchrun --standalone --nproc_per_node=2 ${BASE}/train_reward_model.py \ --pretrain 'roberta-base' --model 'roberta' \ - --strategy colossalai_zero2 --loss_fn 'log_exp'\ + --strategy colossalai_zero2 --loss_fn 'log_sig'\ --dataset 'Anthropic/hh-rlhf' --subset 'harmless-base'\ --test True --lora_rank 4 From 512f70e801fd6079482828ec8a9b282ed8d4a15d Mon Sep 17 00:00:00 2001 From: Camille Zhong <44392324+Camille7777@users.noreply.github.com> Date: Thu, 23 Mar 2023 20:58:59 +0800 Subject: [PATCH 10/14] Revert "Update test_ci.sh" This reverts commit 9c7352b81766f3177d31eeec0ec178a301df966a. --- applications/Chat/examples/test_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/Chat/examples/test_ci.sh b/applications/Chat/examples/test_ci.sh index 2a5b1b50b989..64cf68a0a13f 100755 --- a/applications/Chat/examples/test_ci.sh +++ b/applications/Chat/examples/test_ci.sh @@ -110,7 +110,7 @@ torchrun --standalone --nproc_per_node=2 ${BASE}/train_reward_model.py \ torchrun --standalone --nproc_per_node=2 ${BASE}/train_reward_model.py \ --pretrain 'roberta-base' --model 'roberta' \ - --strategy colossalai_zero2 --loss_fn 'log_sig'\ + --strategy colossalai_zero2 --loss_fn 'log_exp'\ --dataset 'Anthropic/hh-rlhf' --subset 'harmless-base'\ --test True --lora_rank 4 From d643e91a49000a43caabb3c70359567c7de49aa1 Mon Sep 17 00:00:00 2001 From: Camille Zhong <44392324+Camille7777@users.noreply.github.com> Date: Thu, 30 Mar 2023 10:52:35 +0800 Subject: [PATCH 11/14] update roberta with coati --- .../chatgpt/models/roberta/__init__.py | 5 --- .../chatgpt/models/roberta/roberta_actor.py | 35 ----------------- .../chatgpt/models/roberta/roberta_critic.py | 38 ------------------ .../chatgpt/models/roberta/roberta_rm.py | 39 ------------------- 4 files changed, 117 deletions(-) delete mode 100644 applications/ChatGPT/chatgpt/models/roberta/__init__.py delete mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py delete mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py delete mode 100644 applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py diff --git a/applications/ChatGPT/chatgpt/models/roberta/__init__.py b/applications/ChatGPT/chatgpt/models/roberta/__init__.py deleted file mode 100644 index 0f4a8de067b1..000000000000 --- a/applications/ChatGPT/chatgpt/models/roberta/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .roberta_actor import RoBERTaActor -from .roberta_critic import RoBERTaCritic -from .roberta_rm import RoBERTaRM - -__all__ = ['RoBERTaActor', 'RoBERTaCritic', 'RoBERTaRM'] \ No newline at end of file diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py deleted file mode 100644 index e35fa6eb19a8..000000000000 --- a/applications/ChatGPT/chatgpt/models/roberta/roberta_actor.py +++ /dev/null @@ -1,35 +0,0 @@ -from typing import Optional - -from transformers.models.roberta.configuration_roberta import RobertaConfig -from transformers.models.roberta.modeling_roberta import RobertaForCausalLM - -from ..base import Actor - -class RoBERTaActor(Actor): - """ - RoBERTa Actor model. - - Args: - pretrained (str): Pretrained model name or path. - config (RoBERTaConfig): Model config. - checkpoint (bool): Enable gradient checkpointing. - lora_rank (int): Rank of the low-rank approximation. - lora_train_bias (str): LoRA bias training mode. - """ - - - def __init__(self, - pretrained: Optional[str] = None, - config: Optional[RobertaConfig] = None, - checkpoint: bool = False, - lora_rank: int = 0, - lora_train_bias: str = 'none') -> None: - if pretrained is not None: - model = RobertaForCausalLM.from_pretrained(pretrained) - elif config is not None: - model = RobertaForCausalLM(config) - else: - model = RobertaForCausalLM(RobertaConfig()) - if checkpoint: - model.gradient_checkpointing_enable() - super().__init__(model, lora_rank, lora_train_bias) diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py deleted file mode 100644 index 47b083435f03..000000000000 --- a/applications/ChatGPT/chatgpt/models/roberta/roberta_critic.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Optional - -import torch.nn as nn -from transformers.models.roberta.configuration_roberta import RobertaConfig -from transformers.models.roberta.modeling_roberta import RobertaModel - -from ..base import Critic - - -class RoBERTaCritic(Critic): - """ - RoBERTa Critic model. - - Args: - pretrained (str): Pretrained model name or path. - config (RoBERTa Config): Model config. - checkpoint (bool): Enable gradient checkpointing. - lora_rank (int): Rank of the low-rank approximation. - lora_train_bias (str): LoRA bias training mode. - """ - - def __init__(self, - pretrained: Optional[str] = None, - config: Optional[RobertaConfig] = None, - checkpoint: bool = False, - lora_rank: int = 0, - lora_train_bias: str = 'none', - **kwargs) -> None: - if pretrained is not None: - model = RobertaModel.from_pretrained(pretrained, add_pooling_layer = False) - elif config is not None: - model = RobertaModel(config) - else: - model = RobertaModel(RobertaConfig()) - if checkpoint: - model.gradient_checkpointing_enable() - value_head = nn.Linear(model.config.hidden_size, 1) - super().__init__(model, value_head, lora_rank, lora_train_bias, **kwargs) diff --git a/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py b/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py deleted file mode 100644 index 1fb89deaf2a7..000000000000 --- a/applications/ChatGPT/chatgpt/models/roberta/roberta_rm.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import Optional - -import torch.nn as nn -from transformers import RobertaConfig, RobertaModel - - -from ..base import RewardModel - - -class RoBERTaRM(RewardModel): - """ - RoBERTa Reward model. - - Args: - pretrained (str): Pretrained model name or path. - config (RoBERTaConfig): Model config. - checkpoint (bool): Enable gradient checkpointing. - lora_rank (int): Rank of the low-rank approximation. - lora_train_bias (str): LoRA bias training mode. - """ - - def __init__(self, - pretrained: Optional[str] = None, - config: Optional[RobertaConfig] = None, - checkpoint: bool = False, - lora_rank: int = 0, - lora_train_bias: str = 'none') -> None: - if pretrained is not None: - model = RobertaModel.from_pretrained(pretrained, add_pooling_layer = False) - elif config is not None: - model = RobertaModel(config) - else: - model = RobertaModel(RobertaConfig()) - if checkpoint: - model.gradient_checkpointing_enable() - - value_head = nn.Linear(model.config.hidden_size, 1) - value_head.weight.data.normal_(mean=0.0, std=1/(model.config.hidden_size + 1)) - super().__init__(model, value_head, lora_rank, lora_train_bias) \ No newline at end of file From 7c4dad6981c666a701492fa0ebb446a35e363c83 Mon Sep 17 00:00:00 2001 From: Camille Zhong <44392324+Camille7777@users.noreply.github.com> Date: Tue, 4 Apr 2023 10:13:24 +0800 Subject: [PATCH 12/14] chat ci update --- .github/workflows/run_chatgpt_examples.yml | 19 +++++++++++----- .github/workflows/run_chatgpt_unit_tests.yml | 23 +++++++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/.github/workflows/run_chatgpt_examples.yml b/.github/workflows/run_chatgpt_examples.yml index 51bb9d074644..9b3161b6a20d 100644 --- a/.github/workflows/run_chatgpt_examples.yml +++ b/.github/workflows/run_chatgpt_examples.yml @@ -4,10 +4,10 @@ on: pull_request: types: [synchronize, opened, reopened] paths: - - 'applications/ChatGPT/chatgpt/**' - - 'applications/ChatGPT/requirements.txt' - - 'applications/ChatGPT/setup.py' - - 'applications/ChatGPT/examples/**' + - 'applications/Chat/coati/**' + - 'applications/Chat/requirements.txt' + - 'applications/Chat/setup.py' + - 'applications/Chat/examples/**' jobs: @@ -28,13 +28,20 @@ jobs: - name: Install ColossalAI and ChatGPT run: | pip install -v . - cd applications/ChatGPT + cd applications/Chat pip install -v . pip install -r examples/requirements.txt + - name: Install Transformers + run: | + cd applications/Chat + git clone https://github.com/hpcaitech/transformers + cd transformers + pip install -v . + - name: Execute Examples run: | - cd applications/ChatGPT + cd applications/Chat rm -rf ~/.cache/colossalai ./examples/test_ci.sh env: diff --git a/.github/workflows/run_chatgpt_unit_tests.yml b/.github/workflows/run_chatgpt_unit_tests.yml index 4e539bfe06fd..882cb2fdfa45 100644 --- a/.github/workflows/run_chatgpt_unit_tests.yml +++ b/.github/workflows/run_chatgpt_unit_tests.yml @@ -4,12 +4,12 @@ on: pull_request: types: [synchronize, opened, reopened] paths: - - 'applications/ChatGPT/chatgpt/**' - - 'applications/ChatGPT/requirements.txt' - - 'applications/ChatGPT/setup.py' - - 'applications/ChatGPT/requirements-test.txt' - - 'applications/ChatGPT/tests/**' - - 'applications/ChatGPT/pytest.ini' + - 'applications/Chat/coati/**' + - 'applications/Chat/requirements.txt' + - 'applications/Chat/setup.py' + - 'applications/Chat/requirements-test.txt' + - 'applications/Chat/tests/**' + - 'applications/Chat/pytest.ini' jobs: tests: @@ -29,13 +29,20 @@ jobs: - name: Install ColossalAI and ChatGPT run: | pip install -v . - cd applications/ChatGPT + cd applications/Chat pip install -v . pip install -r requirements-test.txt + - name: Install Transformers + run: | + cd applications/Chat + git clone https://github.com/hpcaitech/transformers + cd transformers + pip install -v . + - name: Execute Unit Testing run: | - cd applications/ChatGPT + cd applications/Chat rm -rf ~/.cache/colossalai pytest tests/ env: From 7327cf33b477d2579c34611e535df8461b23dd1e Mon Sep 17 00:00:00 2001 From: Camille Zhong <44392324+Camille7777@users.noreply.github.com> Date: Tue, 4 Apr 2023 10:18:35 +0800 Subject: [PATCH 13/14] Revert "chat ci update" This reverts commit 17ae7ae01fa752bd3289fc39069868fde99cf846. --- .github/workflows/run_chatgpt_examples.yml | 19 +++++----------- .github/workflows/run_chatgpt_unit_tests.yml | 23 +++++++------------- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/.github/workflows/run_chatgpt_examples.yml b/.github/workflows/run_chatgpt_examples.yml index 9b3161b6a20d..51bb9d074644 100644 --- a/.github/workflows/run_chatgpt_examples.yml +++ b/.github/workflows/run_chatgpt_examples.yml @@ -4,10 +4,10 @@ on: pull_request: types: [synchronize, opened, reopened] paths: - - 'applications/Chat/coati/**' - - 'applications/Chat/requirements.txt' - - 'applications/Chat/setup.py' - - 'applications/Chat/examples/**' + - 'applications/ChatGPT/chatgpt/**' + - 'applications/ChatGPT/requirements.txt' + - 'applications/ChatGPT/setup.py' + - 'applications/ChatGPT/examples/**' jobs: @@ -28,20 +28,13 @@ jobs: - name: Install ColossalAI and ChatGPT run: | pip install -v . - cd applications/Chat + cd applications/ChatGPT pip install -v . pip install -r examples/requirements.txt - - name: Install Transformers - run: | - cd applications/Chat - git clone https://github.com/hpcaitech/transformers - cd transformers - pip install -v . - - name: Execute Examples run: | - cd applications/Chat + cd applications/ChatGPT rm -rf ~/.cache/colossalai ./examples/test_ci.sh env: diff --git a/.github/workflows/run_chatgpt_unit_tests.yml b/.github/workflows/run_chatgpt_unit_tests.yml index 882cb2fdfa45..4e539bfe06fd 100644 --- a/.github/workflows/run_chatgpt_unit_tests.yml +++ b/.github/workflows/run_chatgpt_unit_tests.yml @@ -4,12 +4,12 @@ on: pull_request: types: [synchronize, opened, reopened] paths: - - 'applications/Chat/coati/**' - - 'applications/Chat/requirements.txt' - - 'applications/Chat/setup.py' - - 'applications/Chat/requirements-test.txt' - - 'applications/Chat/tests/**' - - 'applications/Chat/pytest.ini' + - 'applications/ChatGPT/chatgpt/**' + - 'applications/ChatGPT/requirements.txt' + - 'applications/ChatGPT/setup.py' + - 'applications/ChatGPT/requirements-test.txt' + - 'applications/ChatGPT/tests/**' + - 'applications/ChatGPT/pytest.ini' jobs: tests: @@ -29,20 +29,13 @@ jobs: - name: Install ColossalAI and ChatGPT run: | pip install -v . - cd applications/Chat + cd applications/ChatGPT pip install -v . pip install -r requirements-test.txt - - name: Install Transformers - run: | - cd applications/Chat - git clone https://github.com/hpcaitech/transformers - cd transformers - pip install -v . - - name: Execute Unit Testing run: | - cd applications/Chat + cd applications/ChatGPT rm -rf ~/.cache/colossalai pytest tests/ env: From 98b0742ee33861c422929ee84c4db4f9ecbe24df Mon Sep 17 00:00:00 2001 From: Camille Zhong <44392324+Camille7777@users.noreply.github.com> Date: Wed, 5 Apr 2023 20:11:19 +0800 Subject: [PATCH 14/14] [Chat] fix the tokenizer "int too big to convert" error in SFT training fix the tokenizer error during SFT training using Bloom and OPT --- applications/Chat/coati/dataset/sft_dataset.py | 11 ++++++----- applications/Chat/examples/train_sft.py | 9 ++++++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/applications/Chat/coati/dataset/sft_dataset.py b/applications/Chat/coati/dataset/sft_dataset.py index 76ae6b158822..91e38f06daba 100644 --- a/applications/Chat/coati/dataset/sft_dataset.py +++ b/applications/Chat/coati/dataset/sft_dataset.py @@ -78,14 +78,14 @@ def __getitem__(self, idx): # return dict(self.prompts[idx], self.prompts[idx]) -def _tokenize_fn(strings: Sequence[str], tokenizer: transformers.PreTrainedTokenizer) -> Dict: +def _tokenize_fn(strings: Sequence[str], tokenizer: transformers.PreTrainedTokenizer, max_length: int) -> Dict: """Tokenize a list of strings.""" tokenized_list = [ tokenizer( text, return_tensors="pt", padding="longest", - max_length=tokenizer.model_max_length, + max_length=max_length, truncation=True, ) for text in strings ] @@ -105,10 +105,11 @@ def preprocess( sources: Sequence[str], targets: Sequence[str], tokenizer: transformers.PreTrainedTokenizer, + max_length: int, ) -> Dict: """Preprocess the data by tokenizing.""" examples = [s + t for s, t in zip(sources, targets)] - examples_tokenized, sources_tokenized = [_tokenize_fn(strings, tokenizer) for strings in (examples, sources)] + examples_tokenized, sources_tokenized = [_tokenize_fn(strings, tokenizer, max_length) for strings in (examples, sources)] input_ids = examples_tokenized["input_ids"] labels = copy.deepcopy(input_ids) for label, source_len in zip(labels, sources_tokenized["input_ids_lens"]): @@ -119,7 +120,7 @@ def preprocess( class SupervisedDataset(Dataset): """Dataset for supervised fine-tuning.""" - def __init__(self, data_path: str, tokenizer: transformers.PreTrainedTokenizer, max_datasets_size: int = None): + def __init__(self, data_path: str, tokenizer: transformers.PreTrainedTokenizer, max_datasets_size: int = None, max_length: int = 512): super(SupervisedDataset, self).__init__() logger.info("Loading data...") list_data_dict = jload(data_path) @@ -138,7 +139,7 @@ def __init__(self, data_path: str, tokenizer: transformers.PreTrainedTokenizer, targets = [f"{example['output']}{tokenizer.eos_token}" for example in list_data_dict] logger.info("Tokenizing inputs... This may take some time...") - data_dict = preprocess(sources, targets, tokenizer) + data_dict = preprocess(sources, targets, tokenizer, max_length) self.input_ids = data_dict["input_ids"] self.labels = data_dict["labels"] diff --git a/applications/Chat/examples/train_sft.py b/applications/Chat/examples/train_sft.py index 035d5a1ded1d..c0ac7b177694 100644 --- a/applications/Chat/examples/train_sft.py +++ b/applications/Chat/examples/train_sft.py @@ -71,6 +71,7 @@ def train(args): else: raise ValueError(f'Unsupported model "{args.model}"') tokenizer.pad_token = tokenizer.eos_token + max_len = args.max_len if args.model == 'llama': tokenizer = prepare_llama_tokenizer_and_embedding(tokenizer, model) @@ -99,13 +100,14 @@ def train(args): train_data = load_dataset(args.dataset, 'super_natural_instructions', split='train') eval_data = load_dataset(args.dataset, 'super_natural_instructions', split='test') - train_dataset = SFTDataset(train_data, tokenizer) - eval_dataset = SFTDataset(eval_data, tokenizer) + train_dataset = SFTDataset(train_data, tokenizer, max_len) + eval_dataset = SFTDataset(eval_data, tokenizer, max_len) else: train_dataset = SupervisedDataset(tokenizer=tokenizer, data_path=args.dataset, - max_datasets_size=args.max_datasets_size) + max_datasets_size=args.max_datasets_size, + max_length=max_len) eval_dataset = None data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer) @@ -176,6 +178,7 @@ def train(args): parser.add_argument('--need_optim_ckpt', type=bool, default=False) parser.add_argument('--max_epochs', type=int, default=3) parser.add_argument('--batch_size', type=int, default=4) + parser.add_argument('--max_len', type=int, default=512) parser.add_argument('--lora_rank', type=int, default=0, help="low-rank adaptation matrices rank") parser.add_argument('--log_interval', type=int, default=100, help="how many steps to log") parser.add_argument('--lr', type=float, default=5e-6)