From b04636830fc37374f75f16e2e9bfc5bb7766b02d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Somoza?= Date: Sun, 16 Jun 2024 09:42:26 -0400 Subject: [PATCH 1/7] fix --- src/diffusers/loaders/lora.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/diffusers/loaders/lora.py b/src/diffusers/loaders/lora.py index 120a89533db6..ef075f02a84a 100644 --- a/src/diffusers/loaders/lora.py +++ b/src/diffusers/loaders/lora.py @@ -30,6 +30,7 @@ _get_model_file, convert_state_dict_to_diffusers, convert_state_dict_to_peft, + convert_unet_state_dict_to_peft, delete_adapter_layers, get_adapter_name, get_peft_kwargs, @@ -1540,6 +1541,7 @@ def load_lora_into_transformer(cls, state_dict, transformer, adapter_name=None, state_dict = { k.replace(f"{cls.transformer_name}.", ""): v for k, v in state_dict.items() if k in transformer_keys } + state_dict = convert_unet_state_dict_to_peft(state_dict) if len(state_dict.keys()) > 0: if adapter_name in getattr(transformer, "peft_config", {}): From 049c55d03481e282349c7c5bc906b03632823578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Somoza?= Date: Thu, 20 Jun 2024 13:10:49 -0400 Subject: [PATCH 2/7] add check --- src/diffusers/loaders/lora.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/diffusers/loaders/lora.py b/src/diffusers/loaders/lora.py index a727a35a6614..82317227ca23 100644 --- a/src/diffusers/loaders/lora.py +++ b/src/diffusers/loaders/lora.py @@ -1542,9 +1542,13 @@ def load_lora_into_transformer(cls, state_dict, transformer, adapter_name=None, state_dict = { k.replace(f"{cls.transformer_name}.", ""): v for k, v in state_dict.items() if k in transformer_keys } - state_dict = convert_unet_state_dict_to_peft(state_dict) if len(state_dict.keys()) > 0: + # check with first key if is not in peft format + first_key = next(iter(state_dict.keys()), None) + if "lora_A" not in first_key: + state_dict = convert_unet_state_dict_to_peft(state_dict) + if adapter_name in getattr(transformer, "peft_config", {}): raise ValueError( f"Adapter name {adapter_name} already in use in the transformer - please select a new adapter name." From 58173a435edca05afe1ab364ddae7ba3dbb37405 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Somoza?= Date: Thu, 20 Jun 2024 13:13:52 -0400 Subject: [PATCH 3/7] key present is checked before --- src/diffusers/loaders/lora.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/loaders/lora.py b/src/diffusers/loaders/lora.py index 82317227ca23..94910956d022 100644 --- a/src/diffusers/loaders/lora.py +++ b/src/diffusers/loaders/lora.py @@ -1545,7 +1545,7 @@ def load_lora_into_transformer(cls, state_dict, transformer, adapter_name=None, if len(state_dict.keys()) > 0: # check with first key if is not in peft format - first_key = next(iter(state_dict.keys()), None) + first_key = next(iter(state_dict.keys())) if "lora_A" not in first_key: state_dict = convert_unet_state_dict_to_peft(state_dict) From 64dd78041d55208409c67b5d1d969b2799eaaf3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Somoza?= Date: Thu, 20 Jun 2024 15:22:20 -0400 Subject: [PATCH 4/7] test case draft --- tests/lora/test_lora_layers_sd3.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/lora/test_lora_layers_sd3.py b/tests/lora/test_lora_layers_sd3.py index e76c65778174..108c5c32278a 100644 --- a/tests/lora/test_lora_layers_sd3.py +++ b/tests/lora/test_lora_layers_sd3.py @@ -287,3 +287,19 @@ def test_simple_inference_with_transformer_fuse_unfuse(self): self.assertTrue( np.allclose(ouput_fused, output_unfused_lora, atol=1e-3, rtol=1e-3), "Fused lora should change the output" ) + + def test_sd3_lora(self): + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + lora_model_id = "OzzyGT/lora_test" + lora_filename = "lora_diffusers_format.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + pipe.unload_lora_weights() + + lora_model_id = "OzzyGT/lora_test" + lora_filename = "lora_peft_format.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) From f268e1162a39e56a1c5579f2d70d8154bbf47333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Somoza?= Date: Fri, 21 Jun 2024 09:37:21 -0400 Subject: [PATCH 5/7] aply suggestions --- tests/lora/test_lora_layers_sd3.py | 111 ++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 1 deletion(-) diff --git a/tests/lora/test_lora_layers_sd3.py b/tests/lora/test_lora_layers_sd3.py index 108c5c32278a..3ad81aca2852 100644 --- a/tests/lora/test_lora_layers_sd3.py +++ b/tests/lora/test_lora_layers_sd3.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import gc import os import sys import tempfile @@ -27,7 +28,13 @@ SD3Transformer2DModel, StableDiffusion3Pipeline, ) -from diffusers.utils.testing_utils import is_peft_available, require_peft_backend, torch_device +from diffusers.utils.testing_utils import ( + is_peft_available, + require_peft_backend, + require_torch_gpu, + slow, + torch_device, +) if is_peft_available(): @@ -288,7 +295,109 @@ def test_simple_inference_with_transformer_fuse_unfuse(self): np.allclose(ouput_fused, output_unfused_lora, atol=1e-3, rtol=1e-3), "Fused lora should change the output" ) + +@require_peft_backend +@slow +@require_torch_gpu +class SD3LoRASlowTests(unittest.TestCase): + pipeline_class = StableDiffusion3Pipeline + params = frozenset( + [ + "prompt", + "height", + "width", + "guidance_scale", + "negative_prompt", + "prompt_embeds", + "negative_prompt_embeds", + ] + ) + batch_params = frozenset(["prompt", "negative_prompt"]) + + def setUp(self): + super().setUp() + gc.collect() + torch.cuda.empty_cache() + + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_dummy_components(self): + torch.manual_seed(0) + transformer = SD3Transformer2DModel( + sample_size=32, + patch_size=1, + in_channels=4, + num_layers=1, + attention_head_dim=8, + num_attention_heads=4, + caption_projection_dim=32, + joint_attention_dim=32, + pooled_projection_dim=64, + out_channels=4, + ) + clip_text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + hidden_act="gelu", + projection_dim=32, + ) + + torch.manual_seed(0) + text_encoder = CLIPTextModelWithProjection(clip_text_encoder_config) + + torch.manual_seed(0) + text_encoder_2 = CLIPTextModelWithProjection(clip_text_encoder_config) + + text_encoder_3 = T5EncoderModel.from_pretrained("hf-internal-testing/tiny-random-t5") + + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + tokenizer_3 = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-t5") + + torch.manual_seed(0) + vae = AutoencoderKL( + sample_size=32, + in_channels=3, + out_channels=3, + block_out_channels=(4,), + layers_per_block=1, + latent_channels=4, + norm_num_groups=1, + use_quant_conv=False, + use_post_quant_conv=False, + shift_factor=0.0609, + scaling_factor=1.5035, + ) + + scheduler = FlowMatchEulerDiscreteScheduler() + + return { + "scheduler": scheduler, + "text_encoder": text_encoder, + "text_encoder_2": text_encoder_2, + "text_encoder_3": text_encoder_3, + "tokenizer": tokenizer, + "tokenizer_2": tokenizer_2, + "tokenizer_3": tokenizer_3, + "transformer": transformer, + "vae": vae, + } + def test_sd3_lora(self): + """ + Test loading the loras that are saved with the diffusers and peft formats. + Related PR: https://github.com/huggingface/diffusers/pull/8584 + """ components = self.get_dummy_components() pipe = self.pipeline_class(**components) From 701f60234ff78df11675974e2491cb63d04f5ade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Somoza?= Date: Fri, 21 Jun 2024 10:26:37 -0400 Subject: [PATCH 6/7] changed testing repo, back to old class --- tests/lora/test_lora_layers_sd3.py | 116 +---------------------------- 1 file changed, 4 insertions(+), 112 deletions(-) diff --git a/tests/lora/test_lora_layers_sd3.py b/tests/lora/test_lora_layers_sd3.py index 3ad81aca2852..226634df1a25 100644 --- a/tests/lora/test_lora_layers_sd3.py +++ b/tests/lora/test_lora_layers_sd3.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import gc import os import sys import tempfile @@ -28,13 +27,7 @@ SD3Transformer2DModel, StableDiffusion3Pipeline, ) -from diffusers.utils.testing_utils import ( - is_peft_available, - require_peft_backend, - require_torch_gpu, - slow, - torch_device, -) +from diffusers.utils.testing_utils import is_peft_available, require_peft_backend, require_torch_gpu, torch_device if is_peft_available(): @@ -295,120 +288,19 @@ def test_simple_inference_with_transformer_fuse_unfuse(self): np.allclose(ouput_fused, output_unfused_lora, atol=1e-3, rtol=1e-3), "Fused lora should change the output" ) - -@require_peft_backend -@slow -@require_torch_gpu -class SD3LoRASlowTests(unittest.TestCase): - pipeline_class = StableDiffusion3Pipeline - params = frozenset( - [ - "prompt", - "height", - "width", - "guidance_scale", - "negative_prompt", - "prompt_embeds", - "negative_prompt_embeds", - ] - ) - batch_params = frozenset(["prompt", "negative_prompt"]) - - def setUp(self): - super().setUp() - gc.collect() - torch.cuda.empty_cache() - - def tearDown(self): - super().tearDown() - gc.collect() - torch.cuda.empty_cache() - - def get_dummy_components(self): - torch.manual_seed(0) - transformer = SD3Transformer2DModel( - sample_size=32, - patch_size=1, - in_channels=4, - num_layers=1, - attention_head_dim=8, - num_attention_heads=4, - caption_projection_dim=32, - joint_attention_dim=32, - pooled_projection_dim=64, - out_channels=4, - ) - clip_text_encoder_config = CLIPTextConfig( - bos_token_id=0, - eos_token_id=2, - hidden_size=32, - intermediate_size=37, - layer_norm_eps=1e-05, - num_attention_heads=4, - num_hidden_layers=5, - pad_token_id=1, - vocab_size=1000, - hidden_act="gelu", - projection_dim=32, - ) - - torch.manual_seed(0) - text_encoder = CLIPTextModelWithProjection(clip_text_encoder_config) - - torch.manual_seed(0) - text_encoder_2 = CLIPTextModelWithProjection(clip_text_encoder_config) - - text_encoder_3 = T5EncoderModel.from_pretrained("hf-internal-testing/tiny-random-t5") - - tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") - tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") - tokenizer_3 = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-t5") - - torch.manual_seed(0) - vae = AutoencoderKL( - sample_size=32, - in_channels=3, - out_channels=3, - block_out_channels=(4,), - layers_per_block=1, - latent_channels=4, - norm_num_groups=1, - use_quant_conv=False, - use_post_quant_conv=False, - shift_factor=0.0609, - scaling_factor=1.5035, - ) - - scheduler = FlowMatchEulerDiscreteScheduler() - - return { - "scheduler": scheduler, - "text_encoder": text_encoder, - "text_encoder_2": text_encoder_2, - "text_encoder_3": text_encoder_3, - "tokenizer": tokenizer, - "tokenizer_2": tokenizer_2, - "tokenizer_3": tokenizer_3, - "transformer": transformer, - "vae": vae, - } - + @require_torch_gpu def test_sd3_lora(self): - """ - Test loading the loras that are saved with the diffusers and peft formats. - Related PR: https://github.com/huggingface/diffusers/pull/8584 - """ components = self.get_dummy_components() pipe = self.pipeline_class(**components) pipe = pipe.to(torch_device) pipe.set_progress_bar_config(disable=None) - lora_model_id = "OzzyGT/lora_test" + lora_model_id = "hf-internal-testing/tiny-sd3-loras" + lora_filename = "lora_diffusers_format.safetensors" pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) pipe.unload_lora_weights() - lora_model_id = "OzzyGT/lora_test" lora_filename = "lora_peft_format.safetensors" pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) From b41cf759d3552f14bf88d8ffb7fb5108a9c2f456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Somoza?= Date: Fri, 21 Jun 2024 10:38:55 -0400 Subject: [PATCH 7/7] forgot docstring --- tests/lora/test_lora_layers_sd3.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/lora/test_lora_layers_sd3.py b/tests/lora/test_lora_layers_sd3.py index 226634df1a25..2589625ec6ac 100644 --- a/tests/lora/test_lora_layers_sd3.py +++ b/tests/lora/test_lora_layers_sd3.py @@ -290,6 +290,10 @@ def test_simple_inference_with_transformer_fuse_unfuse(self): @require_torch_gpu def test_sd3_lora(self): + """ + Test loading the loras that are saved with the diffusers and peft formats. + Related PR: https://github.com/huggingface/diffusers/pull/8584 + """ components = self.get_dummy_components() pipe = self.pipeline_class(**components)