From d86fd5e4329a69f8be0566bb59f2a7161ad45c76 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 18 Jul 2023 05:09:27 +0000 Subject: [PATCH 01/23] initial --- src/diffusers/__init__.py | 3 + src/diffusers/pipelines/__init__.py | 1 + src/diffusers/pipelines/auto_pipeline.py | 144 +++++++++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 src/diffusers/pipelines/auto_pipeline.py diff --git a/src/diffusers/__init__.py b/src/diffusers/__init__.py index 0c0869fb52bd..4a3afa69874e 100644 --- a/src/diffusers/__init__.py +++ b/src/diffusers/__init__.py @@ -66,6 +66,9 @@ DDIMPipeline, DDPMPipeline, DiffusionPipeline, + AutoPipelineForInpainting, + AutoPipelineForImage2Image, + AutoPipelineForText2Image, DiTPipeline, ImagePipelineOutput, KarrasVePipeline, diff --git a/src/diffusers/pipelines/__init__.py b/src/diffusers/pipelines/__init__.py index aa09e7e81130..7d4a2d2ffb20 100644 --- a/src/diffusers/pipelines/__init__.py +++ b/src/diffusers/pipelines/__init__.py @@ -25,6 +25,7 @@ from .latent_diffusion import LDMSuperResolutionPipeline from .latent_diffusion_uncond import LDMPipeline from .pipeline_utils import AudioPipelineOutput, DiffusionPipeline, ImagePipelineOutput + from .auto_pipeline import AutoPipelineForText2Image, AutoPipelineForImage2Image, AutoPipelineForInpainting from .pndm import PNDMPipeline from .repaint import RePaintPipeline from .score_sde_ve import ScoreSdeVePipeline diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py new file mode 100644 index 000000000000..60e7408b8d41 --- /dev/null +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -0,0 +1,144 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..configuration_utils import ConfigMixin +from collections import OrderedDict + +from .stable_diffusion import ( + StableDiffusionPipeline, + StableDiffusionImg2ImgPipeline, + StableDiffusionInpaintPipeline, +) +from .stable_diffusion_xl import ( + StableDiffusionXLPipeline, + StableDiffusionXLImg2ImgPipeline, + StableDiffusionXLInpaintPipeline, +) +from .deepfloyd_if import IFPipeline, IFImg2ImgPipeline, IFInpaintingPipeline +from .kandinsky import KandinskyPipeline, KandinskyImg2ImgPipeline, KandinskyInpaintPipeline +from .kandinsky2_2 import KandinskyV22Pipeline, KandinskyV22Img2ImgPipeline, KandinskyV22InpaintPipeline + +from .pipeline_utils import DiffusionPipeline + + +AUTO_TEXT2IMAGE_PIPELINES_MAPPING = OrderedDict( + [ + ("stable-diffusion", StableDiffusionPipeline), + ("stable-diffusion-xl", StableDiffusionXLPipeline), + ("if", IFPipeline), + ("kandinsky", KandinskyPipeline), + ("kdnsinskyv22", KandinskyV22Pipeline), + ] +) + +AUTO_IMAGE2IMAGE_PIPELINES_MAPPING = OrderedDict( + [ + ("stable-diffusion", StableDiffusionImg2ImgPipeline), + ("stable-diffusion-xl", StableDiffusionXLImg2ImgPipeline), + ("if", IFImg2ImgPipeline), + ("kandinsky", KandinskyImg2ImgPipeline), + ("kdnsinskyv22", KandinskyV22Img2ImgPipeline), + ] +) + +AUTO_INPAINTING_PIPELINES_MAPPING = OrderedDict( + [ + ("stable-diffusion", StableDiffusionInpaintPipeline), + ("stable-diffusion-xl", StableDiffusionXLInpaintPipeline), + ("if", IFInpaintingPipeline), + ("kandinsky", KandinskyInpaintPipeline), + ("kdnsinskyv22", KandinskyV22InpaintPipeline), + ] +) + +SUPPORTED_TASKS_MAPPINGS = [AUTO_TEXT2IMAGE_PIPELINES_MAPPING, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, AUTO_INPAINTING_PIPELINES_MAPPING] + + +def _get_task_class(mapping, pipeline_class_name): + + def get_model(pipeline_class_name): + + for task_mapping in SUPPORTED_TASKS_MAPPINGS: + for model_name, pipeline in task_mapping.items(): + if pipeline.__name__ == pipeline_class_name: + return model_name + + model_name = get_model(pipeline_class_name) + + if model_name is not None: + task_class = mapping.get(model_name, None) + if task_class is not None: + return task_class + raise ValueError( + f"AutoPipeline can't find a pipeline linked to {pipeline_class_name} for {model_name} in {mapping}" + ) + + +class AutoPipelineForText2Image(ConfigMixin): + + config_name = "model_index.json" + + @classmethod + def from_pretrained(cls, pretrained_model_or_path, **kwargs): + config = cls.load_config(pretrained_model_or_path) + + text_2_image_cls = _get_task_class(AUTO_TEXT2IMAGE_PIPELINES_MAPPING, config["_class_name"]) + + return text_2_image_cls.from_pretrained(pretrained_model_or_path, **kwargs) + + @classmethod + def from_pipe(cls, pipeline: DiffusionPipeline, **kwargs): + print(pipeline.config.keys()) + text_2_image_cls = _get_task_class(AUTO_TEXT2IMAGE_PIPELINES_MAPPING, pipeline.__class__.__name__) + + return text_2_image_cls(**pipeline.components) + + +class AutoPipelineForImage2Image(ConfigMixin): + + config_name = "model_index.json" + + @classmethod + def from_pretrained(cls, pretrained_model_or_path, **kwargs): + config = cls.load_config(pretrained_model_or_path) + + image_2_image_cls = _get_task_class(AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, config["_class_name"]) + + return image_2_image_cls.from_pretrained(pretrained_model_or_path, **kwargs) + + @classmethod + def from_pipe(cls, pipeline: DiffusionPipeline, **kwargs): + image_2_image_cls = _get_task_class(AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, pipeline.__class__.__name__) + + return image_2_image_cls(**pipeline.components, **kwargs) + +class AutoPipelineForInpainting(ConfigMixin): + + config_name = "model_index.json" + + @classmethod + def from_pretrained(cls, pretrained_model_or_path, **kwargs): + config = cls.load_config(pretrained_model_or_path) + + inpainting_cls = _get_task_class(AUTO_INPAINTING_PIPELINES_MAPPING, config["_class_name"]) + + return inpainting_cls.from_pretrained(pretrained_model_or_path, **kwargs) + + @classmethod + def from_pipe(cls, pipeline: DiffusionPipeline, **kwargs): + inpainting_cls = _get_task_class(AUTO_INPAINTING_PIPELINES_MAPPING, pipeline.__class__.__name__) + + return inpainting_cls(**pipeline.components) \ No newline at end of file From 7f9be69022e591eb0aa71500b4644c4926338f39 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 18 Jul 2023 05:10:36 +0000 Subject: [PATCH 02/23] style --- src/diffusers/__init__.py | 6 ++-- src/diffusers/pipelines/__init__.py | 2 +- src/diffusers/pipelines/auto_pipeline.py | 37 ++++++++++++------------ 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/diffusers/__init__.py b/src/diffusers/__init__.py index 4a3afa69874e..1a7cdf84b63f 100644 --- a/src/diffusers/__init__.py +++ b/src/diffusers/__init__.py @@ -61,14 +61,14 @@ ) from .pipelines import ( AudioPipelineOutput, + AutoPipelineForImage2Image, + AutoPipelineForInpainting, + AutoPipelineForText2Image, ConsistencyModelPipeline, DanceDiffusionPipeline, DDIMPipeline, DDPMPipeline, DiffusionPipeline, - AutoPipelineForInpainting, - AutoPipelineForImage2Image, - AutoPipelineForText2Image, DiTPipeline, ImagePipelineOutput, KarrasVePipeline, diff --git a/src/diffusers/pipelines/__init__.py b/src/diffusers/pipelines/__init__.py index 7d4a2d2ffb20..3708f1bf25e8 100644 --- a/src/diffusers/pipelines/__init__.py +++ b/src/diffusers/pipelines/__init__.py @@ -17,6 +17,7 @@ except OptionalDependencyNotAvailable: from ..utils.dummy_pt_objects import * # noqa F403 else: + from .auto_pipeline import AutoPipelineForImage2Image, AutoPipelineForInpainting, AutoPipelineForText2Image from .consistency_models import ConsistencyModelPipeline from .dance_diffusion import DanceDiffusionPipeline from .ddim import DDIMPipeline @@ -25,7 +26,6 @@ from .latent_diffusion import LDMSuperResolutionPipeline from .latent_diffusion_uncond import LDMPipeline from .pipeline_utils import AudioPipelineOutput, DiffusionPipeline, ImagePipelineOutput - from .auto_pipeline import AutoPipelineForText2Image, AutoPipelineForImage2Image, AutoPipelineForInpainting from .pndm import PNDMPipeline from .repaint import RePaintPipeline from .score_sde_ve import ScoreSdeVePipeline diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index 60e7408b8d41..711d9f07ac76 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -14,24 +14,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -from ..configuration_utils import ConfigMixin from collections import OrderedDict +from ..configuration_utils import ConfigMixin +from .deepfloyd_if import IFImg2ImgPipeline, IFInpaintingPipeline, IFPipeline +from .kandinsky import KandinskyImg2ImgPipeline, KandinskyInpaintPipeline, KandinskyPipeline +from .kandinsky2_2 import KandinskyV22Img2ImgPipeline, KandinskyV22InpaintPipeline, KandinskyV22Pipeline +from .pipeline_utils import DiffusionPipeline from .stable_diffusion import ( - StableDiffusionPipeline, - StableDiffusionImg2ImgPipeline, + StableDiffusionImg2ImgPipeline, StableDiffusionInpaintPipeline, + StableDiffusionPipeline, ) from .stable_diffusion_xl import ( - StableDiffusionXLPipeline, StableDiffusionXLImg2ImgPipeline, StableDiffusionXLInpaintPipeline, + StableDiffusionXLPipeline, ) -from .deepfloyd_if import IFPipeline, IFImg2ImgPipeline, IFInpaintingPipeline -from .kandinsky import KandinskyPipeline, KandinskyImg2ImgPipeline, KandinskyInpaintPipeline -from .kandinsky2_2 import KandinskyV22Pipeline, KandinskyV22Img2ImgPipeline, KandinskyV22InpaintPipeline - -from .pipeline_utils import DiffusionPipeline AUTO_TEXT2IMAGE_PIPELINES_MAPPING = OrderedDict( @@ -64,13 +63,15 @@ ] ) -SUPPORTED_TASKS_MAPPINGS = [AUTO_TEXT2IMAGE_PIPELINES_MAPPING, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, AUTO_INPAINTING_PIPELINES_MAPPING] +SUPPORTED_TASKS_MAPPINGS = [ + AUTO_TEXT2IMAGE_PIPELINES_MAPPING, + AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, + AUTO_INPAINTING_PIPELINES_MAPPING, +] def _get_task_class(mapping, pipeline_class_name): - def get_model(pipeline_class_name): - for task_mapping in SUPPORTED_TASKS_MAPPINGS: for model_name, pipeline in task_mapping.items(): if pipeline.__name__ == pipeline_class_name: @@ -88,7 +89,6 @@ def get_model(pipeline_class_name): class AutoPipelineForText2Image(ConfigMixin): - config_name = "model_index.json" @classmethod @@ -98,7 +98,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): text_2_image_cls = _get_task_class(AUTO_TEXT2IMAGE_PIPELINES_MAPPING, config["_class_name"]) return text_2_image_cls.from_pretrained(pretrained_model_or_path, **kwargs) - + @classmethod def from_pipe(cls, pipeline: DiffusionPipeline, **kwargs): print(pipeline.config.keys()) @@ -108,7 +108,6 @@ def from_pipe(cls, pipeline: DiffusionPipeline, **kwargs): class AutoPipelineForImage2Image(ConfigMixin): - config_name = "model_index.json" @classmethod @@ -118,15 +117,15 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): image_2_image_cls = _get_task_class(AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, config["_class_name"]) return image_2_image_cls.from_pretrained(pretrained_model_or_path, **kwargs) - + @classmethod def from_pipe(cls, pipeline: DiffusionPipeline, **kwargs): image_2_image_cls = _get_task_class(AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, pipeline.__class__.__name__) return image_2_image_cls(**pipeline.components, **kwargs) -class AutoPipelineForInpainting(ConfigMixin): +class AutoPipelineForInpainting(ConfigMixin): config_name = "model_index.json" @classmethod @@ -136,9 +135,9 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): inpainting_cls = _get_task_class(AUTO_INPAINTING_PIPELINES_MAPPING, config["_class_name"]) return inpainting_cls.from_pretrained(pretrained_model_or_path, **kwargs) - + @classmethod def from_pipe(cls, pipeline: DiffusionPipeline, **kwargs): inpainting_cls = _get_task_class(AUTO_INPAINTING_PIPELINES_MAPPING, pipeline.__class__.__name__) - return inpainting_cls(**pipeline.components) \ No newline at end of file + return inpainting_cls(**pipeline.components) From 3b405eb1a9c9b3b37e01d513b7774548cd208932 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 18 Jul 2023 05:43:55 +0000 Subject: [PATCH 03/23] from ...pipelines -> from ..pipeline_util --- src/diffusers/experimental/rl/value_guided_sampling.py | 2 +- src/diffusers/pipelines/auto_pipeline.py | 7 +++---- src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py | 3 +-- .../pipelines/kandinsky/pipeline_kandinsky_img2img.py | 3 +-- .../pipelines/kandinsky/pipeline_kandinsky_inpaint.py | 3 +-- .../pipelines/kandinsky/pipeline_kandinsky_prior.py | 2 +- .../pipelines/kandinsky2_2/pipeline_kandinsky2_2.py | 3 +-- .../kandinsky2_2/pipeline_kandinsky2_2_controlnet.py | 3 +-- .../pipeline_kandinsky2_2_controlnet_img2img.py | 3 +-- .../kandinsky2_2/pipeline_kandinsky2_2_img2img.py | 3 +-- .../kandinsky2_2/pipeline_kandinsky2_2_inpainting.py | 3 +-- .../pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py | 2 +- .../kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py | 2 +- src/diffusers/pipelines/shap_e/pipeline_shap_e.py | 2 +- src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py | 2 +- .../pipeline_stable_diffusion_k_diffusion.py | 2 +- src/diffusers/pipelines/unclip/pipeline_unclip.py | 3 +-- .../pipelines/unclip/pipeline_unclip_image_variation.py | 2 +- 18 files changed, 20 insertions(+), 30 deletions(-) diff --git a/src/diffusers/experimental/rl/value_guided_sampling.py b/src/diffusers/experimental/rl/value_guided_sampling.py index e4af4986faad..a0ba9c456dd1 100644 --- a/src/diffusers/experimental/rl/value_guided_sampling.py +++ b/src/diffusers/experimental/rl/value_guided_sampling.py @@ -17,7 +17,7 @@ import tqdm from ...models.unet_1d import UNet1DModel -from ...pipelines import DiffusionPipeline +from ..pipeline_utils import DiffusionPipeline from ...utils import randn_tensor from ...utils.dummy_pt_objects import DDPMScheduler diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index 711d9f07ac76..1307beea45b3 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -20,7 +20,6 @@ from .deepfloyd_if import IFImg2ImgPipeline, IFInpaintingPipeline, IFPipeline from .kandinsky import KandinskyImg2ImgPipeline, KandinskyInpaintPipeline, KandinskyPipeline from .kandinsky2_2 import KandinskyV22Img2ImgPipeline, KandinskyV22InpaintPipeline, KandinskyV22Pipeline -from .pipeline_utils import DiffusionPipeline from .stable_diffusion import ( StableDiffusionImg2ImgPipeline, StableDiffusionInpaintPipeline, @@ -100,7 +99,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): return text_2_image_cls.from_pretrained(pretrained_model_or_path, **kwargs) @classmethod - def from_pipe(cls, pipeline: DiffusionPipeline, **kwargs): + def from_pipe(cls, pipeline, **kwargs): print(pipeline.config.keys()) text_2_image_cls = _get_task_class(AUTO_TEXT2IMAGE_PIPELINES_MAPPING, pipeline.__class__.__name__) @@ -119,7 +118,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): return image_2_image_cls.from_pretrained(pretrained_model_or_path, **kwargs) @classmethod - def from_pipe(cls, pipeline: DiffusionPipeline, **kwargs): + def from_pipe(cls, pipeline, **kwargs): image_2_image_cls = _get_task_class(AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, pipeline.__class__.__name__) return image_2_image_cls(**pipeline.components, **kwargs) @@ -137,7 +136,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): return inpainting_cls.from_pretrained(pretrained_model_or_path, **kwargs) @classmethod - def from_pipe(cls, pipeline: DiffusionPipeline, **kwargs): + def from_pipe(cls, pipeline, **kwargs): inpainting_cls = _get_task_class(AUTO_INPAINTING_PIPELINES_MAPPING, pipeline.__class__.__name__) return inpainting_cls(**pipeline.components) diff --git a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py index 2b4c6b963598..63970d303569 100644 --- a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py +++ b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py @@ -20,8 +20,7 @@ ) from ...models import UNet2DConditionModel, VQModel -from ...pipelines import DiffusionPipeline -from ...pipelines.pipeline_utils import ImagePipelineOutput +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import DDIMScheduler, DDPMScheduler from ...utils import ( is_accelerate_available, diff --git a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py index 8d8d85ebb7b6..918d0a9221e0 100644 --- a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py +++ b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py @@ -23,8 +23,7 @@ ) from ...models import UNet2DConditionModel, VQModel -from ...pipelines import DiffusionPipeline -from ...pipelines.pipeline_utils import ImagePipelineOutput +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import DDIMScheduler from ...utils import ( is_accelerate_available, diff --git a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py index cc41e9eb4fc8..31c48279ff8e 100644 --- a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py +++ b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py @@ -25,8 +25,7 @@ ) from ...models import UNet2DConditionModel, VQModel -from ...pipelines import DiffusionPipeline -from ...pipelines.pipeline_utils import ImagePipelineOutput +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import DDIMScheduler from ...utils import ( is_accelerate_available, diff --git a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py index 9322ff6080d1..7ebcbd2aed68 100644 --- a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py +++ b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py @@ -21,7 +21,7 @@ from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection from ...models import PriorTransformer -from ...pipelines import DiffusionPipeline +from ..pipeline_utils import DiffusionPipeline from ...schedulers import UnCLIPScheduler from ...utils import ( BaseOutput, diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py index 4a116e1e600b..480a1bc553b5 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py @@ -17,8 +17,7 @@ import torch from ...models import UNet2DConditionModel, VQModel -from ...pipelines import DiffusionPipeline -from ...pipelines.pipeline_utils import ImagePipelineOutput +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import DDPMScheduler from ...utils import ( is_accelerate_available, diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py index 73fc20b5e0f2..a7e3e7fab03c 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py @@ -17,8 +17,7 @@ import torch from ...models import UNet2DConditionModel, VQModel -from ...pipelines import DiffusionPipeline -from ...pipelines.pipeline_utils import ImagePipelineOutput +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import DDPMScheduler from ...utils import ( is_accelerate_available, diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py index 3e001e89e490..25ecc8683715 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py @@ -20,8 +20,7 @@ from PIL import Image from ...models import UNet2DConditionModel, VQModel -from ...pipelines import DiffusionPipeline -from ...pipelines.pipeline_utils import ImagePipelineOutput +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import DDPMScheduler from ...utils import ( is_accelerate_available, diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py index 0a5f77b0ff2d..5b2b60f877e0 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py @@ -20,8 +20,7 @@ from PIL import Image from ...models import UNet2DConditionModel, VQModel -from ...pipelines import DiffusionPipeline -from ...pipelines.pipeline_utils import ImagePipelineOutput +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import DDPMScheduler from ...utils import ( is_accelerate_available, diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py index 151312979f81..34d32a53f428 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py @@ -22,8 +22,7 @@ from PIL import Image from ...models import UNet2DConditionModel, VQModel -from ...pipelines import DiffusionPipeline -from ...pipelines.pipeline_utils import ImagePipelineOutput +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import DDPMScheduler from ...utils import ( is_accelerate_available, diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py index 3b9974a5dd70..e41cc13591f9 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py @@ -5,7 +5,7 @@ from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection from ...models import PriorTransformer -from ...pipelines import DiffusionPipeline +from ..pipeline_utils import DiffusionPipeline from ...schedulers import UnCLIPScheduler from ...utils import ( is_accelerate_available, diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py index ae46af9c4551..63ce34bc034f 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py @@ -5,7 +5,7 @@ from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection from ...models import PriorTransformer -from ...pipelines import DiffusionPipeline +from ..pipeline_utils import DiffusionPipeline from ...schedulers import UnCLIPScheduler from ...utils import ( is_accelerate_available, diff --git a/src/diffusers/pipelines/shap_e/pipeline_shap_e.py b/src/diffusers/pipelines/shap_e/pipeline_shap_e.py index 5d96fc7bb9f4..1a3ab8160ba1 100644 --- a/src/diffusers/pipelines/shap_e/pipeline_shap_e.py +++ b/src/diffusers/pipelines/shap_e/pipeline_shap_e.py @@ -22,7 +22,7 @@ from transformers import CLIPTextModelWithProjection, CLIPTokenizer from ...models import PriorTransformer -from ...pipelines import DiffusionPipeline +from ..pipeline_utils import DiffusionPipeline from ...schedulers import HeunDiscreteScheduler from ...utils import ( BaseOutput, diff --git a/src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py b/src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py index b99b808e5953..42db4648a21e 100644 --- a/src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py +++ b/src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py @@ -21,7 +21,7 @@ from transformers import CLIPImageProcessor, CLIPVisionModel from ...models import PriorTransformer -from ...pipelines import DiffusionPipeline +from ..pipeline_utils import DiffusionPipeline from ...schedulers import HeunDiscreteScheduler from ...utils import ( BaseOutput, diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_k_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_k_diffusion.py index 283ddb6f19b3..56e044db4e7a 100755 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_k_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_k_diffusion.py @@ -23,7 +23,7 @@ from ...image_processor import VaeImageProcessor from ...loaders import LoraLoaderMixin, TextualInversionLoaderMixin -from ...pipelines import DiffusionPipeline +from ..pipeline_utils import DiffusionPipeline from ...schedulers import LMSDiscreteScheduler from ...utils import is_accelerate_available, is_accelerate_version, logging, randn_tensor from . import StableDiffusionPipelineOutput diff --git a/src/diffusers/pipelines/unclip/pipeline_unclip.py b/src/diffusers/pipelines/unclip/pipeline_unclip.py index abbb48ce8f46..b1e0b6015636 100644 --- a/src/diffusers/pipelines/unclip/pipeline_unclip.py +++ b/src/diffusers/pipelines/unclip/pipeline_unclip.py @@ -21,8 +21,7 @@ from transformers.models.clip.modeling_clip import CLIPTextModelOutput from ...models import PriorTransformer, UNet2DConditionModel, UNet2DModel -from ...pipelines import DiffusionPipeline -from ...pipelines.pipeline_utils import ImagePipelineOutput +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import UnCLIPScheduler from ...utils import is_accelerate_available, logging, randn_tensor from .text_proj import UnCLIPTextProjModel diff --git a/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py b/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py index 30d74cd36bb0..cbf828c88863 100644 --- a/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py +++ b/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py @@ -26,7 +26,7 @@ ) from ...models import UNet2DConditionModel, UNet2DModel -from ...pipelines import DiffusionPipeline, ImagePipelineOutput +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import UnCLIPScheduler from ...utils import is_accelerate_available, logging, randn_tensor from .text_proj import UnCLIPTextProjModel From 4964233bbb8125690d969d856db3c7cb9f4d16be Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 18 Jul 2023 05:46:37 +0000 Subject: [PATCH 04/23] make style --- src/diffusers/experimental/rl/value_guided_sampling.py | 2 +- src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py | 2 +- src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py | 2 +- src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py | 2 +- src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py | 2 +- src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py | 2 +- .../pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py | 2 +- .../kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py | 2 +- .../pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py | 2 +- .../pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py | 2 +- .../pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py | 2 +- .../kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py | 2 +- src/diffusers/pipelines/shap_e/pipeline_shap_e.py | 2 +- src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py | 2 +- .../stable_diffusion/pipeline_stable_diffusion_k_diffusion.py | 2 +- src/diffusers/pipelines/unclip/pipeline_unclip.py | 2 +- .../pipelines/unclip/pipeline_unclip_image_variation.py | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/diffusers/experimental/rl/value_guided_sampling.py b/src/diffusers/experimental/rl/value_guided_sampling.py index a0ba9c456dd1..2a8bebc6e66b 100644 --- a/src/diffusers/experimental/rl/value_guided_sampling.py +++ b/src/diffusers/experimental/rl/value_guided_sampling.py @@ -17,9 +17,9 @@ import tqdm from ...models.unet_1d import UNet1DModel -from ..pipeline_utils import DiffusionPipeline from ...utils import randn_tensor from ...utils.dummy_pt_objects import DDPMScheduler +from ..pipeline_utils import DiffusionPipeline class ValueGuidedRLPipeline(DiffusionPipeline): diff --git a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py index 63970d303569..88f0b564a88d 100644 --- a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py +++ b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py @@ -20,7 +20,6 @@ ) from ...models import UNet2DConditionModel, VQModel -from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import DDIMScheduler, DDPMScheduler from ...utils import ( is_accelerate_available, @@ -29,6 +28,7 @@ randn_tensor, replace_example_docstring, ) +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from .text_encoder import MultilingualCLIP diff --git a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py index 918d0a9221e0..10ad3a71a33d 100644 --- a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py +++ b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py @@ -23,7 +23,6 @@ ) from ...models import UNet2DConditionModel, VQModel -from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import DDIMScheduler from ...utils import ( is_accelerate_available, @@ -32,6 +31,7 @@ randn_tensor, replace_example_docstring, ) +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from .text_encoder import MultilingualCLIP diff --git a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py index 31c48279ff8e..5217f9910c5e 100644 --- a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py +++ b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py @@ -25,7 +25,6 @@ ) from ...models import UNet2DConditionModel, VQModel -from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import DDIMScheduler from ...utils import ( is_accelerate_available, @@ -34,6 +33,7 @@ randn_tensor, replace_example_docstring, ) +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from .text_encoder import MultilingualCLIP diff --git a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py index 7ebcbd2aed68..e76ba0799693 100644 --- a/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py +++ b/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py @@ -21,7 +21,6 @@ from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection from ...models import PriorTransformer -from ..pipeline_utils import DiffusionPipeline from ...schedulers import UnCLIPScheduler from ...utils import ( BaseOutput, @@ -30,6 +29,7 @@ randn_tensor, replace_example_docstring, ) +from ..pipeline_utils import DiffusionPipeline logger = logging.get_logger(__name__) # pylint: disable=invalid-name diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py index 480a1bc553b5..baee4aaf6c7d 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py @@ -17,7 +17,6 @@ import torch from ...models import UNet2DConditionModel, VQModel -from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import DDPMScheduler from ...utils import ( is_accelerate_available, @@ -26,6 +25,7 @@ randn_tensor, replace_example_docstring, ) +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput logger = logging.get_logger(__name__) # pylint: disable=invalid-name diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py index a7e3e7fab03c..008d73041f2b 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py @@ -17,7 +17,6 @@ import torch from ...models import UNet2DConditionModel, VQModel -from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import DDPMScheduler from ...utils import ( is_accelerate_available, @@ -26,6 +25,7 @@ randn_tensor, replace_example_docstring, ) +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput logger = logging.get_logger(__name__) # pylint: disable=invalid-name diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py index 25ecc8683715..b386420448b6 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py @@ -20,7 +20,6 @@ from PIL import Image from ...models import UNet2DConditionModel, VQModel -from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import DDPMScheduler from ...utils import ( is_accelerate_available, @@ -29,6 +28,7 @@ randn_tensor, replace_example_docstring, ) +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput logger = logging.get_logger(__name__) # pylint: disable=invalid-name diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py index 5b2b60f877e0..de2b2eb7a3e1 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py @@ -20,7 +20,6 @@ from PIL import Image from ...models import UNet2DConditionModel, VQModel -from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import DDPMScheduler from ...utils import ( is_accelerate_available, @@ -29,6 +28,7 @@ randn_tensor, replace_example_docstring, ) +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput logger = logging.get_logger(__name__) # pylint: disable=invalid-name diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py index 34d32a53f428..5b10e57f4083 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py @@ -22,7 +22,6 @@ from PIL import Image from ...models import UNet2DConditionModel, VQModel -from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import DDPMScheduler from ...utils import ( is_accelerate_available, @@ -31,6 +30,7 @@ randn_tensor, replace_example_docstring, ) +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput logger = logging.get_logger(__name__) # pylint: disable=invalid-name diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py index e41cc13591f9..3ae37215d36b 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py @@ -5,7 +5,6 @@ from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection from ...models import PriorTransformer -from ..pipeline_utils import DiffusionPipeline from ...schedulers import UnCLIPScheduler from ...utils import ( is_accelerate_available, @@ -14,6 +13,7 @@ replace_example_docstring, ) from ..kandinsky import KandinskyPriorPipelineOutput +from ..pipeline_utils import DiffusionPipeline logger = logging.get_logger(__name__) # pylint: disable=invalid-name diff --git a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py index 63ce34bc034f..a8e3a5dc6372 100644 --- a/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py +++ b/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py @@ -5,7 +5,6 @@ from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection from ...models import PriorTransformer -from ..pipeline_utils import DiffusionPipeline from ...schedulers import UnCLIPScheduler from ...utils import ( is_accelerate_available, @@ -14,6 +13,7 @@ replace_example_docstring, ) from ..kandinsky import KandinskyPriorPipelineOutput +from ..pipeline_utils import DiffusionPipeline logger = logging.get_logger(__name__) # pylint: disable=invalid-name diff --git a/src/diffusers/pipelines/shap_e/pipeline_shap_e.py b/src/diffusers/pipelines/shap_e/pipeline_shap_e.py index 1a3ab8160ba1..5dfacb8b0dcb 100644 --- a/src/diffusers/pipelines/shap_e/pipeline_shap_e.py +++ b/src/diffusers/pipelines/shap_e/pipeline_shap_e.py @@ -22,7 +22,6 @@ from transformers import CLIPTextModelWithProjection, CLIPTokenizer from ...models import PriorTransformer -from ..pipeline_utils import DiffusionPipeline from ...schedulers import HeunDiscreteScheduler from ...utils import ( BaseOutput, @@ -32,6 +31,7 @@ randn_tensor, replace_example_docstring, ) +from ..pipeline_utils import DiffusionPipeline from .renderer import ShapERenderer diff --git a/src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py b/src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py index 42db4648a21e..f8700eadc938 100644 --- a/src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py +++ b/src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py @@ -21,7 +21,6 @@ from transformers import CLIPImageProcessor, CLIPVisionModel from ...models import PriorTransformer -from ..pipeline_utils import DiffusionPipeline from ...schedulers import HeunDiscreteScheduler from ...utils import ( BaseOutput, @@ -30,6 +29,7 @@ randn_tensor, replace_example_docstring, ) +from ..pipeline_utils import DiffusionPipeline from .renderer import ShapERenderer diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_k_diffusion.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_k_diffusion.py index 56e044db4e7a..0671d5dc66b7 100755 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_k_diffusion.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_k_diffusion.py @@ -23,9 +23,9 @@ from ...image_processor import VaeImageProcessor from ...loaders import LoraLoaderMixin, TextualInversionLoaderMixin -from ..pipeline_utils import DiffusionPipeline from ...schedulers import LMSDiscreteScheduler from ...utils import is_accelerate_available, is_accelerate_version, logging, randn_tensor +from ..pipeline_utils import DiffusionPipeline from . import StableDiffusionPipelineOutput diff --git a/src/diffusers/pipelines/unclip/pipeline_unclip.py b/src/diffusers/pipelines/unclip/pipeline_unclip.py index b1e0b6015636..6fb136a5db4f 100644 --- a/src/diffusers/pipelines/unclip/pipeline_unclip.py +++ b/src/diffusers/pipelines/unclip/pipeline_unclip.py @@ -21,9 +21,9 @@ from transformers.models.clip.modeling_clip import CLIPTextModelOutput from ...models import PriorTransformer, UNet2DConditionModel, UNet2DModel -from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import UnCLIPScheduler from ...utils import is_accelerate_available, logging, randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from .text_proj import UnCLIPTextProjModel diff --git a/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py b/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py index cbf828c88863..dec91fdb0dd0 100644 --- a/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py +++ b/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py @@ -26,9 +26,9 @@ ) from ...models import UNet2DConditionModel, UNet2DModel -from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from ...schedulers import UnCLIPScheduler from ...utils import is_accelerate_available, logging, randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from .text_proj import UnCLIPTextProjModel From fda4d70282a1138ac4d9db1141902c155436c2e3 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 18 Jul 2023 05:50:01 +0000 Subject: [PATCH 05/23] fix-copies --- src/diffusers/utils/dummy_pt_objects.py | 45 +++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/diffusers/utils/dummy_pt_objects.py b/src/diffusers/utils/dummy_pt_objects.py index b955ec5320de..a0e3bee76aa9 100644 --- a/src/diffusers/utils/dummy_pt_objects.py +++ b/src/diffusers/utils/dummy_pt_objects.py @@ -240,6 +240,51 @@ def from_pretrained(cls, *args, **kwargs): requires_backends(cls, ["torch"]) +class AutoPipelineForImage2Image(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class AutoPipelineForInpainting(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class AutoPipelineForText2Image(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + class ConsistencyModelPipeline(metaclass=DummyObject): _backends = ["torch"] From 7ff4247acde77fea9ce43af23a83405cd132c684 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 18 Jul 2023 05:57:54 +0000 Subject: [PATCH 06/23] fix value_guided_sampling oops --- src/diffusers/experimental/rl/value_guided_sampling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/experimental/rl/value_guided_sampling.py b/src/diffusers/experimental/rl/value_guided_sampling.py index 2a8bebc6e66b..e4af4986faad 100644 --- a/src/diffusers/experimental/rl/value_guided_sampling.py +++ b/src/diffusers/experimental/rl/value_guided_sampling.py @@ -17,9 +17,9 @@ import tqdm from ...models.unet_1d import UNet1DModel +from ...pipelines import DiffusionPipeline from ...utils import randn_tensor from ...utils.dummy_pt_objects import DDPMScheduler -from ..pipeline_utils import DiffusionPipeline class ValueGuidedRLPipeline(DiffusionPipeline): From 8d7de4517a9b8d6a2f3ca13efd00238ccc4a4763 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Wed, 19 Jul 2023 05:41:55 +0000 Subject: [PATCH 07/23] style --- src/diffusers/pipelines/unclip/pipeline_unclip.py | 2 +- .../pipelines/unclip/pipeline_unclip_image_variation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/diffusers/pipelines/unclip/pipeline_unclip.py b/src/diffusers/pipelines/unclip/pipeline_unclip.py index 09f41a73f37c..1d400a23c972 100644 --- a/src/diffusers/pipelines/unclip/pipeline_unclip.py +++ b/src/diffusers/pipelines/unclip/pipeline_unclip.py @@ -22,7 +22,7 @@ from ...models import PriorTransformer, UNet2DConditionModel, UNet2DModel from ...schedulers import UnCLIPScheduler -from ...utils import is_accelerate_available, logging, randn_tensor +from ...utils import logging, randn_tensor from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from .text_proj import UnCLIPTextProjModel diff --git a/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py b/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py index 57ecee767f49..e24c6af15a01 100644 --- a/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py +++ b/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py @@ -27,7 +27,7 @@ from ...models import UNet2DConditionModel, UNet2DModel from ...schedulers import UnCLIPScheduler -from ...utils import is_accelerate_available, logging, randn_tensor +from ...utils import logging, randn_tensor from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput from .text_proj import UnCLIPTextProjModel From 31e34f7a141945b6b5d42159438a553bc3aa99d0 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Wed, 19 Jul 2023 08:46:51 +0000 Subject: [PATCH 08/23] add test --- tests/pipelines/test_pipelines_auto.py | 65 ++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 tests/pipelines/test_pipelines_auto.py diff --git a/tests/pipelines/test_pipelines_auto.py b/tests/pipelines/test_pipelines_auto.py new file mode 100644 index 000000000000..894a4bb4021c --- /dev/null +++ b/tests/pipelines/test_pipelines_auto.py @@ -0,0 +1,65 @@ +# coding=utf-8 +# Copyright 2023 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from collections import OrderedDict + +from diffusers import ( + AutoPipelineForImage2Image, + AutoPipelineForInpainting, + AutoPipelineForText2Image, +) +from diffusers.pipelines.auto_pipeline import ( + AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, + AUTO_INPAINTING_PIPELINES_MAPPING, + AUTO_TEXT2IMAGE_PIPELINES_MAPPING, +) +from diffusers.utils import slow + + +PRETRAINED_MODEL_REPO_MAPPING = OrderedDict( + [ + ("stable-diffusion", "runwayml/stable-diffusion-v1-5"), + ("if", "DeepFloyd/IF-I-XL-v1.0"), + ("kandinsky", "kandinsky-community/kandinsky-2-1"), + ("kdnsinskyv22", "kandinsky-community/kandinsky-2-2-decoder"), + ] +) + + +@slow +class AutoPipelineTest(unittest.TestCase): + def test_pipe_auto(self): + for model_name, model_repo in PRETRAINED_MODEL_REPO_MAPPING.items(): + # test from_pretrained + pipe_txt2img = AutoPipelineForText2Image.from_pretrained(model_repo) + self.assertIsInstance(pipe_txt2img, AUTO_TEXT2IMAGE_PIPELINES_MAPPING[model_name]) + + pipe_img2img = AutoPipelineForImage2Image.from_pretrained(model_repo) + self.assertIsInstance(pipe_img2img, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING[model_name]) + + pipe_inpaint = AutoPipelineForInpainting.from_pretrained(model_repo) + self.assertIsInstance(pipe_inpaint, AUTO_INPAINTING_PIPELINES_MAPPING[model_name]) + + # test from_pipe + for pipe_from in [pipe_txt2img, pipe_img2img, pipe_inpaint]: + pipe_to = AutoPipelineForText2Image.from_pipe(pipe_from) + self.assertIsInstance(pipe_to, AUTO_TEXT2IMAGE_PIPELINES_MAPPING[model_name]) + + pipe_to = AutoPipelineForImage2Image.from_pipe(pipe_from) + self.assertIsInstance(pipe_to, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING[model_name]) + + pipe_to = AutoPipelineForInpainting.from_pipe(pipe_from) + self.assertIsInstance(pipe_to, AUTO_INPAINTING_PIPELINES_MAPPING[model_name]) From 06aa0779da3cb6d28942fd627e7b2e3b29ac01be Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 19 Jul 2023 20:11:46 +0200 Subject: [PATCH 09/23] Show failing test --- tests/pipelines/test_pipelines_auto.py | 31 +++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/tests/pipelines/test_pipelines_auto.py b/tests/pipelines/test_pipelines_auto.py index 894a4bb4021c..bf272e974a0f 100644 --- a/tests/pipelines/test_pipelines_auto.py +++ b/tests/pipelines/test_pipelines_auto.py @@ -38,9 +38,19 @@ ] ) +class AutoPipelineFastTest(unittest.TestCase): + + def test_from_pipe_consistent(self): + pipe = AutoPipelineForText2Image.from_pretrained("hf-internal-testing/tiny-stable-diffusion-pipe", requires_safety_checker=False) + original_config = dict(pipe.config) + + pipe = AutoPipelineForImage2Image.from_pipe(pipe) + pipe = AutoPipelineForText2Image.from_pipe(pipe) + + assert dict(pipe.config) == original_config @slow -class AutoPipelineTest(unittest.TestCase): +class AutoPipelineIntegrationTest(unittest.TestCase): def test_pipe_auto(self): for model_name, model_repo in PRETRAINED_MODEL_REPO_MAPPING.items(): # test from_pretrained @@ -63,3 +73,22 @@ def test_pipe_auto(self): pipe_to = AutoPipelineForInpainting.from_pipe(pipe_from) self.assertIsInstance(pipe_to, AUTO_INPAINTING_PIPELINES_MAPPING[model_name]) + + def test_from_pipe_consistent(self): + for model_name, model_repo in PRETRAINED_MODEL_REPO_MAPPING.items(): + # test from_pretrained + pipe_txt2img = AutoPipelineForText2Image.from_pretrained(model_repo) + pipe_txt2img_config = dict(pipe_txt2img.config) + pipe_img2img = AutoPipelineForImage2Image.from_pretrained(model_repo) + pipe_img2img_config = dict(pipe_txt2img.config) + pipe_inpaint = AutoPipelineForInpainting.from_pretrained(model_repo) + pipe_inpaint_config = dict(pipe_txt2img.config) + + # test from_pipe + for pipe_from in [pipe_txt2img, pipe_img2img, pipe_inpaint]: + pipe_to = AutoPipelineForText2Image.from_pipe(pipe_from) + pipe_to.config == pipe_txt2img_config + pipe_to = AutoPipelineForImage2Image.from_pipe(pipe_from) + pipe_to.config == pipe_img2img_config + pipe_to = AutoPipelineForInpainting.from_pipe(pipe_from) + pipe_to.config == pipe_inpaint_config From 03d5d05c4cbff52ed13c11cf1c5838077fcfa07e Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Thu, 20 Jul 2023 07:45:21 +0000 Subject: [PATCH 10/23] update from_pipe --- src/diffusers/pipelines/auto_pipeline.py | 154 +++++++++++++++++++++-- tests/pipelines/test_pipelines_auto.py | 15 ++- 2 files changed, 157 insertions(+), 12 deletions(-) diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index 1307beea45b3..80e8d30cc9a7 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import inspect from collections import OrderedDict from ..configuration_utils import ConfigMixin @@ -87,9 +88,24 @@ def get_model(pipeline_class_name): ) +def _get_signature_keys(obj): + parameters = inspect.signature(obj.__init__).parameters + required_parameters = {k: v for k, v in parameters.items() if v.default == inspect._empty} + optional_parameters = set({k for k, v in parameters.items() if v.default != inspect._empty}) + expected_modules = set(required_parameters.keys()) - {"self"} + return expected_modules, optional_parameters + + class AutoPipelineForText2Image(ConfigMixin): config_name = "model_index.json" + def __init__(self, *args, **kwargs): + raise EnvironmentError( + f"{self.__class__.__name__} is designed to be instantiated " + f"using the `{self.__class__.__name__}.from_pretrained(pretrained_model_name_or_path)` or " + f"`{self.__class__.__name__}.from_pipe(pipeline)` methods." + ) + @classmethod def from_pretrained(cls, pretrained_model_or_path, **kwargs): config = cls.load_config(pretrained_model_or_path) @@ -100,15 +116,58 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): @classmethod def from_pipe(cls, pipeline, **kwargs): - print(pipeline.config.keys()) - text_2_image_cls = _get_task_class(AUTO_TEXT2IMAGE_PIPELINES_MAPPING, pipeline.__class__.__name__) + original_config = dict(pipeline.config) + original_cls_name = pipeline.__class__.__name__ + + # derive the pipeline class to instantiate + text_2_image_cls = _get_task_class(AUTO_TEXT2IMAGE_PIPELINES_MAPPING, original_cls_name) + + # define expected module and optional kwargs given the pipeline signature + expected_modules, optional_kwargs = _get_signature_keys(text_2_image_cls) + + pretrained_model_name_or_path = original_config.pop("_name_or_path", None) + + # allow users pass modules in `kwargs` to overwrite the original pipeline's components + passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs} + original_class_obj = { + k: pipeline.components[k] + for k, v in pipeline.components.items() + if k in expected_modules and k not in passed_class_obj + } + + # allow users pass optional kwargs to overwrite the original pipelines config attribute + passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs} + original_pipe_kwargs = { + k: original_config[k] + for k, v in original_config.items() + if k in optional_kwargs and k not in passed_pipe_kwargs + } - return text_2_image_cls(**pipeline.components) + text_2_image_kwargs = {**passed_class_obj, **original_class_obj, **passed_pipe_kwargs, **original_pipe_kwargs} + + missing_modules = set(expected_modules) - set(pipeline._optional_components) - set(text_2_image_kwargs.keys()) + + if len(missing_modules) > 0: + raise ValueError( + f"Pipeline {text_2_image_cls} expected {expected_modules}, but only {set(passed_class_obj.keys()) + set(original_class_obj.keys())} were passed" + ) + + model = text_2_image_cls(**text_2_image_kwargs) + model.register_to_config(_name_or_path=pretrained_model_name_or_path) + + return model class AutoPipelineForImage2Image(ConfigMixin): config_name = "model_index.json" + def __init__(self, *args, **kwargs): + raise EnvironmentError( + f"{self.__class__.__name__} is designed to be instantiated " + f"using the `{self.__class__.__name__}.from_pretrained(pretrained_model_name_or_path)` or " + f"`{self.__class__.__name__}.from_pipe(pipeline)` methods." + ) + @classmethod def from_pretrained(cls, pretrained_model_or_path, **kwargs): config = cls.load_config(pretrained_model_or_path) @@ -119,14 +178,58 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): @classmethod def from_pipe(cls, pipeline, **kwargs): - image_2_image_cls = _get_task_class(AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, pipeline.__class__.__name__) + original_config = dict(pipeline.config) + original_cls_name = pipeline.__class__.__name__ + + # derive the pipeline class to instantiate + image_2_image_cls = _get_task_class(AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, original_cls_name) + + # define expected module and optional kwargs given the pipeline signature + expected_modules, optional_kwargs = _get_signature_keys(image_2_image_cls) + + pretrained_model_name_or_path = original_config.pop("_name_or_path", None) - return image_2_image_cls(**pipeline.components, **kwargs) + # allow users pass modules in `kwargs` to overwrite the original pipeline's components + passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs} + original_class_obj = { + k: pipeline.components[k] + for k, v in pipeline.components.items() + if k in expected_modules and k not in passed_class_obj + } + + # allow users pass optional kwargs to overwrite the original pipelines config attribute + passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs} + original_pipe_kwargs = { + k: original_config[k] + for k, v in original_config.items() + if k in optional_kwargs and k not in passed_pipe_kwargs + } + + image_2_image_kwargs = {**passed_class_obj, **original_class_obj, **passed_pipe_kwargs, **original_pipe_kwargs} + + missing_modules = set(expected_modules) - set(pipeline._optional_components) - set(image_2_image_kwargs.keys()) + + if len(missing_modules) > 0: + raise ValueError( + f"Pipeline {image_2_image_cls} expected {expected_modules}, but only {set(passed_class_obj.keys()) + set(original_class_obj.keys())} were passed" + ) + + model = image_2_image_cls(**image_2_image_kwargs) + model.register_to_config(_name_or_path=pretrained_model_name_or_path) + + return model class AutoPipelineForInpainting(ConfigMixin): config_name = "model_index.json" + def __init__(self, *args, **kwargs): + raise EnvironmentError( + f"{self.__class__.__name__} is designed to be instantiated " + f"using the `{self.__class__.__name__}.from_pretrained(pretrained_model_name_or_path)` or " + f"`{self.__class__.__name__}.from_pipe(pipeline)` methods." + ) + @classmethod def from_pretrained(cls, pretrained_model_or_path, **kwargs): config = cls.load_config(pretrained_model_or_path) @@ -137,6 +240,43 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): @classmethod def from_pipe(cls, pipeline, **kwargs): - inpainting_cls = _get_task_class(AUTO_INPAINTING_PIPELINES_MAPPING, pipeline.__class__.__name__) + original_config = dict(pipeline.config) + original_cls_name = pipeline.__class__.__name__ + + # derive the pipeline class to instantiate + inpainting_cls = _get_task_class(AUTO_INPAINTING_PIPELINES_MAPPING, original_cls_name) + + # define expected module and optional kwargs given the pipeline signature + expected_modules, optional_kwargs = _get_signature_keys(inpainting_cls) + + pretrained_model_name_or_path = original_config.pop("_name_or_path", None) + + # allow users pass modules in `kwargs` to overwrite the original pipeline's components + passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs} + original_class_obj = { + k: pipeline.components[k] + for k, v in pipeline.components.items() + if k in expected_modules and k not in passed_class_obj + } + + # allow users pass optional kwargs to overwrite the original pipelines config attribute + passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs} + original_pipe_kwargs = { + k: original_config[k] + for k, v in original_config.items() + if k in optional_kwargs and k not in passed_pipe_kwargs + } + + inpainting_kwargs = {**passed_class_obj, **original_class_obj, **passed_pipe_kwargs, **original_pipe_kwargs} + + missing_modules = set(expected_modules) - set(pipeline._optional_components) - set(inpainting_kwargs.keys()) + + if len(missing_modules) > 0: + raise ValueError( + f"Pipeline {inpainting_cls} expected {expected_modules}, but only {set(passed_class_obj.keys()) + set(original_class_obj.keys())} were passed" + ) + + model = inpainting_cls(**inpainting_kwargs) + model.register_to_config(_name_or_path=pretrained_model_name_or_path) - return inpainting_cls(**pipeline.components) + return model diff --git a/tests/pipelines/test_pipelines_auto.py b/tests/pipelines/test_pipelines_auto.py index bf272e974a0f..5f05bc45f0bd 100644 --- a/tests/pipelines/test_pipelines_auto.py +++ b/tests/pipelines/test_pipelines_auto.py @@ -38,10 +38,12 @@ ] ) -class AutoPipelineFastTest(unittest.TestCase): +class AutoPipelineFastTest(unittest.TestCase): def test_from_pipe_consistent(self): - pipe = AutoPipelineForText2Image.from_pretrained("hf-internal-testing/tiny-stable-diffusion-pipe", requires_safety_checker=False) + pipe = AutoPipelineForText2Image.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-pipe", requires_safety_checker=False + ) original_config = dict(pipe.config) pipe = AutoPipelineForImage2Image.from_pipe(pipe) @@ -49,6 +51,7 @@ def test_from_pipe_consistent(self): assert dict(pipe.config) == original_config + @slow class AutoPipelineIntegrationTest(unittest.TestCase): def test_pipe_auto(self): @@ -87,8 +90,10 @@ def test_from_pipe_consistent(self): # test from_pipe for pipe_from in [pipe_txt2img, pipe_img2img, pipe_inpaint]: pipe_to = AutoPipelineForText2Image.from_pipe(pipe_from) - pipe_to.config == pipe_txt2img_config + self.assertEqual(dict(pipe_to.config), pipe_txt2img_config) + pipe_to = AutoPipelineForImage2Image.from_pipe(pipe_from) - pipe_to.config == pipe_img2img_config + self.assertEqual(dict(pipe_to.config), pipe_img2img_config) + pipe_to = AutoPipelineForInpainting.from_pipe(pipe_from) - pipe_to.config == pipe_inpaint_config + self.assertEqual(dict(pipe_to.config), pipe_inpaint_config) From 95ac809976a379e2c7b0eb5eb4e06b9d5a0fb4a3 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Thu, 20 Jul 2023 08:12:05 +0000 Subject: [PATCH 11/23] fix --- src/diffusers/pipelines/auto_pipeline.py | 12 ++++++------ tests/pipelines/test_pipelines_auto.py | 15 ++++++++++++++- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index 80e8d30cc9a7..73e6cacb8a1a 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -127,7 +127,7 @@ def from_pipe(cls, pipeline, **kwargs): pretrained_model_name_or_path = original_config.pop("_name_or_path", None) - # allow users pass modules in `kwargs` to overwrite the original pipeline's components + # allow users pass modules in `kwargs` to override the original pipeline's components passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs} original_class_obj = { k: pipeline.components[k] @@ -135,7 +135,7 @@ def from_pipe(cls, pipeline, **kwargs): if k in expected_modules and k not in passed_class_obj } - # allow users pass optional kwargs to overwrite the original pipelines config attribute + # allow users pass optional kwargs to override the original pipelines config attribute passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs} original_pipe_kwargs = { k: original_config[k] @@ -189,7 +189,7 @@ def from_pipe(cls, pipeline, **kwargs): pretrained_model_name_or_path = original_config.pop("_name_or_path", None) - # allow users pass modules in `kwargs` to overwrite the original pipeline's components + # allow users pass modules in `kwargs` to override the original pipeline's components passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs} original_class_obj = { k: pipeline.components[k] @@ -197,7 +197,7 @@ def from_pipe(cls, pipeline, **kwargs): if k in expected_modules and k not in passed_class_obj } - # allow users pass optional kwargs to overwrite the original pipelines config attribute + # allow users pass optional kwargs to override the original pipelines config attribute passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs} original_pipe_kwargs = { k: original_config[k] @@ -251,7 +251,7 @@ def from_pipe(cls, pipeline, **kwargs): pretrained_model_name_or_path = original_config.pop("_name_or_path", None) - # allow users pass modules in `kwargs` to overwrite the original pipeline's components + # allow users pass modules in `kwargs` to override the original pipeline's components passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs} original_class_obj = { k: pipeline.components[k] @@ -259,7 +259,7 @@ def from_pipe(cls, pipeline, **kwargs): if k in expected_modules and k not in passed_class_obj } - # allow users pass optional kwargs to overwrite the original pipelines config attribute + # allow users pass optional kwargs to override the original pipelines config attribute passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs} original_pipe_kwargs = { k: original_config[k] diff --git a/tests/pipelines/test_pipelines_auto.py b/tests/pipelines/test_pipelines_auto.py index 5f05bc45f0bd..f55837d3e808 100644 --- a/tests/pipelines/test_pipelines_auto.py +++ b/tests/pipelines/test_pipelines_auto.py @@ -47,10 +47,23 @@ def test_from_pipe_consistent(self): original_config = dict(pipe.config) pipe = AutoPipelineForImage2Image.from_pipe(pipe) - pipe = AutoPipelineForText2Image.from_pipe(pipe) + assert dict(pipe.config) == original_config + pipe = AutoPipelineForText2Image.from_pipe(pipe) assert dict(pipe.config) == original_config + def test_from_pipe_override(self): + pipe = AutoPipelineForText2Image.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-pipe", requires_safety_checker=False + ) + dict(pipe.config) + + pipe = AutoPipelineForImage2Image.from_pipe(pipe, requires_safety_checker=True) + assert pipe.config.requires_safety_checker is True + + pipe = AutoPipelineForText2Image.from_pipe(pipe, requires_safety_checker=True) + assert pipe.config.requires_safety_checker is True + @slow class AutoPipelineIntegrationTest(unittest.TestCase): From c4b4d9331323d937c8938daad9b1f641252dcabb Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Thu, 20 Jul 2023 16:36:11 +0000 Subject: [PATCH 12/23] add controlnet, additional test and register unused original config --- src/diffusers/pipelines/auto_pipeline.py | 14 ++++++++++++++ tests/pipelines/test_pipelines_auto.py | 15 +++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index 73e6cacb8a1a..edb97207ab61 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -18,6 +18,10 @@ from collections import OrderedDict from ..configuration_utils import ConfigMixin +from .controlnet import ( + StableDiffusionControlNetImg2ImgPipeline, + StableDiffusionControlNetPipeline, +) from .deepfloyd_if import IFImg2ImgPipeline, IFInpaintingPipeline, IFPipeline from .kandinsky import KandinskyImg2ImgPipeline, KandinskyInpaintPipeline, KandinskyPipeline from .kandinsky2_2 import KandinskyV22Img2ImgPipeline, KandinskyV22InpaintPipeline, KandinskyV22Pipeline @@ -40,6 +44,7 @@ ("if", IFPipeline), ("kandinsky", KandinskyPipeline), ("kdnsinskyv22", KandinskyV22Pipeline), + ("contronet", StableDiffusionControlNetPipeline), ] ) @@ -50,6 +55,7 @@ ("if", IFImg2ImgPipeline), ("kandinsky", KandinskyImg2ImgPipeline), ("kdnsinskyv22", KandinskyV22Img2ImgPipeline), + ("controlnet", StableDiffusionControlNetImg2ImgPipeline), ] ) @@ -60,6 +66,7 @@ ("if", IFInpaintingPipeline), ("kandinsky", KandinskyInpaintPipeline), ("kdnsinskyv22", KandinskyV22InpaintPipeline), + ("controlnet", StableDiffusionControlNetImg2ImgPipeline), ] ) @@ -143,6 +150,12 @@ def from_pipe(cls, pipeline, **kwargs): if k in optional_kwargs and k not in passed_pipe_kwargs } + unused_original_config = { + k: original_config[k] + for k, v in original_config.items() + if k not in optional_kwargs and k not in passed_pipe_kwargs + } + text_2_image_kwargs = {**passed_class_obj, **original_class_obj, **passed_pipe_kwargs, **original_pipe_kwargs} missing_modules = set(expected_modules) - set(pipeline._optional_components) - set(text_2_image_kwargs.keys()) @@ -154,6 +167,7 @@ def from_pipe(cls, pipeline, **kwargs): model = text_2_image_cls(**text_2_image_kwargs) model.register_to_config(_name_or_path=pretrained_model_name_or_path) + model.register_to_config(**unused_original_config) return model diff --git a/tests/pipelines/test_pipelines_auto.py b/tests/pipelines/test_pipelines_auto.py index f55837d3e808..5746cfbc660e 100644 --- a/tests/pipelines/test_pipelines_auto.py +++ b/tests/pipelines/test_pipelines_auto.py @@ -64,6 +64,21 @@ def test_from_pipe_override(self): pipe = AutoPipelineForText2Image.from_pipe(pipe, requires_safety_checker=True) assert pipe.config.requires_safety_checker is True + def test_from_pipe_consistent_sdxl(self): + pipe = AutoPipelineForImage2Image.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-xl-pipe", + requires_aesthetics_score=True, + force_zeros_for_empty_prompt=False, + ) + + original_config = dict(pipe.config) + + pipe = AutoPipelineForImage2Image.from_pipe(pipe) + assert dict(pipe.config) == original_config + + pipe = AutoPipelineForText2Image.from_pipe(pipe) + assert dict(pipe.config) == original_config + @slow class AutoPipelineIntegrationTest(unittest.TestCase): From c1c063c79d52c95a9b06f2c7e5f45336676a1e22 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Thu, 20 Jul 2023 17:51:38 +0000 Subject: [PATCH 13/23] update for controlnet --- src/diffusers/pipelines/auto_pipeline.py | 26 ++++++++++----- tests/pipelines/test_pipelines_auto.py | 40 +++++++++++++++++++++++- 2 files changed, 57 insertions(+), 9 deletions(-) diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index edb97207ab61..cebf24d83d0d 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -20,6 +20,7 @@ from ..configuration_utils import ConfigMixin from .controlnet import ( StableDiffusionControlNetImg2ImgPipeline, + StableDiffusionControlNetInpaintPipeline, StableDiffusionControlNetPipeline, ) from .deepfloyd_if import IFImg2ImgPipeline, IFInpaintingPipeline, IFPipeline @@ -44,7 +45,7 @@ ("if", IFPipeline), ("kandinsky", KandinskyPipeline), ("kdnsinskyv22", KandinskyV22Pipeline), - ("contronet", StableDiffusionControlNetPipeline), + ("controlnet", StableDiffusionControlNetPipeline), ] ) @@ -66,7 +67,7 @@ ("if", IFInpaintingPipeline), ("kandinsky", KandinskyInpaintPipeline), ("kdnsinskyv22", KandinskyV22InpaintPipeline), - ("controlnet", StableDiffusionControlNetImg2ImgPipeline), + ("controlnet", StableDiffusionControlNetInpaintPipeline), ] ) @@ -115,9 +116,12 @@ def __init__(self, *args, **kwargs): @classmethod def from_pretrained(cls, pretrained_model_or_path, **kwargs): - config = cls.load_config(pretrained_model_or_path) + if "controlnet" in kwargs: + text_2_image_cls = AUTO_TEXT2IMAGE_PIPELINES_MAPPING["controlnet"] - text_2_image_cls = _get_task_class(AUTO_TEXT2IMAGE_PIPELINES_MAPPING, config["_class_name"]) + else: + config = cls.load_config(pretrained_model_or_path) + text_2_image_cls = _get_task_class(AUTO_TEXT2IMAGE_PIPELINES_MAPPING, config["_class_name"]) return text_2_image_cls.from_pretrained(pretrained_model_or_path, **kwargs) @@ -184,9 +188,12 @@ def __init__(self, *args, **kwargs): @classmethod def from_pretrained(cls, pretrained_model_or_path, **kwargs): - config = cls.load_config(pretrained_model_or_path) + if "controlnet" in kwargs: + image_2_image_cls = AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["controlnet"] - image_2_image_cls = _get_task_class(AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, config["_class_name"]) + else: + config = cls.load_config(pretrained_model_or_path) + image_2_image_cls = _get_task_class(AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, config["_class_name"]) return image_2_image_cls.from_pretrained(pretrained_model_or_path, **kwargs) @@ -246,9 +253,12 @@ def __init__(self, *args, **kwargs): @classmethod def from_pretrained(cls, pretrained_model_or_path, **kwargs): - config = cls.load_config(pretrained_model_or_path) + if "controlnet" in kwargs: + inpainting_cls = AUTO_INPAINTING_PIPELINES_MAPPING["controlnet"] - inpainting_cls = _get_task_class(AUTO_INPAINTING_PIPELINES_MAPPING, config["_class_name"]) + else: + config = cls.load_config(pretrained_model_or_path) + inpainting_cls = _get_task_class(AUTO_INPAINTING_PIPELINES_MAPPING, config["_class_name"]) return inpainting_cls.from_pretrained(pretrained_model_or_path, **kwargs) diff --git a/tests/pipelines/test_pipelines_auto.py b/tests/pipelines/test_pipelines_auto.py index 5746cfbc660e..b484de832644 100644 --- a/tests/pipelines/test_pipelines_auto.py +++ b/tests/pipelines/test_pipelines_auto.py @@ -16,10 +16,13 @@ import unittest from collections import OrderedDict +import torch + from diffusers import ( AutoPipelineForImage2Image, AutoPipelineForInpainting, AutoPipelineForText2Image, + ControlNetModel, ) from diffusers.pipelines.auto_pipeline import ( AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, @@ -56,7 +59,6 @@ def test_from_pipe_override(self): pipe = AutoPipelineForText2Image.from_pretrained( "hf-internal-testing/tiny-stable-diffusion-pipe", requires_safety_checker=False ) - dict(pipe.config) pipe = AutoPipelineForImage2Image.from_pipe(pipe, requires_safety_checker=True) assert pipe.config.requires_safety_checker is True @@ -125,3 +127,39 @@ def test_from_pipe_consistent(self): pipe_to = AutoPipelineForInpainting.from_pipe(pipe_from) self.assertEqual(dict(pipe_to.config), pipe_inpaint_config) + + def test_controlnet(self): + # test from_pretrained + model_repo = "runwayml/stable-diffusion-v1-5" + controlnet_repo = "lllyasviel/sd-controlnet-canny" + + controlnet = ControlNetModel.from_pretrained(controlnet_repo, torch_dtype=torch.float16) + + pipe_txt2img = AutoPipelineForText2Image.from_pretrained( + model_repo, controlnet=controlnet, torch_dtype=torch.float16 + ) + self.assertIsInstance(pipe_txt2img, AUTO_TEXT2IMAGE_PIPELINES_MAPPING["controlnet"]) + + pipe_img2img = AutoPipelineForImage2Image.from_pretrained( + model_repo, controlnet=controlnet, torch_dtype=torch.float16 + ) + self.assertIsInstance(pipe_img2img, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["controlnet"]) + + pipe_inpaint = AutoPipelineForInpainting.from_pretrained( + model_repo, controlnet=controlnet, torch_dtype=torch.float16 + ) + self.assertIsInstance(pipe_inpaint, AUTO_INPAINTING_PIPELINES_MAPPING["controlnet"]) + + # test from_pipe + for pipe_from in [pipe_txt2img, pipe_img2img, pipe_inpaint]: + pipe_to = AutoPipelineForText2Image.from_pipe(pipe_from) + self.assertIsInstance(pipe_to, AUTO_TEXT2IMAGE_PIPELINES_MAPPING["controlnet"]) + self.assertEqual(dict(pipe_to.config), dict(pipe_txt2img.config)) + + pipe_to = AutoPipelineForImage2Image.from_pipe(pipe_from) + self.assertIsInstance(pipe_to, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["controlnet"]) + self.assertEqual(dict(pipe_to.config), dict(pipe_img2img.config)) + + pipe_to = AutoPipelineForInpainting.from_pipe(pipe_from) + self.assertIsInstance(pipe_to, AUTO_INPAINTING_PIPELINES_MAPPING["controlnet"]) + self.assertEqual(dict(pipe_to.config), dict(pipe_inpaint.config)) From de17db257a99c1f80a2a818b2faa02b3c81fa345 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Fri, 21 Jul 2023 12:51:22 -1000 Subject: [PATCH 14/23] Apply suggestions from code review Co-authored-by: Patrick von Platen --- tests/pipelines/test_pipelines_auto.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/pipelines/test_pipelines_auto.py b/tests/pipelines/test_pipelines_auto.py index b484de832644..bbed7ac23047 100644 --- a/tests/pipelines/test_pipelines_auto.py +++ b/tests/pipelines/test_pipelines_auto.py @@ -75,11 +75,12 @@ def test_from_pipe_consistent_sdxl(self): original_config = dict(pipe.config) - pipe = AutoPipelineForImage2Image.from_pipe(pipe) - assert dict(pipe.config) == original_config pipe = AutoPipelineForText2Image.from_pipe(pipe) assert dict(pipe.config) == original_config + + pipe = AutoPipelineForImage2Image.from_pipe(pipe) + assert dict(pipe.config) == original_config @slow From 51e2a56d3aac0d455ec544ebbccad1ba96569458 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Sat, 22 Jul 2023 01:31:24 +0000 Subject: [PATCH 15/23] store unused config as private attribute and pass if can --- src/diffusers/pipelines/auto_pipeline.py | 55 ++++++++++++++++++++++-- tests/pipelines/test_pipelines_auto.py | 4 +- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index cebf24d83d0d..f5e6eec486d7 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -154,14 +154,25 @@ def from_pipe(cls, pipeline, **kwargs): if k in optional_kwargs and k not in passed_pipe_kwargs } + # config that were not expected by original pipeline is stored as private attribute + # we will pass them as optional arguments if they can be accepted by the pipeline + additional_pipe_kwargs = [ + k[1:] + for k in original_config.keys() + if k.startswith("_") and k[1:] in optional_kwargs and k[1:] not in passed_pipe_kwargs + ] + for k in additional_pipe_kwargs: + original_pipe_kwargs[k] = original_config.pop(f"_{k}") + + text_2_image_kwargs = {**passed_class_obj, **original_class_obj, **passed_pipe_kwargs, **original_pipe_kwargs} + + # store unused config as private attribute unused_original_config = { - k: original_config[k] + f"{'' if k.startswith('_') else '_'}{k}": original_config[k] for k, v in original_config.items() - if k not in optional_kwargs and k not in passed_pipe_kwargs + if k not in text_2_image_kwargs } - text_2_image_kwargs = {**passed_class_obj, **original_class_obj, **passed_pipe_kwargs, **original_pipe_kwargs} - missing_modules = set(expected_modules) - set(pipeline._optional_components) - set(text_2_image_kwargs.keys()) if len(missing_modules) > 0: @@ -226,8 +237,25 @@ def from_pipe(cls, pipeline, **kwargs): if k in optional_kwargs and k not in passed_pipe_kwargs } + # config attribute that were not expected by original pipeline is stored as its private attribute + # we will pass them as optional arguments if they can be accepted by the pipeline + additional_pipe_kwargs = [ + k[1:] + for k in original_config.keys() + if k.startswith("_") and k[1:] in optional_kwargs and k[1:] not in passed_pipe_kwargs + ] + for k in additional_pipe_kwargs: + original_pipe_kwargs[k] = original_config.pop(f"_{k}") + image_2_image_kwargs = {**passed_class_obj, **original_class_obj, **passed_pipe_kwargs, **original_pipe_kwargs} + # store unused config as private attribute + unused_original_config = { + f"{'' if k.startswith('_') else '_'}{k}": original_config[k] + for k, v in original_config.items() + if k not in image_2_image_kwargs + } + missing_modules = set(expected_modules) - set(pipeline._optional_components) - set(image_2_image_kwargs.keys()) if len(missing_modules) > 0: @@ -237,6 +265,7 @@ def from_pipe(cls, pipeline, **kwargs): model = image_2_image_cls(**image_2_image_kwargs) model.register_to_config(_name_or_path=pretrained_model_name_or_path) + model.register_to_config(**unused_original_config) return model @@ -291,8 +320,25 @@ def from_pipe(cls, pipeline, **kwargs): if k in optional_kwargs and k not in passed_pipe_kwargs } + # config that were not expected by original pipeline is stored as private attribute + # we will pass them as optional arguments if they can be accepted by the pipeline + additional_pipe_kwargs = [ + k[1:] + for k in original_config.keys() + if k.startswith("_") and k[1:] in optional_kwargs and k[1:] not in passed_pipe_kwargs + ] + for k in additional_pipe_kwargs: + original_pipe_kwargs[k] = original_config.pop(f"_{k}") + inpainting_kwargs = {**passed_class_obj, **original_class_obj, **passed_pipe_kwargs, **original_pipe_kwargs} + # store unused config as private attribute + unused_original_config = { + f"{'' if k.startswith('_') else '_'}{k}": original_config[k] + for k, v in original_config.items() + if k not in inpainting_kwargs + } + missing_modules = set(expected_modules) - set(pipeline._optional_components) - set(inpainting_kwargs.keys()) if len(missing_modules) > 0: @@ -302,5 +348,6 @@ def from_pipe(cls, pipeline, **kwargs): model = inpainting_cls(**inpainting_kwargs) model.register_to_config(_name_or_path=pretrained_model_name_or_path) + model.register_to_config(**unused_original_config) return model diff --git a/tests/pipelines/test_pipelines_auto.py b/tests/pipelines/test_pipelines_auto.py index bbed7ac23047..2e6c4d7896c7 100644 --- a/tests/pipelines/test_pipelines_auto.py +++ b/tests/pipelines/test_pipelines_auto.py @@ -75,11 +75,9 @@ def test_from_pipe_consistent_sdxl(self): original_config = dict(pipe.config) - pipe = AutoPipelineForText2Image.from_pipe(pipe) - assert dict(pipe.config) == original_config - pipe = AutoPipelineForImage2Image.from_pipe(pipe) + assert dict(pipe.config) == original_config From 955c827fbf324aa79de58011640360c7bb19ab18 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Sat, 22 Jul 2023 06:40:02 +0000 Subject: [PATCH 16/23] add doc --- docs/source/en/_toctree.yml | 2 + .../source/en/api/pipelines/auto_pipeline.mdx | 41 ++ src/diffusers/pipelines/auto_pipeline.py | 557 ++++++++++++++++++ 3 files changed, 600 insertions(+) create mode 100644 docs/source/en/api/pipelines/auto_pipeline.mdx diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index 4cf4cf36e4e4..5abee475d9c9 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -186,6 +186,8 @@ title: Audio Diffusion - local: api/pipelines/audioldm title: AudioLDM + - local: api/pipelines/auto_pipeline + title: AutoPipeline - local: api/pipelines/consistency_models title: Consistency Models - local: api/pipelines/controlnet diff --git a/docs/source/en/api/pipelines/auto_pipeline.mdx b/docs/source/en/api/pipelines/auto_pipeline.mdx new file mode 100644 index 000000000000..a74a59de6e1c --- /dev/null +++ b/docs/source/en/api/pipelines/auto_pipeline.mdx @@ -0,0 +1,41 @@ + + +# AutoPipeline + +In many cases, one checkpoint can be used for multiple tasks. You may be able the same checkpoint for Text-to-Image, Image-to-Image and Inpainting, as long as you know which pipelines are linked to your checkpoint. + +AutoPipeline is designed to help you switch between different tasks seamlessly using the same checkpoint. It automatically retrieve the relevant pipeline for the task your want to perform, given the name/path to the pretrained weights. + +## AutoPipelineForText2Image + +[[autodoc]] AutoPipelineForText2Image + - all + - from_pretrained + - from_pipe + + +## AutoPipelineForImage2Image + +[[autodoc]] AutoPipelineForImage2Image + - all + - from_pretrained + - from_pipe + +## AutoPipelineForInpainting + +[[autodoc]] AutoPipelineForInpainting + - all + - from_pretrained + - from_pipe + + diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index f5e6eec486d7..48b370459865 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -105,6 +105,25 @@ def _get_signature_keys(obj): class AutoPipelineForText2Image(ConfigMixin): + r""" + + AutoPipeline for text-to-image generation. + + [`AutoPipelineForText2Image`] is a generic pipeline class that will be instantiated as one of the text-to-image + pipeline class in diffusers. + + The pipeline type (for example [`StableDiffusionPipeline`]) is automatically selected when created with the + AutoPipelineForText2Image.from_pretrained(pretrained_model_name_or_path) or + AutoPipelineForText2Image.from_pipe(pipeline) class methods . + + This class cannot be instantiated using __init__() (throws an error). + + Class attributes: + + - **config_name** (`str`) -- The configuration filename that stores the class and module names of all the + diffusion pipeline's components. + + """ config_name = "model_index.json" def __init__(self, *args, **kwargs): @@ -116,6 +135,146 @@ def __init__(self, *args, **kwargs): @classmethod def from_pretrained(cls, pretrained_model_or_path, **kwargs): + r""" + Instantiates a text-to-image Pytorch diffusion pipeline from pretrained pipeline weight. + + The from_pretrained() method takes care of returning the correct pipeline class instance by: + 1. Detect the pipeline class of the pretrained_model_or_path based on the _class_name property of its + config object + 2. Find the text-to-image pipeline linked to the pipeline class using pattern matching on pipeline class + name. + + If a `controlnet` argument is passed, it will instantiate a StableDiffusionControlNetPipeline object. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + If you get the error message below, you need to finetune the weights for your downstream task: + + ``` + Some weights of UNet2DConditionModel were not initialized from the model checkpoint at runwayml/stable-diffusion-v1-5 and are newly initialized because the shapes did not match: + - conv_in.weight: found shape torch.Size([320, 4, 3, 3]) in the checkpoint and torch.Size([320, 9, 3, 3]) in the model instantiated + You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. + ``` + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *repo id* (for example `CompVis/ldm-text2im-large-256`) of a pretrained pipeline + hosted on the Hub. + - A path to a *directory* (for example `./my_pipeline_directory/`) containing pipeline weights + saved using + [`~DiffusionPipeline.save_pretrained`]. + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model with another dtype. If "auto" is passed, the + dtype is automatically derived from the model's weights. + custom_pipeline (`str`, *optional*): + + + + 🧪 This is an experimental feature and may change in the future. + + + + Can be either: + + - A string, the *repo id* (for example `hf-internal-testing/diffusers-dummy-pipeline`) of a custom + pipeline hosted on the Hub. The repository must contain a file called pipeline.py that defines + the custom pipeline. + - A string, the *file name* of a community pipeline hosted on GitHub under + [Community](https://github.com/huggingface/diffusers/tree/main/examples/community). Valid file + names must match the file name and not the pipeline script (`clip_guided_stable_diffusion` + instead of `clip_guided_stable_diffusion.py`). Community pipelines are always loaded from the + current main branch of GitHub. + - A path to a directory (`./my_pipeline_directory/`) containing a custom pipeline. The directory + must contain a file called `pipeline.py` that defines the custom pipeline. + + + For more information on how to load and create custom pipelines, please have a look at [Loading and + Adding Custom + Pipelines](https://huggingface.co/docs/diffusers/using-diffusers/custom_pipeline_overview) + + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + use_auth_token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + custom_revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id similar to + `revision` when loading a custom pipeline from the Hub. It can be a 🤗 Diffusers version when loading a + custom pipeline from GitHub, otherwise it defaults to `"main"` when loading from the Hub. + mirror (`str`, *optional*): + Mirror source to resolve accessibility issues if you’re downloading a model in China. We do not + guarantee the timeliness or safety of the source, and you should refer to the mirror site for more + information. + device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn’t need to be defined for each + parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the + same device. + + Set `device_map="auto"` to have 🤗 Accelerate automatically compute the most optimized `device_map`. For + more information about each option see [designing a device + map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). + max_memory (`Dict`, *optional*): + A dictionary device identifier for the maximum memory. Will default to the maximum memory available for + each GPU and the available CPU RAM if unset. + offload_folder (`str` or `os.PathLike`, *optional*): + The path to offload weights if device_map contains the value `"disk"`. + offload_state_dict (`bool`, *optional*): + If `True`, temporarily offloads the CPU state dict to the hard drive to avoid running out of CPU RAM if + the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to `True` + when there is some disk offload. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading only loading the pretrained weights and not initializing the weights. This also + tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model. + Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this + argument to `True` will raise an error. + use_safetensors (`bool`, *optional*, defaults to `None`): + If set to `None`, the safetensors weights are downloaded if they're available **and** if the + safetensors library is installed. If set to `True`, the model is forcibly loaded from safetensors + weights. If set to `False`, safetensors weights are not loaded. + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to overwrite load and saveable variables (the pipeline components of the specific pipeline + class). The overwritten components are passed directly to the pipelines `__init__` method. See example + below for more information. + variant (`str`, *optional*): + Load weights from a specified variant filename such as `"fp16"` or `"ema"`. This is ignored when + loading `from_flax`. + + + + To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with + `huggingface-cli login`. + + + + Examples: + + ```py + >>> from diffusers import AutoPipelineForTextToImage + + >>> pipeline = AutoPipelineForTextToImage.from_pretrained("runwayml/stable-diffusion-v1-5") + >>> print(pipeline.__class__) + ``` + """ if "controlnet" in kwargs: text_2_image_cls = AUTO_TEXT2IMAGE_PIPELINES_MAPPING["controlnet"] @@ -127,6 +286,32 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): @classmethod def from_pipe(cls, pipeline, **kwargs): + r""" + Instantiates a text-to-image Pytorch diffusion pipeline from another instantiated diffusion pipeline class. + + The from_pipe() method takes care of returning the correct pipeline class instance by find the text-to-image + pipeline linked to the pipeline class using pattern matching on pipeline class name. + + All the modules the pipeline contains will be used to initialize the new pipeline without reallocating + additional memoery. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + Parameters: + pipeline (`DiffusionPipeline`): + an instantiated `DiffusionPipeline` object + + ```py + >>> from diffusers import AutoPipelineForTextToImage, AutoPipelineForImageToImage + + >>> pipe_i2i = AutoPipelineForImage2Image.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", requires_safety_checker=False + ... ) + + >>> pipe_t2i = AutoPipelineForTextToImage.from_pipe(pipe_t2i) + ``` + """ + original_config = dict(pipeline.config) original_cls_name = pipeline.__class__.__name__ @@ -188,6 +373,25 @@ def from_pipe(cls, pipeline, **kwargs): class AutoPipelineForImage2Image(ConfigMixin): + r""" + + AutoPipeline for image-to-image generation. + + [`AutoPipelineForImage2Image`] is a generic pipeline class that will be instantiated as one of the image-to-image + pipeline class in diffusers. + + The pipeline type (for example [`StableDiffusionImg2ImgPipeline`]) is automatically selected when created with the + AutoPipelineForImage2Image.from_pretrained(pretrained_model_name_or_path) or + AutoPipelineForImage2Image.from_pipe(pipeline) class methods . + + This class cannot be instantiated using __init__() (throws an error). + + Class attributes: + + - **config_name** (`str`) -- The configuration filename that stores the class and module names of all the + diffusion pipeline's components. + + """ config_name = "model_index.json" def __init__(self, *args, **kwargs): @@ -199,6 +403,146 @@ def __init__(self, *args, **kwargs): @classmethod def from_pretrained(cls, pretrained_model_or_path, **kwargs): + r""" + Instantiates a image-to-image Pytorch diffusion pipeline from pretrained pipeline weight. + + The from_pretrained() method takes care of returning the correct pipeline class instance by: + 1. Detect the pipeline class of the pretrained_model_or_path based on the _class_name property of its + config object + 2. Find the image-to-image pipeline linked to the pipeline class using pattern matching on pipeline class + name. + + If a `controlnet` argument is passed, it will instantiate a StableDiffusionControlNetImg2ImgPipeline object. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + If you get the error message below, you need to finetune the weights for your downstream task: + + ``` + Some weights of UNet2DConditionModel were not initialized from the model checkpoint at runwayml/stable-diffusion-v1-5 and are newly initialized because the shapes did not match: + - conv_in.weight: found shape torch.Size([320, 4, 3, 3]) in the checkpoint and torch.Size([320, 9, 3, 3]) in the model instantiated + You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. + ``` + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *repo id* (for example `CompVis/ldm-text2im-large-256`) of a pretrained pipeline + hosted on the Hub. + - A path to a *directory* (for example `./my_pipeline_directory/`) containing pipeline weights + saved using + [`~DiffusionPipeline.save_pretrained`]. + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model with another dtype. If "auto" is passed, the + dtype is automatically derived from the model's weights. + custom_pipeline (`str`, *optional*): + + + + 🧪 This is an experimental feature and may change in the future. + + + + Can be either: + + - A string, the *repo id* (for example `hf-internal-testing/diffusers-dummy-pipeline`) of a custom + pipeline hosted on the Hub. The repository must contain a file called pipeline.py that defines + the custom pipeline. + - A string, the *file name* of a community pipeline hosted on GitHub under + [Community](https://github.com/huggingface/diffusers/tree/main/examples/community). Valid file + names must match the file name and not the pipeline script (`clip_guided_stable_diffusion` + instead of `clip_guided_stable_diffusion.py`). Community pipelines are always loaded from the + current main branch of GitHub. + - A path to a directory (`./my_pipeline_directory/`) containing a custom pipeline. The directory + must contain a file called `pipeline.py` that defines the custom pipeline. + + + For more information on how to load and create custom pipelines, please have a look at [Loading and + Adding Custom + Pipelines](https://huggingface.co/docs/diffusers/using-diffusers/custom_pipeline_overview) + + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + use_auth_token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + custom_revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id similar to + `revision` when loading a custom pipeline from the Hub. It can be a 🤗 Diffusers version when loading a + custom pipeline from GitHub, otherwise it defaults to `"main"` when loading from the Hub. + mirror (`str`, *optional*): + Mirror source to resolve accessibility issues if you’re downloading a model in China. We do not + guarantee the timeliness or safety of the source, and you should refer to the mirror site for more + information. + device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn’t need to be defined for each + parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the + same device. + + Set `device_map="auto"` to have 🤗 Accelerate automatically compute the most optimized `device_map`. For + more information about each option see [designing a device + map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). + max_memory (`Dict`, *optional*): + A dictionary device identifier for the maximum memory. Will default to the maximum memory available for + each GPU and the available CPU RAM if unset. + offload_folder (`str` or `os.PathLike`, *optional*): + The path to offload weights if device_map contains the value `"disk"`. + offload_state_dict (`bool`, *optional*): + If `True`, temporarily offloads the CPU state dict to the hard drive to avoid running out of CPU RAM if + the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to `True` + when there is some disk offload. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading only loading the pretrained weights and not initializing the weights. This also + tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model. + Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this + argument to `True` will raise an error. + use_safetensors (`bool`, *optional*, defaults to `None`): + If set to `None`, the safetensors weights are downloaded if they're available **and** if the + safetensors library is installed. If set to `True`, the model is forcibly loaded from safetensors + weights. If set to `False`, safetensors weights are not loaded. + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to overwrite load and saveable variables (the pipeline components of the specific pipeline + class). The overwritten components are passed directly to the pipelines `__init__` method. See example + below for more information. + variant (`str`, *optional*): + Load weights from a specified variant filename such as `"fp16"` or `"ema"`. This is ignored when + loading `from_flax`. + + + + To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with + `huggingface-cli login`. + + + + Examples: + + ```py + >>> from diffusers import AutoPipelineForTextToImage + + >>> pipeline = AutoPipelineForImageToImage.from_pretrained("runwayml/stable-diffusion-v1-5") + >>> print(pipeline.__class__) + ``` + """ if "controlnet" in kwargs: image_2_image_cls = AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["controlnet"] @@ -210,6 +554,34 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): @classmethod def from_pipe(cls, pipeline, **kwargs): + r""" + Instantiates a image-to-image Pytorch diffusion pipeline from another instantiated diffusion pipeline class. + + The from_pipe() method takes care of returning the correct pipeline class instance by find the image-to-image + pipeline linked to the pipeline class using pattern matching on pipeline class name. + + All the modules the pipeline contains will be used to initialize the new pipeline without reallocating + additional memoery. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + Parameters: + pipeline (`DiffusionPipeline`): + an instantiated `DiffusionPipeline` object + + Examples: + + ```py + >>> from diffusers import AutoPipelineForTextToImage, AutoPipelineForImageToImage + + >>> pipe_t2i = AutoPipelineForText2Image.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", requires_safety_checker=False + ... ) + + >>> pipe_i2i = AutoPipelineForImageToImage.from_pipe(pipe_t2i) + ``` + """ + original_config = dict(pipeline.config) original_cls_name = pipeline.__class__.__name__ @@ -271,6 +643,25 @@ def from_pipe(cls, pipeline, **kwargs): class AutoPipelineForInpainting(ConfigMixin): + r""" + + AutoPipeline for inpainting generation. + + [`AutoPipelineForInpainting`] is a generic pipeline class that will be instantiated as one of the inpainting + pipeline class in diffusers. + + The pipeline type (for example [`IFInpaintingPipeline`]) is automatically selected when created with the + AutoPipelineForInpainting.from_pretrained(pretrained_model_name_or_path) or + AutoPipelineForInpainting.from_pipe(pipeline) class methods . + + This class cannot be instantiated using __init__() (throws an error). + + Class attributes: + + - **config_name** (`str`) -- The configuration filename that stores the class and module names of all the + diffusion pipeline's components. + + """ config_name = "model_index.json" def __init__(self, *args, **kwargs): @@ -282,6 +673,145 @@ def __init__(self, *args, **kwargs): @classmethod def from_pretrained(cls, pretrained_model_or_path, **kwargs): + r""" + Instantiates a inpainting Pytorch diffusion pipeline from pretrained pipeline weight. + + The from_pretrained() method takes care of returning the correct pipeline class instance by: + 1. Detect the pipeline class of the pretrained_model_or_path based on the _class_name property of its + config object + 2. Find the inpainting pipeline linked to the pipeline class using pattern matching on pipeline class name. + + If a `controlnet` argument is passed, it will instantiate a StableDiffusionControlNetInpaintPipeline object. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + If you get the error message below, you need to finetune the weights for your downstream task: + + ``` + Some weights of UNet2DConditionModel were not initialized from the model checkpoint at runwayml/stable-diffusion-v1-5 and are newly initialized because the shapes did not match: + - conv_in.weight: found shape torch.Size([320, 4, 3, 3]) in the checkpoint and torch.Size([320, 9, 3, 3]) in the model instantiated + You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. + ``` + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *repo id* (for example `CompVis/ldm-text2im-large-256`) of a pretrained pipeline + hosted on the Hub. + - A path to a *directory* (for example `./my_pipeline_directory/`) containing pipeline weights + saved using + [`~DiffusionPipeline.save_pretrained`]. + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model with another dtype. If "auto" is passed, the + dtype is automatically derived from the model's weights. + custom_pipeline (`str`, *optional*): + + + + 🧪 This is an experimental feature and may change in the future. + + + + Can be either: + + - A string, the *repo id* (for example `hf-internal-testing/diffusers-dummy-pipeline`) of a custom + pipeline hosted on the Hub. The repository must contain a file called pipeline.py that defines + the custom pipeline. + - A string, the *file name* of a community pipeline hosted on GitHub under + [Community](https://github.com/huggingface/diffusers/tree/main/examples/community). Valid file + names must match the file name and not the pipeline script (`clip_guided_stable_diffusion` + instead of `clip_guided_stable_diffusion.py`). Community pipelines are always loaded from the + current main branch of GitHub. + - A path to a directory (`./my_pipeline_directory/`) containing a custom pipeline. The directory + must contain a file called `pipeline.py` that defines the custom pipeline. + + + For more information on how to load and create custom pipelines, please have a look at [Loading and + Adding Custom + Pipelines](https://huggingface.co/docs/diffusers/using-diffusers/custom_pipeline_overview) + + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + use_auth_token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + custom_revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id similar to + `revision` when loading a custom pipeline from the Hub. It can be a 🤗 Diffusers version when loading a + custom pipeline from GitHub, otherwise it defaults to `"main"` when loading from the Hub. + mirror (`str`, *optional*): + Mirror source to resolve accessibility issues if you’re downloading a model in China. We do not + guarantee the timeliness or safety of the source, and you should refer to the mirror site for more + information. + device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn’t need to be defined for each + parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the + same device. + + Set `device_map="auto"` to have 🤗 Accelerate automatically compute the most optimized `device_map`. For + more information about each option see [designing a device + map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). + max_memory (`Dict`, *optional*): + A dictionary device identifier for the maximum memory. Will default to the maximum memory available for + each GPU and the available CPU RAM if unset. + offload_folder (`str` or `os.PathLike`, *optional*): + The path to offload weights if device_map contains the value `"disk"`. + offload_state_dict (`bool`, *optional*): + If `True`, temporarily offloads the CPU state dict to the hard drive to avoid running out of CPU RAM if + the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to `True` + when there is some disk offload. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading only loading the pretrained weights and not initializing the weights. This also + tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model. + Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this + argument to `True` will raise an error. + use_safetensors (`bool`, *optional*, defaults to `None`): + If set to `None`, the safetensors weights are downloaded if they're available **and** if the + safetensors library is installed. If set to `True`, the model is forcibly loaded from safetensors + weights. If set to `False`, safetensors weights are not loaded. + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to overwrite load and saveable variables (the pipeline components of the specific pipeline + class). The overwritten components are passed directly to the pipelines `__init__` method. See example + below for more information. + variant (`str`, *optional*): + Load weights from a specified variant filename such as `"fp16"` or `"ema"`. This is ignored when + loading `from_flax`. + + + + To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with + `huggingface-cli login`. + + + + Examples: + + ```py + >>> from diffusers import AutoPipelineForTextToImage + + >>> pipeline = AutoPipelineForImageToImage.from_pretrained("runwayml/stable-diffusion-v1-5") + >>> print(pipeline.__class__) + ``` + """ if "controlnet" in kwargs: inpainting_cls = AUTO_INPAINTING_PIPELINES_MAPPING["controlnet"] @@ -293,6 +823,33 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): @classmethod def from_pipe(cls, pipeline, **kwargs): + r""" + Instantiates a inpainting Pytorch diffusion pipeline from another instantiated diffusion pipeline class. + + The from_pipe() method takes care of returning the correct pipeline class instance by find the inpainting + pipeline linked to the pipeline class using pattern matching on pipeline class name. + + All the modules the pipeline class contain will be used to initialize the new pipeline without reallocating + additional memoery. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + Parameters: + pipeline (`DiffusionPipeline`): + an instantiated `DiffusionPipeline` object + + Examples: + + ```py + >>> from diffusers import AutoPipelineForTextToImage, AutoPipelineForInpainting + + >>> pipe_t2i = AutoPipelineForText2Image.from_pretrained( + ... "DeepFloyd/IF-I-XL-v1.0", requires_safety_checker=False + ... ) + + >>> pipe_inpaint = AutoPipelineForInpainting.from_pipe(pipe_t2i) + ``` + """ original_config = dict(pipeline.config) original_cls_name = pipeline.__class__.__name__ From 5d5296d94dfdaa7ad5e6f1d100c435763dfc247c Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Sat, 22 Jul 2023 20:13:08 +0000 Subject: [PATCH 17/23] kandinsky inpaint pipeline does not work with decoder checkpoint --- src/diffusers/pipelines/auto_pipeline.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index 48b370459865..5984162584fe 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -65,8 +65,8 @@ ("stable-diffusion", StableDiffusionInpaintPipeline), ("stable-diffusion-xl", StableDiffusionXLInpaintPipeline), ("if", IFInpaintingPipeline), - ("kandinsky", KandinskyInpaintPipeline), - ("kdnsinskyv22", KandinskyV22InpaintPipeline), + ("kandinsky-inpaint", KandinskyInpaintPipeline), + ("kdnsinskyv22-inpaint", KandinskyV22InpaintPipeline), ("controlnet", StableDiffusionControlNetInpaintPipeline), ] ) @@ -91,9 +91,7 @@ def get_model(pipeline_class_name): task_class = mapping.get(model_name, None) if task_class is not None: return task_class - raise ValueError( - f"AutoPipeline can't find a pipeline linked to {pipeline_class_name} for {model_name} in {mapping}" - ) + raise ValueError(f"AutoPipeline can't find a pipeline linked to {pipeline_class_name} for {model_name}") def _get_signature_keys(obj): From 30a64250c90fa303e4f165f5b6fd0113cb8fcbc3 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Mon, 24 Jul 2023 01:11:53 +0000 Subject: [PATCH 18/23] update doc --- .../source/en/api/pipelines/auto_pipeline.mdx | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/docs/source/en/api/pipelines/auto_pipeline.mdx b/docs/source/en/api/pipelines/auto_pipeline.mdx index a74a59de6e1c..4ae2b86ac269 100644 --- a/docs/source/en/api/pipelines/auto_pipeline.mdx +++ b/docs/source/en/api/pipelines/auto_pipeline.mdx @@ -12,9 +12,36 @@ specific language governing permissions and limitations under the License. # AutoPipeline -In many cases, one checkpoint can be used for multiple tasks. You may be able the same checkpoint for Text-to-Image, Image-to-Image and Inpainting, as long as you know which pipelines are linked to your checkpoint. +In many cases, one checkpoint can be used for multiple tasks. For example, you may be able to use the same checkpoint for Text-to-Image, Image-to-Image, and Inpainting. However, you'll need to know the pipeline class names linked to your checkpoint. + +AutoPipeline is designed to make it easy for you to use multiple pipelines in your workflow. We currently provide 3 AutoPipeline classes to perform three different tasks, i.e. [`AutoPipelineForText2Image`], [`AutoPipelineForImage2Image`], and [`AutoPipelineForInpainting`]. You'll need to choose the AutoPipeline class based on the task you want to perform and use it to automatically retrieve the relevant pipeline given the name/path to the pre-trained weights. + +For example, to perform Image-to-Image with the SD1.5 checkpoint, you can do + +```python +from diffusers import PipelineForImageToImage + +pipe_i2i = PipelineForImageoImage.from_pretrained("runwayml/stable-diffusion-v1-5") +``` + +It will also help you switch between tasks seamlessly using the same checkpoint without reallocating additional memory. For example, to re-use the Image-to-Image pipeline we just created for inpainting, you can do + +```python +from diffusers import PipelineForInpainting + +pipe_inpaint = AutoPipelineForInpainting.from_pipe(pipe_i2i) +``` +All the components will be transferred to the inpainting pipeline with zero cost. + + +Currently AutoPipeline support the Text-to-Image, Image-to-Image, and Inpainting tasks for below diffusion models: +- [stable Diffusion](./stable_diffusion) +- [Stable Diffusion Controlnet](./api/pipelines/controlnet) +- [Stable Diffusion XL](./stable_diffusion/stable_diffusion_xl) +- [IF](./if) +- [Kandinsky](./kandinsky)(./kandinsky)(./kandinsky)(./kandinsky)(./kandinsky) +- [Kandinsky 2.2]()(./kandinsky) -AutoPipeline is designed to help you switch between different tasks seamlessly using the same checkpoint. It automatically retrieve the relevant pipeline for the task your want to perform, given the name/path to the pretrained weights. ## AutoPipelineForText2Image From eca40b678e91908ced7be95ca36d1f1388c631cd Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Sun, 23 Jul 2023 15:21:28 -1000 Subject: [PATCH 19/23] Apply suggestions from code review Co-authored-by: Sayak Paul --- src/diffusers/pipelines/auto_pipeline.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index 5984162584fe..47c2db03d505 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -142,7 +142,7 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): 2. Find the text-to-image pipeline linked to the pipeline class using pattern matching on pipeline class name. - If a `controlnet` argument is passed, it will instantiate a StableDiffusionControlNetPipeline object. + If a `controlnet` argument is passed, it will instantiate a [`StableDiffusionControlNetPipeline`] object. The pipeline is set in evaluation mode (`model.eval()`) by default. @@ -287,7 +287,7 @@ def from_pipe(cls, pipeline, **kwargs): r""" Instantiates a text-to-image Pytorch diffusion pipeline from another instantiated diffusion pipeline class. - The from_pipe() method takes care of returning the correct pipeline class instance by find the text-to-image + The from_pipe() method takes care of returning the correct pipeline class instance by finding the text-to-image pipeline linked to the pipeline class using pattern matching on pipeline class name. All the modules the pipeline contains will be used to initialize the new pipeline without reallocating @@ -376,11 +376,11 @@ class AutoPipelineForImage2Image(ConfigMixin): AutoPipeline for image-to-image generation. [`AutoPipelineForImage2Image`] is a generic pipeline class that will be instantiated as one of the image-to-image - pipeline class in diffusers. + pipeline classes in diffusers. The pipeline type (for example [`StableDiffusionImg2ImgPipeline`]) is automatically selected when created with the - AutoPipelineForImage2Image.from_pretrained(pretrained_model_name_or_path) or - AutoPipelineForImage2Image.from_pipe(pipeline) class methods . + `AutoPipelineForImage2Image.from_pretrained(pretrained_model_name_or_path)` or + `AutoPipelineForImage2Image.from_pipe(pipeline)` class methods. This class cannot be instantiated using __init__() (throws an error). @@ -555,7 +555,7 @@ def from_pipe(cls, pipeline, **kwargs): r""" Instantiates a image-to-image Pytorch diffusion pipeline from another instantiated diffusion pipeline class. - The from_pipe() method takes care of returning the correct pipeline class instance by find the image-to-image + The from_pipe() method takes care of returning the correct pipeline class instance by finding the image-to-image pipeline linked to the pipeline class using pattern matching on pipeline class name. All the modules the pipeline contains will be used to initialize the new pipeline without reallocating @@ -824,7 +824,7 @@ def from_pipe(cls, pipeline, **kwargs): r""" Instantiates a inpainting Pytorch diffusion pipeline from another instantiated diffusion pipeline class. - The from_pipe() method takes care of returning the correct pipeline class instance by find the inpainting + The from_pipe() method takes care of returning the correct pipeline class instance by finding the inpainting pipeline linked to the pipeline class using pattern matching on pipeline class name. All the modules the pipeline class contain will be used to initialize the new pipeline without reallocating From 9a877d15435deb203e956bc1739e3efaeceadcb5 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Mon, 24 Jul 2023 02:36:13 +0000 Subject: [PATCH 20/23] style --- src/diffusers/pipelines/auto_pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index 47c2db03d505..6b9865c0e599 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -555,8 +555,8 @@ def from_pipe(cls, pipeline, **kwargs): r""" Instantiates a image-to-image Pytorch diffusion pipeline from another instantiated diffusion pipeline class. - The from_pipe() method takes care of returning the correct pipeline class instance by finding the image-to-image - pipeline linked to the pipeline class using pattern matching on pipeline class name. + The from_pipe() method takes care of returning the correct pipeline class instance by finding the + image-to-image pipeline linked to the pipeline class using pattern matching on pipeline class name. All the modules the pipeline contains will be used to initialize the new pipeline without reallocating additional memoery. From 2f506db4b8164b653b7c42a945a8fcf58a63c3b9 Mon Sep 17 00:00:00 2001 From: YiYi Xu Date: Mon, 24 Jul 2023 17:54:41 -1000 Subject: [PATCH 21/23] Apply suggestions from code review Co-authored-by: Patrick von Platen --- src/diffusers/pipelines/auto_pipeline.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index 6b9865c0e599..5e89f1a57281 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -1,6 +1,5 @@ # coding=utf-8 # Copyright 2023 The HuggingFace Inc. team. -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -44,8 +43,8 @@ ("stable-diffusion-xl", StableDiffusionXLPipeline), ("if", IFPipeline), ("kandinsky", KandinskyPipeline), - ("kdnsinskyv22", KandinskyV22Pipeline), - ("controlnet", StableDiffusionControlNetPipeline), + ("kandinsky22", KandinskyV22Pipeline), + ("stable-diffusion-controlnet", StableDiffusionControlNetPipeline), ] ) @@ -55,17 +54,17 @@ ("stable-diffusion-xl", StableDiffusionXLImg2ImgPipeline), ("if", IFImg2ImgPipeline), ("kandinsky", KandinskyImg2ImgPipeline), - ("kdnsinskyv22", KandinskyV22Img2ImgPipeline), + ("kandinsky22", KandinskyV22Img2ImgPipeline), ("controlnet", StableDiffusionControlNetImg2ImgPipeline), ] ) -AUTO_INPAINTING_PIPELINES_MAPPING = OrderedDict( +AUTO_INPAINT_PIPELINES_MAPPING = OrderedDict( [ ("stable-diffusion", StableDiffusionInpaintPipeline), ("stable-diffusion-xl", StableDiffusionXLInpaintPipeline), ("if", IFInpaintingPipeline), - ("kandinsky-inpaint", KandinskyInpaintPipeline), + ("kandinsky", KandinskyInpaintPipeline), ("kdnsinskyv22-inpaint", KandinskyV22InpaintPipeline), ("controlnet", StableDiffusionControlNetInpaintPipeline), ] @@ -541,12 +540,14 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): >>> print(pipeline.__class__) ``` """ + orig_class_name = config["_class_name"] + if "controlnet" in kwargs: - image_2_image_cls = AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["controlnet"] - - else: - config = cls.load_config(pretrained_model_or_path) - image_2_image_cls = _get_task_class(AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, config["_class_name"]) + orig_class_name = config["_class_name"].replace("Pipeline", "ControlNetPipeline") + + + config = cls.load_config(pretrained_model_or_path) + image_2_image_cls = _get_task_class(AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, config["_class_name"]) return image_2_image_cls.from_pretrained(pretrained_model_or_path, **kwargs) From d8121ccdb75c143284ebd17a2b941d132ae7a378 Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 25 Jul 2023 06:16:28 +0000 Subject: [PATCH 22/23] fix --- src/diffusers/pipelines/auto_pipeline.py | 117 ++++------------------- tests/pipelines/test_pipelines_auto.py | 105 +++++++++++++------- 2 files changed, 92 insertions(+), 130 deletions(-) diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index 5e89f1a57281..447d87fd8572 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -21,6 +21,7 @@ StableDiffusionControlNetImg2ImgPipeline, StableDiffusionControlNetInpaintPipeline, StableDiffusionControlNetPipeline, + StableDiffusionXLControlNetPipeline, ) from .deepfloyd_if import IFImg2ImgPipeline, IFInpaintingPipeline, IFPipeline from .kandinsky import KandinskyImg2ImgPipeline, KandinskyInpaintPipeline, KandinskyPipeline @@ -45,6 +46,7 @@ ("kandinsky", KandinskyPipeline), ("kandinsky22", KandinskyV22Pipeline), ("stable-diffusion-controlnet", StableDiffusionControlNetPipeline), + ("stable-diffusion-xl-controlnet", StableDiffusionXLControlNetPipeline), ] ) @@ -55,7 +57,7 @@ ("if", IFImg2ImgPipeline), ("kandinsky", KandinskyImg2ImgPipeline), ("kandinsky22", KandinskyV22Img2ImgPipeline), - ("controlnet", StableDiffusionControlNetImg2ImgPipeline), + ("stable-diffusion-controlnet", StableDiffusionControlNetImg2ImgPipeline), ] ) @@ -65,15 +67,15 @@ ("stable-diffusion-xl", StableDiffusionXLInpaintPipeline), ("if", IFInpaintingPipeline), ("kandinsky", KandinskyInpaintPipeline), - ("kdnsinskyv22-inpaint", KandinskyV22InpaintPipeline), - ("controlnet", StableDiffusionControlNetInpaintPipeline), + ("kdnsinsky22", KandinskyV22InpaintPipeline), + ("stable-diffusion-controlnet", StableDiffusionControlNetInpaintPipeline), ] ) SUPPORTED_TASKS_MAPPINGS = [ AUTO_TEXT2IMAGE_PIPELINES_MAPPING, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, - AUTO_INPAINTING_PIPELINES_MAPPING, + AUTO_INPAINT_PIPELINES_MAPPING, ] @@ -165,32 +167,6 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): torch_dtype (`str` or `torch.dtype`, *optional*): Override the default `torch.dtype` and load the model with another dtype. If "auto" is passed, the dtype is automatically derived from the model's weights. - custom_pipeline (`str`, *optional*): - - - - 🧪 This is an experimental feature and may change in the future. - - - - Can be either: - - - A string, the *repo id* (for example `hf-internal-testing/diffusers-dummy-pipeline`) of a custom - pipeline hosted on the Hub. The repository must contain a file called pipeline.py that defines - the custom pipeline. - - A string, the *file name* of a community pipeline hosted on GitHub under - [Community](https://github.com/huggingface/diffusers/tree/main/examples/community). Valid file - names must match the file name and not the pipeline script (`clip_guided_stable_diffusion` - instead of `clip_guided_stable_diffusion.py`). Community pipelines are always loaded from the - current main branch of GitHub. - - A path to a directory (`./my_pipeline_directory/`) containing a custom pipeline. The directory - must contain a file called `pipeline.py` that defines the custom pipeline. - - - For more information on how to load and create custom pipelines, please have a look at [Loading and - Adding Custom - Pipelines](https://huggingface.co/docs/diffusers/using-diffusers/custom_pipeline_overview) - force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. @@ -272,12 +248,13 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): >>> print(pipeline.__class__) ``` """ + config = cls.load_config(pretrained_model_or_path) + orig_class_name = config["_class_name"] + if "controlnet" in kwargs: - text_2_image_cls = AUTO_TEXT2IMAGE_PIPELINES_MAPPING["controlnet"] + orig_class_name = config["_class_name"].replace("Pipeline", "ControlNetPipeline") - else: - config = cls.load_config(pretrained_model_or_path) - text_2_image_cls = _get_task_class(AUTO_TEXT2IMAGE_PIPELINES_MAPPING, config["_class_name"]) + text_2_image_cls = _get_task_class(AUTO_TEXT2IMAGE_PIPELINES_MAPPING, orig_class_name) return text_2_image_cls.from_pretrained(pretrained_model_or_path, **kwargs) @@ -433,32 +410,6 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): torch_dtype (`str` or `torch.dtype`, *optional*): Override the default `torch.dtype` and load the model with another dtype. If "auto" is passed, the dtype is automatically derived from the model's weights. - custom_pipeline (`str`, *optional*): - - - - 🧪 This is an experimental feature and may change in the future. - - - - Can be either: - - - A string, the *repo id* (for example `hf-internal-testing/diffusers-dummy-pipeline`) of a custom - pipeline hosted on the Hub. The repository must contain a file called pipeline.py that defines - the custom pipeline. - - A string, the *file name* of a community pipeline hosted on GitHub under - [Community](https://github.com/huggingface/diffusers/tree/main/examples/community). Valid file - names must match the file name and not the pipeline script (`clip_guided_stable_diffusion` - instead of `clip_guided_stable_diffusion.py`). Community pipelines are always loaded from the - current main branch of GitHub. - - A path to a directory (`./my_pipeline_directory/`) containing a custom pipeline. The directory - must contain a file called `pipeline.py` that defines the custom pipeline. - - - For more information on how to load and create custom pipelines, please have a look at [Loading and - Adding Custom - Pipelines](https://huggingface.co/docs/diffusers/using-diffusers/custom_pipeline_overview) - force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. @@ -540,14 +491,13 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): >>> print(pipeline.__class__) ``` """ + config = cls.load_config(pretrained_model_or_path) orig_class_name = config["_class_name"] - + if "controlnet" in kwargs: orig_class_name = config["_class_name"].replace("Pipeline", "ControlNetPipeline") - - - config = cls.load_config(pretrained_model_or_path) - image_2_image_cls = _get_task_class(AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, config["_class_name"]) + + image_2_image_cls = _get_task_class(AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, orig_class_name) return image_2_image_cls.from_pretrained(pretrained_model_or_path, **kwargs) @@ -704,32 +654,6 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): torch_dtype (`str` or `torch.dtype`, *optional*): Override the default `torch.dtype` and load the model with another dtype. If "auto" is passed, the dtype is automatically derived from the model's weights. - custom_pipeline (`str`, *optional*): - - - - 🧪 This is an experimental feature and may change in the future. - - - - Can be either: - - - A string, the *repo id* (for example `hf-internal-testing/diffusers-dummy-pipeline`) of a custom - pipeline hosted on the Hub. The repository must contain a file called pipeline.py that defines - the custom pipeline. - - A string, the *file name* of a community pipeline hosted on GitHub under - [Community](https://github.com/huggingface/diffusers/tree/main/examples/community). Valid file - names must match the file name and not the pipeline script (`clip_guided_stable_diffusion` - instead of `clip_guided_stable_diffusion.py`). Community pipelines are always loaded from the - current main branch of GitHub. - - A path to a directory (`./my_pipeline_directory/`) containing a custom pipeline. The directory - must contain a file called `pipeline.py` that defines the custom pipeline. - - - For more information on how to load and create custom pipelines, please have a look at [Loading and - Adding Custom - Pipelines](https://huggingface.co/docs/diffusers/using-diffusers/custom_pipeline_overview) - force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download of the model weights and configuration files, overriding the cached versions if they exist. @@ -811,12 +735,13 @@ def from_pretrained(cls, pretrained_model_or_path, **kwargs): >>> print(pipeline.__class__) ``` """ + config = cls.load_config(pretrained_model_or_path) + orig_class_name = config["_class_name"] + if "controlnet" in kwargs: - inpainting_cls = AUTO_INPAINTING_PIPELINES_MAPPING["controlnet"] + orig_class_name = config["_class_name"].replace("Pipeline", "ControlNetPipeline") - else: - config = cls.load_config(pretrained_model_or_path) - inpainting_cls = _get_task_class(AUTO_INPAINTING_PIPELINES_MAPPING, config["_class_name"]) + inpainting_cls = _get_task_class(AUTO_INPAINT_PIPELINES_MAPPING, orig_class_name) return inpainting_cls.from_pretrained(pretrained_model_or_path, **kwargs) @@ -853,7 +778,7 @@ def from_pipe(cls, pipeline, **kwargs): original_cls_name = pipeline.__class__.__name__ # derive the pipeline class to instantiate - inpainting_cls = _get_task_class(AUTO_INPAINTING_PIPELINES_MAPPING, original_cls_name) + inpainting_cls = _get_task_class(AUTO_INPAINT_PIPELINES_MAPPING, original_cls_name) # define expected module and optional kwargs given the pipeline signature expected_modules, optional_kwargs = _get_signature_keys(inpainting_cls) diff --git a/tests/pipelines/test_pipelines_auto.py b/tests/pipelines/test_pipelines_auto.py index 2e6c4d7896c7..595a7a5f25ff 100644 --- a/tests/pipelines/test_pipelines_auto.py +++ b/tests/pipelines/test_pipelines_auto.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import gc import unittest from collections import OrderedDict @@ -26,7 +27,7 @@ ) from diffusers.pipelines.auto_pipeline import ( AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, - AUTO_INPAINTING_PIPELINES_MAPPING, + AUTO_INPAINT_PIPELINES_MAPPING, AUTO_TEXT2IMAGE_PIPELINES_MAPPING, ) from diffusers.utils import slow @@ -37,7 +38,7 @@ ("stable-diffusion", "runwayml/stable-diffusion-v1-5"), ("if", "DeepFloyd/IF-I-XL-v1.0"), ("kandinsky", "kandinsky-community/kandinsky-2-1"), - ("kdnsinskyv22", "kandinsky-community/kandinsky-2-2-decoder"), + ("kandinsky22", "kandinsky-community/kandinsky-2-2-decoder"), ] ) @@ -85,47 +86,83 @@ def test_from_pipe_consistent_sdxl(self): class AutoPipelineIntegrationTest(unittest.TestCase): def test_pipe_auto(self): for model_name, model_repo in PRETRAINED_MODEL_REPO_MAPPING.items(): - # test from_pretrained - pipe_txt2img = AutoPipelineForText2Image.from_pretrained(model_repo) + # test txt2img + pipe_txt2img = AutoPipelineForText2Image.from_pretrained( + model_repo, variant="fp16", torch_dtype=torch.float16 + ) self.assertIsInstance(pipe_txt2img, AUTO_TEXT2IMAGE_PIPELINES_MAPPING[model_name]) - pipe_img2img = AutoPipelineForImage2Image.from_pretrained(model_repo) + pipe_to = AutoPipelineForText2Image.from_pipe(pipe_txt2img) + self.assertIsInstance(pipe_to, AUTO_TEXT2IMAGE_PIPELINES_MAPPING[model_name]) + + pipe_to = AutoPipelineForImage2Image.from_pipe(pipe_txt2img) + self.assertIsInstance(pipe_to, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING[model_name]) + + if "kandinsky" not in model_name: + pipe_to = AutoPipelineForInpainting.from_pipe(pipe_txt2img) + self.assertIsInstance(pipe_to, AUTO_INPAINT_PIPELINES_MAPPING[model_name]) + + del pipe_txt2img, pipe_to + gc.collect() + + # test img2img + + pipe_img2img = AutoPipelineForImage2Image.from_pretrained( + model_repo, variant="fp16", torch_dtype=torch.float16 + ) self.assertIsInstance(pipe_img2img, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING[model_name]) - pipe_inpaint = AutoPipelineForInpainting.from_pretrained(model_repo) - self.assertIsInstance(pipe_inpaint, AUTO_INPAINTING_PIPELINES_MAPPING[model_name]) + pipe_to = AutoPipelineForText2Image.from_pipe(pipe_img2img) + self.assertIsInstance(pipe_to, AUTO_TEXT2IMAGE_PIPELINES_MAPPING[model_name]) + + pipe_to = AutoPipelineForImage2Image.from_pipe(pipe_img2img) + self.assertIsInstance(pipe_to, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING[model_name]) - # test from_pipe - for pipe_from in [pipe_txt2img, pipe_img2img, pipe_inpaint]: - pipe_to = AutoPipelineForText2Image.from_pipe(pipe_from) + if "kandinsky" not in model_name: + pipe_to = AutoPipelineForInpainting.from_pipe(pipe_img2img) + self.assertIsInstance(pipe_to, AUTO_INPAINT_PIPELINES_MAPPING[model_name]) + + del pipe_img2img, pipe_to + gc.collect() + + # test inpaint + + if "kandinsky" not in model_name: + pipe_inpaint = AutoPipelineForInpainting.from_pretrained( + model_repo, variant="fp16", torch_dtype=torch.float16 + ) + self.assertIsInstance(pipe_inpaint, AUTO_INPAINT_PIPELINES_MAPPING[model_name]) + + pipe_to = AutoPipelineForText2Image.from_pipe(pipe_inpaint) self.assertIsInstance(pipe_to, AUTO_TEXT2IMAGE_PIPELINES_MAPPING[model_name]) - pipe_to = AutoPipelineForImage2Image.from_pipe(pipe_from) + pipe_to = AutoPipelineForImage2Image.from_pipe(pipe_inpaint) self.assertIsInstance(pipe_to, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING[model_name]) - pipe_to = AutoPipelineForInpainting.from_pipe(pipe_from) - self.assertIsInstance(pipe_to, AUTO_INPAINTING_PIPELINES_MAPPING[model_name]) + pipe_to = AutoPipelineForInpainting.from_pipe(pipe_inpaint) + self.assertIsInstance(pipe_to, AUTO_INPAINT_PIPELINES_MAPPING[model_name]) + + del pipe_inpaint, pipe_to + gc.collect() def test_from_pipe_consistent(self): for model_name, model_repo in PRETRAINED_MODEL_REPO_MAPPING.items(): - # test from_pretrained - pipe_txt2img = AutoPipelineForText2Image.from_pretrained(model_repo) - pipe_txt2img_config = dict(pipe_txt2img.config) - pipe_img2img = AutoPipelineForImage2Image.from_pretrained(model_repo) - pipe_img2img_config = dict(pipe_txt2img.config) - pipe_inpaint = AutoPipelineForInpainting.from_pretrained(model_repo) - pipe_inpaint_config = dict(pipe_txt2img.config) + if model_name in ["kandinsky", "kandinsky22"]: + auto_pipes = [AutoPipelineForText2Image, AutoPipelineForImage2Image] + else: + auto_pipes = [AutoPipelineForText2Image, AutoPipelineForImage2Image, AutoPipelineForInpainting] - # test from_pipe - for pipe_from in [pipe_txt2img, pipe_img2img, pipe_inpaint]: - pipe_to = AutoPipelineForText2Image.from_pipe(pipe_from) - self.assertEqual(dict(pipe_to.config), pipe_txt2img_config) + # test from_pretrained + for pipe_from_class in auto_pipes: + pipe_from = pipe_from_class.from_pretrained(model_repo, variant="fp16", torch_dtype=torch.float16) + pipe_from_config = dict(pipe_from.config) - pipe_to = AutoPipelineForImage2Image.from_pipe(pipe_from) - self.assertEqual(dict(pipe_to.config), pipe_img2img_config) + for pipe_to_class in auto_pipes: + pipe_to = pipe_to_class.from_pipe(pipe_from) + self.assertEqual(dict(pipe_to.config), pipe_from_config) - pipe_to = AutoPipelineForInpainting.from_pipe(pipe_from) - self.assertEqual(dict(pipe_to.config), pipe_inpaint_config) + del pipe_from, pipe_to + gc.collect() def test_controlnet(self): # test from_pretrained @@ -137,28 +174,28 @@ def test_controlnet(self): pipe_txt2img = AutoPipelineForText2Image.from_pretrained( model_repo, controlnet=controlnet, torch_dtype=torch.float16 ) - self.assertIsInstance(pipe_txt2img, AUTO_TEXT2IMAGE_PIPELINES_MAPPING["controlnet"]) + self.assertIsInstance(pipe_txt2img, AUTO_TEXT2IMAGE_PIPELINES_MAPPING["stable-diffusion-controlnet"]) pipe_img2img = AutoPipelineForImage2Image.from_pretrained( model_repo, controlnet=controlnet, torch_dtype=torch.float16 ) - self.assertIsInstance(pipe_img2img, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["controlnet"]) + self.assertIsInstance(pipe_img2img, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["stable-diffusion-controlnet"]) pipe_inpaint = AutoPipelineForInpainting.from_pretrained( model_repo, controlnet=controlnet, torch_dtype=torch.float16 ) - self.assertIsInstance(pipe_inpaint, AUTO_INPAINTING_PIPELINES_MAPPING["controlnet"]) + self.assertIsInstance(pipe_inpaint, AUTO_INPAINT_PIPELINES_MAPPING["stable-diffusion-controlnet"]) # test from_pipe for pipe_from in [pipe_txt2img, pipe_img2img, pipe_inpaint]: pipe_to = AutoPipelineForText2Image.from_pipe(pipe_from) - self.assertIsInstance(pipe_to, AUTO_TEXT2IMAGE_PIPELINES_MAPPING["controlnet"]) + self.assertIsInstance(pipe_to, AUTO_TEXT2IMAGE_PIPELINES_MAPPING["stable-diffusion-controlnet"]) self.assertEqual(dict(pipe_to.config), dict(pipe_txt2img.config)) pipe_to = AutoPipelineForImage2Image.from_pipe(pipe_from) - self.assertIsInstance(pipe_to, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["controlnet"]) + self.assertIsInstance(pipe_to, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["stable-diffusion-controlnet"]) self.assertEqual(dict(pipe_to.config), dict(pipe_img2img.config)) pipe_to = AutoPipelineForInpainting.from_pipe(pipe_from) - self.assertIsInstance(pipe_to, AUTO_INPAINTING_PIPELINES_MAPPING["controlnet"]) + self.assertIsInstance(pipe_to, AUTO_INPAINT_PIPELINES_MAPPING["stable-diffusion-controlnet"]) self.assertEqual(dict(pipe_to.config), dict(pipe_inpaint.config)) From ec2cf57b1bf29f414f6b74ed26696bc2863ae026 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Tue, 25 Jul 2023 12:44:08 +0200 Subject: [PATCH 23/23] Apply suggestions from code review --- src/diffusers/pipelines/auto_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/auto_pipeline.py b/src/diffusers/pipelines/auto_pipeline.py index 447d87fd8572..c827231ada7d 100644 --- a/src/diffusers/pipelines/auto_pipeline.py +++ b/src/diffusers/pipelines/auto_pipeline.py @@ -67,7 +67,7 @@ ("stable-diffusion-xl", StableDiffusionXLInpaintPipeline), ("if", IFInpaintingPipeline), ("kandinsky", KandinskyInpaintPipeline), - ("kdnsinsky22", KandinskyV22InpaintPipeline), + ("kandinsky22", KandinskyV22InpaintPipeline), ("stable-diffusion-controlnet", StableDiffusionControlNetInpaintPipeline), ] )