diff --git a/.gitignore b/.gitignore index 9b33e071646..4b1aa81a0bd 100644 --- a/.gitignore +++ b/.gitignore @@ -233,5 +233,3 @@ installer/install.sh installer/update.bat installer/update.sh -# no longer stored in source directory -models diff --git a/ldm/invoke/generator/diffusers_pipeline.py b/ldm/invoke/generator/diffusers_pipeline.py index e1d860f309e..022bcf600b5 100644 --- a/ldm/invoke/generator/diffusers_pipeline.py +++ b/ldm/invoke/generator/diffusers_pipeline.py @@ -400,8 +400,15 @@ def device(self) -> torch.device: @property def _submodels(self) -> Sequence[torch.nn.Module]: module_names, _, _ = self.extract_init_dict(dict(self.config)) - values = [getattr(self, name) for name in module_names.keys()] - return [m for m in values if isinstance(m, torch.nn.Module)] + submodels = [] + for name in module_names.keys(): + if hasattr(self, name): + value = getattr(self, name) + else: + value = getattr(self.config, name) + if isinstance(value, torch.nn.Module): + submodels.append(value) + return submodels def image_from_embeddings(self, latents: torch.Tensor, num_inference_steps: int, conditioning_data: ConditioningData, @@ -472,7 +479,7 @@ def generate_latents_from_embeddings(self, latents: torch.Tensor, timesteps, step_count=len(self.scheduler.timesteps) ): - yield PipelineIntermediateState(run_id=run_id, step=-1, timestep=self.scheduler.num_train_timesteps, + yield PipelineIntermediateState(run_id=run_id, step=-1, timestep=self.scheduler.config.num_train_timesteps, latents=latents) batch_size = latents.shape[0] @@ -756,7 +763,7 @@ def _tokenize(self, prompt: Union[str, List[str]]): @property def channels(self) -> int: """Compatible with DiffusionWrapper""" - return self.unet.in_channels + return self.unet.config.in_channels def decode_latents(self, latents): # Explicit call to get the vae loaded, since `decode` isn't the forward method. diff --git a/ldm/models/diffusion/cross_attention_control.py b/ldm/models/diffusion/cross_attention_control.py index d7125e9a3ea..8c0e495df5b 100644 --- a/ldm/models/diffusion/cross_attention_control.py +++ b/ldm/models/diffusion/cross_attention_control.py @@ -14,7 +14,6 @@ from compel.cross_attention_control import Arguments from diffusers.models.unet_2d_condition import UNet2DConditionModel -from diffusers.models.cross_attention import AttnProcessor from ldm.invoke.devices import torch_dtype @@ -163,7 +162,7 @@ def offload_saved_attention_slices_to_cpu(self): class InvokeAICrossAttentionMixin: """ - Enable InvokeAI-flavoured CrossAttention calculation, which does aggressive low-memory slicing and calls + Enable InvokeAI-flavoured Attention calculation, which does aggressive low-memory slicing and calls through both to an attention_slice_wrangler and a slicing_strategy_getter for custom attention map wrangling and dymamic slicing strategy selection. """ @@ -178,7 +177,7 @@ def set_attention_slice_wrangler(self, wrangler: Optional[Callable[[nn.Module, t Set custom attention calculator to be called when attention is calculated :param wrangler: Callback, with args (module, suggested_attention_slice, dim, offset, slice_size), which returns either the suggested_attention_slice or an adjusted equivalent. - `module` is the current CrossAttention module for which the callback is being invoked. + `module` is the current Attention module for which the callback is being invoked. `suggested_attention_slice` is the default-calculated attention slice `dim` is -1 if the attenion map has not been sliced, or 0 or 1 for dimension-0 or dimension-1 slicing. If `dim` is >= 0, `offset` and `slice_size` specify the slice start and length. @@ -326,7 +325,7 @@ def setup_cross_attention_control_attention_processors(unet: UNet2DConditionMode def get_cross_attention_modules(model, which: CrossAttentionType) -> list[tuple[str, InvokeAICrossAttentionMixin]]: - from ldm.modules.attention import CrossAttention # avoid circular import + from ldm.modules.attention import CrossAttention # avoid circular import # TODO: rename as in diffusers? cross_attention_class: type = InvokeAIDiffusersCrossAttention if isinstance(model,UNet2DConditionModel) else CrossAttention which_attn = "attn1" if which is CrossAttentionType.SELF else "attn2" attention_module_tuples = [(name,module) for name, module in model.named_modules() if @@ -432,7 +431,7 @@ def get_mem_free_total(device): -class InvokeAIDiffusersCrossAttention(diffusers.models.attention.CrossAttention, InvokeAICrossAttentionMixin): +class InvokeAIDiffusersCrossAttention(diffusers.models.attention.Attention, InvokeAICrossAttentionMixin): def __init__(self, **kwargs): super().__init__(**kwargs) @@ -457,8 +456,8 @@ def _attention(self, query, key, value, attention_mask=None): """ # base implementation -class CrossAttnProcessor: - def __call__(self, attn: CrossAttention, hidden_states, encoder_hidden_states=None, attention_mask=None): +class AttnProcessor: + def __call__(self, attn: Attention, hidden_states, encoder_hidden_states=None, attention_mask=None): batch_size, sequence_length, _ = hidden_states.shape attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length) @@ -487,7 +486,7 @@ def __call__(self, attn: CrossAttention, hidden_states, encoder_hidden_states=No import torch -from diffusers.models.cross_attention import CrossAttention, CrossAttnProcessor, SlicedAttnProcessor +from diffusers.models.attention_processor import Attention, AttnProcessor, SlicedAttnProcessor @dataclass @@ -532,7 +531,7 @@ class SlicedSwapCrossAttnProcesser(SlicedAttnProcessor): # TODO: dynamically pick slice size based on memory conditions - def __call__(self, attn: CrossAttention, hidden_states, encoder_hidden_states=None, attention_mask=None, + def __call__(self, attn: Attention, hidden_states, encoder_hidden_states=None, attention_mask=None, # kwargs swap_cross_attn_context: SwapCrossAttnContext=None): diff --git a/ldm/models/diffusion/shared_invokeai_diffusion.py b/ldm/models/diffusion/shared_invokeai_diffusion.py index 0926b948cc3..61a78147160 100644 --- a/ldm/models/diffusion/shared_invokeai_diffusion.py +++ b/ldm/models/diffusion/shared_invokeai_diffusion.py @@ -5,7 +5,6 @@ import numpy as np import torch - from diffusers import UNet2DConditionModel from typing_extensions import TypeAlias diff --git a/pyproject.toml b/pyproject.toml index 898ca078679..f0b0cb6b6c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "clip_anytorch", "compel~=1.1.0", "datasets", - "diffusers[torch]==0.14", + "diffusers[torch]~=0.15.0", "dnspython==2.2.1", "einops", "eventlet",