From 5eef2ecac5b681e868abd355930dfd2388b47418 Mon Sep 17 00:00:00 2001 From: bironsecret <44505718+bironsecret@users.noreply.github.com> Date: Sun, 4 Sep 2022 10:33:39 +0200 Subject: [PATCH 1/2] optimized attention --- ldm/modules/attention.py | 43 +++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/ldm/modules/attention.py b/ldm/modules/attention.py index f4eff39cc..0698c2f70 100644 --- a/ldm/modules/attention.py +++ b/ldm/modules/attention.py @@ -1,9 +1,10 @@ -from inspect import isfunction import math +from inspect import isfunction + import torch import torch.nn.functional as F -from torch import nn, einsum from einops import rearrange, repeat +from torch import nn, einsum from ldm.modules.diffusionmodules.util import checkpoint @@ -13,7 +14,7 @@ def exists(val): def uniq(arr): - return{el: True for el in arr}.keys() + return {el: True for el in arr}.keys() def default(val, d): @@ -82,14 +83,14 @@ def __init__(self, dim, heads=4, dim_head=32): super().__init__() self.heads = heads hidden_dim = dim_head * heads - self.to_qkv = nn.Conv2d(dim, hidden_dim * 3, 1, bias = False) + self.to_qkv = nn.Conv2d(dim, hidden_dim * 3, 1, bias=False) self.to_out = nn.Conv2d(hidden_dim, dim, 1) def forward(self, x): b, c, h, w = x.shape qkv = self.to_qkv(x) - q, k, v = rearrange(qkv, 'b (qkv heads c) h w -> qkv b heads c (h w)', heads = self.heads, qkv=3) - k = k.softmax(dim=-1) + q, k, v = rearrange(qkv, 'b (qkv heads c) h w -> qkv b heads c (h w)', heads=self.heads, qkv=3) + k = k.softmax(dim=-1) context = torch.einsum('bhdn,bhen->bhde', k, v) out = torch.einsum('bhde,bhdn->bhen', context, q) out = rearrange(out, 'b heads c (h w) -> b (heads c) h w', heads=self.heads, h=h, w=w) @@ -131,12 +132,12 @@ def forward(self, x): v = self.v(h_) # compute attention - b,c,h,w = q.shape + b, c, h, w = q.shape q = rearrange(q, 'b c h w -> b (h w) c') k = rearrange(k, 'b c h w -> b c (h w)') w_ = torch.einsum('bij,bjk->bik', q, k) - w_ = w_ * (int(c)**(-0.5)) + w_ = w_ * (int(c) ** (-0.5)) w_ = torch.nn.functional.softmax(w_, dim=2) # attend to values @@ -146,7 +147,7 @@ def forward(self, x): h_ = rearrange(h_, 'b c (h w) -> b c h w', h=h) h_ = self.proj_out(h_) - return x+h_ + return x + h_ class CrossAttention(nn.Module): @@ -174,29 +175,34 @@ def forward(self, x, context=None, mask=None): context = default(context, x) k = self.to_k(context) v = self.to_v(context) + del context, x q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> (b h) n d', h=h), (q, k, v)) - sim = einsum('b i d, b j d -> b i j', q, k) * self.scale + sim = einsum('b i d, b j d -> b i j', q, k) * self.scale # (8, 4096, 40) + del q, k if exists(mask): mask = rearrange(mask, 'b ... -> b (...)') max_neg_value = -torch.finfo(sim.dtype).max mask = repeat(mask, 'b j -> (b h) () j', h=h) sim.masked_fill_(~mask, max_neg_value) + del mask - # attention, what we cannot get enough of - attn = sim.softmax(dim=-1) + # attention, what we cannot get enough of, by halves + sim[4:] = sim[4:].softmax(dim=-1) + sim[:4] = sim[:4].softmax(dim=-1) - out = einsum('b i j, b j d -> b i d', attn, v) - out = rearrange(out, '(b h) n d -> b n (h d)', h=h) - return self.to_out(out) + sim = einsum('b i j, b j d -> b i d', sim, v) + sim = rearrange(sim, '(b h) n d -> b n (h d)', h=h) + return self.to_out(sim) class BasicTransformerBlock(nn.Module): def __init__(self, dim, n_heads, d_head, dropout=0., context_dim=None, gated_ff=True, checkpoint=True): super().__init__() - self.attn1 = CrossAttention(query_dim=dim, heads=n_heads, dim_head=d_head, dropout=dropout) # is a self-attention + self.attn1 = CrossAttention(query_dim=dim, heads=n_heads, dim_head=d_head, + dropout=dropout) # is a self-attention self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff) self.attn2 = CrossAttention(query_dim=dim, context_dim=context_dim, heads=n_heads, dim_head=d_head, dropout=dropout) # is self-attn if context is none @@ -223,6 +229,7 @@ class SpatialTransformer(nn.Module): Then apply standard transformer action. Finally, reshape to image """ + def __init__(self, in_channels, n_heads, d_head, depth=1, dropout=0., context_dim=None): super().__init__() @@ -238,7 +245,7 @@ def __init__(self, in_channels, n_heads, d_head, self.transformer_blocks = nn.ModuleList( [BasicTransformerBlock(inner_dim, n_heads, d_head, dropout=dropout, context_dim=context_dim) - for d in range(depth)] + for d in range(depth)] ) self.proj_out = zero_module(nn.Conv2d(inner_dim, @@ -258,4 +265,4 @@ def forward(self, x, context=None): x = block(x, context=context) x = rearrange(x, 'b (h w) c -> b c h w', h=h, w=w) x = self.proj_out(x) - return x + x_in \ No newline at end of file + return x + x_in From 7e8543633289926fc8a31bcd7792e119c33ef638 Mon Sep 17 00:00:00 2001 From: bironsecret <44505718+bironsecret@users.noreply.github.com> Date: Mon, 5 Sep 2022 09:55:08 +0200 Subject: [PATCH 2/2] rolled back the cosmetic stuff --- ldm/modules/attention.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/ldm/modules/attention.py b/ldm/modules/attention.py index 0698c2f70..2642bd5bb 100644 --- a/ldm/modules/attention.py +++ b/ldm/modules/attention.py @@ -1,10 +1,9 @@ -import math from inspect import isfunction - +import math import torch import torch.nn.functional as F -from einops import rearrange, repeat from torch import nn, einsum +from einops import rearrange, repeat from ldm.modules.diffusionmodules.util import checkpoint @@ -14,7 +13,7 @@ def exists(val): def uniq(arr): - return {el: True for el in arr}.keys() + return{el: True for el in arr}.keys() def default(val, d): @@ -83,13 +82,13 @@ def __init__(self, dim, heads=4, dim_head=32): super().__init__() self.heads = heads hidden_dim = dim_head * heads - self.to_qkv = nn.Conv2d(dim, hidden_dim * 3, 1, bias=False) + self.to_qkv = nn.Conv2d(dim, hidden_dim * 3, 1, bias = False) self.to_out = nn.Conv2d(hidden_dim, dim, 1) def forward(self, x): b, c, h, w = x.shape qkv = self.to_qkv(x) - q, k, v = rearrange(qkv, 'b (qkv heads c) h w -> qkv b heads c (h w)', heads=self.heads, qkv=3) + q, k, v = rearrange(qkv, 'b (qkv heads c) h w -> qkv b heads c (h w)', heads = self.heads, qkv=3) k = k.softmax(dim=-1) context = torch.einsum('bhdn,bhen->bhde', k, v) out = torch.einsum('bhde,bhdn->bhen', context, q) @@ -132,12 +131,12 @@ def forward(self, x): v = self.v(h_) # compute attention - b, c, h, w = q.shape + b,c,h,w = q.shape q = rearrange(q, 'b c h w -> b (h w) c') k = rearrange(k, 'b c h w -> b c (h w)') w_ = torch.einsum('bij,bjk->bik', q, k) - w_ = w_ * (int(c) ** (-0.5)) + w_ = w_ * (int(c)**(-0.5)) w_ = torch.nn.functional.softmax(w_, dim=2) # attend to values @@ -147,7 +146,7 @@ def forward(self, x): h_ = rearrange(h_, 'b c (h w) -> b c h w', h=h) h_ = self.proj_out(h_) - return x + h_ + return x+h_ class CrossAttention(nn.Module): @@ -201,8 +200,7 @@ def forward(self, x, context=None, mask=None): class BasicTransformerBlock(nn.Module): def __init__(self, dim, n_heads, d_head, dropout=0., context_dim=None, gated_ff=True, checkpoint=True): super().__init__() - self.attn1 = CrossAttention(query_dim=dim, heads=n_heads, dim_head=d_head, - dropout=dropout) # is a self-attention + self.attn1 = CrossAttention(query_dim=dim, heads=n_heads, dim_head=d_head, dropout=dropout) # is a self-attention self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff) self.attn2 = CrossAttention(query_dim=dim, context_dim=context_dim, heads=n_heads, dim_head=d_head, dropout=dropout) # is self-attn if context is none @@ -229,7 +227,6 @@ class SpatialTransformer(nn.Module): Then apply standard transformer action. Finally, reshape to image """ - def __init__(self, in_channels, n_heads, d_head, depth=1, dropout=0., context_dim=None): super().__init__() @@ -245,7 +242,7 @@ def __init__(self, in_channels, n_heads, d_head, self.transformer_blocks = nn.ModuleList( [BasicTransformerBlock(inner_dim, n_heads, d_head, dropout=dropout, context_dim=context_dim) - for d in range(depth)] + for d in range(depth)] ) self.proj_out = zero_module(nn.Conv2d(inner_dim,