From 8670ec76bb6f23feadb569be2a658f3b6b9bbe7c Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 3 Sep 2024 10:10:51 +0200 Subject: [PATCH 1/2] update --- src/diffusers/models/embeddings.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/diffusers/models/embeddings.py b/src/diffusers/models/embeddings.py index 5e9863ab0d0a..dbb3769e0a81 100644 --- a/src/diffusers/models/embeddings.py +++ b/src/diffusers/models/embeddings.py @@ -550,8 +550,11 @@ def get_1d_rotary_pos_embed( pos = torch.from_numpy(pos) # type: ignore # [S] theta = theta * ntk_factor - freqs = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=freqs_dtype)[: (dim // 2)] / dim)) / linear_factor # [D/2] - freqs = freqs.to(pos.device) + freqs = ( + 1.0 + / (theta ** (torch.arange(0, dim, 2, dtype=freqs_dtype, device=pos.device))[: (dim // 2)] / dim) + / linear_factor + ) # [D/2] freqs = torch.outer(pos, freqs) # type: ignore # [S, D/2] if use_real and repeat_interleave_real: # flux, hunyuan-dit, cogvideox From 29275cd9d9ef9a986d17c657143f95c94f9109ec Mon Sep 17 00:00:00 2001 From: yiyixuxu Date: Tue, 3 Sep 2024 11:09:49 +0200 Subject: [PATCH 2/2] fix --- src/diffusers/models/embeddings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/models/embeddings.py b/src/diffusers/models/embeddings.py index dbb3769e0a81..debeedf6626e 100644 --- a/src/diffusers/models/embeddings.py +++ b/src/diffusers/models/embeddings.py @@ -552,7 +552,7 @@ def get_1d_rotary_pos_embed( theta = theta * ntk_factor freqs = ( 1.0 - / (theta ** (torch.arange(0, dim, 2, dtype=freqs_dtype, device=pos.device))[: (dim // 2)] / dim) + / (theta ** (torch.arange(0, dim, 2, dtype=freqs_dtype, device=pos.device)[: (dim // 2)] / dim)) / linear_factor ) # [D/2] freqs = torch.outer(pos, freqs) # type: ignore # [S, D/2]