From a1e1806575f8f38c3603c7efa99cc17c7f97bdcc Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Thu, 16 Jun 2022 17:45:31 +0200 Subject: [PATCH 1/3] render latex in readme --- README.md | 37 +++++++++++++++++----------------- src/diffusers/models/README.md | 2 +- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index cbbd973b8170..18f4fb7836f4 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ More precisely, 🤗 Diffusers offers: ## Definitions -**Models**: Neural network that models **p_θ(x_t-1|x_t)** (see image below) and is trained end-to-end to *denoise* a noisy input to an image. +**Models**: Neural network that models $p_\theta(x_{t-1}|x_t)$ (see image below) and is trained end-to-end to *denoise* a noisy input to an image. *Examples*: UNet, Conditioned UNet, 3D UNet, Transformer UNet ![model_diff_1_50](https://user-images.githubusercontent.com/23423619/171610307-dab0cd8b-75da-4d4e-9f5a-5922072e2bb5.png) @@ -44,7 +44,6 @@ The class provides functionality to compute previous image according to alpha, b ![imagen](https://user-images.githubusercontent.com/23423619/171609001-c3f2c1c9-f597-4a16-9843-749bf3f9431c.png) - ## Philosophy - Readability and clarity is prefered over highly optimized code. A strong importance is put on providing readable, intuitive and elementary code desgin. *E.g.*, the provided [schedulers](https://github.com/huggingface/diffusers/tree/main/src/diffusers/schedulers) are separated from the provided [models](https://github.com/huggingface/diffusers/tree/main/src/diffusers/models) and provide well-commented code that can be read alongside the original paper. @@ -59,7 +58,7 @@ The class provides functionality to compute previous image according to alpha, b pip install diffusers # should install diffusers 0.0.4 ``` -### 1. `diffusers` as a toolbox for schedulers and models. +### 1. `diffusers` as a toolbox for schedulers and models `diffusers` is more modularized than `transformers`. The idea is that researchers and engineers can use only parts of the library easily for the own use cases. It could become a central place for all kinds of models, schedulers, training utils and processors that one can mix and match for one's own use case. @@ -137,8 +136,8 @@ unet = UNetModel.from_pretrained("fusing/ddpm-celeba-hq").to(torch_device) # 2. Sample gaussian noise image = torch.randn( - (1, unet.in_channels, unet.resolution, unet.resolution), - generator=generator, + (1, unet.in_channels, unet.resolution, unet.resolution), + generator=generator, ) image = image.to(torch_device) @@ -147,22 +146,22 @@ num_inference_steps = 50 eta = 0.0 # <- deterministic sampling for t in tqdm.tqdm(reversed(range(num_inference_steps)), total=num_inference_steps): - # 1. predict noise residual - orig_t = noise_scheduler.get_orig_t(t, num_inference_steps) - with torch.no_grad(): - residual = unet(image, orig_t) + # 1. predict noise residual + orig_t = noise_scheduler.get_orig_t(t, num_inference_steps) + with torch.no_grad(): + residual = unet(image, orig_t) - # 2. predict previous mean of image x_t-1 - pred_prev_image = noise_scheduler.step(residual, image, t, num_inference_steps, eta) + # 2. predict previous mean of image x_t-1 + pred_prev_image = noise_scheduler.step(residual, image, t, num_inference_steps, eta) - # 3. optionally sample variance - variance = 0 - if eta > 0: - noise = torch.randn(image.shape, generator=generator).to(image.device) - variance = noise_scheduler.get_variance(t).sqrt() * eta * noise + # 3. optionally sample variance + variance = 0 + if eta > 0: + noise = torch.randn(image.shape, generator=generator).to(image.device) + variance = noise_scheduler.get_variance(t).sqrt() * eta * noise - # 4. set current image to prev_image: x_t -> x_t-1 - image = pred_prev_image + variance + # 4. set current image to prev_image: x_t -> x_t-1 + image = pred_prev_image + variance # 5. process image to PIL image_processed = image.cpu().permute(0, 2, 3, 1) @@ -233,7 +232,7 @@ image_pil = PIL.Image.fromarray(image_processed[0]) image_pil.save("test.png") ``` - #### **Text to speech with BDDM** +#### **Text to speech with BDDM** _Follow the instructions [here](https://pytorch.org/hub/nvidia_deeplearningexamples_tacotron2/) to load tacotron2 model._ diff --git a/src/diffusers/models/README.md b/src/diffusers/models/README.md index 35f09b294fb5..6b5b94b5edeb 100644 --- a/src/diffusers/models/README.md +++ b/src/diffusers/models/README.md @@ -1,6 +1,6 @@ # Models -- Models: Neural network that models p_θ(x_t-1|x_t) (see image below) and is trained end-to-end to denoise a noisy input to an image. Examples: UNet, Conditioned UNet, 3D UNet, Transformer UNet +- Models: Neural network that models $p_\theta(x_{t-1}|x_t)$ (see image below) and is trained end-to-end to denoise a noisy input to an image. Examples: UNet, Conditioned UNet, 3D UNet, Transformer UNet ## API From 13f003c9bd57653b5a1d18d2560434b4a177ce1f Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Thu, 16 Jun 2022 17:49:35 +0200 Subject: [PATCH 2/3] use bold --- README.md | 38 +++++++++++++++++----------------- src/diffusers/models/README.md | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 18f4fb7836f4..05078a8cbe45 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ More precisely, 🤗 Diffusers offers: ## Definitions -**Models**: Neural network that models $p_\theta(x_{t-1}|x_t)$ (see image below) and is trained end-to-end to *denoise* a noisy input to an image. +**Models**: Neural network that models $p_\theta(\mathbf{x}_{t-1}|\mathbf{x}_t)$ (see image below) and is trained end-to-end to *denoise* a noisy input to an image. *Examples*: UNet, Conditioned UNet, 3D UNet, Transformer UNet ![model_diff_1_50](https://user-images.githubusercontent.com/23423619/171610307-dab0cd8b-75da-4d4e-9f5a-5922072e2bb5.png) @@ -136,8 +136,8 @@ unet = UNetModel.from_pretrained("fusing/ddpm-celeba-hq").to(torch_device) # 2. Sample gaussian noise image = torch.randn( - (1, unet.in_channels, unet.resolution, unet.resolution), - generator=generator, + (1, unet.in_channels, unet.resolution, unet.resolution), + generator=generator, ) image = image.to(torch_device) @@ -146,22 +146,22 @@ num_inference_steps = 50 eta = 0.0 # <- deterministic sampling for t in tqdm.tqdm(reversed(range(num_inference_steps)), total=num_inference_steps): - # 1. predict noise residual - orig_t = noise_scheduler.get_orig_t(t, num_inference_steps) - with torch.no_grad(): - residual = unet(image, orig_t) - - # 2. predict previous mean of image x_t-1 - pred_prev_image = noise_scheduler.step(residual, image, t, num_inference_steps, eta) - - # 3. optionally sample variance - variance = 0 - if eta > 0: - noise = torch.randn(image.shape, generator=generator).to(image.device) - variance = noise_scheduler.get_variance(t).sqrt() * eta * noise - - # 4. set current image to prev_image: x_t -> x_t-1 - image = pred_prev_image + variance + # 1. predict noise residual + orig_t = noise_scheduler.get_orig_t(t, num_inference_steps) + with torch.no_grad(): + residual = unet(image, orig_t) + + # 2. predict previous mean of image x_t-1 + pred_prev_image = noise_scheduler.step(residual, image, t, num_inference_steps, eta) + + # 3. optionally sample variance + variance = 0 + if eta > 0: + noise = torch.randn(image.shape, generator=generator).to(image.device) + variance = noise_scheduler.get_variance(t).sqrt() * eta * noise + + # 4. set current image to prev_image: x_t -> x_t-1 + image = pred_prev_image + variance # 5. process image to PIL image_processed = image.cpu().permute(0, 2, 3, 1) diff --git a/src/diffusers/models/README.md b/src/diffusers/models/README.md index 6b5b94b5edeb..e786fe518f03 100644 --- a/src/diffusers/models/README.md +++ b/src/diffusers/models/README.md @@ -1,6 +1,6 @@ # Models -- Models: Neural network that models $p_\theta(x_{t-1}|x_t)$ (see image below) and is trained end-to-end to denoise a noisy input to an image. Examples: UNet, Conditioned UNet, 3D UNet, Transformer UNet +- Models: Neural network that models $p_\theta(\mathbf{x}_{t-1}|\mathbf{x}_t)$ (see image below) and is trained end-to-end to denoise a noisy input to an image. Examples: UNet, Conditioned UNet, 3D UNet, Transformer UNet ## API From cf3fdb84796ebf471ff2baaf35a4ef1e707500cb Mon Sep 17 00:00:00 2001 From: Kashif Rasul Date: Thu, 16 Jun 2022 17:54:47 +0200 Subject: [PATCH 3/3] use inference_mode --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7c3671c04be6..45922b97e7b8 100644 --- a/README.md +++ b/README.md @@ -147,9 +147,9 @@ eta = 0.0 # <- deterministic sampling for t in tqdm.tqdm(reversed(range(num_inference_steps)), total=num_inference_steps): # 1. predict noise residual - orig_t = noise_scheduler.get_orig_t(t, num_inference_steps) - with torch.no_grad(): - residual = unet(image, orig_t) + orig_t = noise_scheduler.get_orig_t(t, num_inference_steps) + with torch.inference_mode(): + residual = unet(image, orig_t) # 2. predict previous mean of image x_t-1 pred_prev_image = noise_scheduler.step(residual, image, t, num_inference_steps, eta)