From ee15e3dbb623e00de1f34389735c1901040f9968 Mon Sep 17 00:00:00 2001 From: sidthekidder Date: Sat, 10 Sep 2022 21:23:55 -0700 Subject: [PATCH 1/8] [WIP] add LMSDiscreteSchedulerTest --- tests/test_scheduler.py | 72 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index d4aef24a989a..26235a519793 100755 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -19,7 +19,7 @@ import numpy as np import torch -from diffusers import DDIMScheduler, DDPMScheduler, PNDMScheduler, ScoreSdeVeScheduler +from diffusers import DDIMScheduler, DDPMScheduler, LMSDiscreteScheduler, PNDMScheduler, ScoreSdeVeScheduler torch.backends.cuda.matmul.allow_tf32 = False @@ -853,3 +853,73 @@ def test_step_shape(self): self.assertEqual(output_0.shape, sample.shape) self.assertEqual(output_0.shape, output_1.shape) + + +class LMSDiscreteSchedulerTest(SchedulerCommonTest): + scheduler_classes = (LMSDiscreteScheduler,) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1100, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + "trained_betas": None, + "timestep_values": None, + "tensor_format": "pt", + } + + config.update(**kwargs) + return config + + def test_timesteps(self): + for timesteps in [10, 50, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_betas(self): + for beta_start, beta_end in zip([0.0001, 0.001, 0.01, 0.1], [0.002, 0.02, 0.2, 2]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear"]: + self.check_over_configs(beta_schedule=schedule) + + def test_time_indices(self): + for t in [0, 500, 800]: + self.check_over_forward(time_step=t) + + def test_pytorch_equal_numpy(self): + pass + + def test_full_loop_no_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_trained_timesteps = len(scheduler) + + model = self.dummy_model() + sample = self.dummy_sample_deter + + for t in reversed(range(num_trained_timesteps - 1)): + # 1. predict noise residual + residual = model(sample, t) + # print("residual: ") + # print(residual) + + # 2. predict previous mean of sample x_t-1 + pred_prev_sample = scheduler.step(residual, t, sample).prev_sample + + # if t > 0: + # noise = self.dummy_sample_deter + # variance = scheduler.get_variance(t) ** (0.5) * noise + # + # sample = pred_prev_sample + variance + sample = pred_prev_sample + print("Result sample: ") + print(sample) + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 259.0883) < 1e-2 + assert abs(result_mean.item() - 0.3374) < 1e-3 From c159bca98953a820ea4ffd47b8ad9ec5e90d8497 Mon Sep 17 00:00:00 2001 From: sidthekidder Date: Tue, 13 Sep 2022 01:35:07 -0700 Subject: [PATCH 2/8] fixes for comments --- tests/test_scheduler.py | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 26235a519793..d6ddb9c06b8d 100755 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -881,7 +881,7 @@ def test_betas(self): self.check_over_configs(beta_start=beta_start, beta_end=beta_end) def test_schedules(self): - for schedule in ["linear"]: + for schedule in ["linear", "scaled_linear"]: self.check_over_configs(beta_schedule=schedule) def test_time_indices(self): @@ -896,30 +896,22 @@ def test_full_loop_no_noise(self): scheduler_config = self.get_scheduler_config() scheduler = scheduler_class(**scheduler_config) - num_trained_timesteps = len(scheduler) + num_inference_steps = 10 + scheduler.set_timesteps(num_inference_steps) model = self.dummy_model() - sample = self.dummy_sample_deter + sample = self.dummy_sample_deter * scheduler.sigmas[0] - for t in reversed(range(num_trained_timesteps - 1)): - # 1. predict noise residual - residual = model(sample, t) - # print("residual: ") - # print(residual) + for i, t in enumerate(scheduler.timesteps): + sample = sample / ((scheduler.sigmas[i] ** 2 + 1) ** 0.5) - # 2. predict previous mean of sample x_t-1 - pred_prev_sample = scheduler.step(residual, t, sample).prev_sample + model_output = model(sample, t) + + output = scheduler.step(model_output, i, sample) + sample = output.prev_sample - # if t > 0: - # noise = self.dummy_sample_deter - # variance = scheduler.get_variance(t) ** (0.5) * noise - # - # sample = pred_prev_sample + variance - sample = pred_prev_sample - print("Result sample: ") - print(sample) result_sum = torch.sum(torch.abs(sample)) result_mean = torch.mean(torch.abs(sample)) - assert abs(result_sum.item() - 259.0883) < 1e-2 - assert abs(result_mean.item() - 0.3374) < 1e-3 + assert abs(result_sum.item() - 1006.388) < 1e-2 + assert abs(result_mean.item() - 1.31) < 1e-3 From af9701a5c706a5daa0fcd3fce72f3f9ea79774e2 Mon Sep 17 00:00:00 2001 From: sidthekidder Date: Thu, 15 Sep 2022 02:02:26 -0700 Subject: [PATCH 3/8] add torch numpy test --- tests/test_scheduler.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index d6ddb9c06b8d..27710fe1a8fe 100755 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -857,6 +857,7 @@ def test_step_shape(self): class LMSDiscreteSchedulerTest(SchedulerCommonTest): scheduler_classes = (LMSDiscreteScheduler,) + num_inference_steps = 10 def get_scheduler_config(self, **kwargs): config = { @@ -889,15 +890,39 @@ def test_time_indices(self): self.check_over_forward(time_step=t) def test_pytorch_equal_numpy(self): - pass + for scheduler_class in self.scheduler_classes: + sample_pt = self.dummy_sample + residual_pt = 0.1 * sample_pt + dummy_past_residuals_pt = [residual_pt + 0.2, residual_pt + 0.15, residual_pt + 0.1, residual_pt + 0.05] + + sample = sample_pt.numpy() + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.1, residual + 0.05] + + scheduler_config = self.get_scheduler_config() + scheduler_config["tensor_format"] = "np" + scheduler = scheduler_class(**scheduler_config) + + scheduler_config["tensor_format"] = "pt" + scheduler_pt = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + scheduler_pt.set_timesteps(self.num_inference_steps) + + # copy over dummy past residuals (must be done after set_timesteps) + scheduler.ets = dummy_past_residuals[:] + scheduler_pt.ets = dummy_past_residuals_pt[:] + + output = scheduler.step(residual, 1, sample).prev_sample + output_pt = scheduler_pt.step(residual_pt, 1, sample_pt).prev_sample + assert np.sum(np.abs(output - output_pt.numpy())) < 1e-4, "Scheduler outputs are not identical" def test_full_loop_no_noise(self): scheduler_class = self.scheduler_classes[0] scheduler_config = self.get_scheduler_config() scheduler = scheduler_class(**scheduler_config) - num_inference_steps = 10 - scheduler.set_timesteps(num_inference_steps) + scheduler.set_timesteps(self.num_inference_steps) model = self.dummy_model() sample = self.dummy_sample_deter * scheduler.sigmas[0] From 629b934faf95f99168d08f63e0039b7f16afcabb Mon Sep 17 00:00:00 2001 From: sidthekidder Date: Thu, 15 Sep 2022 02:12:11 -0700 Subject: [PATCH 4/8] rebase --- tests/test_scheduler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 27710fe1a8fe..844b4e9fd263 100755 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -866,7 +866,6 @@ def get_scheduler_config(self, **kwargs): "beta_end": 0.02, "beta_schedule": "linear", "trained_betas": None, - "timestep_values": None, "tensor_format": "pt", } From ab880e1b610aaf5a4eceee7613292bcda081d20a Mon Sep 17 00:00:00 2001 From: Anton Lozhkov Date: Fri, 16 Sep 2022 18:56:23 +0200 Subject: [PATCH 5/8] Update tests/test_scheduler.py --- tests/test_scheduler.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 844b4e9fd263..586109b446d8 100755 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -891,12 +891,7 @@ def test_time_indices(self): def test_pytorch_equal_numpy(self): for scheduler_class in self.scheduler_classes: sample_pt = self.dummy_sample - residual_pt = 0.1 * sample_pt - dummy_past_residuals_pt = [residual_pt + 0.2, residual_pt + 0.15, residual_pt + 0.1, residual_pt + 0.05] - sample = sample_pt.numpy() - residual = 0.1 * sample - dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.1, residual + 0.05] scheduler_config = self.get_scheduler_config() scheduler_config["tensor_format"] = "np" From b404452b6bb3966385296b2777167ee003c0c10c Mon Sep 17 00:00:00 2001 From: Anton Lozhkov Date: Fri, 16 Sep 2022 18:56:30 +0200 Subject: [PATCH 6/8] Update tests/test_scheduler.py --- tests/test_scheduler.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 586109b446d8..ce18ac9f1fc1 100755 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -902,11 +902,7 @@ def test_pytorch_equal_numpy(self): scheduler.set_timesteps(self.num_inference_steps) scheduler_pt.set_timesteps(self.num_inference_steps) - - # copy over dummy past residuals (must be done after set_timesteps) - scheduler.ets = dummy_past_residuals[:] - scheduler_pt.ets = dummy_past_residuals_pt[:] - + output = scheduler.step(residual, 1, sample).prev_sample output_pt = scheduler_pt.step(residual_pt, 1, sample_pt).prev_sample assert np.sum(np.abs(output - output_pt.numpy())) < 1e-4, "Scheduler outputs are not identical" From 7c3170738aab1efd7d61b5a77bfdfae7eb819d80 Mon Sep 17 00:00:00 2001 From: anton-l Date: Fri, 16 Sep 2022 19:03:08 +0200 Subject: [PATCH 7/8] style --- tests/test_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 001e5f05d64f..db51495c1e1f 100755 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -902,7 +902,7 @@ def test_pytorch_equal_numpy(self): scheduler.set_timesteps(self.num_inference_steps) scheduler_pt.set_timesteps(self.num_inference_steps) - + output = scheduler.step(residual, 1, sample).prev_sample output_pt = scheduler_pt.step(residual_pt, 1, sample_pt).prev_sample assert np.sum(np.abs(output - output_pt.numpy())) < 1e-4, "Scheduler outputs are not identical" From bd713b70d49ee598dc933e8f4f08439caaffd9b2 Mon Sep 17 00:00:00 2001 From: anton-l Date: Fri, 16 Sep 2022 19:06:31 +0200 Subject: [PATCH 8/8] return residuals --- tests/test_scheduler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index db51495c1e1f..e78f100409f8 100755 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -891,7 +891,10 @@ def test_time_indices(self): def test_pytorch_equal_numpy(self): for scheduler_class in self.scheduler_classes: sample_pt = self.dummy_sample + residual_pt = 0.1 * sample_pt + sample = sample_pt.numpy() + residual = 0.1 * sample scheduler_config = self.get_scheduler_config() scheduler_config["tensor_format"] = "np"