From 19eac791c3e559cb40e6970af9fbae4ddecb9d42 Mon Sep 17 00:00:00 2001 From: Walter Hugo Lopez Pinaya Date: Mon, 5 Dec 2022 20:32:37 +0000 Subject: [PATCH 1/3] [WIP] Add classifier-free guidance tutorial Signed-off-by: Walter Hugo Lopez Pinaya --- ..._ddpm_classifier_free_guidance_tutorial.py | 326 ++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100644 tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.py diff --git a/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.py b/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.py new file mode 100644 index 00000000..16549b46 --- /dev/null +++ b/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.py @@ -0,0 +1,326 @@ +# --- +# jupyter: +# jupytext: +# formats: ipynb,py:percent +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# jupytext_version: 1.14.1 +# kernelspec: +# display_name: Python 3 +# language: python +# name: python3 +# --- + +# %% [markdown] +# # Denoising Diffusion Probabilistic Models with MedNIST Dataset +# +# This tutorial illustrates how to use MONAI for training a denoising diffusion probabilistic model (DDPM)[1] to create +# synthetic 2D images. +# +# [1] - Ho et al. "Denoising Diffusion Probabilistic Models" https://arxiv.org/abs/2006.11239 +# +# TODO: Add Open in Colab +# +# ## Setup environment + +# %% +# !python -c "import monai" || pip install -q "monai-weekly[pillow, tqdm, einops]" +# !python -c "import matplotlib" || pip install -q matplotlib +# %matplotlib inline + +# %% [markdown] +# ## Setup imports + +# %% jupyter={"outputs_hidden": false} +# Copyright 2020 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import shutil +import tempfile +import time + +import matplotlib.pyplot as plt +import numpy as np +import torch +import torch.nn.functional as F +from monai import transforms +from monai.apps import MedNISTDataset +from monai.config import print_config +from monai.data import CacheDataset, DataLoader +from monai.utils import first, set_determinism +from torch.cuda.amp import GradScaler, autocast +from tqdm import tqdm + +from generative.inferers import DiffusionInferer + +# TODO: Add right import reference after deployed +from generative.networks.nets import DiffusionModelUNet +from generative.schedulers import DDPMScheduler + +print_config() + +# %% [markdown] +# ## Setup data directory +# +# You can specify a directory with the MONAI_DATA_DIRECTORY environment variable. +# +# This allows you to save results and reuse downloads. +# +# If not specified a temporary directory will be used. + +# %% jupyter={"outputs_hidden": false} +directory = os.environ.get("MONAI_DATA_DIRECTORY") +root_dir = tempfile.mkdtemp() if directory is None else directory +print(root_dir) + +# %% [markdown] +# ## Set deterministic training for reproducibility + +# %% jupyter={"outputs_hidden": false} +set_determinism(42) + +# %% [markdown] +# ## Setup MedNIST Dataset and training and validation dataloaders +# In this tutorial, we will train our models on the MedNIST dataset available on MONAI +# (https://docs.monai.io/en/stable/apps.html#monai.apps.MedNISTDataset). In order to train faster, we will select just +# one of the available classes ("Hand"), resulting in a training set with 7999 2D images. + +# %% jupyter={"outputs_hidden": false} +train_data = MedNISTDataset(root_dir=root_dir, section="training", download=True, progress=False, seed=0) +train_datalist = [] +for item in train_data.data: + if item["class_name"] in ["Hand", "HeadCT"]: + train_datalist.append({"image": item["image"], "class": 1 if item["class_name"] == "Hand" else 2}) + +# %% [markdown] +# Here we use transforms to augment the training dataset: +# +# 1. `LoadImaged` loads the hands images from files. +# 1. `EnsureChannelFirstd` ensures the original data to construct "channel first" shape. +# 1. `ScaleIntensityRanged` extracts intensity range [0, 255] and scales to [0, 1]. +# 1. `RandAffined` efficiently performs rotate, scale, shear, translate, etc. together based on PyTorch affine transform. + +# %% jupyter={"outputs_hidden": false} +train_transforms = transforms.Compose( + [ + transforms.LoadImaged(keys=["image"]), + transforms.EnsureChannelFirstd(keys=["image"]), + transforms.ScaleIntensityRanged(keys=["image"], a_min=0.0, a_max=255.0, b_min=0.0, b_max=1.0, clip=True), + transforms.RandAffined( + keys=["image"], + rotate_range=[(-np.pi / 36, np.pi / 36), (-np.pi / 36, np.pi / 36)], + translate_range=[(-1, 1), (-1, 1)], + scale_range=[(-0.05, 0.05), (-0.05, 0.05)], + spatial_size=[64, 64], + padding_mode="zeros", + prob=0.5, + ), + transforms.RandLambdad(keys=["class"], prob=0.15, func=lambda x: -1 * torch.ones_like(x)), + transforms.EnsureTyped(keys=["class"], dtype=torch.float), + ] +) +train_ds = CacheDataset(data=train_datalist, transform=train_transforms) +train_loader = DataLoader(train_ds, batch_size=128, shuffle=True, num_workers=4, persistent_workers=True) + +# %% jupyter={"outputs_hidden": false} +val_data = MedNISTDataset(root_dir=root_dir, section="validation", download=True, progress=False, seed=0) +val_datalist = [] +for item in val_data.data: + if item["class_name"] in ["Hand", "HeadCT"]: + val_datalist.append({"image": item["image"], "class": 1 if item["class_name"] == "Hand" else 2}) + + +val_transforms = transforms.Compose( + [ + transforms.LoadImaged(keys=["image"]), + transforms.EnsureChannelFirstd(keys=["image"]), + transforms.ScaleIntensityRanged(keys=["image"], a_min=0.0, a_max=255.0, b_min=0.0, b_max=1.0, clip=True), + transforms.EnsureTyped(keys=["class"], dtype=torch.float), + ] +) +val_ds = CacheDataset(data=val_datalist, transform=val_transforms) +val_loader = DataLoader(val_ds, batch_size=128, shuffle=False, num_workers=4, persistent_workers=True) + +# %% [markdown] +# ### Visualisation of the training images + +# %% jupyter={"outputs_hidden": false} +check_data = first(train_loader) +print(f"batch shape: {check_data['image'].shape}") +image_visualisation = torch.cat( + [check_data["image"][0, 0], check_data["image"][1, 0], check_data["image"][2, 0], check_data["image"][3, 0]], dim=1 +) +plt.figure("training images", (12, 6)) +plt.imshow(image_visualisation, vmin=0, vmax=1, cmap="gray") +plt.axis("off") +plt.tight_layout() +plt.show() + +# %% [markdown] +# ### Define network, scheduler, optimizer, and inferer +# At this step, we instantiate the MONAI components to create a DDPM, the UNET, the noise scheduler, and the inferer used for training and sampling. We are using +# the original DDPM scheduler containing 1000 timesteps in its Markov chain, and a 2D UNET with attention mechanisms +# in the 2nd and 3rd levels, each with 1 attention head. + +# %% jupyter={"outputs_hidden": false} +device = torch.device("cuda") + +model = DiffusionModelUNet( + spatial_dims=2, + in_channels=1, + out_channels=1, + num_channels=(64, 64, 64), + attention_levels=(False, False, True), + num_res_blocks=1, + num_head_channels=64, + with_conditioning=True, + cross_attention_dim=1, +) +model.to(device) + +scheduler = DDPMScheduler( + num_train_timesteps=1000, +) + +optimizer = torch.optim.Adam(params=model.parameters(), lr=2.5e-5) + +inferer = DiffusionInferer(scheduler) +# %% [markdown] +# ### Model training +# Here, we are training our model for 75 epochs (training time: ~50 minutes). + +# %% jupyter={"outputs_hidden": false} +n_epochs = 75 +val_interval = 5 +epoch_loss_list = [] +val_epoch_loss_list = [] + +scaler = GradScaler() +total_start = time.time() +for epoch in range(n_epochs): + model.train() + epoch_loss = 0 + progress_bar = tqdm(enumerate(train_loader), total=len(train_loader), ncols=70) + progress_bar.set_description(f"Epoch {epoch}") + for step, batch in progress_bar: + images = batch["image"].to(device) + classes = batch["class"].to(device) + optimizer.zero_grad(set_to_none=True) + + with autocast(enabled=True): + # Generate random noise + noise = torch.randn_like(images).to(device) + + # Get model prediction + noise_pred = inferer( + inputs=images, diffusion_model=model, noise=noise, condition=classes.unsqueeze(-1).unsqueeze(-1) + ) + + loss = F.mse_loss(noise_pred.float(), noise.float()) + + scaler.scale(loss).backward() + scaler.step(optimizer) + scaler.update() + + epoch_loss += loss.item() + + progress_bar.set_postfix( + { + "loss": epoch_loss / (step + 1), + } + ) + epoch_loss_list.append(epoch_loss / (step + 1)) + + if (epoch + 1) % val_interval == 0: + model.eval() + val_epoch_loss = 0 + for step, batch in enumerate(val_loader): + images = batch["image"].to(device) + classes = batch["class"].to(device) + with torch.no_grad(): + with autocast(enabled=True): + noise = torch.randn_like(images).to(device) + noise_pred = inferer( + inputs=images, diffusion_model=model, noise=noise, condition=classes.unsqueeze(-1).unsqueeze(-1) + ) + val_loss = F.mse_loss(noise_pred.float(), noise.float()) + + val_epoch_loss += val_loss.item() + progress_bar.set_postfix( + { + "val_loss": val_epoch_loss / (step + 1), + } + ) + val_epoch_loss_list.append(val_epoch_loss / (step + 1)) + +total_time = time.time() - total_start +print(f"train completed, total time: {total_time}.") +# %% [markdown] +# ### Learning curves + +# %% jupyter={"outputs_hidden": false} +plt.style.use("seaborn-v0_8") +plt.title("Learning Curves", fontsize=20) +plt.plot(np.linspace(1, n_epochs, n_epochs), epoch_loss_list, color="C0", linewidth=2.0, label="Train") +plt.plot( + np.linspace(val_interval, n_epochs, int(n_epochs / val_interval)), + val_epoch_loss_list, + color="C1", + linewidth=2.0, + label="Validation", +) +plt.yticks(fontsize=12) +plt.xticks(fontsize=12) +plt.xlabel("Epochs", fontsize=16) +plt.ylabel("Loss", fontsize=16) +plt.legend(prop={"size": 14}) +plt.show() + +# %% [markdown] +# ### Plotting sampling process along DDPM's Markov chain + +# %% jupyter={"outputs_hidden": false} +model.eval() +guidance_scale = 0.7 +conditioning = torch.cat([-1 * torch.ones(1, 1, 1, 1).float(), torch.ones(1, 1, 1, 1).float()], dim=0).to(device) + +noise = torch.randn((1, 1, 64, 64)) +noise = noise.to(device) +scheduler.set_timesteps(num_inference_steps=1000) +progress_bar = tqdm(scheduler.timesteps) +for t in progress_bar: + with autocast(enabled=True): + with torch.no_grad(): + noise_input = torch.cat([noise] * 2) + + model_output = model(noise_input, timesteps=torch.Tensor((t,)).to(noise.device), context=conditioning) + noise_pred_uncond, noise_pred_text = model_output.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # 2. compute previous image: x_t -> x_t-1 + noise, _ = scheduler.step(noise_pred, t, noise) + +plt.style.use("default") +plt.imshow(noise[0, 0].cpu(), vmin=0, vmax=1, cmap="gray") +plt.tight_layout() +plt.axis("off") +plt.show() + +# %% [markdown] +# ### Cleanup data directory +# +# Remove directory if a temporary was used. + +# %% +if directory is None: + shutil.rmtree(root_dir) From bd2d5e063a4dfed5d377be4a3d9be1fef8eeea1c Mon Sep 17 00:00:00 2001 From: Walter Hugo Lopez Pinaya Date: Sat, 10 Dec 2022 12:28:21 +0000 Subject: [PATCH 2/3] Add classifier free-guidance tutorial Signed-off-by: Walter Hugo Lopez Pinaya --- ...pm_classifier_free_guidance_tutorial.ipynb | 778 ++++++++++++++++++ ..._ddpm_classifier_free_guidance_tutorial.py | 66 +- 2 files changed, 817 insertions(+), 27 deletions(-) create mode 100644 tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.ipynb diff --git a/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.ipynb b/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.ipynb new file mode 100644 index 00000000..7b418b8d --- /dev/null +++ b/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.ipynb @@ -0,0 +1,778 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "63d95da6", + "metadata": {}, + "source": [ + "# Classifier-free Guidance\n", + "\n", + "This tutorial illustrates how to use MONAI for training a denoising diffusion probabilistic model (DDPM)[1] to create synthetic 2D images using the classifier-free guidance technique [2] to perform conditioning.\n", + "\n", + "\n", + "[1] - Ho et al. \"Denoising Diffusion Probabilistic Models\" https://arxiv.org/abs/2006.11239\n", + "[2] - Ho and Salimans \"Classifier-Free Diffusion Guidance\" https://arxiv.org/abs/2207.12598\n", + "\n", + "\n", + "TODO: Add Open in Colab\n", + "\n", + "## Setup environment" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "75f2d5f3", + "metadata": {}, + "outputs": [], + "source": [ + "!python -c \"import monai\" || pip install -q \"monai-weekly[pillow, tqdm, einops]\"\n", + "!python -c \"import matplotlib\" || pip install -q matplotlib\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "id": "6b766027", + "metadata": {}, + "source": [ + "## Setup imports" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "972ed3f3", + "metadata": { + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "MONAI version: 1.1.dev2239\n", + "Numpy version: 1.23.3\n", + "Pytorch version: 1.8.0+cu111\n", + "MONAI flags: HAS_EXT = False, USE_COMPILED = False, USE_META_DICT = False\n", + "MONAI rev id: 13b24fa92b9d98bd0dc6d5cdcb52504fd09e297b\n", + "MONAI __file__: /media/walter/Storage/Projects/GenerativeModels/venv/lib/python3.8/site-packages/monai/__init__.py\n", + "\n", + "Optional dependencies:\n", + "Pytorch Ignite version: 0.4.10\n", + "Nibabel version: 4.0.2\n", + "scikit-image version: NOT INSTALLED or UNKNOWN VERSION.\n", + "Pillow version: 9.2.0\n", + "Tensorboard version: 2.11.0\n", + "gdown version: NOT INSTALLED or UNKNOWN VERSION.\n", + "TorchVision version: 0.9.0+cu111\n", + "tqdm version: 4.64.1\n", + "lmdb version: NOT INSTALLED or UNKNOWN VERSION.\n", + "psutil version: 5.9.3\n", + "pandas version: NOT INSTALLED or UNKNOWN VERSION.\n", + "einops version: 0.6.0\n", + "transformers version: NOT INSTALLED or UNKNOWN VERSION.\n", + "mlflow version: NOT INSTALLED or UNKNOWN VERSION.\n", + "pynrrd version: NOT INSTALLED or UNKNOWN VERSION.\n", + "\n", + "For details about installing the optional dependencies, please visit:\n", + " https://docs.monai.io/en/latest/installation.html#installing-the-recommended-dependencies\n", + "\n" + ] + } + ], + "source": [ + "# Copyright 2020 MONAI Consortium\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License.\n", + "import os\n", + "import shutil\n", + "import tempfile\n", + "import time\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import torch\n", + "import torch.nn.functional as F\n", + "from monai import transforms\n", + "from monai.apps import MedNISTDataset\n", + "from monai.config import print_config\n", + "from monai.data import CacheDataset, DataLoader\n", + "from monai.utils import first, set_determinism\n", + "from torch.cuda.amp import GradScaler, autocast\n", + "from tqdm import tqdm\n", + "\n", + "from generative.inferers import DiffusionInferer\n", + "\n", + "# TODO: Add right import reference after deployed\n", + "from generative.networks.nets import DiffusionModelUNet\n", + "from generative.schedulers import DDPMScheduler\n", + "\n", + "print_config()" + ] + }, + { + "cell_type": "markdown", + "id": "7d4ff515", + "metadata": {}, + "source": [ + "## Setup data directory" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "8b4323e7", + "metadata": { + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/tmp/tmp142o2qtd\n" + ] + } + ], + "source": [ + "directory = os.environ.get(\"MONAI_DATA_DIRECTORY\")\n", + "root_dir = tempfile.mkdtemp() if directory is None else directory\n", + "print(root_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "99175d50", + "metadata": {}, + "source": [ + "## Set deterministic training for reproducibility" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "34ea510f", + "metadata": { + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "set_determinism(42)" + ] + }, + { + "cell_type": "markdown", + "id": "fac55e9d", + "metadata": {}, + "source": [ + "## Setup MedNIST Dataset and training and validation dataloaders\n", + "In this tutorial, we will train our models on the MedNIST dataset available on MONAI\n", + "(https://docs.monai.io/en/stable/apps.html#monai.apps.MedNISTDataset). \n", + "Here, we will use the \"Hand\" and \"HeadCT\", where our conditioning variable `class` will specify the modality." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "da1927b0", + "metadata": { + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-12-10 11:50:40,187 - INFO - Downloaded: /tmp/tmp142o2qtd/MedNIST.tar.gz\n", + "2022-12-10 11:50:40,255 - INFO - Verified 'MedNIST.tar.gz', md5: 0bc7306e7427e00ad1c5526a6677552d.\n", + "2022-12-10 11:50:40,256 - INFO - Writing into directory: /tmp/tmp142o2qtd.\n" + ] + } + ], + "source": [ + "train_data = MedNISTDataset(root_dir=root_dir, section=\"training\", download=True, progress=False, seed=0)\n", + "train_datalist = []\n", + "for item in train_data.data:\n", + " if item[\"class_name\"] in [\"Hand\", \"HeadCT\"]:\n", + " train_datalist.append({\"image\": item[\"image\"], \"class\": 1 if item[\"class_name\"] == \"Hand\" else 2})" + ] + }, + { + "cell_type": "markdown", + "id": "6986f55c", + "metadata": {}, + "source": [ + "Here we use transforms to augment the training dataset, as usual:\n", + "\n", + "1. `LoadImaged` loads the hands images from files.\n", + "1. `EnsureChannelFirstd` ensures the original data to construct \"channel first\" shape.\n", + "1. `ScaleIntensityRanged` extracts intensity range [0, 255] and scales to [0, 1].\n", + "1. `RandAffined` efficiently performs rotate, scale, shear, translate, etc. together based on PyTorch affine transform.\n", + "\n", + "### Classifier-free guidance during training\n", + "\n", + "In order to use the classifier-free guidance during training time, we need to not just have the `class` variable saying the modality of the image (`1` for Hands and `2` for HeadCTs) but we also need to train the model with an \"unconditional\" class.\n", + "Here we specify the \"unconditional\" class with the value `-1` with a probability of training on unconditional being 15%. Specified in the following line using MONAI's RandLambdad:\n", + "\n", + "`transforms.RandLambdad(keys=[\"class\"], prob=0.15, func=lambda x: -1 * torch.ones_like(x))`\n", + "\n", + "Finally, our conditioning variable need to have the format (batch_size, 1, cross_attention_dim) when feeding into the model. For this reason, we use Lambdad to reshape our variables in the right format." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "e3184009", + "metadata": { + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading dataset: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 15990/15990 [00:08<00:00, 1784.85it/s]\n" + ] + } + ], + "source": [ + "train_transforms = transforms.Compose(\n", + " [\n", + " transforms.LoadImaged(keys=[\"image\"]),\n", + " transforms.EnsureChannelFirstd(keys=[\"image\"]),\n", + " transforms.ScaleIntensityRanged(keys=[\"image\"], a_min=0.0, a_max=255.0, b_min=0.0, b_max=1.0, clip=True),\n", + " transforms.RandAffined(\n", + " keys=[\"image\"],\n", + " rotate_range=[(-np.pi / 36, np.pi / 36), (-np.pi / 36, np.pi / 36)],\n", + " translate_range=[(-1, 1), (-1, 1)],\n", + " scale_range=[(-0.05, 0.05), (-0.05, 0.05)],\n", + " spatial_size=[64, 64],\n", + " padding_mode=\"zeros\",\n", + " prob=0.5,\n", + " ),\n", + " transforms.RandLambdad(keys=[\"class\"], prob=0.15, func=lambda x: -1 * torch.ones_like(x)),\n", + " transforms.Lambdad(\n", + " keys=[\"class\"], func=lambda x: torch.tensor(x, dtype=torch.float32).unsqueeze(0).unsqueeze(0)\n", + " ),\n", + " ]\n", + ")\n", + "train_ds = CacheDataset(data=train_datalist, transform=train_transforms)\n", + "train_loader = DataLoader(train_ds, batch_size=128, shuffle=True, num_workers=4, persistent_workers=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "4c11b93f", + "metadata": { + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-12-10 11:51:08,067 - INFO - Verified 'MedNIST.tar.gz', md5: 0bc7306e7427e00ad1c5526a6677552d.\n", + "2022-12-10 11:51:08,067 - INFO - File exists: /tmp/tmp142o2qtd/MedNIST.tar.gz, skipped downloading.\n", + "2022-12-10 11:51:08,068 - INFO - Non-empty folder exists in /tmp/tmp142o2qtd/MedNIST, skipped extracting.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading dataset: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1977/1977 [00:01<00:00, 1545.02it/s]\n" + ] + } + ], + "source": [ + "val_data = MedNISTDataset(root_dir=root_dir, section=\"validation\", download=True, progress=False, seed=0)\n", + "val_datalist = []\n", + "for item in val_data.data:\n", + " if item[\"class_name\"] in [\"Hand\", \"HeadCT\"]:\n", + " val_datalist.append({\"image\": item[\"image\"], \"class\": 1 if item[\"class_name\"] == \"Hand\" else 2})\n", + "\n", + "\n", + "val_transforms = transforms.Compose(\n", + " [\n", + " transforms.LoadImaged(keys=[\"image\"]),\n", + " transforms.EnsureChannelFirstd(keys=[\"image\"]),\n", + " transforms.ScaleIntensityRanged(keys=[\"image\"], a_min=0.0, a_max=255.0, b_min=0.0, b_max=1.0, clip=True),\n", + " transforms.Lambdad(\n", + " keys=[\"class\"], func=lambda x: torch.tensor(x, dtype=torch.float32).unsqueeze(0).unsqueeze(0)\n", + " ),\n", + " ]\n", + ")\n", + "val_ds = CacheDataset(data=val_datalist, transform=val_transforms)\n", + "val_loader = DataLoader(val_ds, batch_size=128, shuffle=False, num_workers=4, persistent_workers=True)" + ] + }, + { + "cell_type": "markdown", + "id": "7f108ebb", + "metadata": {}, + "source": [ + "### Visualisation of the training images" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "4105a01f", + "metadata": { + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_16221/734547315.py:17: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n", + " keys=[\"class\"], func=lambda x: torch.tensor(x, dtype=torch.float32).unsqueeze(0).unsqueeze(0)\n", + "/tmp/ipykernel_16221/734547315.py:17: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n", + " keys=[\"class\"], func=lambda x: torch.tensor(x, dtype=torch.float32).unsqueeze(0).unsqueeze(0)\n", + "/tmp/ipykernel_16221/734547315.py:17: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n", + " keys=[\"class\"], func=lambda x: torch.tensor(x, dtype=torch.float32).unsqueeze(0).unsqueeze(0)\n", + "/tmp/ipykernel_16221/734547315.py:17: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n", + " keys=[\"class\"], func=lambda x: torch.tensor(x, dtype=torch.float32).unsqueeze(0).unsqueeze(0)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "batch shape: (128, 1, 64, 64)\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABKUAAAE4CAYAAACKfUBxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAABtbUlEQVR4nO3dWcxe1132/1XaJLYTu57nebZjO3HiTE3akgpBqYDSqoflBM6R4ICzAgdUHCIVkFAlhBAHtEWIilKavm2aqRnt2Ek8z3Y8D/EQx3Zip/Q9+L9/ve+6fld8/7qy7/08tr+fs7W17j2utfZ6tp597U/88pe//GUBAAAAAAAAevRrI70DAAAAAAAAuP3wUAoAAAAAAAC946EUAAAAAAAAesdDKQAAAAAAAPSOh1IAAAAAAADoHQ+lAAAAAAAA0DseSgEAAAAAAKB3PJQCAAAAAABA73goBQAAAAAAgN59Kltx6dKlYdkvf/nLTnfm//eJT3yi03qD/NqvDX429z//8z8D63z44Ydh2Z133jlwPfo7tz+ZYx3m+XDr1mWZ3/3iF7/obPstdZxPfvKTA+tcvnw5LBs3blxVvnbtWqijx+KOTduEO4677rorLNPtjRkzJtS5evVqVb7jjjtCnYxPfSoOFZlj03ObOTa3rY0bN1bl6dOnhzqub2l7c2OW/s6tR3+XbWu6Lrf9zDiqdTLH0bruTF93dbL7NEjm3GbPY+a8tdRx+hx/W69/5rpl1u3quP3WMcptX8ckN45NmDChKuvYW0op8+bNC8suXbpUlT/96U+HOjqWXL9+PdTR/T537lyo88Mf/jAsA/qSGTdcncx9vKXOMLff1Vh7s3j//fer8vnz50doT4CRl51/ZujY4uZRLfPf7P5k5naZfczUGWmZc8J/SgEAAAAAAKB3PJQCAAAAAABA73goBQAAAAAAgN6lM6UyWUB9a3mH0r2v3pK74rbntu+yiJTL0FHu/GfyQjJ5WYPWm1135neZOq3vwmaO1R3HBx98UJVdxlQm08llimX2Seu47V+5cmXgelydrt6XHqZMplamjWQydTK/G2amVJ+5P9nfqWFmSmXHlkF1WjOlnK5yvzLnLaM192qYbUvHkUx+YimlLFiwoCq7sW3WrFlVWfNTSill/PjxVdndM10W1Fe/+tWq/Mgjj4Q6f/VXf1WVNYeqlDjWu2wqjKzMHGWYmUp9bv92y1S63bXM44FbVWY+5vJz3Ryh5XeZ7bu5TuuzBuXG/9GYIdWCkQ4AAAAAAAC946EUAAAAAAAAesdDKQAAAAAAAPRuVGZKtea1tP4ms73Wd1gzdPuZ/CinyyycDPfOrOpqe5l36jPnyNHj0IypUnyGiWafuEyVTKaQrjublzRmzJiqnMm0crlTun2Xn+XWnbn+g/anlFzuUyZTyu3j7ZQp1ZrppLrMlGq5tpl9yvb11iymQetx+sx5ac2Ual239i3Xj9euXRuWTZs2rSqfP38+1Jk3b94Ny6WUMmfOnKq8fPnyUGfixIlhmY7RDz30UKizYsWKqvzyyy+HOjpGurFmpA0z00jXnVlP39sHblW0d+D/cn9rae6jy49yf8fpvbyrbKbWbFJH98n97TUac79bMNIBAAAAAACgdzyUAgAAAAAAQO94KAUAAAAAAIDe8VAKAAAAAAAAvRsVQed9BsR2ua1MiK/WcYGFGsjmzrULaMsEZGeCflX2HLW0iZZw7I/aVlfXUtfjAvKcTPikhui566jXzdXJhK9nwnfvvvvusEyPNxv0PqwQZbctlQ11zgSNt4SRO649arsdZoiio9tvPY7WDw20BJ239uvWYMm+r8kgrR/e6Cro3K1HQ0THjRsX6hw9ejQsO3PmTFW+fPlyqPOXf/mXVfmJJ54YuP2dO3eGOqtWrQrLvvWtb1Xls2fPhjp//Md/XJV3794d6rz77rtV2d0jJkyYEJYNK0Sc4GPg9kF/B/4vnQ+UEsPPXZ3WuZ7+TeLmVV3N9VvD2HWM6Gpe2zdGOgAAAAAAAPSOh1IAAAAAAADoHQ+lAAAAAAAA0Lt0plTr+4mt+RjD2lar1iwWPW9uPZlMFXf+M++wdvUuutunluvWmk3mjkO333r99d3jO+64I7V9fc/X5aVkrv/7778/cPsui0tzTdzx6z5dvXo11BkzZkxYlqHHksn9ylzHVpmcnUzuUDavR2X6bSZTyWnJhvs4v1OZ4289t131464ypfrMOHQybSSTjVZKd+dWx4grV66EOm7dmo/gxrZ//ud/rsrf/va3Qx3NedKMp1JK+fd///ewTPfz+eefD3XGjx9flZ988slQ5zvf+U5VdplaLq8PAD4uMqWAG9P5j7tHu7+jLl26NHDd2v9cXpWu2/095Lav9Vqfmei6yZQCAAAAAAAAkngoBQAAAAAAgN7xUAoAAAAAAAC946EUAAAAAAAAetdp0PlIB8Q6uk+Z4NuugpdLiSFmLvzszjvvrMoaDltKDNUuJYatuRC1TIhvyzkapmzQe0YmIFLX7c6/hpE799xzT1g2c+bMgevRNuGutbsmGtDrfjdt2rSqrG2tlHi8LiBw48aNYdmwuOuv1zHbHlqCtlsDs1uD1vsMOm+t07qtPsMWu7q2rdeoK8Nso610jHLjyMWLFweu5+DBg2GZrsuFiE6fPr0q33XXXaGOu//p+OfC0Dds2FCVP/vZz4Y6Ov6dOnUq1AGAPnT14SHgZpT5W/eDDz4IdaZMmRKWfeYzn6nKx44dC3V02fLly0OdM2fO3LBciv8bUedN7u+4jNaPiI02/KcUAAAAAAAAesdDKQAAAAAAAPSOh1IAAAAAAADoXTpTajS+r6z7lMlLallvdj0ui0jzea5evRrqXLlypSq7d0NdNpLuk6uj68qco+z76l2d28x6W6+tvmfszpHuk8s0uX79elimuSYTJ04MdRYtWlSVL1y4EOo89thjVXnevHmhjvPuu+9W5WXLloU6+g71n/3Zn4U6d9xxR1XOvpucySvTc5vJGOoy02lYmVLZ8bDleFvPY0s2VWsdx+3TsDLtsm2k5dpmzmN2jGxZd2sO1zDv0Tr+ZXLf3O90PCwl3iPHjh0b6mg+hKvzN3/zN2GZnpNVq1aFOs8880xVdrkPuo/z588Pdfbu3RuWAUDXMnN94Fbl2vqYMWOqssvvdff/v//7v6/K7m/0kydPVuVJkyaFOt/4xjeq8lNPPRXqtM5tM3Pk0fiMpgX/KQUAAAAAAIDe8VAKAAAAAAAAveOhFAAAAAAAAHrHQykAAAAAAAD0rveg867CeH+Ver+qbIiYHsuKFStCnalTp1bls2fPhjrbt2+vyu+9916oo6HajgvjzgSkZQLTnZbz3xKO3uW6WwOi77zzzoF1Tp06FZbpddNw8lJK+aM/+qOq7EJ0z58/H5Z96UtfqsqbNm0KdTTo/P777w91fvazn1XlmTNnhjqOXn8XPqjn1n0MQH/n1tMakN5Vndag8z7HqK7WlQ3xztTpat0t23LLWgMiuwqa7DuMsiVo3e2jBn3rxzlKiWHkpZQya9asquzuLRos7j4YoSGmn/zkJ0MdF2x68eLFquyO7fjx41XZ3aPfeeedquyOVfcRAIaBoHPcztw8xt3/1c9//vOwbOnSpVV52rRpoc63vvWtqvzSSy+FOhs2bKjKP/rRj0Kdy5cvD9zHjC6fo4w2/KcUAAAAAAAAesdDKQAAAAAAAPSOh1IAAAAAAADoHQ+lAAAAAAAA0Lt00HkmaNiFb2UCubROdj2ZQO6uwrgzwbYaqlpKDHpdvXp1qKPB1idPngx19u7dG5adPn26Krswbg2EdWFwd9xxR1X+8MMPB67HceseO3bswN9pQKM71277uixzrV2b0bbtAiP1HLnfaWBvKTEQ2J2jv/iLv/A7+/+4du1aWPbtb3+7Ku/ZsyfU0RD93//93w91NLTPBeY7Ws+dW12WGUdcncy1dr/TZe53mfaXCbrOhOhntt8yZpbi+20m6Fu378LoB633o37n9kllrm1riGsm6Lu1TXalJbSy9WMcrR96cOOPcvefSZMmVeXx48cPXPeFCxdCHf34h6vjxtbM+KvnxJ0PbaPZj4EAv6rWDya4Npm5/+jv3Firv3Njves3OkfI7ndm3bczxp/hcX9rtMw/uvqoi1tXZj2tx+H+1sn+TdCXzPzTjVFu/qPnxB2//k3+n//5n6HOH/7hH1ZlN9e5dOlSWKbzJjfX0mvp5tV6vJm592jEyAYAAAAAAIDe8VAKAAAAAAAAveOhFAAAAAAAAHqXzpTK5FVkfzeoTjZTquX93NZ3ejMOHjw4cHuLFy8OdWbOnFmVH3jggVBn0aJFYdnmzZur8o4dO0IdfRfYvWeceafVLRszZkxVdu/w6vbdO736fnw2r0DfBc7kpbh38XV7mW2VEnNOXH7W1KlTq/KUKVNCnUymw1133RWWPf/881V5165doY5ub+nSpaHOo48+WpUPHz4c6pw4cSIsa71uKpOX06eR3n5Gdh/1vfY5c+aEOpqFd+rUqVAnk2mQyaJz/e+DDz644bbcerLHn8lUatFlXoRy+9i67sy9VWXOv7vW77zzTli2adOmqpy9t6jWTBvdb3f/07GVvJaR1TrXbO2TmUzT1v6XyTTM3P+1veuYWUqcj7l1u99phkqmH7n1uHFL7z8ur0VzVtzY4vLqbmeMUd3J9D+VyV3K5qe1jFGOriczjrh1j7b8KMf9PaRjkuZZlhL/ZiullOnTp1dll1f5zW9+84bbKqWUt956qyq7v/U047KU3Pwn099v1gwpxcgGAAAAAACA3vFQCgAAAAAAAL3joRQAAAAAAAB6x0MpAAAAAAAA9O4Tv0wmOLrQsJbwyS5DLLsKrR1msLEGy7nwMw0xX7VqVajjAtLff//9qrxnz55Q59VXX63KLsRYg9Zaz7U7jxoI6IJmM9c/E/7r1p3ZfiYM3h2bhtbNmjUr1Fm5cmVVdgF9GRoGWkrc70z44tWrV0MdPf8uVPTYsWNhmQbruRBFPZfuWuvvXGDr3r17q7I7jy6gUcMe3XXUtpVp667OMEO0lQuxdEGHc+fOrcoPPvhgqKMhikePHg11NPzxzJkzoc758+fDMj0nrUGfKhtGnAkazgQdd/VRj4zMBxuyMkHnek3cNcqEvzotfcuNYzqOZD6Y4Za5Oq3zD3X33XcPrHO7afnQjZO5j2T6f6ZvZ9bjZNpf5iMqbvt6v3NBu5kwXPc7d99Wek9258ONEXpP1jlrKbHfuPu/2+/b2bvvvhuWXb58eQT25Nbj+lGm/+uHjtxvumrHmQ82uflgV3PUkZa5RtnrmPnQl9IPOJQS/0Zz8wE3RmY+2KV1Wj9GM9Iy93r+UwoAAAAAAAC946EUAAAAAAAAesdDKQAAAAAAAPRu8MvkN5DJq9A67p3ClvV0KZMpkOGyiDTDx2Xz6PvhLq/FvUOueUXr168PdebPn1+VNWOqlFK2bt1alS9evBjquHOix5vJYnLvuWYyXVqzyPT94My1de/9utwFzT5xmTobN26syi5TQY/DnSO3fT22TF6VW3cmC8XlTGXe4VaZNtJ6rTNjS6uWcayUtiygrnJYSolt9OTJk6GO5gXOmzcv1NG8NJeN57KodLzLZIpkshhcf3DnRPuya3/DNKz7Vus9srWPZPLjMut2v9NjyeS1ubGuqzEikw3Udzu6WWm7cedN20Qmd8ndozPXVnNfXB237swcMTP+dDVHzmbz6fZacx91/M1kQ7p6mT6amSPd7jJzLbRxY1Qmi0nnyG5e7f6Oe++996pyZoxw+6N/D7Teo9zvWjMlh8UdfyZTy80bMjlf48aNq8ou41fPm8umcrm3Ldw1Go0ZUi0Y2QAAAAAAANA7HkoBAAAAAACgdzyUAgAAAAAAQO94KAUAAAAAAIDepYPOMyGKwwoVdtv6qGUt625dbybEWwPJXEChBjsePHgw1Llw4UJYpoHoq1evDnVWrFhRlSdOnDiwjoZzlxLD0Esp5cqVK1V5zJgxoY4G0rlzq+ckG+KYuU6ZEM9M0Klbpvvp2lYmfFBD9LJheJm2pUHXmeN37TgT9OhkQlwzIbaD1vtRy1rqZGTXM8wPNGScOnWqKrvAem037kMHGhC8fPnyUEc/qlBKHEvc2Kbjn2truo+ujbjfdRWQ21XbytTJjH9dfgwkE6LqQoxVpt9mA5qVjnUuDDkTou3qtASkt46Ht5uu5mit/Uavt7u3Zq5lJjS4dW6py9w+6jzCHWvmODIh8pmPkbg6LjBYj0U/qlFK3G8XBp35iMvthOD37ugcOTP/dvdDDb92QedujqR/x7kPXWX+RtJ5e+aDDaXEvnwzBGa748jstwsfV+7c6t+67vrrOObaUeYelZnbZj4YNtJ/e7TiP6UAAAAAAADQOx5KAQAAAAAAoHc8lAIAAAAAAEDveCgFAAAAAACA3qWDzh0N0moJDHWyIa7DCvLqcr16LJkwXrf9EydOhGUaiOaCJjVE24Whr1q1qiovW7Ys1PnJT34Slr322mtV2QVU6vFqqGAp8fizofZ6nlyIXSYgMNOOs+H7g+q46+/Om3LnLRMir+fEhehpiKgLw3YyIfJ9Bt1m1zVIlyHSfYYPuv0+f/58VXbXVtuR+xiC0uDHUkoZP358WPbII49UZReQu3PnzqrszpH+LhNYWUouxLQ1WDyjq+vdVTtq/WBJpk7mPGY+IuHOtQaLuqBR17Yy+5T5GANB5210vHHnLRNQq23CXdfWENmW9TiZgNrM2Obav67HjePuPOo46cKXdd1uHNf7iDtHkydPDst0bnPu3LlQRz+Q4+4/V69eDctuZ9mPAWGwrj5+pf3IfTDGtW39e8v10SNHjlRl19d1THBzpJs1/LqFG7Mz9+3MRyS6DIPPftjqdsHIBgAAAAAAgN7xUAoAAAAAAAC946EUAAAAAAAAepfOlMpkIWV0lWnxUfVa6mR0lSnjcgcymUru/J86daoqX758OdTR94rfeeedUOf++++vyi53av78+WHZ8uXLq/Irr7wS6hw+fLgqnzlzJtTR93Pd8WvuUSkxLyZz3tz513ePM7lTbt2ZLAqXBaHXKNuvMm1Ss1fce9aZTJXWvDhd1pLx0lqnb6Mtd6iU2N5cfplmH7hMg2nTplVll1eg+XWllDJz5syqvG7dulBH36l/++23Q51MFoxzM9wjMutpyT1yWu+jmTHa9e1Mn9BxPDNGZDNVWjIk3Loz4ygiPZfuerTcR12bdTkfOrZkMvXcWKPryW5f67lsSK2TuUdn5yja/+bMmRPqTJgwoSrrWF9KKXv37q3KOq8rxc/RFi9eXJVPnz4d6uickL41GJlS3dF+6/IKtY7r63pN3N9jLhv4nnvuqcrr168fuO4DBw6EOpm/IzJ/I3WZqTosw8yYdXXGjh1blTMZd9m/4zP0vpFpf61z5pHGyAYAAAAAAIDe8VAKAAAAAAAAveOhFAAAAAAAAHrHQykAAAAAAAD0Lh10PsygM113NmhttAXUuhCzTBh3JpBszJgxA3936dKlUOfll1+uyi5o/ODBg1X5wQcfDHXWrl0blm3YsKEqa6hxKaUcOnSoKu/atSvUOXLkSFU+evRoqONCAzUQ0wUUah0XoqnnMRsQp9c2E5Dq9jETYurajbaJ999//6N39v9wYdSZ0D4XrJeh+93VBxNuZZnA2uwYqe09E77owtCvXLkycFsamOvWvXDhwlBH25YLUddxJBu03dJuWwPDu9IadO50tZ861rl+nPkYhKuj4dNunz/44IMb7s9H/a6l37h91LE9c69H7H+ZD724e6T27UyoeClx/MvcozMfjHDzCLffur2ugoZd+3PHpvf7lStXhjq67Pjx46HOsmXLqvJbb70V6jz//PNhmZ4TN0fUvv3ee++FOpm+1dXHWLIB0SOJoPPhGWZAtJtb7dmzpyq7v/XWrFlTlSdOnBjqbNu2rSq7eVRm3pD9iMJIyvwdl6VjlLu36N9ImTD87DnTsU3vR26Zu0Y3a7C5YmQDAAAAAABA73goBQAAAAAAgN7xUAoAAAAAAAC9+1iZUpn3/DPvPuu63fui7h36ljqZ9/WzmT76Dmcmi6L1vXf3nqmu66677gp19H19zWYpJb4v6zKdXBaV5ky57X/lK1+pyjt27Ah1Lly4UJX37dsX6rgsqhMnTlTlixcvhjqas+Tes86cR0fbWyYvyl1HzavIvi+s23f7rdvX9uC257aVec/c/U7fj3fHr8eRyYvJZJO4ZW6M0PGmNRvH7bdrb4O278ZRXY8bI7UdlZIbozK5U7qe/fv3D6xTSilPPvlkVXaZZpppkskr0hy6j9p+Ji9g0LaydbrK9Mrkrrnr35q7mMld0nObzZ3Qdbk+osfrrlEm0yqTReRoncwcYbRlzIxWXeX1ZDK93P1v0qRJNyyXEu9Rbj2Z+5ibI2k+UyYvKdOP3X3c/W7p0qVV+aGHHgp1tm7dWpV/9rOfhTp6/Pfcc0+oM3Xq1IH75MYtPf7Mfex2z1S6WY8/87dd5h7pZDJenczcTs93po9m79H698/hw4dDHR235s2bF+ro8b/55puhTiZ31rWtlty7LN1e5lpnrm32b4SWLKYu85vcvWSQzDnK0nMy0vlhN+fIBgAAAAAAgJsaD6UAAAAAAADQOx5KAQAAAAAAoHc8lAIAAAAAAEDv0kHnTkuIaiYM3MmE37qARA1NzYSxu3Bit48aLOeC7XRdmRBpF9jqZM6/7qMLejt9+vTAOj//+c/DskuXLlXlr371q6GOXrdFixaFOpMnT67K9913X6izffv2sOzkyZNV+dSpU6HO22+/XZXPnz8/cD0uDDATPu+u29ixY6tyJmjPtXUXPqhhp6796TLX/jJhzE5m3VeuXKnK7jh0Pa7/ab/N9hE9lkyIvAse1OvvApv1WF09d251e66OBoS7NnL58uWB22/9GISeN3eO9MMDpZSycePGqnz33XeHOtr/ly1bNnAf3fV34evabtx1awmazMp86CLzoY3MRwUyMmHkbh8zYfyZ+5Hr23q8bhzTOu5en/lAgZOZI7R+DON21xIQnJnruT4yf/78sOyBBx6oyrNmzQp1NMTbGT9+fFWeMGFCqLNnz56w7Lvf/e7AdWto8bFjx0IdHdtd/58zZ05YtmLFiqrsPhjz/PPPV2UXxq73H3fO3PY1tNldfx3LW8PwWwN6W9Y90h86uFmDzrsMiFat9/FM0HZLsHT2WLX9698spcQPCzz88MOhjvZ1N9fZvHlzWKYfusr8jdzlx1gy50mPpfVadzVGZD9G1cW2Wuu0hqGP+Ng2olsHAAAAAADAbYmHUgAAAAAAAOgdD6UAAAAAAADQu3SmVCYLoDUvQN+pnTJlSqij7/SXEvMp3HuemXevdZ9c7oV7hzXzzqbWce+dZupofpD7nTtWPRa3Hs09cu8Uu3OiGTKvvfZaqDNz5syq/Pjjj4c6em5dXse6devCstWrV99wf0qJ72e73J133333hr8pJeZOue25LIZMH9H3pd274Jm8ssz70plMMyfznr3bR92ey6LIjBGaIZMZa0rp7t1rzRCZNm3awN+UEtvWuXPnQp3MdcxkWmRybjJ5PZlMLddmXBbb8ePHq/KmTZtCnQcffLAqT506NdTRvBh3zXQcK6WUI0eOVOWLFy+GOhl63jK5N447b3otW9fdmrvSch93ddyx6b299Tj0nuTGscy6XRaVzi2mT58e6ui9Zvny5aHOP/7jPw7cft8ymZ7a3jLX1l2jTM6IayP6u8wc0Y2/mTmCGyP0nuTy6jQv0N3r3e9chqaaNGlSVXZjlM5bMrmDpcTxz81tdN2//du/HeosXry4Krvjd+P/jBkzBm5f83LcelrukcPUOtfoykjnvrTKzBFb72OZsc71Ub23ZP6OdH8jDfqN28fsunQetXv37lBH72NPPvlkqOPGv23btlVl17fdMqXnO3Ncpfg2oTLXSLVmKjmZTM/MPrXOrTLrUdlMry7PUxf4TykAAAAAAAD0jodSAAAAAAAA6B0PpQAAAAAAANA7HkoBAAAAAACgd+mgcxfslQmk09+5oC0NaJw3b16oo0GLpcRgNw2jdHVcGKSGqLmgaReQquFvLjBM1+WOX4MdXdDlqVOnwjIN2na/0+25oO+77767KrswwMmTJ4dlGpq3a9euUEdDk13QsV7vWbNmhTouNE+XLVy4MNR54IEHqvKBAwdCHQ0RXLBgQajjAuI1/NyFeB49erQqHzt2LNTRfuS25YLmtU1mgq5dP9bz6IKuXZ/Q/XR9JNP/M0H32m41HNVtq5Xb/ty5c6vyqlWrQp0LFy6EZTomuD6q7ejMmTOhjo5t2RBJ7ctujNLz5q6/hjhmAkNLiedE+1opcWxxoZoaRuzuBxqqW0ocI904qsfmgna1rWc+WFFK7Lfu+uuyTPCkC7F09+NMn9DfuX6sx58J9SzFj1stdPzRe1YppUycODEs0367ZMmSUEdD9PXjHKXEe5IL2h6NQefallwb0bbk6mSCzjMhqq7d6lji+p+O9/fdd1+o8+ijj4Zl+oEKN7ZpW3Ljj47RV69eDXVcW9fz9s4774Q62rcyQe8uQF0/GFFK3O9MQLKbI+s4vmPHjlDHWblyZVXW4PVSYrtxfTvzMaDWjx+0BhSPNnocoy3AOKt1v/X4M3PdUuJY1nofHbTeUtrbkc4RtmzZEurouPXFL34x1Fm6dGlYpn/HufO/b9++qnzp0qVQR8cIN0ZmPuKSuf6jsT+Otv7W+vdQ60cUMv0og/+UAgAAAAAAQO94KAUAAAAAAIDe8VAKAAAAAAAAveOhFAAAAAAAAHqXDjrvigso00AuF0bnQiQzwWIafqeBuaXE8E0X6j19+vSB23Ihthoa68K/tI4LmnQBoRos50I0NVjYBRRr0KYG35WSC5Z0QbMaiOeCxvfv31+VP/3pT4c67vzrMndtJ02aVJXXrVsX6qxZs6YquzBOF+x3/vz5qqyhnqXEYFX9jVu3C+x3Idp6vC6wU6+t63+6PbeeTIizCyjVNuJClDXEWK9ZKaW8+uqrVdmFgWe448iE8WqfdPs4ZcqUsEzDJzVUtpQYrOwCIrUfu8Bu9zu9/q5OJmhSz4kLg8+0EXfdNETTtVHtoy6M2vURbVvuIwoarOzGaD1e10bc8Wv/d+df+5/7YMLp06ersgYYu205mfBV96EFlbmPumVubNf7xtSpU0MdvW4aYF6K75N6L3cfSNBlrh9riL6rMxrp9c4E5Lt7fSbo37Ut7X9u/qH9z13/tWvXVuXHHnss1HHzj3fffbcq61zDbd+NrRrs7cbxr33ta2GZfnzlv/7rv0IdHbfcR0X04ytf+cpXQp177703LNP7pgtI1jnJnj17Qh0d29wcyV1/vZZuHrd3796q7Ob/ek5aQ4Vbw3dvBjdD0HnLhzdKifcod//RPpn9O1LX7ebfGTq2ZoOm3Zio9FjcHOngwYNV+aWXXgp13N8o69evr8ruQxM6/zl37lyok/kYj/sbWeeE7vzr8WY+auD+jnFtq6WfuPM/zLElEyKuy7IfQ9Lz5OYImfOf+ahKan+afgUAAAAAAAB8DDyUAgAAAAAAQO94KAUAAAAAAIDefaxMKfdepdL3Fd1vLl++XJXde/8uQ0O5vAjNGXBZAPo7976yy8vRdblMC63j1qNcplQmZ8m9G6vv8GrGQikxi2rVqlUD97GU+F7x4sWLQx3NOXGZMrpP7lq7vCzdvsur0uyRFStWhDqa++PeRXY5F5l2o+3PZZFopoR7p9qdN30X3bX/TBaDXiN3/O7Y9J1h17f03WO3Hm3vro/83d/9XVX+t3/7t1DH0fHGjT96TlzugOZuuEwN128zY4SeI3ce9b1/926+y0LRPnL27NlQR9fl2prWcefItRt9P137mttHd/w6Jrh25LJYZs+eXZXde+56/3Hv4mfeu3fL9Hdu3ZrXM2fOnFBHxwjX/tz4p+fN5U7puOHO//Lly6uyy+aaNm1aWKZ92fVtvW6ZcdRl07lMIe1v7j6aqeO2dzPQNpnJ9HT3CK3j1uPav3J5Jbq9RYsWhTq/9Vu/VZXdfVT7cSmxvel4UEophw8frspPPvlkqPOnf/qnVXn+/Pmhjus3r7/+elV+6qmnQh2d27h+9NBDD1VlzTgrpZRjx46FZTr+LFu2LNTZuXPnDcuOyxRx2Tiac+PmlpncUT2OLvNi3LpU5m+dkZY5jpGWyYvK5OW4uVYmY9atW+/JbhzTscXlV+p93N1r3fin229tj9qPdu/eHeq4cUPHW5cX+tnPfrYqZ/5GcfNId050ey53V8cR94xA8zq7zFTL5CNl2nHmd611lGtHmfW4/da+5O51mXlUxugfxQAAAAAAAHDL4aEUAAAAAAAAesdDKQAAAAAAAPSOh1IAAAAAAADoXTro3AVk6TIXfpgJ+9KgNw2+LSUX9OyCbjPBXhpi6oLuXLCeBnu5oGkNrXSB2RoINnbs2FDHLdPwMRdQN2HChKrsAsp0+y6M2AW9arCgW7eGmC9YsCDU0fOYCfErJRc+rQHNu3btCnWWLl1alV1gbibYeObMmaGOtomFCxeGOtr+XYigC3rX43UBqXqNNIyxlHhtXWBpJqDY0bBBF76XqaPtODMefdQypdtzYYwnTpyoyi6wct68eWGZtj8Xvpnp/9qOXHt0fVTbmwvo1HPkQtQ1WPL48eOhjguo1PBJd2za3lxg8aFDh6qyO373EQP90IFrD268G8QFlrvwd23bmaBpd4/Q+6gLDHb3Tb2Wrm/r+OvGP723uKBpF9Cu/dbRdut+o9ufPHnywPU4mQ8dOJkQb7eekQ4f1naamaO586jjVvY8tgTEuvFHz7+7j7m2rf1/zZo1oY62LRcGrOvOfLChlHgs7t6u9+1777031FmyZElVdvNh1/9+/dd//YbrKSXe77Zt2xbqvPzyy1U5O0fX8+TaiB7/0aNHQx29J4x0vxqNboZzkpmPZcZj14+0r7mxx/2Npn/buT6iv3MfjNG/UXTuU4r/GI2uy41/Ot65+UdmjHbb1w89uLlF5u8Inf+68dgdm44bbmzVa+Lm2vr3kBuP3H7rvC1zH3P3Ub0m7u9x9xxBl7k5stZx7Vh/5+ZR7t6uf7e73+m63T7qetw+Zoz+UQwAAAAAAAC3HB5KAQAAAAAAoHc8lAIAAAAAAEDvPlamlL576d7F1Pcs3Xu++u6pexc1k8Xh3hfV96zduvXdX/e+qMtiOXbsWFV2uROZvAbN8Mhkarh1u2PT9zpdXsL8+fOr8rp160Id9360nif3Trsuc9dI3w9376u6DBPNgnrooYdCHc200rLj8kpczo2eW5expO3GHb9eW3f9XRaRtj/3Dq+eN5fN5d5zVq7fan93OUtax7VRPbfuWms7cvvTmiml++Te19f33F2mgXuHXfOBXF6Pvovt3rvX3CmX3+bajbblzHvurv+tXr26KmvbKyXmTpUSs6fc2Krn37V1HVvdOTp58mRYpufJZaPpPrlxXPfJ9ZlMpqFrN9on3Tiq++2ukaPbd/0vk/ujv3NjzfLly8MybX/uPqLnzV1/PY7WTCN3rHq93XoybWQ0Zkq15GVl5l/uOrpxU6+ty1TUe7LLZnvqqacGrsdtX/f7K1/5SqijbSKTe+LmEa7/nz59uiq7LJTHHnusKrtsPj0Od101P6uUUpYtW1aVXV6ijj9PPPFEqPPII49U5RdeeCHUee6558IyvZe7+5/m1bi59pYtW6qy61eZsc0df6b/6/l3/T8z1ximkR5rMjKZoplz6+aamkWmfa+UmJ9YSsw0c3Mknce5e5Teo91cJ3OPfPPNN0MdzcJ1Y5T+befaoxsjdS7n5t96/C73Upe58+jofrp59MGDB6uyG0d3795dlTWHtJTc3yiu/WXyc3VOmM2UymRD6+/cPFbnhO44MnnBbm6p+5TJxnV/o2SM/lEMAAAAAAAAtxweSgEAAAAAAKB3PJQCAAAAAABA73goBQAAAAAAgN6lg85daJvKBL269WgYoQtjy/zO0RAztx4N32sNDHTr1rAxF3StYcCnTp0Kddw+ZQKyNURPA5tLiaGd586dC3XmzJkTlunxuqBhDVFz4XsabOqC1jRErZQYyLdkyZJQR/fbBbRpO3Ihrm6fMmHDek1cG9HwTdeP3O9mzZo1cH/0fLt16/G6UHcXPquheZkxwoXvadt2QaP6u0w/dsvc7zL7rcfq2rELUXz77ber8uHDhweu27VRDYh1bX3x4sVhmYZGaqhnKbHduDDETIihCx/WQE7Xt3QfXRi49hEX9Oj2OxOirOfEfehAx1p3HK7dakCl9tlSYtt2/U+PP/NxglJif3cB5XpOXBiwjmPugx1ujNa+5X6n59ZtPxM06mTmCKr1/p8JFe+bG2+VHq+bf+nYlmkjpcQQa7fuRYsWVeUFCxaEOgcOHKjKrj24fvOjH/2oKruxXj+Q4sZx7SPuWrtzomG7Lnz3M5/5TFV2x6Zjmxv/Nm/eHJbpfC9zb88EFLuxfu/evWGZzgnd3PKBBx6oyu48atCxW4+7J2v7d327JaB8pEPNnZsh6DzzMYrMOJr50I2717u/rXT+4drxzJkzq7J+ZKmUONa5eZybW+j4px+VKSWGeLv5h34gwY0jJ06cCMv0XGpgfClx3HYfVdCxxo0R7kNbuv3M37HuYwh6/l2ovDtv7u9WpfMmF5ivY427/o7Ov93fcdq3XZ3MB4vc2K776fY7E3Suy1rnbKN/FAMAAAAAAMAth4dSAAAAAAAA6B0PpQAAAAAAANC7dKaUy4LQ90zdO7xdvdPtshH0d24fM3kJut/Z98UzWTiZ96N1v91xuH3Sdzbd7zR3Sd9fLiVmOMyePTvUce85K81vKSXmA7h91JwHlynjshBOnz5dlS9duhTqaPaAy92ZOnVqVc5kGrl6mfwStx5d5q61ez9Y3+t2fUvfodb350uJ59b148z7yS7TJ9O29bxlsqHctlpzHnT7mbwCty23T9pGXNvSMUlzmEqJeQHa9kspZceOHWGZZh+4vq3tSMeMUuL1z2SjlRLfxXfvouv2XRvVDIFMNlEpub6leSVnz54NdXSMcNlc7rxpm3Bjm15vza8oJfZJ19bcNdHsGZcNqH3L5Q5qFqK7j7p+q/co10ZUa//L/K4lYyprNGZKZc5by7l1+WHLli0Ly+bNm1eV3X1k1apVN/xNKbFNutyT1157beD2XaaM9ncdax13HK7faM7T/v37Q51t27ZVZTe26Di6c+fOUMf1SR1v3fxHj9/dfzTnZf78+aGO2289Nrffuk+uHT366KNV+Zlnngl13PnP5JzomJTJZsqOP326GTKlMmN0RmaOmBnrSsnN0TSLKTNHcOOYu/+/8847Vdndo3Xe5vqo/v3l/mZbt25dWKbzf3eP1nPrMp00i8/lV7lxY82aNQO3r8fr7j86Juv1KMXPrXTe6DLF9L5x7733hjo6trjcVTdv0ywol6ml8103jo30+NOV0T+KAQAAAAAA4JbDQykAAAAAAAD0jodSAAAAAAAA6B0PpQAAAAAAANC7dNC5C4jLBMu1hG+5oLlMaJ0Lem0JH3VhZJmAdg2VLSWGmLkwXg0tc2HALjRNQ+OWLFkS6syaNasqr127NtTREDUXxucCMjWQzYX/aWioCwidNGlSVdZzVkouDDFTx11HPX7Xjlq5YPPMPmVou3F9TetoeyglBoS6oPNMsLE7b25dKnOO9DgygZVun9w+trQttx53HfWaZILuM+fRBf9r0GQpMWxy4sSJoc7ChQursgva1GDJ6dOnD6xTSjwW1x704wtu/NV9dGOtGyMz9x9tW24fNejbBc278FE9J67/aRh95j7mxkgXEJoJFtfz7e5Ruk+u/bkPXejYqh9ecHXc9ddr5II++5QN0R1pOra4cVzPt5t/aBivW4/7+ICGX3/xi18MdXRuoaHepcSx1YWRP/HEE2GZtlv3MRRt7zofKSXeI916tm7dGpZp0LeeR1fny1/+cqjzxhtvVOXjx4+HOn/+538elulY7s6tHtv3vve9UEfHPzfWu/DfQ4cOVWUXIqzH//jjj4c6+jEMN466dev1d21Ux1Z3j9b7iGv/mXnMMN0MQcduH3VZ5m82Vydz/JkxOnPPdHMEvbe5v4cOHz4clukHUtw+ami6zodKiX3CzZHcHEXHBPc7/btN+2Mp8fjdfMCdk40bN1bl+++/P9TROaK7RjqOu7mG+9CXHpubR+sYmQlRd2Nt5m/L1rlNZt2Z/peR6Uet2+I/pQAAAAAAANA7HkoBAAAAAACgdzyUAgAAAAAAQO94KAUAAAAAAIDepYPOXbCYBmS6oF/9XSbEzq0nEyLsZAJSNTTWBV26YEUNO9MwulJi+K4LqNWgNReQ5gIaNdhNg8dLiYGgbvt6Ht05ciFqGnbrrq0GYro6Z86cqcouaNcFa2q9TNCkC7/TYFfXrrJtUuk+ZQLi3LnO7FMm/NG1o8mTJw/cJ0e35/ZbA0Hd8WtopIaqumWtgeWtofLKhTG7gMTMNVKZwPrsRyW0nmv/O3bsqMpHjhwJdTQw1401btmMGTOqsuvHGr7prr9u3wW2ZwKa3TnS/c58sMKNNe4eqe3f7aPukxv/dD1uH13b1nOZ+YiIG/913HAfrHDb1+25Onou3Ril/W00hvq2fFRl2DSg1fVR3W8X4qv91gU9uzb5pS99qSo/+OCDoY7Od9zYpuO96/8LFiwIy86dO1eVX3/99VBHx0Sdj5QSxxEX4vvcc8+FZTr/Wr9+faijfdT1v3Xr1lXlr33ta6GOCz/W8ca1UZ3vumPTD2asWbMm1HHXX7mgYw1xd210xYoVVdkFFh89ejQs0zE584GCTEDvaBx/uprbDFPr/KflYzTZvxlb9sn1UR0T3cdA3LILFy5UZXf/P3nyZFV2H1pZvnx5VXbzKDduzp49uyq786H3X3f8+sEK14/dHEmPzc2j9f7j6ujf2pmPM5US73fugzl6jtzfHzr+ZbZVSrxOmbbu/v5Qbh7VOrbpsQxzrBn9oxgAAAAAAABuOTyUAgAAAAAAQO94KAUAAAAAAIDepTOl3HuG+s6mq6PZEy5TQzMlXF6Fe89TM50yeSHTpk0buG6Xu+SyUPRdfPcu7vTp02/4m1LiO8TufV33frDmRbj3VfW9Uvcu7pUrV6qye1/Zveeqx6L5CaXEfBrNeCglvp+beae2lHje5s+fH+roe87uOur2s++iZ/dzkJZMg6xMpkumH2fyuly7dcuUjgluH9278CpzPVwf1WWtuRPuHGk9t2737nuLzPbdu/BK81NKibkHLj9v5syZYZmOJXPnzh24PZdNov3WnUe9H5QSj9/lHOj9JpON6N7pd2N7JgtM153JInC5L+7YMrmPeo5ce9T1uPuoyzDI5Hy0yGaqDUtrNknfNK8p07Zdpon2kd/93d8Ndf7gD/4gLNN7spv/qUymojuOS5cuhWW7d++uyi+//HKos3bt2qrsxj/NnXLzKLffOk6uXr061Mkcm2YqPfDAA6GOa396T3D3CM3ZcpmCOm9zcz23TPfJZYrpPPbVV18NdTTTxeWuujG55d7aev8faTdDplRmHp05jswYkbmOTiYb0f2tpVx7dNvXccuNLdr+Xbs+cOBAVXb5Vfr3aClxbufo38R79+4duO6VK1eGOkuWLAnLdN7ijk3HZPf3QCab0J1bXeYydnVdbv517NixquyOQ//WLiW2E3eP1DqZjOXseJDJlGoZ71rnaKN/FAMAAAAAAMAth4dSAAAAAAAA6B0PpQAAAAAAANA7HkoBAAAAAACgd+mgcxeQqGFvLqBLg81cQG4m6GvGjBlhmYbIuvAv/Z0LSNOgs4sXL4Y6LqBL99MFxmlApgs/0+25wEi3bheapjREzgXGa2jgmTNnQp09e/aEZRpsmwmadOFrmfBB1yY0/P2tt94KdTR8zoXvaZtw4dyu/et5c0Gj2m7cevScZEPVMyHmGXrdsgHumWBfvf7u+LWOC0zWc5IJLHcyQc+Z43fXKBO+mQmMd9vPBJS73+kY7UKMNejYbUt/54IeHQ0fd2OLjr/uHJ06daoqu/Ews0/ugxHHjx+vyi5o3bVJ5cYoDT93Yex633Ljup7/bKivtgl3bnWZO0c6Jrp25LR8DCITUDsaQ31HY9B5ZvzVgNjPfvazoc7Xv/71qrxhw4ZQJxM+fujQoVBHQ2wzwbtuWxqYXUoM+818MMOFaOtxuPX83u/9Xlh28uTJqjxlypRQJ9O39Hh1XlmK/0DQa6+9VpU1DLmUUv7jP/6jKu/cuTPU0THR3f90rlVKvLYuIF8/kPHUU0+FOrrMfVSjNehdx9LM3CIbYt2n0TgmqtZzlPkYTuZau2ur/S9zj3TXX7n7caaNug+W6Hjj2r/+zo1R7m+bzLxJ+7Gb6+zbt68q79q1K9Rx458+E1i3bl2oo3MSN0fVc6vz2lJ8QL22JTf/0+27OZKeI7d9d/01WN1tX6+ba3+67i4/mNWidVujfxQDAAAAAADALYeHUgAAAAAAAOgdD6UAAAAAAADQu3SmVCYvwb2vqu+5u/dl9f3ITO5OKTHnw71n7zIMlL776d4FzWzf2b17d1V2792fOHGiKrvjcO+w6jXJ7M+sWbPCMs05cddx2rRpA9ftzpFe20xek3sX2x2bHot7X1mXuePXLAr3Tm/mHXL3vnzLe7XZvIjM+/n6DrV7p1ozNLL7rNe7tY/o9tz2dT1uW26M0nfPM+9ZZzKdMtty68pcs0zumqvj+pZe29ZMDd2+y+9zGQa6bpeXov0tk1fkrqPLRtB1nTt3LtTRvuUynbTfuP7ochbGjRtXld04kmlbeh3dtXZ03ZncKXduM9vLtO1h5r71mZeQyfgbDbS9aXssJd4T77333lBn4cKFVdm1I5ch8uabb1Zlzd0oJY5Jbh91vHHXWrPpSonH7/KiNm3aVJXdPVKzUd31f+ONN8IyHUsyuT9u+w888EBVdufItT/NnnHXSOcbLi9MM/3cWKu5W6XEvFTN7yullEWLFlVlbWulxCwsd63dHFnHZDf+6jJ3/pW71490ptNIbz8jM0d0f2voPN7NEbRPuL8HXKau9lHXt3Uf3VxH5wSZe20p8Vjc3ELbu/s7Rn/n7tku01n30/UtvW6ur+nY6uZR7pwsWLCgKmfmEZlsVsfN0fQ8ufOm9x/XtrSNuHPk8qIzfyPoeJPJmM387VVKbt7U1dwmM0aN/lEMAAAAAAAAtxweSgEAAAAAAKB3PJQCAAAAAABA73goBQAAAAAAgN6lg85dsJYGZLmAtMOHD1fly5cvhzqZYFsXUKYBhS5YTEOs3T5q+Fom1LqUGEjmjk1D0134m4aIuRA3F76ov3Phl0uXLr3h/pQSw+c1VLOUUtavXx+WZUJ0dd0uRE6XuWvtAko1INmt+8KFC1X55MmToY6eb3f9XfhgJmxP+0gm6M0F3bm2rW3ZhThq2F0mxDrTjl09N0a4dSndJ3etNSDVtTV3bJmgWz1vrv1lrrWONaW0BRK6fqztz9Vxx6brzpy3TB91MuHrbvzVMcJ9aGDu3LlVef78+aGO6zeTJk0aWEf30d2P9Py7Oi5YUs+Ja0e6Lrce3Ud3HTNt1J1/7duujvYJNx65NpIZ7zJ9JBN+2lX4eWZbzs0QdD5jxoxQ5/HHH6/Kn//850MdDaPevHlzqPO9730vLNN5y+rVq0OdyZMnV2XXt7Rtuzbj+o1u331oZvv27VXZjaM6/3HbP3LkSFim4edbtmwJdebMmVOVv/rVr4Y6Gqzr5iiurWuIsJsj6BzRrUf7xNmzZ0Mdd0/S6/bKK6+EOnv37q3KOmcrJTfX0LG+lBjingnaduOf3pPcvCbzEZVhuhmCzt0YqW3E3f/dB7KUXkfXHmbPnh2W6d9k7t6q7cjd63SM1H5Vir9GGn7tPgag7c0dm46b7j7mlukHCtw8Qo/NjSO6fTeOu76tH03IfDDGrbv1g2l6j3AfbNA24cY6PbduH6dPnx6WqcwHk9z4r/M419cyY9SIj2O9bQkAAAAAAAD4P3goBQAAAAAAgN7xUAoAAAAAAAC9S2dKOfqeoXuH8cyZMzcsu/U47l1Y3V5mPV3mPuh7nZncrcw+uvW495z1HWr3vrS+i/3lL3851Pn0pz9dlV3ug3unW48/c201P6KU+A65O1Z3To4ePVqV9d1kt26XFzFx4sSwTOk75aXEc+Lalv7OveesOVenTp0Kddy70HpOXO6YLnN5Ce+8884Ny6X4vCTNlXGZHnq8mf7nztGBAweqsnun2h2/Xtt169aFOvoOu8vP0j7iMiVcH9H2596X12viMhWUO9Zly5aFZXpN3HnTTAV3jXQfs31Uz79rWzomuCwGzVRx2Xyuj2o9148y47hy63HtRs+Ty6LQ8+1yFzTDwGUauPFP99Pdf/R43XnM5CVkuLaVyULJ5C46emxu+zqOubwsXZbNtBtp2t5mzpwZ6ixcuPCG5VLi8bt7lDtvmk/psuC037h+pO3W9SM3Jmo70Wy6UuJ1c2OL3v9cm3U5K7rf7rzpsbhsKs1wmjZtWqjjxq1MXuC5c+eqsuvbuh53H7t48WJYptfStRE9R5m8vEw2jNsnN9fTOaEb//Q43FynNYvudqdzC9f+NXcpkzHr2uPu3bvDMu23mdwf10c0i2jr1q2hjrtHaNty93at4+ZR2kdc7pSb/+vc1s0RdIxwmXLaJ9y9JpOX6fqxzhFd/9f9duNh5m/0TKZhJr8yOx5k7m26LneNWudkow3/KQUAAAAAAIDe8VAKAAAAAAAAveOhFAAAAAAAAHrHQykAAAAAAAD07mMFnffJhYZp2JkLkdNAsEyoqgtRddt3gYxKw8dc0JxyQccutG7OnDlVedGiRaHOk08+WZXvu+++UEfD5zTUrhR/bjV82AX06fnfvn17qKOhfS4M1IWoa7Cgu7YabOnC92bMmFGV582bF+q48FcNH3Qh0hrIuHHjxlBny5YtVXnXrl2hjmsTuj0Xvq2BhC4gUYOVsyHSLrS0hfY3FxirofYuDNHR8+8+BqBtxPXRsWPHVmV3PrSO254Lejx27FhVzoTouu27ZTr+uLFN25Eb13SZa2uu/Wv4uwto1HHEhZjq+OuukdtvHTenTp0a6mRCszMhlpnz5tqtXjcXIqzn24VaunOr63ZtS5e5oGkd/zOBxaX49q50ey7oU9d9+vTpUMe1fw27dWHw+jt3HfV6u/vhaAw617bk5gh6vO6a6T3BtRG9j5YS+5/7YIb2CXf/yYTGujraT9wYpeHnbv5z/PjxquzGKBeQrHMJN0fSD8u4vq33P3cd3dii93vXR7X/uzmahiEvWbIk1HEfaNBgYfcRi8cff7wqu/nf008/XZVXrFgxcB9LKWXz5s1V2fVt7SOu/ev9JvPBilIIP1eZD224uYUuc+O/rtv1dbdMxyjXjvR6u7meziPcWJf5YJe7/2t/dx9s0Dnizp07Qx03/9HtuT6SmW/rmOg+KuA+BqQfv3Af7NHz7fZH7+3ub0a3T/q3lWujeo4yHyxzdTK/czLzj5ZnHa3rzshuP/yu6VcAAAAAAADAx8BDKQAAAAAAAPSOh1IAAAAAAADoHQ+lAAAAAAAA0LuhB51riFYmMCsbDpgJ0tLtZQJK3T5mwnBd+KEGJLoQRQ3IdGHcLnx73bp1NyyXEoNeXfibrtsFjU6ePDks06BxF6y3Z8+equzC/3SZCyN0NHzOhYHr+XZBoxp06wJzXZvQsF8NbCwlBjS6EE8NbXSBoRq06n7nrq2uyx2bBhu6/pe5Jq4/ZsL3tP+5EF091y4M1v1Ol7n2p1yItm7fhUG68EsNmnUhlnq+XTvSvuaO1Y0tui73u8zHF3Tdrj+4c6LtzYWYZvZRx0gXhu/OW2b8031yx5a5j7k2qeOmC4PWOq6v6f3HhbFqGyklHr/bvvZJF/Ss599dI3dOdLx1AaU6jrkwcr0n6XGV4tuf9ls3/umxuHak++2OYzSGGut4d+DAgVBHw6cPHToU6miwtAu6dh/o0DmBC7qdNm1aVXbjqF6jzJhVSrz+bh6n8zZ3bfWcuA9muPBh7duzZs0KdXS+59qx9i1XJxPsrKHCpZTy4IMPVmXXt/RjFBrOXor/iIp+xOOhhx4KdZYvX16VT548Gepo0LO71504cSIs07HMfaBB1+Xm8ZmPumTuG7e7zPnIzCMzf8e5fuz6jc6t3VxbuXu97rc71swczX3USo9FP2BUSry3u48xuIB4nUu485/pIzpuu7HWHb/O5dw4ptfN3SN03e5e74LOtd24eZSu280RdK6T+aiEW5b5YErmeYRbT6ZvZdY9TPynFAAAAAAAAHrHQykAAAAAAAD0jodSAAAAAAAA6N3HypTK5ENl6rTSnAG3LV2WyYty72K6d0j1/Uy3bs0LeOyxx0KdxYsXV2WXTaDv3Tsui0PfK3Z5AfoO744dO0KdjRs3hmW6LpcXo9fIZeoM2p+PkskiyryLrtffvXfs8qqUy4J68cUXq7LLS9i/f39VfuONN1LrPn/+fFV2x6/vfrt3uvWauHfh3XnTdbl3kXWZu7aa1+Dee9Z3+N3+uOPXTIndu3eHOnptXT/OvHfv8or0/fTMu/hu3Xpsro5ms5Ti36tXmjvjjl/3WzNGSvHXVs+Jq6PH78YRHWsy2VxumWv/eryur+uY4HIf3Lihx+bOreZFuHuNnkd3/O7c6vG7vAxtIy73TtuIa39u+3q+3TimORcuv0WPw23LZYjo9lwWh9Zx91Gt43InMu1vmPMhR8+lu7dopqQ7t5qN5DKl1q5dG5Zt3ry5Kh89ejTU0Zwrl5em45/rxzqPKiXmfLj2p9fb9T89/mz71/vUvn37Qh3NVHH5obo9dxxu3qjH77Kw9Hy79Sg3/r3++uthWSYvUcc2l02m1//RRx8NdVzOoLaTrVu3hjqaV+Ouo55/N9Zk8lowWCZ3qsv8rpbfZTLtMtlIpeTmdgsWLKjKCxcuDHV0TrBy5cpQx2UxZbJ59Ry5bF5d5vIDXV6yzi0zeZFurqt/a7hjdeOPjv+ZLDJ3/d19S7m/UXS+6zKddY7m2pFeo8zfUaXE8+TurZlMWd1e9u/4sJ6mXwEAAAAAAAAfAw+lAAAAAAAA0DseSgEAAAAAAKB3PJQCAAAAAABA79JJVKMxsM+FBisXLKY0xMuFoTkaIu7CF1evXl2V16xZE+rMmDHjhvtTig/j04A0F+KqoZEu6FbXraG2H7VMw9ZciJqG37njmDp1alV2bc0F67kgN6Uhfu7aaoicO//vvPNOWKZhn25/9HgPHToU6rz88stV2V0jd040SM6FH+s+ufOf+RiAO7bMBwLcskHbzwQmuzDATB/RUN0sbduuHbllLpBQ6Tly578laNDVy4QPuvVoYLKOWaX48HNtkxoYXEouIFL7fyawsZQY2unCN3Ucz4SBurbmAjJdaKfSa+TWrW3LXUd3HjXE040/GjTuwvF1/HPH6gLKtS25/qD3Fhciqv1YP2BQig8oP3bsWFU+ceLEwN+5fqx9NNOv3e/cxyD65O4RGn7uzr/2t89//vOhzpNPPhmWaSC6C4jXsVznA6XEMGy9rqX4cUvPvwsI13W5oHEN6HXt37WtF154oSq7jwjs3bu3KruA4jlz5lRl9zEINx/WMdndNzT8XMfDUmJAvbuObvzX7f2v//W/Qh29Jxw5ciTUmT9/flV2oeYu/Fh/d/DgwVBHx1J3b9HznQ06R83dt3S8zXwwolXLfNTJzKPdWO/+jtG/I9wYvXTp0qq8bt26getxgeWuj+qYqKHqpZQyffr0quzmGq5PKnf+dUx093E9FjdG6TVx9zE3/9G5jftgjT5HcH8P65zEzf3c3+j6Ozf/0vPm+pGO/24e69qfzondtdU6maB7t3133RSjKAAAAAAAAHrHQykAAAAAAAD0jodSAAAAAAAA6F06U8ppffe2Zb1uPfqep3sXXN+9dFkQuh73vuTixYvDsvXr11fl5cuXhzqaveKyMPQdTvdu+iuvvBKWaRbBvn37Qh3NC3GZCvoOr3un2b2fq+c2856rO4+6PXf93bXV8+QyFfS9bpd7ou9Lu+Nw7/nqfu/YsSPU2bZtW1X+/ve/H+ocPny4Krt3cV2byORFZfLR9By5d+Ez7/ln+rqro8fhtp95X9+1EX0X27VjPUfuXOv1d1l1mYw79059Jp9G9zszHpYSz5PLwtB1uePQdbttuX6jbTmTTeayCWbNmnXD8kf9TrfvxjbNC8nkdbk+6tqkti03/ug1cbkzmqmQza/S8d/ltWheTybTwG3f5czoutw+6vbdPUpzH/Q3HyWTIaLt1rV/bROZbEK3/ZHOlDp//nxYpm3SZfM8++yzVXnz5s2hzpe+9KWw7OGHH67KLq9Rz60bI3TekJ0j6Dj1wx/+MNTRedOyZctCHZ0juWwWl8Wh2SM/+clPQh3N9HL9SOek2dxTHdtd39Jr4sZ23Sc3RmVyL1370z7ickc008tlA2bm/66Pan93WTzab929zvXtzPhzO2nNdMrkdbXmTum6Xd/SOm4c02w0zXgqxecVaYaT5vC5fXLr1nbsco8y5yjzd5zLvdV9zOau6TJXR7efyfR02VTubzTNmXPXSMcNN4/TbET3t4Ybf3S8zeSVZfJjHTdG6pzYjX86JrtnJDr/dvPx3/md3xm4j/ynFAAAAAAAAHrHQykAAAAAAAD0jodSAAAAAAAA6B0PpQAAAAAAANC7dNB5JkQrUyfzu0zQVykxkM2FjykXULlo0aKq7IIuXUC3rsuFuGr4oQs6e+mll6ryyZMnQ50tW7aEZRrk5kIkNfwtExjoZELEXUCmht9pqFwpMeh53rx5oY4LFtSwYRc0qet2x7Fw4cKq7IIO3e9OnDhRld01+vnPf16V3fFr+FwmsNstc/0mE2Ku3HXMtJtMiGEmxDITGJoN+suMSZnt67l116g1DN6tS+nY5o7fBRRq2KKro+Po1KlTQx0NOnRhyBr0WUr8iIALsZ09e3ZVXrFiRaijwZ4aKlmKDwjWQFwXYqnr1uDzUmKwr9u+u/9ou3XjiC5z11ZDS929xu2TBoIfO3Ys1NH7iAso1XuSCzp2Y5t+2KP1QwN6Hl0Ya2tAqPb31jEqE3Q+0tzYriHy7mMsem3dtd65c2dY9oUvfKEqf+5znwt11qxZU5UzYayOO9fatzZs2BDqaLvRcPZS4sdI3BjhwofnzJlTld34+y//8i9V2R2rHseBAwdCHXf8jz76aFXWD6+UUsqqVauqshsjdGx1Y4SbN+k45fqWjgn33XdfqPPII49UZXf/cWOizv/cfis3tugyd65H+iMGN4PWoHPto63jauYauT6qcxT3oZWlS5fesFyKn3/o9tz4q/dfF+KtbdvNtTIfyHnmmWdCncwHo/Q43N9sbllm/qPn2wVt6/HrR15KyZ1b/buulDj/ch8n0vHH3Wvd+KPrcuOotvdM+898HKyUeE3c/ScTdK4f+nAfoyDoHAAAAAAAAKMSD6UAAAAAAADQOx5KAQAAAAAAoHc8lAIAAAAAAEDv0kHnTmuweVc0tE2Dr0spZcGCBVV55cqVoc78+fOrsgvaduFfGiyugaGlxPBPV+fo0aNV2YXIuoBYXdYa9KxBZy7ELRMs68LXdN0uoE+P1wWGumubCWjTQFIX/qfhgy6gzoUfvvDCC1X5+eefH1jHBf1lQgTd+dfQOhdiqKF5rj3o9jPbcjJ1XNvS7WdC9d05cuvOBB2rzHl058iFGGqbdOvW32UC011gpzs2Hcvmzp0b6mjfcmGcuszVcfuk4ecahuh+566tjnXu/LsxwvV3tWnTpoHb1/Pvxhq3fT02N0bq9lavXh3q6Lhx6NChUMeFH2tosQsxfvvtt6vym2++Gerofcv1NXdsLXOEzAcTslr6vwsazYyjzmgLOndh+Dre6nyklFxgvwvR1vB3Dd4vJd6jly9fHuroBxPcdXTh4xpsq4HlpcSxxQXk6nG4EFc3t5s5c2ZV/uxnPxvq6MdQzp49G+osWbKkKm/evDnUcQG9n/nMZ6py5kMLbq6p44/rI+68adhuZm7hxmy9R+7duzfUcfP2TP/PfERCx3t3j3B9y833bmet86/MOKpta/z48aGOa1v694aONaXEj2G5dqx/I7i/Y1xAtLYb1450/NmzZ0+oo3MNN45kPmLkQtz1nLj5j/Z1N9a48G09NldHP0bl5h8XLlyoyu4cHTlyJCw7fvx4VXYfGtNluq1S4piQ+ThVKbm/o7VO5oMt7lo7Let2c31tE66tf+Mb3xi4P/ynFAAAAAAAAHrHQykAAAAAAAD0jodSAAAAAAAA6F06U8q956vvFWZyJtw71rpul83i3mHUfBJ9776UUhYvXlyVZ8yYEepoPorLBnDvomoWkr6bWkp8P11zqEqJ7/C7c+0yLPS95q5yN9y5dvuk3LXNZPHoOdJ3/EvxGTa63+5dbM15efzxxwduX98fL8VnOPz3f/93VX7xxRcH7qN7X1rf/c3kd7l1u+vfUse1Nbf9THvTY3Hr0XfP3bvQmWyezLFlcqcymVKurbtlmUyNzPWfMmVKVXbZUO4dbs0n0vWUEvNRXBaAjsluHNNsgFJiX9aMlVJizpHmwJRSyrRp06qyGw8czVlybUvfhXftSPMpsmOtjkmu3bqcw0Hrcdl8LgtFs6c0v6aUUnbs2FGV3fifyVTqKmPS9VGVPf/alzPzmEymQuv2R5pmvJUSszBcFolrE4PWU0qcI+3bty/U+du//duqfN9994U6mjPlcjddzpDmY7ksEO1/c+bMCXV0HHeZKi53UsdEt+7Zs2dX5e3bt4c6OrYfPHgw1HFZXOqRRx4Jy15//fWqfOnSpVBHx5GNGzeGOprNV0rsy+68aZ3nnnsu1NFzollZpfi5vY7/mf7v7m06j3T9n/yowTLztsyY6e7/U6dOrcpurHM5U/fee29VdvcxHdsyuWsuv85l6u3fv78qu/6nY7L7O0LnbZn7eClx3HLnVq+b276eN3et3d92mUxNzXRy11bHepcppRm/pcT5j5tbZXKfbnfu2rbgP6UAAAAAAADQOx5KAQAAAAAAoHc8lAIAAAAAAEDveCgFAAAAAACA3qWDzjMhpi5YLRMirEGHLmhXQ81LKWXDhg1V2YX4akChC4M9ceJEVXaB5Rr0WEopp0+frsouRM4tUxr054KOW2VCrIdJt+eurYaBuqDjpUuXDly3C9H8jd/4jarswkg1aM+FiD711FNh2csvv1yVXUCmBuK5a6vX312jYYaYt9Rxy1z/12WZwPRM0LA7jy5od9B63O9cG501a1ZVdqHiGsbt1uXOo65Lg8dLiQGdLozS7beey0mTJoU699xzT1V2QdOZMToz1rnwSQ0NdWO0fnzCnSMXvqnB6i6MVs+ROzYNGnfX3wVk6jlxH1FQGnxaShy3NBy+FD9uvfbaa1V5586doY72pUzQ+GgL8P44Wo4l+5vRFojqAnozYfB6HG4cc8eqffmtt94KdbS/6XyslPgRETfWz58/Pyxz80b1uc99rirrx3FKiWOLu4+4MUE/EKHz0VLiHEX7bCml/Mmf/ElVfuCBB0IdF2K+devWquzmyK+88kpV1uDzUmKwvPvwj7v+OidzH5rQ+Z4Lo9bw+2XLloU6u3btCsu2bdtWlV271XuCG/+0jutH7v6X+UDA7aT1fGj/c/MfbduuP7r7v95L3Ye2dJm71tpu3IeX3NxGA6Ld73Ru4fpa5p7kzr/OpVwf0Tpu/qXH7+bo7iNiDz/8cFV2fUtpOHkpcRx75plnQh33EQkXLI+Rw39KAQAAAAAAoHc8lAIAAAAAAEDveCgFAAAAAACA3nWaKeVk3iHWd2Fdpox7F1h/595X1fdFz5w5E+ro+7r6jm8puSwGd6yZvKBM7k7mHeJsFpHqKmcq806zq5PJtHB5LStXrqzKv/mbvxnqaKbE0aNHQx3N1NGsqFJK+clPfhKWaa6Cy7TSc+ves870kcy1bc2UyrRRR/c7m0U1aPvuOHRZJr+olNhv3Lq1vbm8gvXr11dlbTMftW7NS3HjmG7PZSNl3tfP5GW5vATNMHCZBpcvX67Kbox0eQmZdSv3jr/+bsaMGaGOy4/RvDp33rQtufFH8yncvcZZs2ZNVXZZDJoh5fqM3rfefvvtUMdlwWiGT2aMducokzN1K8vca53RlinlZLIBB/3mo36nfcv1bR1L9+7dG+pozpC717vcSbVq1aqwbO3atVXZtX+d/7nco8mTJ4dlek7c2P71r3+9KmsOVCmlfPe7363Kmmdais9Z0fuPHmsppRw+fLgqP/vss6GOHr8bf7/whS+EZXq8mvFUSikLFy6syu48aj9y87if/vSnYZne/1ymqI7lmh9ZSswrc/lV7t6KWnbepvTcuvmH9rVsfpXOW9y9LvM3qu6jzplK8cev442rk/k7btB6S/H3Iz2XbvzVDDc3/9U50oULF0KdJ598MizTeZubR+q5ffrpp0MdzeJ74403Qp1MXpVrN5l5081wr78Z8J9SAAAAAAAA6B0PpQAAAAAAANA7HkoBAAAAAACgdzyUAgAAAAAAQO8+VtB5S7CXCxHTZS7ozQVkbtmypSq7MDINOtTg21JisJ1bj9tvDS3OhJE7+rvMb9w+ZQLaugo1z65b6+j1KCWeRxfq50KMlYZ6lhLD/lzQ8LFjx6ryj3/841DHhf/p9tz2MyHuev3ddWwNEe9KS9DiMLlQURcifv369aqcCWx2IY7Tp0+vyi4w1Y0tmTEy00YyIZqub+nvzp07N/B3LmhSj81ty/1OQzRd0Ln7iIGaOXNmVV6xYkWoM3v27LBMgy1diK0GdLq2pfcf91ED/fCCW/e+ffsG7qNrM9u3b6/K7mMMmzZtCsv0fLuxRftEn+NKlu5T9h6Zkel/g37zUQg/HUzHDddGNQx7yZIloY6O0aXE8+9CtPWjASdPngx19D7i7jXunjB37tyq7ALC9Xf6cYRSSvnZz35WlV0Y+pQpU8Kyhx9+uCq7sVbHRHeO9EMLbvw/depUWKZjstvHQ4cOVWU3/uq8TX/zUTIfmlmwYEFVnjZtWqij46g7j2PHjg3L6P+DZcZ2XebmMRqs7dbj7m0j/RGPlu23Bm9nlrmPuOgYuXz58lBHP2rgPkbh+oiO/+5vvb/+67+uym+++Waoo3/HZULNS4ljQuZ6jLa/h24lnFkAAAAAAAD0jodSAAAAAAAA6B0PpQAAAAAAANC7dKaUexfVvZ896HfunV59P9O90+kyVHSZe89Tt+e2r8fhMq3cssw7q7o9t4+aoZA5Dif7DvVIclkMmk/grr/L+Vm2bFlVdrlj+u6/vvdcSikvvPBCVX7ttdcG7mMpMVcmk3Pk2pFmUWXzSrrKVWl9P125dqu/c+vR9p/JZnPbcuORXrdMHzl//nyoc+DAgaqsWUGl+HarbcTlpeg7/G5c0UyLTH5bKTHTKTOOue3rMpef5fqfy5BSet3c9T979mxVdpkSmp9SShw33HnTvJhJkyaFOpr7omNPKf7aHj16tCq7dqP79OKLL4Y6mhelGTOl+POv/cRdf60z0jkoo+2eVUp7NuNIn8ubQSZTUfNi9u/fH+q4vCLNJ3Fju45tLhtJ+42bj7r7j46Tbr+1L+t8pJTY3jQHqZRS3njjjbBMs1fmz58f6qxdu7Yqu7w8vSZuPuTmTXpt3di6aNGiquzO7ebNm2+43lJ8ppeO/y53UMdkvde4ZW4e6zJF3XlCLfM3WmYe2dVYm8kLymwr8/dxKbkMo8z9pvW+qfs5fvz4UEczpNzfOqtWrRq4P65v6XjzzW9+M9R56623qvLBgwdDHbdPKpOpiZHFf0oBAAAAAACgdzyUAgAAAAAAQO94KAUAAAAAAIDe8VAKAAAAAAAAvUsHnbsQXQ2EywTUZYJWXRhZZvuZ8Du3j7rMBdR1GT4+aD3ZwDpdd+YcteoqVNsFjWsY6NSpU0MdF5iswXbTp08PdTRE75/+6Z9CnZ/+9KdVORPGV0ruumn4pWujmTBipzV8t2U9mYDw1vDJloBK10ddYKGu2wWUKheGq0GLEyZMCHVcu9Vz5ELENYzchYjrsuyHFzTEO/sRhxat7T/TjvR6nzhxItT58Y9/PHAfp02bFpbpmDR27NhQR6+tXrNS/PnX0FAX0P6DH/ygKrvA4i1btlRlF2rugnZbQjzd/VevSVf3g1LaQly73L7qct0EnQ+mfcSNrdrfXND/yZMnw7LPfe5zVVlDtUuJ/d/NIzJB3+6+oaHpR44cCXVeeumlqjxnzpxQ54knnqjK+gGXUkr50Y9+FJbphxa0XEocE9evXx/q3HvvvVV59+7doc7x48fDMg02nzFjRqhz5syZquyuo963XRtxH8PReZu7t7799ttV2QWt633UIdS8TVdjZFd/j3W1P9l7b+bepnM093ed/i7zt0YpcbxbvHhxqKNj0po1a0Id12+UCyP/h3/4h6r8+uuvhzo632v5u/qjftfyNzL39eHhP6UAAAAAAADQOx5KAQAAAAAAoHc8lAIAAAAAAEDveCgFAAAAAACA3qWDzl0YmIZ9uTouNHWQTBh5KTH8zdXJhLFntuVktq9aA9rcudVlmWvUpZbw2QsXLoRlEydOHPg7FyKt4ZMvvvhiqKPH/9xzz4U627dvr8rZoFutlwkIdtdWj8OtJ9snWnTVJzJtNBO+2PpRgcxHFDKB7W5bFy9erMousPHw4cNhmQa0uuPX/e4qsN1xbcsFZCvdJ3euMx+6yIxt7vprGLEL1XQhtvrRAhfGqyGeLkRXj0PDcUsp5e677w7LNFj4qaeeCnU0xHzv3r2hjgYbZ4P+M2OUXkvXHjLXsauA8Naxbpjh563bIhB1MA3t17G2lBhi7T584u7tGqJ93333hTpz586tym4+otufMmVKqOPGZA0237NnT6ijYejuoyo6jrig8UOHDoVlOm66cXvnzp1VedWqVaGOhp/rOSsl96GTbdu2hTp633T7qOO9u/6ZD224e0Tm3pqZ62bmFre71o9odDVHdevWe2lmbpOZ67Tej7L3dpX5YNHMmTPDsmXLllXlDRs2hDp63dwYqeO2fhyolFJee+21sOz555+vyu4jDi0f48n2UdX339Go8Z9SAAAAAAAA6B0PpQAAAAAAANA7HkoBAAAAAACgdx8rU0rf83Tva+o73Zn1ZN6fdevKZFFk3nvPZOO4dbfmpajsu8iZ99yH+U57Sz6WZhyUEt/h1/yGUvy70Lou9776d77znaq8devWUEevicsUcBlCmv2QyYsaN25cqJPJJsssG2ad1pwnrZPpW5ntZ9t6JlNC1+XaqF4jt55MFo9rR9pu3Xoy79Rnx03VkoXg9tH9Ttu/o/vtMhU0U8T19RUrVoRlCxYsqMqu/+nxu2v7wQcfVGV3/Dt27AjLNC9B81vc79w9Srfn6rRmWGi/cdcs0/9GWp8Ze1lkUQyWycfT/uf6sdYpJfY3l7uk+VAuL0Xv9ZMnTw513BxFxxLNuCol3hPcGLFr164b7k8ppYwdOzYs07HV9dvz589XZZcFo/vtxt/3338/LHvvvfeqsssL0/EmkzuZpet295YWZEW1ackGKiV3v2m9J7XkNWVk24iuO7M/rv/pelzu3fz588OyWbNmVeXp06cP3P6mTZvCMs29c/lRbmzTfLhsXqnKZGplfufGn5ZMMbThP6UAAAAAAADQOx5KAQAAAAAAoHc8lAIAAAAAAEDveCgFAAAAAACA3qVT/1yIl4YGujDClhBdF+KWDX9Wur3WoGVHj98dmy7LBP21hihmzpHbfibo2YWI6nXKBNS5fdTQ0AkTJoQ6K1euDMu03gsvvBDqvPzyy1XZXSM9DtfWXRhrJlhP24gLMezqereGmGdk+r/Tsv3WDwa4fdQ26fY500cz44hrW5mPKGTWM2i92XVnPvTQOo6432m/cdvXPpEZI5ctWxaWLVmyJCybNGlSVV66dGmoo+OIhgqXEkN8L126FOq8+eabYdnmzZur8vHjx0OdTEBmJjA+o6ug2a5C1d2yLkOEW/pSVx8syW6/T+7YRts+uo8IKDcfcddE5zKuH2mfPHHiRKiT+RiFmyPo9tx+Z+4tyn0wIyMz/3H7+Pbbb1flbNCvjjeZsc7N/1vv/6OtbeP20eU9MjNHXbRoUVV2H35xf1s9/vjjVXnGjBmhzv79+6vy008/Heps2bKlKutHDkrxY4ty/b9l3tJl3+9q/oPB+E8pAAAAAAAA9I6HUgAAAAAAAOgdD6UAAAAAAADQu3SmlNOSD9NlXsTNoOV4s7/JnNtMXoZy7++6ZZlMKTV27NiwbNasWVX5/vvvD3WmTJkSluk7y88880yoo1kwmbyybH5Zaz7SsLbfVR0nm6HSomXdXeautbwf7n4zGse2TO5Z5vy7LLQMHRMyY4TLZpk+fXpVXrBgQajjMqXmzJlTlU+fPh3qaM6Yy0I4cuRIVXa5U9u2bQvLzp49W5Vdu9F8mq7yo7rUZT5Gy7a67O9drLv1fLRmwd2sMudJ62RyR1rH8UymaCYb0O2jWzas9tclPd5MNpM7R+4ekbkmLdmw5EfhVpWZI0ybNi3UWbVqVVWeOnVqqKO5U6WUMmbMmKqsc51SSvnXf/3Xqnzw4MFQR3M2W8e11txL3Bpu3dkQAAAAAAAARi0eSgEAAAAAAKB3PJQCAAAAAABA73goBQAAAAAAgN6lg85bAxu7Ch8cZtBpV7raVmvQeYYL+syESGa27661BosvX7481FmzZk1VdmF8LkR0+/btVXnr1q2hzl133VWV3bHdDEHjw9x+Rldh6K6OrjsTRut0dW6HGZjaEuo6bHpuMyGy7nq4jwho+G0maF1DPUspZfXq1VV54sSJoc7kyZMHbt8d2/Hjx6tyJsR87969oY4GfTruvI3GYPNhaR1/+uwTw5wzjMag85EO2h5tujwffZ7b1j6SCTHPaP2IxqD9KSX2m+zHEEa6bwG/KtdmNYx8xYoVoc6ECROqsvvwy9133x2W7d+/vyp///vfD3U02DzzMQo3H3T9NhNs3uccHSOLERsAAAAAAAC946EUAAAAAAAAesdDKQAAAAAAAPQunSnltOTVZHKnstkww3qvdDTmV7WuW99Pbr1GmSwq9zvNfpkxY0aoM27cuKrs3jF+7733wrLXXnutKl+5ciXU0XyaO++8M9QZbZlSXWY6DFNLhlRXfavLa9TVONJVpkfrbzLH7zJtMnkdrWN0hv7uiSeeCHUmTZpUlVetWhXqTJ06NSy7fPlyVXbHtmnTpqq8ZcuWUEczFa5duxbquP6g44373a2iyzbRldb73bBk8xoxuox0O8rcI7q6t7Vuy2XIZOY2mXsLGTK4Xbh7hM5/9G+mUmKmpptr7Ny5Myx79tlnq/KpU6cG7qPr6/p3m+Z5ltL+9w+ZUrcP/lMKAAAAAAAAveOhFAAAAAAAAHrHQykAAAAAAAD0jodSAAAAAAAA6F066Lw1RLh13S11bhXDPFYXIp4Jo3bBdhrI5343ffr0G5ZLKWXu3LlVec6cOaHOvn37wrJXX321KrvwPw3EcwF5emytIdou6Lg1oLtFNvy6xWgM/89sq8+g+ew+qUwbzaw307YzQeeOtm03jrhlum4N4yyllAcffLAquzFi2rRpVXnWrFkD97GUUsaPH1+Vn3766VDnlVdeqcpHjx4NdVxop3Ln8f333x/4O/0Yw/Xr1wf+BtHNMEcg6PzmNNJtK/PBiszvhvlRj+yyFpm5FuHHuBW4jzFNmDChKi9atCjU0Y+6uLnOkSNHwjL9QJT7W0/nP12Oh5kxivvm7YP/lAIAAAAAAEDveCgFAAAAAACA3vFQCgAAAAAAAL3joRQAAAAAAAB6lw46d7oKOu8q2HekwygzRnofXWBc63XMBJ1PnDixKrvA4PPnz1flT30qNsuNGzeGZe++++7A7Wv48pgxY0Kd1jDyloD+1lD/PtuNCxFtNayg9ex69Vj6Dkxsud6tga2Zbblrq+ckE9h/7dq11D5p0PiGDRtCnRkzZtzwN6WUMnv27Kr83nvvhTouIP3ZZ5+tyi+//HKoc/z48arsxqiWwPosFxA/2g0z6PRWRmArutD6UZOu5ujZD220rNvJ3CNvp3EEty43t9I50oULF0KdH/zgB1VZ/z4qJdfXMh91cWHouu7svIYPFOD/xX9KAQAAAAAAoHc8lAIAAAAAAEDveCgFAAAAAACA3qUzpYb5vnrr9kd6n1oyrfrcHyeTF5PJnSrFv1es7rrrrqp8zz33hDq67K233gp1nnvuubDs+vXrVdll0XT1vnKm/bVe25FuN9omXBvJtJvWTKPMerrKuRrpvK6+cy+0/bs+q/29tc+4/rdixYqq7HKfxo4dW5UnTZoU6uh+u/bw/PPPh2UvvfRSVd6/f3+o0zJuto6RLq/hZsgZuln7yLDyKoaZn4nRp6v80q6uf3Y9Xc1RMtyYrPvZVe5UNtOKnCncbFyb3bVrV1XetGlTqHPx4sWB67777rvDssuXL/8Ke/f/yeROOa1/o+qy1u1j9OM/pQAAAAAAANA7HkoBAAAAAACgdzyUAgAAAAAAQO94KAUAAAAAAIDepYPOndEWIjjMwOhMiGLfgdUt4Zuf+lS85Boid+3atYF13Lp+8YtfDPydW8+YMWOq8t69e0Odc+fOhWUakOy2r8ei23L6DuzP1LlZ90mv9zC3Ncww+kxAY2b7rUGvmf12624JiM2GeKvJkyeHZStXrqzK2mdLKWXZsmVV2Y0/J0+erMqnTp0KdZ5++umw7MiRI35nPyZ3Ptx5c2OSyoSoj7SRvte3tv+MPsPHR+O1xa9upAPrs/fIYX7opWVbI/2hEeBmdOzYsars/o5T+pGpUnKh5pkPDWXmNe6jOpnfuXsk983bB/8pBQAAAAAAgN7xUAoAAAAAAAC946EUAAAAAAAAepfOlHLvi2eyMPR3LlNIf5ep0yqTu+K05sW01OmSbi9zrO69X/d+8AcffFCV3TvM+u7z7NmzQx3Ni3n99ddDnXHjxoVl2ibuuOOOgXVcO9J9bL0erXkJLbk/2XVntOZFuDYxaN2Z99WdzLYy28/UyeyPOx+ubWXWlVl3S36cq5fp227d2tddNptmQ5US+6QbI1xenNq2bVtV3rhxY6hz9uzZsCzT/jI5B5k6Tma8bbm3DTM/JiO7rZHOlGvJS3N9tqUfO2RjoAtdtUcAo0vm3vrhhx8OrKNzto+z/Zb5T+ucCbc37mwAAAAAAADoHQ+lAAAAAAAA0DseSgEAAAAAAKB3PJQCAAAAAABA7z5W0HnGMEO8MyG2w9qWW9Z6joYZUKuBmJnAehcY7oKmMyH2ly9frsrvvvtuqLNjx46qfPXq1VDHBZ1fu3atKmtguVvm6nQVdD9MXQUEtwak9hlQ7PaxNQw+sz+txzasdXcV6lxK7LeZj1G0hjG766Zjyd133x3qXLhwoSpv3bo11NmyZUtVdiGebvvXr1+vyiPdjzNaP2rQVfj5MM/RSJ/rzLG11skg6BwAACDiP6UAAAAAAADQOx5KAQAAAAAAoHc8lAIAAAAAAEDveCgFAAAAAACA3qWDzp1M+GcmoLjPwHLndgpx/cUvfhHqaEBwa2CuhgqXUsqpU6eqsgsxfvPNNwdu3wWta4iyq6P71GWIcJ8BuU4mtDxzbVvW26WW89jVtrLbb92fzMcQumoj7ncarJz50IGrox8IGDNmTKjjQsynTJlyw/WUUsrrr79eld96661Q5/3336/Kro26/q/L3LEN80MTI22YIf5dbT8j81GNVn3etwk6BwAAiPhPKQAAAAAAAPSOh1IAAAAAAADoHQ+lAAAAAAAA0LtOM6VczkNLplRmPX1rzeIZ5n53lQ+U2UeXhaHb//DDD0Ods2fPVuWLFy+GOpcuXarK48aNC3XcujXXJnP+M5k62UyjliyW1vW05Edlt9+SO/Wr1LvRtrpab9Ywc7+GmXvVOo5kMmw058nlzmWysa5duzZw2c6dO0OdN954Y+B6JkyYUJUvX74c6rgx4s4776zKH3zwQajTd4ZaF7rsI31mSnXZtrvSZ6YdmVIAAADRzTcbBwAAAAAAwE2Ph1IAAAAAAADoHQ+lAAAAAAAA0DseSgEAAAAAAKB36aDz1oBgDfbMBC0PMzC87xBl1XocrWG0d9xxR9P2MuvWZZ/85CdDHQ0fdkHDGkbsrr8LP77rrrtuuC1XpzVoezQGnXcVYt5ViLHbR20Tbltap3U9mb7V1fFnw7Fb9qnLMOhMiLlyYcy6zAWN7927NyzTjxhs3bp14PZ1n9163DnScaSUOCZk+pHTOra3fOgjE4Y9zKDzzD66Y+3qIwLDvEdmfjfM4HeCzgEAACL+UwoAAAAAAAC946EUAAAAAAAAesdDKQAAAAAAAPTuY2VKZXNVBq0nk03jciZatp+RzYvoKueqVVdZOJnck0wWi8uU0gwNl/uidVzuxtixYwfuk8vL0e253KlMpllXeSVOV7lPXeVOufVklrk6mg+U2b5rR5n9cbRtZ7JwWjNl3D5lMmRaxrHs2KPrdu0/s496HV3G29GjR8OyEydOVOX33nsv1BkzZkxVdtdf+7ZmxZXiz+P169erssvYy+QeZnSVKeUMMwsxs63MPdrJjG2ZPjpovR+lJQsrO/61IFMKGN3cGKH9NlPHLWuto9vLrAcAbjb8pxQAAAAAAAB6x0MpAAAAAAAA9I6HUgAAAAAAAOgdD6UAAAAAAADQu3TQudNV0LmG+LUGnbcGj490iOwwaUCwO7cu/DijJcQ3c21dHRd0rsfmQoxbQtxbQ3wzQZMuxLk1RLgloLt1+26ZrmuYIc6t5ygTYt0y/rQeq2sjmd9lxja3nszxa4i525bWcdyHBt5///2qrKHmpZRy9erVquz68bhx4wZuK/MRA3f+u/rQQFdB510Gn7fcE7vcfp9B55l1O30GnTuZNgKMpJbA7tbfZULEh7l9AMDI4D+lAAAAAAAA0DseSgEAAAAAAKB3PJQCAAAAAABA79KZUl3lxQwzL6M1Uyqz/dZMl5Y6rdw+6vZcNkxrhofLdRn0O5f7cueddw7cH7ctXabrKaWU69evV2WXTZXJ1HJZTCqTO9LaRlrbTSbTpTVTS7X2v8zvWjOdMsfSkumSPUdd5VVlspEy58S147vuuqsqu/7Q0tfdut0+ap90Y9S1a9cG7o/r23pudT2ldJcplbkmXfXt7D1K62XqZLaXXc+wMqWy5zGz3y3Xv8v7uLabzL0G/eoq06irvKRhbr+reTQAAL8K/lMKAAAAAAAAveOhFAAAAAAAAHrHQykAAAAAAAD0jodSAAAAAAAA6N0nfkmqIQAAAAAAAHrGf0oBAAAAAACgdzyUAgAAAAAAQO94KAUAAAAAAIDe8VAKAAAAAAAAveOhFAAAAAAAAHrHQykAAAAAAAD0jodSAAAAAAAA6B0PpQAAAAAAANA7HkoBAAAAAACgd/8bQYaLGb7jjI0AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "check_data = first(train_loader)\n", + "print(f\"batch shape: {check_data['image'].shape}\")\n", + "image_visualisation = torch.cat(\n", + " [check_data[\"image\"][0, 0], check_data[\"image\"][1, 0], check_data[\"image\"][2, 0], check_data[\"image\"][3, 0]], dim=1\n", + ")\n", + "plt.figure(\"training images\", (12, 6))\n", + "plt.imshow(image_visualisation, vmin=0, vmax=1, cmap=\"gray\")\n", + "plt.axis(\"off\")\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "08428bc6", + "metadata": {}, + "source": [ + "### Define network, scheduler, optimizer, and inferer\n", + "At this step, we instantiate the MONAI components to create a DDPM, the UNET, the noise scheduler, and the inferer used for training and sampling. We are using\n", + "the original DDPM scheduler containing 1000 timesteps in its Markov chain, and a 2D UNET with attention mechanisms\n", + "in the 3rd level, each with 1 attention head (`num_head_channels=64`).\n", + "\n", + "In order to pass conditioning variables with dimension of 1 (just specifying the modality of the image), we use:\n", + "\n", + "`\n", + "with_conditioning=True,\n", + "cross_attention_dim=1,\n", + "`" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "bee5913e", + "metadata": { + "jupyter": { + "outputs_hidden": false + }, + "lines_to_next_cell": 0 + }, + "outputs": [], + "source": [ + "device = torch.device(\"cuda\")\n", + "\n", + "model = DiffusionModelUNet(\n", + " spatial_dims=2,\n", + " in_channels=1,\n", + " out_channels=1,\n", + " num_channels=(64, 64, 64),\n", + " attention_levels=(False, False, True),\n", + " num_res_blocks=1,\n", + " num_head_channels=64,\n", + " with_conditioning=True,\n", + " cross_attention_dim=1,\n", + ")\n", + "model.to(device)\n", + "\n", + "scheduler = DDPMScheduler(\n", + " num_train_timesteps=1000,\n", + ")\n", + "\n", + "optimizer = torch.optim.Adam(params=model.parameters(), lr=2.5e-5)\n", + "\n", + "inferer = DiffusionInferer(scheduler)" + ] + }, + { + "cell_type": "markdown", + "id": "2a4d3ab2", + "metadata": {}, + "source": [ + "### Model training\n", + "Here, we are training our model for 75 epochs (training time: ~50 minutes)." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "6c0ed909", + "metadata": { + "jupyter": { + "outputs_hidden": false + }, + "lines_to_next_cell": 0 + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Epoch 0: 100%|██████████| 125/125 [00:27<00:00, 4.61it/s, loss=0.723]\n", + "Epoch 1: 100%|██████████| 125/125 [00:27<00:00, 4.60it/s, loss=0.276]\n", + "Epoch 2: 100%|█████████| 125/125 [00:27<00:00, 4.57it/s, loss=0.0965]\n", + "Epoch 3: 100%|█████████| 125/125 [00:27<00:00, 4.62it/s, loss=0.0376]\n", + "Epoch 4: 100%|█████████| 125/125 [00:27<00:00, 4.59it/s, loss=0.0224]\n", + "Epoch 5: 100%|█████████| 125/125 [00:27<00:00, 4.47it/s, loss=0.0187]\n", + "Epoch 6: 100%|█████████| 125/125 [00:27<00:00, 4.52it/s, loss=0.0179]\n", + "Epoch 7: 100%|█████████| 125/125 [00:28<00:00, 4.44it/s, loss=0.0169]\n", + "Epoch 8: 100%|█████████| 125/125 [00:27<00:00, 4.56it/s, loss=0.0161]\n", + "Epoch 9: 100%|██████████| 125/125 [00:27<00:00, 4.50it/s, loss=0.016]\n", + "Epoch 10: 100%|████████| 125/125 [00:27<00:00, 4.52it/s, loss=0.0156]\n", + "Epoch 11: 100%|████████| 125/125 [00:27<00:00, 4.57it/s, loss=0.0152]\n", + "Epoch 12: 100%|████████| 125/125 [00:27<00:00, 4.56it/s, loss=0.0152]\n", + "Epoch 13: 100%|████████| 125/125 [00:27<00:00, 4.54it/s, loss=0.0151]\n", + "Epoch 14: 100%|████████| 125/125 [00:27<00:00, 4.57it/s, loss=0.0147]\n", + "Epoch 15: 100%|████████| 125/125 [00:27<00:00, 4.56it/s, loss=0.0151]\n", + "Epoch 16: 100%|████████| 125/125 [00:27<00:00, 4.55it/s, loss=0.0151]\n", + "Epoch 17: 100%|████████| 125/125 [00:27<00:00, 4.52it/s, loss=0.0146]\n", + "Epoch 18: 100%|████████| 125/125 [00:27<00:00, 4.56it/s, loss=0.0144]\n", + "Epoch 19: 100%|████████| 125/125 [00:27<00:00, 4.54it/s, loss=0.0143]\n", + "Epoch 20: 100%|████████| 125/125 [00:27<00:00, 4.52it/s, loss=0.0145]\n", + "Epoch 21: 100%|████████| 125/125 [00:27<00:00, 4.53it/s, loss=0.0143]\n", + "Epoch 22: 100%|████████| 125/125 [00:27<00:00, 4.54it/s, loss=0.0138]\n", + "Epoch 23: 100%|████████| 125/125 [00:27<00:00, 4.57it/s, loss=0.0135]\n", + "Epoch 24: 100%|████████| 125/125 [00:27<00:00, 4.55it/s, loss=0.0134]\n", + "Epoch 25: 100%|████████| 125/125 [00:27<00:00, 4.49it/s, loss=0.0135]\n", + "Epoch 26: 100%|████████| 125/125 [00:27<00:00, 4.57it/s, loss=0.0135]\n", + "Epoch 27: 100%|████████| 125/125 [00:27<00:00, 4.55it/s, loss=0.0136]\n", + "Epoch 28: 100%|████████| 125/125 [00:27<00:00, 4.54it/s, loss=0.0135]\n", + "Epoch 29: 100%|████████| 125/125 [00:27<00:00, 4.57it/s, loss=0.0131]\n", + "Epoch 30: 100%|████████| 125/125 [00:27<00:00, 4.57it/s, loss=0.0128]\n", + "Epoch 31: 100%|████████| 125/125 [00:27<00:00, 4.54it/s, loss=0.0129]\n", + "Epoch 32: 100%|████████| 125/125 [00:27<00:00, 4.55it/s, loss=0.0128]\n", + "Epoch 33: 100%|████████| 125/125 [00:27<00:00, 4.57it/s, loss=0.0135]\n", + "Epoch 34: 100%|████████| 125/125 [00:27<00:00, 4.52it/s, loss=0.0138]\n", + "Epoch 35: 100%|████████| 125/125 [00:27<00:00, 4.56it/s, loss=0.0131]\n", + "Epoch 36: 100%|████████| 125/125 [00:27<00:00, 4.55it/s, loss=0.0132]\n", + "Epoch 37: 100%|████████| 125/125 [00:27<00:00, 4.54it/s, loss=0.0125]\n", + "Epoch 38: 100%|████████| 125/125 [00:27<00:00, 4.54it/s, loss=0.0124]\n", + "Epoch 39: 100%|████████| 125/125 [00:27<00:00, 4.57it/s, loss=0.0124]\n", + "Epoch 40: 100%|████████| 125/125 [00:27<00:00, 4.54it/s, loss=0.0132]\n", + "Epoch 41: 100%|████████| 125/125 [00:27<00:00, 4.54it/s, loss=0.0128]\n", + "Epoch 42: 100%|████████| 125/125 [00:27<00:00, 4.55it/s, loss=0.0122]\n", + "Epoch 43: 100%|████████| 125/125 [00:27<00:00, 4.54it/s, loss=0.0127]\n", + "Epoch 44: 100%|████████| 125/125 [00:27<00:00, 4.56it/s, loss=0.0129]\n", + "Epoch 45: 100%|████████| 125/125 [00:27<00:00, 4.56it/s, loss=0.0132]\n", + "Epoch 46: 100%|████████| 125/125 [00:27<00:00, 4.53it/s, loss=0.0125]\n", + "Epoch 47: 100%|████████| 125/125 [00:27<00:00, 4.56it/s, loss=0.0123]\n", + "Epoch 48: 100%|████████| 125/125 [00:27<00:00, 4.57it/s, loss=0.0123]\n", + "Epoch 49: 100%|████████| 125/125 [00:27<00:00, 4.55it/s, loss=0.0125]\n", + "Epoch 50: 100%|████████| 125/125 [00:27<00:00, 4.54it/s, loss=0.0127]\n", + "Epoch 51: 100%|████████| 125/125 [00:27<00:00, 4.54it/s, loss=0.0125]\n", + "Epoch 52: 100%|████████| 125/125 [00:27<00:00, 4.52it/s, loss=0.0124]\n", + "Epoch 53: 100%|████████| 125/125 [00:27<00:00, 4.55it/s, loss=0.0127]\n", + "Epoch 54: 100%|████████| 125/125 [00:27<00:00, 4.55it/s, loss=0.0123]\n", + "Epoch 55: 100%|████████| 125/125 [00:27<00:00, 4.55it/s, loss=0.0127]\n", + "Epoch 56: 100%|█████████| 125/125 [00:27<00:00, 4.58it/s, loss=0.012]\n", + "Epoch 57: 100%|████████| 125/125 [00:27<00:00, 4.58it/s, loss=0.0126]\n", + "Epoch 58: 100%|████████| 125/125 [00:27<00:00, 4.57it/s, loss=0.0121]\n", + "Epoch 59: 100%|████████| 125/125 [00:27<00:00, 4.59it/s, loss=0.0126]\n", + "Epoch 60: 100%|████████| 125/125 [00:27<00:00, 4.60it/s, loss=0.0119]\n", + "Epoch 61: 100%|████████| 125/125 [00:27<00:00, 4.59it/s, loss=0.0122]\n", + "Epoch 62: 100%|████████| 125/125 [00:27<00:00, 4.57it/s, loss=0.0119]\n", + "Epoch 63: 100%|████████| 125/125 [00:27<00:00, 4.59it/s, loss=0.0125]\n", + "Epoch 64: 100%|████████| 125/125 [00:27<00:00, 4.56it/s, loss=0.0121]\n", + "Epoch 65: 100%|████████| 125/125 [00:27<00:00, 4.58it/s, loss=0.0121]\n", + "Epoch 66: 100%|████████| 125/125 [00:27<00:00, 4.58it/s, loss=0.0117]\n", + "Epoch 67: 100%|████████| 125/125 [00:27<00:00, 4.56it/s, loss=0.0121]\n", + "Epoch 68: 100%|████████| 125/125 [00:27<00:00, 4.59it/s, loss=0.0123]\n", + "Epoch 69: 100%|████████| 125/125 [00:27<00:00, 4.57it/s, loss=0.0121]\n", + "Epoch 70: 100%|█████████| 125/125 [00:27<00:00, 4.57it/s, loss=0.012]\n", + "Epoch 71: 100%|████████| 125/125 [00:27<00:00, 4.59it/s, loss=0.0118]\n", + "Epoch 72: 100%|████████| 125/125 [00:27<00:00, 4.59it/s, loss=0.0117]\n", + "Epoch 73: 100%|████████| 125/125 [00:27<00:00, 4.58it/s, loss=0.0119]\n", + "Epoch 74: 100%|████████| 125/125 [00:27<00:00, 4.56it/s, loss=0.0125]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "train completed, total time: 2074.1517136096954.\n" + ] + } + ], + "source": [ + "n_epochs = 75\n", + "val_interval = 5\n", + "epoch_loss_list = []\n", + "val_epoch_loss_list = []\n", + "\n", + "scaler = GradScaler()\n", + "total_start = time.time()\n", + "for epoch in range(n_epochs):\n", + " model.train()\n", + " epoch_loss = 0\n", + " progress_bar = tqdm(enumerate(train_loader), total=len(train_loader), ncols=70)\n", + " progress_bar.set_description(f\"Epoch {epoch}\")\n", + " for step, batch in progress_bar:\n", + " images = batch[\"image\"].to(device)\n", + " classes = batch[\"class\"].to(device)\n", + " optimizer.zero_grad(set_to_none=True)\n", + "\n", + " with autocast(enabled=True):\n", + " # Generate random noise\n", + " noise = torch.randn_like(images).to(device)\n", + "\n", + " # Get model prediction\n", + " noise_pred = inferer(inputs=images, diffusion_model=model, noise=noise, condition=classes)\n", + "\n", + " loss = F.mse_loss(noise_pred.float(), noise.float())\n", + "\n", + " scaler.scale(loss).backward()\n", + " scaler.step(optimizer)\n", + " scaler.update()\n", + "\n", + " epoch_loss += loss.item()\n", + "\n", + " progress_bar.set_postfix(\n", + " {\n", + " \"loss\": epoch_loss / (step + 1),\n", + " }\n", + " )\n", + " epoch_loss_list.append(epoch_loss / (step + 1))\n", + "\n", + " if (epoch + 1) % val_interval == 0:\n", + " model.eval()\n", + " val_epoch_loss = 0\n", + " for step, batch in enumerate(val_loader):\n", + " images = batch[\"image\"].to(device)\n", + " classes = batch[\"class\"].to(device)\n", + " with torch.no_grad():\n", + " with autocast(enabled=True):\n", + " noise = torch.randn_like(images).to(device)\n", + " noise_pred = inferer(inputs=images, diffusion_model=model, noise=noise, condition=classes)\n", + " val_loss = F.mse_loss(noise_pred.float(), noise.float())\n", + "\n", + " val_epoch_loss += val_loss.item()\n", + " progress_bar.set_postfix(\n", + " {\n", + " \"val_loss\": val_epoch_loss / (step + 1),\n", + " }\n", + " )\n", + " val_epoch_loss_list.append(val_epoch_loss / (step + 1))\n", + "\n", + "total_time = time.time() - total_start\n", + "print(f\"train completed, total time: {total_time}.\")" + ] + }, + { + "cell_type": "markdown", + "id": "a676b3fe", + "metadata": {}, + "source": [ + "### Learning curves" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "f8385176", + "metadata": { + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAArsAAAILCAYAAADoqVT3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAABsNklEQVR4nO3deXwTdf7H8ffk6EFLkVuEolYERESrKKgccixWLu9rVwFBRfFAcFVcF11cZAEX3Z8WVlyVXV1PVNBVAQURUVEsKogKHhSkgCA3PWiTzPz+SJM2toVOm5Jp+3o+Hn00+WZm8s2nQd/55jvfMSzLsgQAAADUQa5YdwAAAACoKYRdAAAA1FmEXQAAANRZhF0AAADUWYRdAAAA1FmEXQAAANRZhF0AAADUWYRdAAAA1FmEXQAAANRZhF0AOMJef/11dejQQR06dFBOTk6suwMAdZon1h0AUHdde+21WrlypTp27Kg33ngj1t1xjEaNGqljx46SJK/XG+PeVE1RUZHeeecdLV++XF9//bV2796tgoICNWjQQG3atFGXLl10wQUXqHv37rHuKoB6zrAsy4p1JwDUTYTdumnhwoX629/+pl9++UVSMLA3b95cDRo00K5du7Rnz57wtmeccYamTp2qtm3bxqq7AOo5RnYBAJX2xBNP6NFHH5UkpaWl6bbbblPv3r2VlJQU3ub777/X008/rTfffFOrVq3SVVddpf/+979KS0uLVbcB1GPM2QUAVMrixYvDQXfAgAF64403NHDgwIigK0nt27fXtGnTNHPmTHk8Hu3atUvjx4+XaZqx6DaAeo6wC8DRsrOz9eCDD+qCCy7QaaedptNOO039+vXTvffeq7Vr1x5y302bNumhhx7SkCFDlJ6erk6dOumss87SH/7wBz3//PPy+/3l7hc6eeztt9/WDz/8oOuuu05du3bVlVdeGd7m2muvVYcOHXT//fdLkpYtW6ZRo0apR48e6ty5s3r06KE777xTGzZsKHP8ik5Qy8nJCbd/9dVXKigo0BNPPKEhQ4bo9NNP16mnnqrBgwcrMzNTRUVF5fbd5/PpmWee0cUXX6z09HR17dpVV111ld58801J0osvvhh+Djt8Pp8mT54sSWrXrp1mzJihuLi4Q+7Tt29fDRs2THFxcTr66KP166+/hh+bMGGCOnTooL59+1a4f2XrlJWVpSuvvFLp6ekaP368/vnPf4Yf37p16yH7OGLECHXo0EEZGRllHluxYoXuuOMO9enTR507d1bXrl01ZMgQPfzwwxGv5bd++eUXTZs2TRdeeKFOP/10de7cWT179tQVV1yhf/3rX9q9e/ch+wQgupjGAMCx5s+fr4kTJ6qoqEiGYahly5ayLEs5OTnKycnR/PnzNX78eN1www1l9l2yZInuuOOOcCg8+uijlZiYqG3btikrK0tZWVl655139PTTTyshIaHc58/NzdX111+vPXv2qE2bNmrYsGG52z399NOaPn26kpOTdfTRR8s0Tf3666966623tGzZMs2bN0+pqam2XntBQYGGDx+u1atXq1WrVjr66KOVk5OjH374QT/88IPWrl2rJ554ImKfwsJCjRo1Sp9//rkkqUGDBmrevLl+/PFH3XXXXfr88891wgkn2OpHyMKFC7Vt2zZJ0h133HHYoBty++2365ZbblFycnKVnvdwtm7dqokTJ8o0TbVu3VqJiYkaPHiw/vGPf0iSFi1apOuuu67cfXft2qWVK1dKki688MJwu2VZ+utf/6rnn39eUnBOcqtWrbR//359//33+v777/XSSy8pMzNTZ599dsQxV69erZEjRyo3N1eS1KJFCx199NHavXu3Vq9erdWrV2vOnDl67rnnqvy3AGAPI7sAHGnVqlW67777VFRUpIyMDC1btkzLli3Thx9+qBUrVujCCy+UaZr6+9//riVLlkTse+DAAd1zzz0qKipS+/bt9d5772nZsmVauHChVq1apXHjxkmSsrKy9NRTT1XYh7lz56pVq1b64IMP9M4775S77Zo1a/R///d/mjRpkj799FO9/fbb+uSTT/Twww+H+/Kvf/3L9ut/+OGHlZeXp3nz5oWff8WKFfrd734nSVq6dKm+/PLLiH2eeOKJcNC97rrr9Omnn2rBggVasWKF7r33Xr366qtasGCB7b5I0vLlyyVJDRs2VJ8+fSq9X2JiYo0FXUl65pln1L17d3300Ud666239NBDDyk1NVXp6emSgmG3IgsXLlQgEJBhGBo6dGi4/V//+peef/55GYahO+64QytXrtR7772nzz77TG+88YZOOeUU5ebm6pZbbtH27dsjjnn//fcrNzdXJ598st577z0tX75c77zzjj799FPNnTtXxx9/vHbt2qUHHnigZgoCoAzCLgBHmjFjhvx+v8444ww9+uijatmyZfixJk2aaPr06Tr33HMlKTyKF/Lxxx+HR9buueeeiJUAPB6PbrrpJp155pmSpHfeeafCPnzzzTeaPn26mjRpUuE23333nW6//XZdddVVEcuIDR06VGeddZYk6bPPPqvkqy7x008/6cknn1SnTp3CbUlJSZowYUL4funjBgIBvfDCC5KCKyBMmDBB8fHxkoIjkyNGjNAtt9yir776ynZfpGCol6RTTjlFHo9zvhTcsGGDpk6dWmbe8JAhQyRJX331VXjViN8KBf+uXbuqdevWkqT9+/eHR8xvvPFG3XzzzWrQoEF4n44dO+qZZ55RkyZNlJeXF/FBZu/evVq3bp0kafTo0WVWoOjSpYv+9re/6cwzz1SbNm0qnIoCILoIuwAcZ+vWrVq1apWk4JxKl6v8/1T94Q9/kBQ8+3/Tpk3h9oyMDK1du1Yffvhhheu8nnLKKZKkzZs3V9iPzp07H3bJLK/Xq6uvvrrcx0Jr6VYUtg6lf//+4QBWWunpFKWPu3btWu3du1eSdPHFF5d7zOuuuy4iuNkRmmd69NFHV2n/mtKrV69yp5dccMEF8ng8siyr3NHd7du3h99jpacwvP/++8rLy5PL5dKIESPKfc6UlJTwSPC7774bbi+9kueOHTvK3Tc9PV3//e9/NXXq1EpPBQFQPYRdAI5T+uv5E088scLtTj311PDtb775JuIxj8ejli1bVjgKGQp9hxpdq8ycyrS0tDKjiiGhr+8PHjx42OP8ViiMH+q4hYWF4bbs7Ozw7ZNPPrnc/ZKSknTGGWfY7osk5eXlSQpOS3CSiv5GTZo0UY8ePSSVP5Vh4cKFMk1T8fHxESenffHFF5Kkpk2bHnJEP/Te2759u3bt2iVJaty4sU466SRJ0tSpUzVjxoxyT1AEcGQ557soAChW+kz38s6SL89v5076/X699dZbWrRokTZs2KDdu3dr//79tvrRuHHjw26TkpJS4WMVjUhXRkUnw5U+bumRxFDgkqRmzZpVuO/xxx8fnn9rR3Jysvbu3asDBw7Y3rcmHSqQDhkyRB988IG++OIL7dixQy1atAg/FprC0Ldv34ha79y5U1LwPVjZFSt++eUXNW3aVFJwrvXIkSO1Y8cOPfnkk3ryySd1zDHHqFu3burRo4f69OlT4YcjADWDsAvAcQoKCsK327dvX6nQWPrr+QMHDuj666+PmJ/avHlztWvXLjzSu3PnznCwqcwxK2IYxmG3qQq7xy09enyor8erOo2hRYsW2rt3b8R0ESc41Ehzv3791KBBA+Xn5+vdd9/VNddcIyk4TSb03rjooosi9gm99+Lj43X88cfb7s+JJ56ohQsX6qWXXtKrr76qDRs2aOvWrZo3b57mzZunpKQkjRw5UmPGjKnWhyEAlUfYBeA4pQPZrFmzbC/bNXXq1HCYGTZsmIYPH642bdpEbPP4448rMzOz2n11itInx1W0frBUtSkVUvBr+++//17ffvutcnNza3SFhZDqXs0+MTFR/fv315tvvqmFCxeGw+6CBQtkWVbEVIeQ0HuvSZMmVb7EdVJSkkaNGqVRo0Zp8+bN+vjjj/XJJ59o+fLlysvL0+OPP66tW7dqypQp1Xp9ACqHj5UAHKf0SVB2T+4KBAJ66623JEk9e/bUfffdVyboStK+ffuq10mHadSoUfj2oS5aUHpurx29e/eWFLy4xGuvvVbp/UzT1OTJk8usAhEauT5UoI3GlInQqgyrVq0Kj+SHVuAYNGhQmTndoffer7/+GpUrvqWmpuqqq67SY489pg8//FDnn3++JOm1115jPi9whBB2AThOly5dwrcPtVSWZVllRjF3794dHr0MLS/2W6ZpVmneqpOVXjXihx9+KHeb/Px8ZWVlVen4ffv2DX9omDVr1iGvIFbas88+q+eee05XXnmlPv7443B7aKpFaIm48nz99ddV6mtp55xzjpo2bSrTNPXBBx8oJycnfOW9305hkEpODPT7/Ye8Qp/P56vwsYpG1hs2bKi//OUv4fvff/99JV4BgOoi7AJwnFatWoVXDXjhhRfCKwH81htvvKGzzz5bd911lwKBgCRFXA2tohHOp59+OmLuaelVDWqrU089NRwg33777XK3+c9//lNhLQ/H7XZr0qRJMgxDe/fu1ejRow972dv58+dr+vTpkqRzzz1X55xzTvix0Ajq/v37tWXLljL7btmyRYsXL65SX0vzeDwaOHCgJOnDDz8MH/OEE05Q586dy2zfv3//8FSGJ598ssLjPvDAA+rTp0/EOrv//ve/1atXL914440V7lc6CHOiGnBkEHYBONKdd94pl8ulrVu36oYbbogIp0VFRXrllVf0wAMPaP/+/UpOTpbb7ZYUHD1r3769pOBXxaWXJNu5c6ceeughZWZm6pZbbgm3V3W000kaNGigQYMGSQpeKvnpp58OfwDw+Xz697//rccff7zC0e7K6NGjhyZMmCDDMPTNN99o8ODBev7557Vnz56I7b777jvdcccdmjBhggKBgDp16qRHHnkk4qS70AU3JGnatGkRJyWuW7dON954Y/jvWF2hqQyffPJJ+Gp7pdfWLS05OVk333yzJOm9997TX/7yl4gpL7t379bf/vY3vfbaa9q6dWvEFJn27dtr+/bt+vjjjzVp0qQya+1u2bIlfFGQJk2aVOtvAaDyOEENQI3Lzs6uMFyUdtVVV4Uv0HDGGWdoypQpmjhxolatWqUBAwaodevW8nq9+uWXX8JTFc455xzdfffdEce58847dfPNN+vAgQO69NJLdcwxx8gwDG3dulVut1vTp09Xenq6Zs+eLZ/Pp5tuukmpqanKzMxUWlpa9AtwhNx5551auXKltmzZounTp2vWrFk6+uij9csvvyg3N1e33XabLMsKX1K4KkaMGKHU1FQ99NBD2rJlix588EFNnjxZzZs3V0pKinbu3BkOv263W5dddpnuueeeMqOYZ5xxhnr37q1ly5Zp0aJF+vDDD9W6dWsVFBRoy5YtOuWUU/THP/5Rw4YNq1ZNpOCod9u2bfXzzz9r5cqVZS4P/FvXX3+9tmzZopdeekkvvviiXn31VbVu3VpFRUXasWNHeHR29OjRuuCCC8L7nXPOObrpppv0xBNP6IUXXtBLL70Ursv+/fvDy+M1aNBAM2bMiPgWAkDNIewCqHGFhYXhy6geym+XArv44ot1xhln6D//+Y9WrFihbdu2yefz6aijjlL37t114YUX6oILLiizTNd5552nZ555Rk8++aS+/vpr/fLLL2rSpImGDBmikSNHhq9s9uCDDyozM1M7duyQZVmOu2CCXc2bN9drr72mWbNmaenSpdq+fbv27dun0047Tdddd5169Oihxx57TFL1lkzr16+fevbsqXfeeUcffvihvvnmG+3atUu7du1ScnKy0tPT1b17d1100UU67rjjKjxOZmamZs+erYULF2rz5s3hkdKxY8fquuuu048//ljlPv7WkCFDNHPmTEnBUeVWrVpVuK3L5dKkSZM0cOBAvfzyy/ryyy+1detWGYYRnmJz9dVX67TTTiuz77hx43Teeedp/vz5WrFihbZv365ff/1VDRo00Mknn6xzzjlH11xzjeOuRAfUZYZV3bVdAAC1xtSpUzVnzhwlJyeHL5cLAHUZc3YBoA6xLOuQS3aFRktbt259pLoEADFF2AWAOuKee+5Renq6LrvssnLXiN22bZs+/fRTSVL37t2PdPcAICYIuwBQR3Tt2lUFBQXauHGj7r333ohVEjZs2KBbbrlFPp9P8fHx4auJAUBdx5xdAKgjLMvSn/70J73++uuSgpcQPuaYY+T3+8Nr2cbFxWnatGnhtWcBoK4j7AJAHbN48WK9+uqrWrt2rfbs2aO4uDi1bNlS3bp107Bhw3TCCSfEuosAcMQQdgEAAFBnMWcXAAAAdRYXlSjHr79WvGxPVbhchpo0SdLu3XkyTQbSD4d62UO97KFe9lEze6iXPdTLHupVonnzhpXajpHdI8DlMmQYhlyuql+xqD6hXvZQL3uol33UzB7qZQ/1sod62UfYBQAAQJ1F2AUAAECdRdgFAABAnUXYBQAAQJ1F2AUAAECdRdgFAABAnUXYBQAAQJ1F2AUAAECdRdgFAABAnUXYBQAAQJ1F2AUAAECdRdgFAABAnUXYBQAAQJ1F2AUAAECdRdgFAABAnUXYBQAAQJ3liXUH6rvVP+7U6x9u0HmnHaM+p7eJdXcAAEA5HnroL1qw4K1KbfunPz2ggQOHVPs5e/ToqtNOO12ZmU9W+1j1GWE3xhZ89rM278jVq8s2EHYBAHCokSNv1KWXXhHRdv31w3Tcccfrz3+eFNHeqtUxUXnOp556Vg0aNIjKseozwm6MBQKmJKmg0C/TsuQyjBj3CAAA/FarVseUG2Lj4xPUsWOnGnnOmjpufcOc3Rjzekr+BH6/GcOeAACAaHj66dnq0aOrvvgiS+PH36Z+/c7Vxx8vDz++YMFbGj36Ov3udz3Vr9+5+v3vL9WTT85Sfn5exHF69OiqW2+9MeK43bufrrVr12rOnKd05ZUXqW/fc3XFFRfqmWeeVCAQOGKvsTZhZDfGvB53+LYvYCrO6z7E1gAAoLZ48slZOv30rho58ka1bh2cqvjKKy/qscdm6Lzz+mnUqNHyer366KMP9eyzz+jnnzdp8uRphz3uww8/rAYNknXHHX+UYbj03HNz9MwzT6p58xYaMuSiGn5VtQ9hN8ZKj+z6GNkFAKDOSElppBtvHBPRtmfPbp199rn6y18ekscTjGHp6Wdo9eovtWzZ+8rPzz/sPF3TNDVlyvTwN8ItWrTUsGFX6oMP3ifsloOwG2OEXQBAXfD5uh2av3yDDhbZ/yrd5TJkmlYN9KpEQpxbF/dMU9eOLWr0eUrr3v2cMm2jR99S7rbHHnus1q37Vtu3/6Ljj0875HEHDhwYcT81ta0kaf/+vVXraB1H2I0xr5uwCwCo/RZ+tknbduXHuhuHtOCzn49o2G3atFmZtt27d+mll57XJ58s1/bt21VQEFkzyzp8FmjZsmXEfa/XK0k1/oGhtnJ02J07d67mzJmjn3/+WY0bN9bgwYM1fvz48B+1tNdff1333ntvhcdasmSJ2rRx3tJejOwCAOqCC7odq3kOH9m9oFvbGn2O3wpNUwgpLDyom28epW3bturyy69W9+7nKCWlkVwuQ0899UTESWyHYrByky2ODbvz58/XxIkTNWHCBPXr10/r16/XxIkTlZ+fr0mTJpXZfuDAgerZs2eZ9lmzZunTTz/V0UcffSS6bRthFwBQF3Tt2KJKo6Yej0uNGydpz568Or8qUVbW59qyJUeXXXaVbrttXMRjBQUFMepV3efYsJuZmalBgwZpxIgRkqTU1FTt3LlTkyZN0pgxY8oM4SckJCghISGibdOmTXr11Vc1c+bMMp+unCIy7LJkCAAAdVVoabDGjRtHtK9d+7VWr/4yYhtEjyPX2d24caM2b96s3r17R7T36tVLpmlq+fLKDfM/9NBDOvvss9WrV6+a6GZURMzZDdTtT7QAANRnnTufosTEBnr99bl6//3FWr36K/3nP09r8uT7w1dnW7jwHW3atDG2Ha1jHDncmZ2dLUlq2zZybk2rVq3k9Xq1YcOGwx5j9erVWrZsmV599dUa6WO0MI0BAID6oUmTppo6dYb++c/HNWXKX5SQkKgzzuiqf/xjljwej774YpVef/0V5efn65577ot1d+sMR4bd3NxcSVJSUlJEu2EYSkpKCj9+KLNnz9Y555yjU045xfbzu1yGXK7oTf52F4/eut1lB9Lj40ouIhGwgnOX6rtD1QtlUS97qJd91Mwe6mVPba7Xp59+UW776NE3a/Tom8t9rFu3burWrVu5j/33vy8d8vijR9+sMWNuUUpKovbvLzjktijhyLBbXZs3b9b777+vf/7zn1Xav0mTpBo50zElJbFMW6NSbXFxHjVunFRmm/qqvHqhYtTLHuplHzWzh3rZQ73soV6V58iwm5KSIkllRnAty1JeXl748Yq8++67SkhI0DnnlF3MuTJ2786L+shu6FNY4Dfzcn1FvvDtvfsLtGdP3m93r3cOVS+URb3soV72UTN7qJc91Mse6lWisgOEjgy7aWnBK4ds2rRJ6enp4facnBz5fD61a9fukPu/99576t69u+Lj46v0/KZp1ch6f4GAWWZZFVepEeTCokCdX3bFjvLqhYpRL3uol33UzB7qZQ/1sod6VZ4jJ8ikpqYqLS1NS5cujWhfsmSJPB5Puevphhw8eFCrV6/W6aefXtPdjApOUAMAAKg5jgy7kjR27FgtWrRIc+bM0ZYtW7R48WLNnDlTw4YNU9OmTbVmzRplZGQoKysrYr+NGzfKNM0yKzk4FWEXAACg5jhyGoMkZWRkaPr06Zo9e7ZmzJihZs2aafjw4RozZoyk4JVGsrOzlZ8feU3pvXv3SpIaNmx4pLtcJayzCwAAUHMcG3YlaejQoRo6dGi5j3Xr1k3r168v0969e/dy253K6ylZeoyRXQAAgOhy7DSG+oJpDAAAADWHsBtjhF0AAICaQ9iNMebsAgAA1BzCbox5vSV/AtbLAwAAiC7CboxFjOz6AzHsCQAAQN1D2I0x5uwCAADUHMJujLldhkJXDGbOLgAAQHQRdmPMMIzw6G4RI7sAADjOPfeMU48eXbVu3XeH3O6HH9arR4+u+uMfb6/Ucbdt26oePbrqoYf+Em677LIhuuyyIZXaf9CgfpXetjK++CJLPXp01dNPz47aMZ2AsOsAoXm7TGMAAMB5LrnkCknSG2+8dsjt3njjdUnSpZdeUeXnmjbtUU2b9miV96+s3Nxc9e7dTV98kRVu69jxJD311LO68MJLavz5jyTCrgOERnYJuwAAOM9ZZ3VXampbLV68SHl5ueVuk5+fr3ffXajWrduoe/dzq/xcJ5zQTiec0K7K+1fWl19mKRCIPDG+QYMkdezYSc2aNa/x5z+SHH254PqCsAsAgHMZhqGLL75cjz02QwsXvlPuyO177y1Ufn6eRo68QYWFhXrxxef03nsLtW3bVsXHx+uYY9ro4osv05AhFx3yuULTEl599X/htnXrvlVm5j/03XffyOv16swzz9SYMWPL3X/Dhh/17LNztGrV5zpwYL8aN26ijh07aeTIG3Xiie0lSQ899BctWPCWJOn222+SJM2d+6a2bduq22+/Sdddd4NGjRodPubq1V/pueee0TffrFVBQb4aN26iM8/sppEjb9TRR7eK6HtyckNNnTpDmZmP6quvvlBRkU/HH5+m0aNv0emnd61EtaOPsOsAXo9bEieoAQDgVAMHDtG//jVLb775erlh9403XldCQoIGDhyqSZP+rI8+WqZhw0bqrLO66+DBg3r55Rc0bdpkFRUV2ZrmsGPHdt1++81KSEjQ2LF/1LHHttXmzdmaMOGPKiryKTGxZNtfftmmMWNuUHJysm67bZxatTpGmzf/rCeeyNTtt9+k//znRbVo0VIjR94oj8er//1vnv74x3vVseNJatasubZt21rm+T/99BPdc884nXhiB40ff7eaN2+hjRs36KmnntBnn63Qv//9gho3bhLePj8/T3/841hlZAzUpZdeqS1bNisz8x/605/+qJdfnq9GjY6yVfdoIOw6QGjOLheVAADUVl/sWKO3NryrwkCh7X1dLkOmadVAr0rEu+M1OG2ATm/RpUr7Jycna8CAC/TGG6/r669X65RTTg0/9t133+j779dpyJCLFB8fJ4/Hoyuu+L1uuOHm8DYnn3yKBg3qpwUL3rIVdufNe1X5+Xm6774H1Lt3X3k8LvXvf548nnhNnjxJjRo1Cm+7efMmdelymi699Ap163a2JOmUU05VQUGBHn10upYvX6ZLL71CrVodo2bNmkmS2rY9Vh07dqrw+TMz/6GEhAQ98sjjSkkJPtdpp52uo45qrD//+R69/PILuummW8Pbb926Rffd9xddcMFgSVJ6+hnatGmTXnzxOWVlrVS/fgMq/dqjhbDrAKFpDAHTUsA05XYxlRoAULss3rRM2/N3xLobh7T452VVDrtS8MSzN954XfPnvxYRdkMnpl1yyRWKj0/QX/86tcy+ycnJatq0mX75ZZut5/z669UyDEPdup0T0d6nTz899NCDEW1nntldZ57Zvcwxjj32OEnS9u32nnvHju3auHGDevXqEw66Ieee20tut1urVn0e0W4Yhvr27R/R1qZNqiRp3759tp4/Wgi7DlD6whJ+vyV3XAw7AwBAFfQ/trfjR3b7t+1drWOkpbXTaaedrqVLl2js2DuVktJIeXm5WrLkXZ1yyqnhObHff79Or776slat+lx79uxWUVFR+BilR2IrY9eunUpKSlJCQkJEe1JSsho0aBDRZlmW3nnnf1q48G1lZ/+k/fv3yzRLvjW2W+MdO4IfXlq0aFnmMa/Xq6OOaqydO3+NaG/YMEXx8Qlltg32LzbfYBN2HSDiKmoBU/Fyx7A3AADYd3qLLlUaNfV4XGrcOEl79uTViul8l1xyhb76aoLeeed/uuqqa7Rw4TsqKCgIT0348ccfdNNNoxQfH68RI0apY8dO4VD6xz+Old/vs/V81iHy6W/D65NPztJzz83Rqaema9y4e9SqVSt5vV6tW/edpk2bbO+FKjhKW9yLSmwTum/7aWocYdcBQnN2JVZkAADAyXr1Ok/NmjXX22+/qauuukZvv/2mmjZtqvPO6ydJWrjwbRUVFeqBByard+8+4f38fr8OHNivxNJnlFVC48aNtWXLZhUWFio+Pj7cvnfvHhUU5EeMFL/55us66qjG+sc/ZoVHU6VgAK+Ko48+WpK0ffsvZR4rLCzU3r171KnTyVU69pHE5FAHiBjZ9QcOsSUAAIglj8ejCy+8RNnZG/TBB0v0/ffrNHToJfJ4guOHobVrGzduHLHfyy8/r6KiojJr2x5Op06dZVmWVqz4KKL9/feXlNk2EAgoOTk5Iuj6fD7NnftiRN+kkhHZQ/WnadNmat++o7KyPtfevXsjHvvoow8VCATKzCV2IsKuA3g8jOwCAFBbDB16sTwej6ZPnxIOvyFnntlNkvTPfz6mzz//TJ9//pn+9rcHtWrV5+ra9Szl5ubq3XcX6NdfK3cy30UXXaq4uHj9/e9T9fbbb2rVqizNnj1bL7/8QpmTxrp27aacnM168slZWrPmKy1evEjXXz9M/fufL0nKyvpMq1d/Kb/fr+bNW0iS3nxznpYte7/c0VtJuu22cfL5ivTHP96upUsX66uvvtDcuS/p73//m9q0SdVll11lu35HGmHXAX47ZxcAADhX06bNdN55/bR//z716tUn4opj55zTQ3fc8Uft2bNH99wzTlOn/lWNGjXS3/72d1177XVq2rSZpk9/SFlZKyv1XKmpbfXoo5lq0yZVM2ZM0913j9eqVav097//Q0cddVTEtnfeOUG/+12G3nxznu688za9/PILGjnyBv3+98N02WVXatu2rbr//gk6ePCg+vUboK5dz9JHHy3T3/721wpXiUhPP0OZmU+qUaOjNG3aQxo79ma99NJ/9bvfna9//vMZJScnV7mOR4phWYea+lw//frrgage73CT719a8oPe/XyzJOnea07XiW2Oiurz1za17WSFWKNe9lAv+6iZPdTLHuplD/Uq0bx5w0ptx8iuA3iZxgAAAFAjCLsOQNgFAACoGYRdB4jzlKyrS9gFAACIHsKuA3CCGgAAQM0g7DoA0xgAAABqBmHXAbiCGgAAQM0g7DoAI7sAAAA1g7DrAB4uFwwAAFAjCLsOwAlqAAAANYOw6wDM2QUAAKgZhF0HYM4uAABAzSDsOgBhFwAAoGYQdh2AObsAAAA1g7DrAMzZBQAAqBmEXQdgGgMAAEDNIOw6AGEXAACgZhB2HYA5uwAAADWDsOsAbpdLLsOQxMguAABANBF2HSI0uusn7AIAAEQNYdchQmGXkV0AAIDocXTYnTt3rgYOHKjOnTurZ8+emjZtmnw+3yH3+fTTT3XllVeqS5cu6tGjhyZPnqyioqIj1OOqC4dd5uwCAABEjWPD7vz58zVx4kRdccUVWrBggR544AHNnz9fkydPrnCf1atX6/rrr9c555yjt99+W3/961/1v//9T3/961+PYM+rJrTWLiO7AAAA0eOJdQcqkpmZqUGDBmnEiBGSpNTUVO3cuVOTJk3SmDFj1LJlyzL7PPLII+rVq5fGjh0b3iczM1N+v/9Idr1KvF7CLgAAQLQ5cmR348aN2rx5s3r37h3R3qtXL5mmqeXLl5fZZ+/evVq5cqUGDx4c0X7mmWfq7LPPrtH+RgMjuwAAANHnyLCbnZ0tSWrbtm1Ee6tWreT1erVhw4Yy+6xfv16maaphw4YaP368zj33XPXp00f/+Mc/DjvP1wlCc3ZNy5KfebsAAABR4chpDLm5uZKkpKSkiHbDMJSUlBR+vLRdu3ZJkiZPnqzrrrtON9xwg1auXKmHH35Y+/fv1/3331/p53e5DLlcRjVeQSR38aht6Hd54rzu8G1LksfjyM8hR0Rl6oUS1Mse6mUfNbOHetlDveyhXvY5MuxWRWj0duDAgbrqqqskSSeddJK2bdum5557TrfeequaNGlSqWM1aZIkw4he2A1JSUms8LEGid7w7aTkBDVKjo/689c2h6oXyqJe9lAv+6iZPdTLHuplD/WqPEeG3ZSUFEkqM4JrWZby8vLCj5fWsGFDSVLnzp0j2rt27ao5c+bohx9+ULdu3Sr1/Lt350V9ZDclJVH79xcoUNEUBcsK3/x1Z65Mn/NPqqsplaoXwqiXPdTLPmpmD/Wyh3rZQ71KNG6cdPiN5NCwm5aWJknatGmT0tPTw+05OTny+Xxq165dmX2OO+44SdK+ffsi2q3iEJmcnFzp5zdNS6ZpHX5DmwIBs8IrpHlKheuDhX6upKZD1wtlUS97qJd91Mwe6mUP9bKHelWeIyd8pKamKi0tTUuXLo1oX7JkiTwej3r27Flmn7S0NKWmpuq9996LaM/KylJ8fHw4DDuVt9QcXVZkAAAAiA5Hhl1JGjt2rBYtWqQ5c+Zoy5YtWrx4sWbOnKlhw4apadOmWrNmjTIyMpSVlRXe54477tD777+vxx57TJs3b9bcuXP14osvavjw4WVOdnMar7vkBDWuogYAABAdjpzGIEkZGRmaPn26Zs+erRkzZqhZs2YaPny4xowZI0kqKChQdna28vPzw/sMHjxYlmVp9uzZevLJJ9W0aVPdeuutuv7662P1MiqNkV0AAIDoc2zYlaShQ4dq6NCh5T7WrVs3rV+/vkz7kCFDNGTIkJruWtR5CLsAAABR59hpDPUNI7sAAADRR9h1CG+pxaGZswsAABAdhF2HiBzZDcSwJwAAAHUHYdchmMYAAAAQfYRdhyDsAgAARB9h1yGYswsAABB9hF2HYGQXAAAg+gi7DkHYBQAAiD7CrkMQdgEAAKKPsOsQcR53+DZzdgEAAKKDsOsQEZcL9hF2AQAAooGw6xAR0xgY2QUAAIgKwq5DRCw9xpxdAACAqCDsOgQnqAEAAEQfYdchIsNuIIY9AQAAqDsIuw7BFdQAAACij7DrEC6XIbfLkMQ0BgAAgGgh7DpIaCoDYRcAACA6CLsOQtgFAACILsKug4TDLnN2AQAAooKw6yChk9T8jOwCAABEBWHXQZjGAAAAEF2EXQcpHXYty4pxbwAAAGo/wq6DhKYxWJICJmEXAACgugi7DsIlgwEAAKKLsOsgXo87fJuwCwAAUH2EXQfxMLILAAAQVYRdBwnN2ZVYaxcAACAaCLsOEudlZBcAACCaCLsOUnpkt8gfiGFPAAAA6gbCroOUXo2Bq6gBAABUH2HXQVh6DAAAILoIuw5C2AUAAIguwq6DsBoDAABAdBF2HYSRXQAAgOgi7DoIF5UAAACILsKugzCyCwAAEF2EXQfxut3h28zZBQAAqD7CroMwsgsAABBdhF0HIewCAABEF2HXQQi7AAAA0eWJdQcOZe7cuZozZ45+/vlnNW7cWIMHD9b48ePl9XrLbJuTk6N+/fqVe5w//OEPuv/++2u6u9XGOrsAAADR5diwO3/+fE2cOFETJkxQv379tH79ek2cOFH5+fmaNGlShfs9/vjjSk9Pj2hLTEys6e5GReTIbiCGPQEAAKgbHBt2MzMzNWjQII0YMUKSlJqaqp07d2rSpEkaM2aMWrZsWe5+jRo1UvPmzY9gT6OHaQwAAADR5cg5uxs3btTmzZvVu3fviPZevXrJNE0tX748Rj2rWYRdAACA6HJk2M3OzpYktW3bNqK9VatW8nq92rBhQyy6VeMiwi5zdgEAAKrNkdMYcnNzJUlJSUkR7YZhKCkpKfx4ed5++23NmDFDP//8s4466ihdcsklGjFihOLi4ir9/C6XIZfLqFrny+EuPvHM7T70Z4vE+JI/RyBgRVw+uD6pbL0QRL3soV72UTN7qJc91Mse6mWfI8NuVbjdbjVr1kwHDx7U3XffrQYNGuijjz7SY489po0bN2rKlCmVPlaTJkkyjOiF3ZCUlEOfKGeaVsltSY0bJ1W8cT1wuHohEvWyh3rZR83soV72UC97qFflOTLspqSkSFKZEVzLspSXlxd+vLRWrVrp448/jmjr1KmT8vLy9MQTT+jWW2/VMcccU6nn3707L+ojuykpidq/v0CBw0xP8Lpd8gVMFRz0ac+evKj1oTaxUy9QL7uol33UzB7qZQ/1sod6lajsoKAjw25aWpokadOmTRHLiOXk5Mjn86ldu3aVPtZJJ50kSdq+fXulw65pWhGjrNESCJjyH+bEM48nGHaLfIfftq6rTL1QgnrZQ73so2b2UC97qJc91KvyHDnhIzU1VWlpaVq6dGlE+5IlS+TxeNSzZ88y+yxevFgTJkyQ3++PaP/666/lcrnKnOzmVKGT1FiNAQAAoPocGXYlaezYsVq0aJHmzJmjLVu2aPHixZo5c6aGDRumpk2bas2aNcrIyFBWVpYkqWXLlnrrrbc0btw4rV27Vps2bdJ///tfPfvss7rsssvUtGnTGL+iygldRY3VGAAAAKrPkdMYJCkjI0PTp0/X7NmzNWPGDDVr1kzDhw/XmDFjJEkFBQXKzs5Wfn6+JOmUU07RnDlzNGvWLF1//fXKzc1V69atdeutt2rUqFGxfCm2MLILAAAQPY4Nu5I0dOhQDR06tNzHunXrpvXr10e0nXnmmZozZ86R6FqNIewCAABEj2OnMdRXobDrD5iyrOifJAcAAFCfEHYdxltqkWg/83YBAACqhbDrMBGXDGYqAwAAQLUQdh2GsAsAABA9hF2HIewCAABED2HXYUrP2WWtXQAAgOoh7DoMI7sAAADRQ9h1GA9hFwAAIGoIuw7DyC4AAED0EHYdhjm7AAAA0UPYdRhGdgEAAKKHsOswXo87fJuwCwAAUD2EXYdhZBcAACB6CLsOEzFn1x+IYU8AAABqP8Kuw8R5GdkFAACIFsKuw7AaAwAAQPQQdh2GObsAAADRQ9h1GMIuAABA9BB2HYbLBQMAAEQPYddhmLMLAAAQPYRdh2EaAwAAQPQQdh2GsAsAABA9hF2H4XLBAAAA0UPYdRjm7AIAAEQPYddhmMYAAAAQPYRdh/G4jfBtwi4AAED1EHYdxjCM8OguYRcAAKB6CLsOFJq3y5xdAACA6iHsOlBoZNfvD8S4JwAAALUbYdeBmMYAAAAQHYRdBwqHXaYxAAAAVAth14HCc3YZ2QUAAKgWwq4DhefsBiyZphXj3gAAANRehF0HiriwBFMZAAAAqoyw60Bejzt8m6kMAAAAVUfYdSAuGQwAABAdhF0HYhoDAABAdBB2HSi0GoPEyC4AAEB1EHYdqPTIrp+wCwAAUGWEXQdizi4AAEB0EHYdKDLsBmLYEwAAgNrN0WF37ty5GjhwoDp37qyePXtq2rRp8vl8ldp37969Ovfcc9W3b98a7mX0RczZ5QQ1AACAKnNs2J0/f74mTpyoK664QgsWLNADDzyg+fPna/LkyZXaf8qUKdq7d2/NdrKGMI0BAAAgOhwbdjMzMzVo0CCNGDFCqamp6t+/v8aOHatXXnlF27dvP+S+H374oRYtWqShQ4ceod5Gl4ewCwAAEBWODLsbN27U5s2b1bt374j2Xr16yTRNLV++vMJ9c3Nz9cADD+i2227TMcccU9NdrRGM7AIAAERHjYbdPXv2yO/3294vOztbktS2bduI9latWsnr9WrDhg0V7jtjxgw1btxY1113ne3ndQrm7AIAAESHp7oHWLZsmebOnavMzMxw2yeffKL77rtPv/zyi5KSknTLLbfYCp+5ubmSpKSkpIh2wzCUlJQUfvy3srKyNHfuXL3yyityu91VeDVBLpchl8uo8v6/5S4Or2535T5bJMSX/FkCphUxraE+sFuv+o562UO97KNm9lAve6iXPdTLvmqF3aysLN1yyy0yDEOmacrlcmnHjh265ZZbVFBQoE6dOiknJ0fTp0/Xcccdpz59+kSr32UUFhbqvvvu04gRI9SpU6dqHatJkyQZRvTCbkhKSmKltmvcqGQ7j9ejxo2TDrF13VXZeiGIetlDveyjZvZQL3uolz3Uq/KqFXafffZZJSYm6vnnn5fLFfyE8fLLL6ugoEC33367xowZo7179+qiiy7SSy+9VOmwm5KSIkllRnAty1JeXl748dIef/xxeTwe3XbbbdV5SZKk3bvzoj6ym5KSqP37CxSoxLSEwoMly6vtP3BQe/bkRa0vtYHdetV31Mse6mUfNbOHetlDveyhXiUqOxhYrbC7Zs0aDRgwQO3btw+3LV26VAkJCRo2bJgk6aijjlL//v21YMGCSh83LS1NkrRp0yalp6eH23NycuTz+dSuXbsy+7zzzjvatm1bxPamacqyLHXq1EljxozRrbfeWqnnN01LpmlVur+VFQiYlbr8b+mcXegL1NtLBle2XgiiXvZQL/uomT3Uyx7qZQ/1qrxqhd1du3bp2GOPDd/ft2+fvvvuO51zzjlKTk4Ot7do0UL79u2r9HFTU1OVlpampUuX6qKLLgq3L1myRB6PRz179iyzz9NPP13mghMvvPCClixZoqefflpNmza18cpiy+spmW/MagwAAABVV63ZzXFxcRFTDT7++GNZlqVzzz03Yrvc3NwyJ5sdztixY7Vo0SLNmTNHW7Zs0eLFizVz5kwNGzZMTZs21Zo1a5SRkaGsrCxJ0vHHH6/27dtH/DRt2lRerzd8u7bgcsEAAADRUa2we8IJJ2jp0qXy+/0yTVPPPvusDMMoMzd35cqVat26ta1jZ2RkaPr06Xr11Vd1/vnna/LkyRo+fLjuuusuSVJBQYGys7OVn59fnZfgSKyzCwAAEB3VmsYwePBgTZkyRQMGDJAkbdu2Tb169dLxxx8vScrPz9fjjz+u1atXV+nEsaFDh1Z4FbRu3bpp/fr1h9z/tttui8oJa0daHGEXAAAgKqoVdq+55hr9+OOPev311+X3+3XKKado6tSp4cd37dqlOXPm6KSTTqrVF3k40rhcMAAAQHRUK+y6XC49+OCD+tOf/qS8vLwy82JTU1N133336ZJLLlFiIuvBVRZXUAMAAIiOal9BTZISEhKUkJBQ7mPXXnttNJ6iXmHOLgAAQHRU+1pz3377raZMmRLRtm7dOl1zzTVKT0/XoEGDtHDhwuo+Tb3idhkKXcCNsAsAAFB11Qq769ev1zXXXKMXXnhBphkMZfv379fIkSOVlZWluLg4bdiwQePHj9eqVaui0uH6wDCM8Ogu0xgAAACqrlph95lnnpHf79esWbPClwueO3eudu/erd///vf67LPPtGjRIqWkpOjZZ5+NSofri9C8XUZ2AQAAqq5aYffzzz/XgAED1KtXr3Dbe++9J4/HE740b9u2bTVgwAB9+eWX1etpPRMe2SXsAgAAVFm1wu7OnTvVrl278P28vDytXbtWp556qpo0aRJub926tXbv3l2dp6p3CLsAAADVV62w63a7VVhYGL6/cuVK+f3+MpcLLigoYOkxm7wetyTm7AIAAFRHtcLuscceqxUrVoTvv/jiizIMQ+edd17Edl9//bVatmxZnaeqd0Jzdv2M7AIAAFRZtdbZHTBggB577DFdddVVcrlc+vLLL3XaaaepU6dOkqRAIKAXX3xRK1as0MiRI6PS4foiNI0hYFoKmKbcrmqvEgcAAFDvVCvsjho1SqtWrdLHH38sSWrVqpWmT58efnzjxo2aPHmyjjnmGMKuTaUvLOH3W3LHxbAzAAAAtVS1wm58fLyefvppbdy4Ufv371fHjh0VF1eSytLS0jRixAhdd911ESes4fAirqIWMBUvdwx7AwAAUDtF5XLBxx13XLnthmFowoQJ0XiKeic0Z1diRQYAAICqikrY/eWXX7Rw4UJ9++232rNnjwzDUNOmTdW5c2cNHDhQjRs3jsbT1CsRI7v+QAx7AgAAUHtVO+z++9//1owZM+T3+2VZVsRj8+fP14wZM/Tggw9q8ODB1X2qesXjYWQXAACguqoVdpctW6apU6cqMTFRF154obp06aImTZrINE3t3r1bq1at0qJFizRhwgS1bdtWXbp0iVa/67zSI7tFhF0AAIAqqVbYfe6559SoUSO98sorOvbYY8s8ftVVV+mGG27Q1VdfraeeekqPPfZYdZ6uXmHOLgAAQPVVa/HWtWvX6vzzzy836Ia0b99e559/vr744ovqPFW989vVGAAAAGBftcJubm6ujj766MNu16ZNG+3du7c6T1XvxDFnFwAAoNqqFXZTUlK0efPmw263detWpaSkVOep6h2vp2RdXS4ZDAAAUDXVCrunnnqq3n33Xa1fv77CbdatW6cFCxbotNNOq85T1TteRnYBAACqrVonqF133XX64IMPdPnll2vQoEFKT08PXylt165dysrK0qJFixQIBDRq1KiodLi+YM4uAABA9VUr7J511ll68MEH9dBDD2nevHmaP39+xOOWZSkxMVGTJ0/WGWecUZ2nqndYjQEAAKD6qn1Ricsvv1x9+vTRO++8o7Vr12rXrl3hK6idcsopGjRoEFdQqwKmMQAAAFRfVC4X3KxZMw0bNqzCx5csWaJ58+YpMzMzGk9XL3i4XDAAAEC1VesEtcratGmTlixZciSeqs5gzi4AAED1HZGwC/uYswsAAFB9hF2HYs4uAABA9RF2HYqwCwAAUH2EXYdizi4AAED1EXYdijm7AAAA1UfYdSimMQAAAFSf7XV2zz77bNtPcvDgQdv71HeEXQAAgOqzHXb37NlTpScyDKNK+9VXzNkFAACoPtthl4tDHBlul0suw5BpWfL5CLsAAABVYTvstm7duib6gXJ4PS4V+gKM7AIAAFQRJ6g5WGgqg88fiHFPAAAAaifCroOVhF1GdgEAAKqCsOtghF0AAIDqIew6WDjsMmcXAACgSgi7Dha6iprPb8qyrBj3BgAAoPZxdNidO3euBg4cqM6dO6tnz56aNm2afD5fhdvv2bNHkydPVt++fdW5c2edd955mjZtWq29qEVoZNeypIBJ2AUAALDL9tJjR8r8+fM1ceJETZgwQf369dP69es1ceJE5efna9KkSWW2N01T119/vfLz8/XQQw+pTZs2ysrK0v33369ff/1Vf//732PwKqrnt1dR87gd/dkEAADAcRwbdjMzMzVo0CCNGDFCkpSamqqdO3dq0qRJGjNmjFq2bBmx/XfffadNmzZp1qxZOuuss8L7ZGVlacGCBbIsq9Zdxc3rjryKWmIM+wIAAFAbOXKocOPGjdq8ebN69+4d0d6rVy+Zpqnly5eX2efkk09WVlZWOOiGuFwuud3uWhd0pciRXT8rMgAAANjmyJHd7OxsSVLbtm0j2lu1aiWv16sNGzYc9hh+v1/vv/++3nrrLd122222nt/lMuRyRS8cu4tHaN02pyHEed3h26Ykj8eRn02irqr1qq+olz3Uyz5qZg/1sod62UO97HNk2M3NzZUkJSUlRbQbhqGkpKTw4xW56qqrtHr1aiUlJelPf/qTLr/8clvP36RJUo2MBKek2JuIkJwUH76d2CBejRsnHWLrusduveo76mUP9bKPmtlDveyhXvZQr8pzZNitrkcffVT79u3TRx99pAcffFA7duzQLbfcUun9d+/Oi/rIbkpKovbvL1DAxpq5ZqDkMsG7dueqUYL7EFvXHVWtV31FveyhXvZRM3uolz3Uyx7qVaKyg4CODLspKSmSVGYE17Is5eXlhR+vSKtWrdSqVSt17NhRhmFoxowZuvzyy9WiRYtKPb9pWjJrYKmvQMC0NffW4yr5iuJgYaDezdu1W6/6jnrZQ73so2b2UC97qJc91KvyHDnhIy0tTZK0adOmiPacnBz5fD61a9euzD4bNmzQm2++Wab9xBNPVCAQCM8Drk08v1l6DAAAAPY4MuympqYqLS1NS5cujWhfsmSJPB6PevbsWWafNWvW6K677tKaNWsi2tetWydJZZYqqw1+u84uAAAA7HFk2JWksWPHatGiRZozZ462bNmixYsXa+bMmRo2bJiaNm2qNWvWKCMjQ1lZWZKkCy64QGlpabr77ru1fPlybd68WW+++ab+9a9/qUePHjruuONi+4Kq4Lfr7AIAAMAeR87ZlaSMjAxNnz5ds2fP1owZM9SsWTMNHz5cY8aMkSQVFBQoOztb+fn5kqT4+Hj9+9//1owZM3T33XcrNzdXxxxzjK6++mqNHj06li+lyiJHdgOH2BIAAADlcWzYlaShQ4dq6NCh5T7WrVs3rV+/PqKtZcuWmj59+pHo2hFROuwWMY0BAADANsdOYwBzdgEAAKqLsOtgpefssrwIAACAfYRdB2NkFwAAoHoIuw4WEXZZjQEAAMA2wq6DxXlKLg/MyC4AAIB9hF0Hi1iNwcfSYwAAAHYRdh2sQULJynB5B/0x7AkAAEDtRNh1sOREb/h2boEvhj0BAAConQi7DpYQ55bHbUiSDuQTdgEAAOwi7DqYYRjh0d3cgqIY9wYAAKD2Iew6XHJinKTgyK5lWTHuDQAAQO1C2HW4hg2CI7sB09LBIlZkAAAAsIOw63ChsCtJBzhJDQAAwBbCrsNFrMjASWoAAAC2EHYdLnL5MU5SAwAAsIOw63ANG8SFb7P8GAAAgD2EXYfjwhIAAABVR9h1uOQGhF0AAICqIuw6XMNSI7tMYwAAALCHsOtwyRFhlxPUAAAA7CDsOlxDpjEAAABUGWHX4bwet+Lj3JIIuwAAAHYRdmuB0Lxd5uwCAADYQ9itBULzdvMO+mSaVox7AwAAUHsQdmuB0PJjliXlF/pj3BsAAIDag7BbCzRkRQYAAIAqIezWAsmJJZcM5iQ1AACAyiPs1gKlr6LGSWoAAACVR9itBVhrFwAAoGoIu7UAc3YBAACqhrBbC5S+ZDAjuwAAAJVH2K0FkhuUOkGNObsAAACVRtitBSKmMTCyCwAAUGmE3VogKdETvs00BgAAgMoj7NYCbpdLSQnBwMs0BgAAgMoj7NYSoXm7BwpYjQEAAKCyCLu1RGjebkFhQP6AGePeAAAA1A6E3VqC5ccAAADsI+zWEqUvGcy8XQAAgMoh7NYSLD8GAABgH2G3logY2SXsAgAAVIqjw+7cuXM1cOBAde7cWT179tS0adPk81Uc9PLz8zVjxgydf/75OvXUU5WRkaEnnnjikPvUFhFzdvNZkQEAAKAyPIffJDbmz5+viRMnasKECerXr5/Wr1+viRMnKj8/X5MmTSp3n/Hjx2v16tWaNGmSOnbsqBUrVujBBx9UQUGBxo0bd4RfQXQ1TCy5ZDDTGAAAACrHsWE3MzNTgwYN0ogRIyRJqamp2rlzpyZNmqQxY8aoZcuWEdv/9NNPWrp0qaZOnaoBAwZIktq2bauVK1fqhRdeqP1hlxPUAAAAbHPkNIaNGzdq8+bN6t27d0R7r169ZJqmli9fXmaf448/Xh999JEGDRoU0d6yZUsVFBTINGv32rSl5+wysgsAAFA5jhzZzc7OlhQcmS2tVatW8nq92rBhQ5l9XC6XmjdvHtHm9/v14YcfqkuXLnK5HJnrK60hc3YBAABsc2TYzc3NlSQlJSVFtBuGoaSkpPDjhzNjxgxt2LBBzz77rK3nd7kMuVyGrX0Oxe12RfyuioZJcXIZhkzLUu5Bvzye2h3eDyUa9apPqJc91Ms+amYP9bKHetlDvexzZNitLsuyNG3aNP373//WpEmT1LVrV1v7N2mSJMOIXtgNSUlJrN7+yXHae6BQeQf9atw46fA71HLVrVd9Q73soV72UTN7qJc91Mse6lV5jgy7KSkpklRmBNeyLOXl5YUfL4/P59OECRO0aNEiTZ8+XUOHDrX9/Lt350V9ZDclJVH79xcoEKj63OGkBI/2HijU/txC7dmTF7X+OU206lVfUC97qJd91Mwe6mUP9bKHepWo7MCfI8NuWlqaJGnTpk1KT08Pt+fk5Mjn86ldu3bl7mdZlu655x598MEH+te//qWzzz67Ss9vmpZM06rSvocSCJjy+6v+xkxOCM7bLfKbyivwKd7rjlbXHKm69apvqJc91Ms+amYP9bKHetlDvSrPkRM+UlNTlZaWpqVLl0a0L1myRB6PRz179ix3v5kzZ2rJkiXVCrpOlszyYwAAALY4MuxK0tixY7Vo0SLNmTNHW7Zs0eLFizVz5kwNGzZMTZs21Zo1a5SRkaGsrCxJ0rZt2/TEE0/ommuuUdu2bfXrr79G/BQV1f4VDBo2KLmwBJcMBgAAODxHTmOQpIyMDE2fPl2zZ8/WjBkz1KxZMw0fPlxjxoyRJBUUFCg7O1v5+fmSpE8//VQ+n09PPfWUnnrqqTLHe/bZZ9WtW7cj+hqirfQlgw+w/BgAAMBhOTbsStLQoUMrPMGsW7duWr9+ffj+xRdfrIsvvvhIdS0mSq+1y4UlAAAADs+x0xhQFnN2AQAA7CHs1iKM7AIAANhD2K1FIkZ2CbsAAACHRditRUqfoJbLCWoAAACHRditRRomsvQYAACAHYTdWiQ+zq04T/BPxpxdAACAwyPs1jKhebsHWI0BAADgsAi7tUxo3m5uvk+WZcW4NwAAAM5G2K1lQsuPmZalgkJ/jHsDAADgbITdWia5QclJaszbBQAAODTCbi0TufwYYRcAAOBQCLu1DFdRAwAAqDzCbi0TcRU1RnYBAAAOibBbyzRswIUlAAAAKouwW8uUnrN7gEsGAwAAHBJht5Zhzi4AAEDlEXZrGebsAgAAVB5ht5aJWHqMkV0AAIBDIuzWMh63S4nxbklMYwAAADgcwm4tFBrdzeUENQAAgEMi7NZCyYnB5cfyD/oVMM0Y9wYAAMC5CLu1UMPik9QsSXkH/bHtDAAAgIMRdmuh0suPsSIDAABAxQi7tVDp5ce4sAQAAEDFCLu1EMuPAQAAVA5htxZq2CAufJvlxwAAACpG2K2FkpmzCwAAUCmE3VqIaQwAAACVQ9ithRpGnKBG2AUAAKgIYbcWKj1nl5FdAACAihF2a6EG8R4ZRvB2bgFLjwEAAFSEsFsLuVyGkhKCUxmYxgAAAFAxwm4tFZq3y9JjAAAAFSPs1lKhFRkKiwLy+QMx7g0AAIAzEXZrqcjlx/wx7AkAAIBzEXZrqcjlxzhJDQAAoDyE3VoqOZHlxwAAAA6HsFtLlR7ZJewCAACUj7BbS5Wes8vyYwAAAOUj7NZSzNkFAAA4PMJuLcWcXQAAgMNzdNidO3euBg4cqM6dO6tnz56aNm2afL5DB7v8/Hzdc8896tChg1588cUj1NMjL5k5uwAAAIfliXUHKjJ//nxNnDhREyZMUL9+/bR+/XpNnDhR+fn5mjRpUrn7rF+/XnfccYcMwzjCvT3yGjJnFwAA4LAcO7KbmZmpQYMGacSIEUpNTVX//v01duxYvfLKK9q+fXu5+8ycOVM9evTQrFmzjnBvj7yEOLfcrmCoZ2QXAACgfI4Muxs3btTmzZvVu3fviPZevXrJNE0tX7683P3uvPNO3XffffJ4HDtgHTWGYYSnMhB2AQAAyufIsJudnS1Jatu2bUR7q1at5PV6tWHDhnL3O/bYY2u8b07SsPgktQP5PlmWFePeAAAAOI8jh0Bzc3MlSUlJSRHthmEoKSkp/HhNcbkMuVzRm/frdrsifkdLSpJX+lXyB0wFLEsJXndUjx8rNVWvuop62UO97KNm9lAve6iXPdTLPkeG3Vhr0iSpRk5yS0lJjOrxmjRKlLRHkmR4PGrcOOnQO9Qy0a5XXUe97KFe9lEze6iXPdTLHupVeY4MuykpKZJUZgTXsizl5eWFH68pu3fnRX1kNyUlUfv3FygQMKN23ARvyae67M17FF9HPuTVVL3qKuplD/Wyj5rZQ73soV72UK8SlR3kc2TYTUtLkyRt2rRJ6enp4facnBz5fD61a9euRp/fNC2ZZvTnwAYCpvz+6L0x2zRPDt9e//MetWvdKGrHdoJo16uuo172UC/7qJk91Mse6mUP9ao8R44FpqamKi0tTUuXLo1oX7JkiTwej3r27BmjnjlL+9Sjwrd/yNkXu44AAAA4lCPDriSNHTtWixYt0pw5c7RlyxYtXrxYM2fO1LBhw9S0aVOtWbNGGRkZysrKCu/z66+/6tdff9Xu3bslBadBhNoCgUCsXkqNadk4USnFy4/9kLO3RkajAQAAajNHTmOQpIyMDE2fPl2zZ8/WjBkz1KxZMw0fPlxjxoyRJBUUFCg7O1v5+fnhfXr06BFxjL///e/6+9//Lik4KtymTZsj9wKOAMMwdGLqUVq1/lcVFAaU82uu2rZsGOtuAQAAOIZjw64kDR06VEOHDi33sW7dumn9+vURbb+9Xx+0bxMMu5L0/ea9hF0AAIBSHDuNAZVTet7u98zbBQAAiEDYreVSWyQrIS54MYkfNu/lSmoAAAClEHZrOZfLCC85ti+vSDv2FsS4RwAAAM5B2K0DTiw9lWHz3pj1AwAAwGkIu3VA+zYlF5P4YTPzdgEAAEIIu3VA2jEp8riDlzdmZBcAAKAEYbcO8HrcOq5ViiRpx94C7c0tjHGPAAAAnIGwW0e0b3NU+DajuwAAAEGE3Tqi9Hq7zNsFAAAIIuzWEe1aN5JRfPv7nL2x7AoAAIBjEHbriAYJHqW2SJYk5ezIVf5BX4x7BAAAEHuE3ToktN6uJenHLUxlAAAAIOzWIe0jLi5B2AUAACDs1iGlLy7BvF0AAADCbp3SKDleLRonSpKyt+5XkS8Q4x4BAADEFmG3jgmttxswLWVv2x/bzgAAAMQYYbeOOTG11FQGLi4BAADqOcJuHRNxkloOJ6kBAID6jbBbx7Q4KlGNkuIkBZcfC5hmjHsEAAAQO4TdOsYwjPB6u4VFAW3ekRvbDgEAAMQQYbcOiliCjPV2AQBAPUbYrYNKz9v9gZPUAABAPUbYrYPaNE9WYrxHUvDiEpZlxbhHAAAAsUHYrYNcLkMnFk9lOJDv0y+782PcIwAAgNgg7NZRJ7ZhvV0AAADCbh1Vet7uup/3xqwfAAAAsUTYraOOOzpFcZ7gn/ezb7drxdpfYtwjAACAI4+wW0d5PS4NPue48P2n3/5OX/24M3YdAgAAiAHCbh026Oxj1ef01pIk07L0z/lrtf7nPTHuFQAAwJFD2K3DDMPQH37XXmed1EKS5PObeuy1Nfp5+4EY9wwAAODIIOzWcS7D0PWDO6lzWhNJUkFhQI+8/JW2sxwZAACoBwi79YDH7dItF52idq2Dy5Htz/fp7y99pT0HCmPcMwAAgJpF2K0n4uPcGnt5F7VpniRJ2rX/oGa8/JVyC3wx7hkAAEDNIezWI0kJXo2/8jQ1a5QgSdq6M08zXvpKn377i/blFcW4dwAAANHniXUHcGQdlRyvP151mqY8/7kOtvlE25P26dmfvbI2eBVnJKhRQpKaJqeoVaNGSklIVpI3UQ08DZTkbaAG3kQleRqogbeBEtzxMgwj1i8HAADgkAi79VCLxg10xaDmej67eBkyd6GMuEL5latd2qldedL3eYc+hiFDCa5EJXoSleRtoOS4BmoYl6QG3kQ18DYoDsXBx4JhOdjewJMol8EXCgAA4Mgg7NZT3Y7toJyic7X21++VW5SvQvOg5ApUen9LlgrMfBUU5Wt30S7pMOG4tDgjXvGuRCW4ExTvSlCCOzH425WoeHeCGngbqFFSknxFPlmy5HJJbsOQyyUZLoV/S5JlWbJkybIsmTIj7gfbim+Xesw81D6hdssstU3JY5LkMdxyuzzyuNzyGB65Xe7iNvdhHiu+7/KUu73biHyMDwUAAFQfYbeecrvcuqLDhbqiQ0nb3vx8fb1xu77N2a6fftmp3fkHZHh8Mjw+yeOT4S7+HWpzh277bT13kVWookChDlQ+W9dLhgy55JbLCAZfd/Ftt+GWS265DVfwt8str8crvz8QDPKlgr4sS5YlWYZVfFRLRvDg4fuh24YR/Am1hbYzwltakmFJVvC2peCHDUXcVvhDgmRJMuR1eeRxexXn8sjr8gZ/3F55Q/fdXsW5vPK4PIoLPxb5uNcV3N/j8iqu9L6ljlXbPxyYpqWAacrjdjFFqB4wLVMBy1TA9MtvBRQwA/KbAQUsf/HvQKnf/rL3zUDJftZh7hcfN2AGZBhG8EN18Y/X5ZHHKP7tKvntcXnlcbnljfhd+vHi/dyR+/Pejb7Q39Jn+uQ3/bKKTBV68pR/0CeZrnDt3Yab+lfAsEL/h0LYr79G96ILHo9LjRsnac+ePPn9ZlSPXZMO5Bcpt8Cng0UBHSz0q6AooIJCf/B+kV/5hX7lH/Qrt6BQBwoLlOvLU4G/QAcDBSqyCiVPUTAMVxCS5fGJf5eIGtMlWS7JckumW4blkmF55JJbhlX8AUEeuQ2PvG6vLMuSywhefMVwBdekdhmh0G+UjPSrVJC3QuP7wSAvWTKMknvBR6zi4Br8FiFgWjJNM9gW+iAS+hAS/hYidMzggdwuyeUygj9Gye3whxEp/E1DyX/CrXDfg6+n5LWUfJAp3j/i351V6pZZ5nWGXpthGDJNyTIV7LslWZYRvm3IJZdhyDCM4lq6gh/YDJdcLkNuwyW3K7iN2+WW2+WS22XI7XJJxccxzdAxreLnMhR6eZZlBHtiSbKMcJtpBesbsII1Ni0z3GYp+E2Q220Ef4pvu1yS22UUf0NklfmQF/5gV/obH5X6m8mSZZnh+2bpb48sK/jh0rDkCwQDZkABmVZApszw71Ct65rgB3G33EbJN1yhkBz88O6RSy655JGs4Ad2Q24leL2SKbndbsW5PfK63YpzuxXn9SjOHQzRhmUoeF598e3i94QhV/H7yRV8zuL3V/C2S6ZlyAxIgeIff8CS3y8F/JYCpoLP5fEo3uOR1xN8/nhv8L7hUvjDhs/0F38Y8ctf/NtUQJYCMg1Tpvwyi7fzl/rxhX4CPhUF/CoyffIF/OHwGvwJhtmIDzdW8PksVS6mGQr+2/IapT+4eIsHHMr/UOM9xAeb0h+IvKU+4FiWS5bpkuU3ZJoumQFDAX/wp3lKstq2bHjEQnfz5g0rtR0ju6hQwwZxatggrkr7BkxTBYUB5Rf6VXDQr4LC4E9+8c/BQr9M05JPRfJZB4OjvdZB+XVQfhVJHlNFRYHwf4xM01IgEPyfYSAQChPB2+HfAYVvS8F/+JIkyxX8n5kVum/ILP4fa+n/aap4r+B/V4yI7cO/DUmGWfxjyXCV3JbLlGFE3pdR3OaywvsF9ym+H94m8n7lj12lP09YyUfd0Osuvi2V1CG8ceknO/z2lmVEvi5XoGY/3LhMSaYkf7g7VnGLLaEdy1PZ/rtUqbVujEMcMvRqquVQr6W6DtX53/ahJr/FMSS5bWwfUM32p54zi4O9zyqy9wYuqLEu1RuWrGBwlj9m73GrKF7nt7hYF55+emw6UAHCLmqE2+VScqJLyYle2/seyZHw0KhacCQoOAoXMC35/KZ8AVN+vxlxO2BZivO4FOdxy+txKc7jktfjCo4GeIOhumQkSlJ4NCgY2H0BU0W+gHx+U0U+U0X+QPC3LyBfwAz3wzRValQw+BMabSv9XUzACkiGqfgEj3w+vwwrODLidgVH1Dzu4MiZIYU/NFimikcYFXy9gWC/fMV98fmD/fL5Q7dNBQKm/KYV/B2w5A+Y4f0i+mVZMq3I2paMLloyXJLhCgSDvNuUjIAsIyAZAZlGMIVYRkCWKyBLpgx3oFRQDv4OBv2S31bx7dBx5ArIMszwbRk1lfZqj/I/1BTflw7/waZko5J6Fo98ylDwQw0khWptSKZR/E1D8ci1Gfxdtq34xzRklXos9E2FFdqn1P5W8fahfcsc+1DHkko+SIf+LZb+4O0q9aHbFQj+bV2B8Dblbx8o9UH+UMeK6Z/G0SzTVfJ3Cr0vituCf293uC34t5X9v90RqL8RV6hN+RskEXYrbe7cuZozZ45+/vlnNW7cWIMHD9b48ePl9ZYfoIqKivToo4/q7bff1u7du5Wamqrrr79el1566RHuOWqL0Fe8Lhn2RoeqKLEGjllbp8kcKQEz+LViob9IBb4i+UyfvAlu7d1foKKigPymGf5mIHTbsoLTAFyGKzzVwVX8AaLkfxhGqekNwUhoWsFvFOLj3IrzuBXvdSvOG/ztcRvhr/aMUsOihlHqW4jQY8VtRvGxfX5LhT6//AGreGqAUfzeNYrvB7f3maaKfJZ8PlOFRQEV+U0V+gIqLP6AFTAtWWbwA0now5RVPN3CLH7NUnBKR+grfcMITgE4qlGiLL+pOI9LCXFuJcR5in+75fG45PMHSj40Ff/4AoHw/UKfX0X+4BSookBARb6ACv1++f2mPG5DHo8hjyf4fva4DXncwas/utzBMG0Ulyw05cAonirgdrnldbvkcbvkcbnkcbvlcbvldQe/3vYFTPl8por8VvBDpc8q/oAZ/OBmWpIZCH3gVfEHzOB0jfBfIfz3CE7NCP0dg1MziqdqGEawv67g7+TkeBUeDF60x+UyiqdshN5HwekXllnyQTv0N7DC9xWe8hL8sKvw9Bd38XO43Ubx6y657zKMkg/qpX6HPrQbMorr6ypzDJfLCL5niqeqFfoCOlgUKL4fPCcgIS74fo73uhVf/PcPvc9NyyqudeSHZZ8/IH/Akscjud2W3B5LLnfwvstlyu215Il3a9+BfBUU+VXo84XfMz6/X75AoPjbruAJy4ah4tvB90HwY5cZnMIS+m2ZsorvG4bkcgenBwVPcrZKfhtW8dxps3gedaD4bxE6hsJToQy5Im4bljscTq2AS6YZ/ABjBoJf7VsBV/HJx57iE5Q98rrc4akFXlewLXhSsqvk/eEN/h2DU3uC059M0wwPfliW5Pa4VVjkC/73KvT+tUr+fYffR5Is0wpP8wnIL6t4UCE4nab4tmFKxVMy5Cr1wSb0rWKpEO1yW3K5zeI6Rn4IOiq+kf5wWv+a+Y95NTg27M6fP18TJ07UhAkT1K9fP61fv14TJ05Ufn6+Jk2aVO4+DzzwgJYuXaopU6bohBNO0AcffKA///nPSkxM1MCBA4/wKwDgBMH5oW4leOLVKKGWfjio2myiqKmVNYsh6mUP9bKHetnn2NOXMzMzNWjQII0YMUKpqanq37+/xo4dq1deeUXbt28vs/2WLVs0b948jRs3Tn379tWxxx6r4cOH64ILLtD//d//xeAVAAAAINYcGXY3btyozZs3q3fv3hHtvXr1kmmaWr58eZl9Pv74Y1mWpfPOO6/MPqHjAQAAoH5xZNjNzs6WJLVt2zaivVWrVvJ6vdqwYUO5+8TFxally5YR7aFjlLcPAAAA6jZHztnNzc2VJCUlJUW0G4ahpKSk8OO/3ee320tScnKyJOnAgcqvnRta0zJa3G5XxG8cGvWyh3rZQ73so2b2UC97qJc91Ms+R4bdWGvSJKlGFkROSamJc/HrLuplD/Wyh3rZR83soV72UC97qFflOTLspqSkSFKZEVzLspSXlxd+vLSGDRsqLy+vTHtoRLe8fSqye3de1Ed2U1IStX9/gQIBzpw8HOplD/Wyh3rZR83soV72UC97qFeJxo3LfqNfHkeG3bS0NEnSpk2blJ6eHm7PycmRz+dTu3btyt2nqKhI27ZtU6tWrcLtGzdulKRy96lIaBH/aAsUr3mIyqFe9lAve6iXfdTMHuplD/Wyh3pVniMnfKSmpiotLU1Lly6NaF+yZIk8Ho969uxZZp+ePXvK5XLp/fffj2hfvHixOnTooGOOOaZG+wwAAADncWTYlaSxY8dq0aJFmjNnjrZs2aLFixdr5syZGjZsmJo2bao1a9YoIyNDWVlZkqSWLVvq97//vR577DG9//772rJli/71r39p6dKlGjduXIxfDQAAAGLBkdMYJCkjI0PTp0/X7NmzNWPGDDVr1kzDhw/XmDFjJEkFBQXKzs5Wfn5+eJ97771XycnJ+stf/qLdu3fr+OOP16OPPqo+ffrE6mUAAAAghgwrdHF3hP36a+WXKasMLu1nD/Wyh3rZQ73so2b2UC97qJc91KtE8+YNK7WdY6cxAAAAANVF2AUAAECdRdgFAABAnUXYBQAAQJ1F2AUAAECdRdgFAABAncXSYwAAAKizGNkFAABAnUXYBQAAQJ1F2AUAAECdRdgFAABAnUXYBQAAQJ1F2AUAAECdRdgFAABAnUXYBQAAQJ1F2AUAAECdRditYXPnztXAgQPVuXNn9ezZU9OmTZPP54t1txzj3//+tzp37qxx48aVeSwrK0t/+MMfdOqpp6pr16664447tH379hj00jleffVVXXjhhUpPT1efPn305z//Wbt27Qo//sMPP+j6669Xenq60tPTdcMNN+inn36KYY9jxzRNPfPMMxo8eLC6dOmibt26aezYsdqyZUt4G95jFRs5cqQ6dOignJyccBv1KtG3b1916NChzM/gwYPD21CvsnJycnTrrbfq9NNP15lnnqkxY8Zo69at4cepWVBOTk6576/Qz+uvvy6JelWahRozb948q0OHDtacOXOsn3/+2Xrvvfes7t27W/fff3+suxZze/bssUaPHm316NHDOv3006077rgj4vGffvrJ6tKli3XPPfdYP/30k5WVlWVdfvnl1uDBg62ioqIY9Tq2nnnmGatjx47W008/bW3cuNFatmyZ1atXL+vqq6+2TNO0du/ebXXv3t0aNWqUtW7dOuvrr7+2Ro8ebZ177rnWvn37Yt39I27KlCnWaaedZs2fP9/6+eefrY8++sjq16+f1bdvX6uwsJD32CHMnTvX6tSpk9W+fXtr8+bNlmXxb/K3+vTpY02dOtXasWNHxM/u3bsty6Je5dm3b5/Vp08f66abbrK+//57a/Xq1dbFF19sZWRkWIFAgJqV4vf7y7y3duzYYb3xxhtW586drU2bNlEvGwi7Nahfv37W+PHjI9pefPFFq2PHjtYvv/wSo145w3PPPWdde+211s6dO60+ffqUCbsTJkywevfubfl8vnDbTz/9ZLVv39763//+d6S7G3OmaVrnnnuuNWHChIj2l19+2Wrfvr313XffWY8//rh16qmnWnv37g0/vnfvXqtLly7WE088caS7HFM+n88677zzrMzMzIj2+fPnW+3bt7fWrFnDe6wC27dvt7p27WpNmjQpIuxSr0h9+vSxHnvssQofp15lZWZmWueee65VUFAQbsvOzrYWLFhgHTx4kJodRlFRkZWRkWE9/PDDlmXxHrODaQw1ZOPGjdq8ebN69+4d0d6rVy+Zpqnly5fHqGfO0Lt3b82ZM0dNmzYt9/GPPvpIPXr0kMfjCbelpaWpTZs2+vDDD49UNx3DMAy99dZb+tOf/hTR3rJlS0lSXl6ePvroI6Wnp6tRo0bhxxs1aqRTTz213tXM4/Fo6dKluuWWWyLaXa7gf/K8Xi/vsQo8+OCDSk9P1/nnnx/RTr3soV5lvfvuu+rfv78SEhLCbccdd5wyMjIUHx9PzQ7jP//5j/bv36+bbrpJEu8xOwi7NSQ7O1uS1LZt24j2Vq1ayev1asOGDbHolmOkpqbK7XaX+1heXp527NhRpnaSdOyxx9bb2h111FFq2LBhRNuSJUvUoEEDtW/fXtnZ2UpNTS2zX32uWWnffvutZs2apT59+ig1NZX3WDkWLFigjz/+WJMmTYpo59+kPdSrLJ/Ppx9//FGpqal65JFH1LdvX5199tm68847tXv3bmp2GPn5+Xrqqac0cuRIJScnUy+bCLs1JDc3V5KUlJQU0W4YhpKSksKPo6yKaidJycnJOnDgwJHukiO9//77euWVVzR69Gg1bNhQeXl51KwcDz/8sDp37qxLL71U5557rh5//HHeY+XYu3evJk+erDvvvFOtWrWKeIx6le+bb77R9ddfrx49eqh37966//77tWvXLupVjn379snv9+s///mPCgsLlZmZqUmTJunzzz/XiBEjqNlhvPLKKzJNU1deeaUk/k3a5Tn8JgCcZsGCBbrrrrs0ZMgQjR49OtbdcbRRo0bp4osv1rfffqtHHnlE2dnZmjJlSqy75ThTpkxRamqqfv/738e6K7VC48aNlZubq5EjR6pNmzb67rvvNGPGDK1atUrPPPNMrLvnOH6/X1LwW717771XktSpUyd5PB7dfPPN+uyzz2LZPcd79tlndemllyo5OTnWXamVCLs1JCUlRZLKjOBalqW8vLzw4ygr9FV9eaPfBw4ciJiTWh8999xzmjJlin7/+9/rvvvuk2EYkhQe3f2t+l6zJk2aqEmTJmrXrp2OP/54XXbZZfrkk08k8R4L+fDDD/Xuu+/qtddeC89rLo1/k2W99tprEffbt2+v5s2b67rrruP9VY5QSOvcuXNE+5lnnilJ+u677yRRs/J8/fXX2rJli/r16xdu49+kPYTdGpKWliZJ2rRpk9LT08PtOTk58vl8ateuXay65ngNGjRQq1attGnTpjKPbdy4Ud27d49Br5zhxRdf1EMPPaQ777xTN9xwQ8RjaWlpFdbshBNOOFJddITdu3fr008/1ZlnnqnmzZuH29u3by8p+O+Q91iJBQsW6ODBgxoyZEi4zbIsSdKAAQN05plnUq9K6NixoyRpx44d1Os3kpOT1bx5c+3bty+i3TRNSVKLFi2oWQUWL16sRo0aRWQJ/j9pD3N2a0hqaqrS0tK0dOnSiPYlS5bI4/GoZ8+eMepZ7dC7d28tX7484gIc3377rbZu3aq+ffvGsGexs2LFCj344IOaMGFCmaArBWv25Zdfas+ePeG2nTt36quvvqp3NSssLNS4ceM0f/78iPZ169ZJCq5iwXusxB133KE333xT8+fPD/9MnjxZkvTkk09q8uTJ1KuUn376SXfffXeZC7Z8/fXXkoIrDFCvsnr16qUPP/xQhYWF4basrCxJUocOHahZBT799FN16dKlzEnd1MuGWK99VpctWLDA6tChg/XMM89YOTk51nvvvWd17drVmjp1aqy7FnN79uwJL5Ldq1cv6+abbw7fLygosH7++WcrPT3duuuuu6wNGzZYq1evtoYOHWpdfvnlViAQiHX3jzjTNK0LLrjAuvrqq8tdaDw3N9fav3+/1bNnT2vkyJHWunXrrHXr1lnDhw+3+vTpY+Xl5cX6JRxxEyZMsNLT061XX33V2rRpk/XJJ59YgwcPDl9kg/fYoX366acR6+xSrxK5ubnWeeedZw0ePNj66KOPwhcNOu+886xBgwZZRUVF1Ksc2dnZVnp6unXTTTdZP/30k/XRRx9Zffr0sa688krLsniPVSS07vVvUa/KMyyr+Lsq1Ig333xTs2fP1qZNm9SsWTNddtllGjNmTLnz4uqTa6+9VitXriz3sb/97W+65JJL9PXXX2vatGlas2aNEhIS1KdPH02YMEGNGzc+wr2NvS1bthzyk/qtt96q2267TZs2bdKUKVO0cuVKGYahs88+W/fee6/atGlzBHvrDEVFRZo5c6beeustbd++Xc2aNdMZZ5yhcePGhevBe6xin332mYYNG6YlS5ZQr3Lk5OTo//7v//TZZ59p9+7dOuqoo9SnTx+NGzdOTZo0kUS9yrN27dpwTeLi4vS73/1Of/rTn8JzeqlZJNM0ddJJJ+mmm27SuHHjyjxOvSqHsAsAAIA6q34PLwIAAKBOI+wCAACgziLsAgAAoM4i7AIAAKDOIuwCAACgziLsAgAAoM4i7AIAAKDOIuwCAA7p2muvVYcOHcKXwwWA2sQT6w4AQF2Vk5Ojfv36VXr70JXwAADRQ9gFgBqWmJhYqRCbnp5+BHoDAPULYRcAalh8fLxGjRoV624AQL1E2AUAh5kwYYLmzZunadOmqXnz5srMzNT69etlWZY6dOigm266Seedd16Z/RYvXqznn39e3377rfLy8tSoUSOlp6dr1KhR5Y4a//LLL5o1a5Y+/PBD7dy5U40aNVKfPn1066236uijjy63bytWrNBjjz2mdevWSZJOPvlkjR8/XqeffnrEdl9++aWeeuoprV69Wnv27FFycrJSU1M1ZMgQXXPNNXK73dUvFABUAmEXABzqs88+04IFC/S73/1OPXr0UE5Ojt58803ddNNNmjVrlvr27Rve9rHHHtPMmTPVuHFjDRgwQC1bttTPP/+sRYsW6f3339eMGTN0wQUXhLffsGGDrrrqKhUUFGjo0KFq06aNfvzxR7322mt67733NHfuXLVt2zaiP5988omeeeYZDR06VL1799aKFSv06aefatSoUXrnnXfUqlUrSVJWVpaGDx+uhIQEXXDBBWrdurUOHDigZcuWacqUKVq9erUeeeSRI1NEALAAADVi8+bNVvv27a2zzjrL1n733HOP1b59e6tDhw7W8uXLIx579dVXrfbt21sZGRnhtm+++cbq0KGDddZZZ1nbtm2L2P7zzz+3OnbsaJ155plWfn5+uP2SSy6x2rdvX+b4//3vf6327dtbo0ePDrddc801Vvv27a3u3btb2dnZ4XbTNK0RI0ZY7du3t+bMmRNuHz9+vNW+fXvrgw8+iDh2UVGRdfXVV1tnnHGGtXXrVls1AYCqYmQXAGqYZVnKyck55DZer1ctW7aMaEtPT1ePHj0i2i666CJNmzZNGzZs0ObNm5Wamqr58+fLsiz9/ve/LzP9oGvXrurWrZtWrFih5cuXa8CAAfruu++0du1adezYsczxL730Um3ZskUtWrQo08crrrhCxx13XPi+YRjq2bOnPvnkE23ZsiXcvm/fPkkqM1XB6/Xq2WeflcfD/3oAHDn8FwcAati+ffsOuwRZx44d9cYbb0S0/XYerBQMkMcff7y++uorbdiwQampqVq7dm2F20tSly5dtGLFCn3zzTcaMGBAeL3ck046qcy2CQkJuvvuu8s9TufOncu0paSkSJJyc3PDbX369NHy5cs1fvx4jRo1Sv3799cJJ5wgSQRdAEcc/9UBgBqWlJSk6dOnH3Kb5OTkMm1NmzYtd9ujjjpKkrR//35J0q5duw65fZMmTSRJe/bsidg+FFQrq7ztXa7gtYksywq3/eEPf1BeXp6eeOIJPfLII3rkkUfUvHlz9ejRQxdffLG6detm63kBoDoIuwBQw7xer/r37297v1CQ/C3TNCUFlzSTgtMJpMjAWd72oe1Cxy0qKrLdp8q68cYbdfXVV+uDDz7QRx99pI8//ljz5s3TvHnzdPnll2vy5Mk19twAUBqXCwYAhwqNxP7W3r17JZWM5IZ+h0Zsf2v37t3lbh9qrykNGzbUkCFDNG3aNC1fvlxPP/20WrZsqblz52rFihU1+twAEELYBQCHWr16dZk2v9+v7OxsSVKbNm0kSaeccookadWqVeUe54svvojYLvQ7KytLgUAgYlvTNHXHHXfo9ttvl9/vr1K/9+3bF3HCmhQcVe7Ro4euv/56SdI333xTpWMDgF2EXQBwqM8++0yff/55RNvrr7+uAwcOqFOnTuHVGy699FK5XC699NJL2rZtW8T2H3/8sVatWqWWLVuGV17o0KGDTj75ZO3atUuvv/56xPbvvPOOFixYoLy8vCqdTLZnzx6dc845uu6668KrMpQWCrmhNXkBoKYxZxcAalhhYaGefvrpw24XHx+va665Jnz/wgsv1I033qh+/frp+OOPD19Uwu1266677gpvd+KJJ+qOO+7QI488oksuuUQZGRlq2rSpNmzYoPfee08JCQmaNm2avF5veJ+HHnpI1157re6//3599tlnOuGEE/TTTz9pwYIFSk5OrnBFhsNp3Lixbr75Zj3++OMaNGiQ+vfvr6OPPloFBQX64osvtHLlSp188sn63e9+V6XjA4BdhF0AqGEFBQWHXY1BCs5xLR12O3furEsvvVSZmZlaunSpTNNUly5ddNttt+mcc86J2Hf06NFq166dnnvuOb311lsqKChQkyZNlJGREX6stJNOOknz5s1TZmamPvnkEy1cuFCNGjXSoEGDdOutt5a5epodt956qzp06KBXXnlFixcv1t69e+X1enXcccfp9ttv1/DhwxUXF1fl4wOAHYZV0em7AICYmDBhgubNm6eJEydGhF8AgH3M2QUAAECdRdgFAABAnUXYBQAAQJ1F2AUAAECdxQlqAAAAqLMY2QUAAECdRdgFAABAnUXYBQAAQJ1F2AUAAECdRdgFAABAnUXYBQAAQJ1F2AUAAECdRdgFAABAnUXYBQAAQJ31/+1MPX1gW6EJAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plt.style.use(\"seaborn-v0_8\")\n", + "plt.title(\"Learning Curves\", fontsize=20)\n", + "plt.plot(np.linspace(1, n_epochs, n_epochs), epoch_loss_list, color=\"C0\", linewidth=2.0, label=\"Train\")\n", + "plt.plot(\n", + " np.linspace(val_interval, n_epochs, int(n_epochs / val_interval)),\n", + " val_epoch_loss_list,\n", + " color=\"C1\",\n", + " linewidth=2.0,\n", + " label=\"Validation\",\n", + ")\n", + "plt.yticks(fontsize=12)\n", + "plt.xticks(fontsize=12)\n", + "plt.xlabel(\"Epochs\", fontsize=16)\n", + "plt.ylabel(\"Loss\", fontsize=16)\n", + "plt.legend(prop={\"size\": 14})\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "0cd48c2d", + "metadata": {}, + "source": [ + "### Sapling process with classifier-free guidance\n", + "In order to sample using classifier-free guidance, for each step of the process we need to have 2 elements, one generated conditioned in the desired class (here we want to condition on Hands `=1`) and one using the unconditional class (`=-1`).\n", + "Instead using directly the predicted class in every step, we use the unconditional plus the direction vector pointing to the condition that we want (`noise_pred_text - noise_pred_uncond`). The effect of the condition is defined by the `guidance_scale` defining the influence of our direction vector." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "f71e4924", + "metadata": { + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1000/1000 [00:14<00:00, 71.08it/s]\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbsAAAG7CAYAAABaaTseAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy89olMNAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAjrUlEQVR4nO3dWXNUydXu8WyEJJDQhAQSiKHVGIfd7THC/ii+8Cfyt3KEHe2wI7rdtrtNt5nMjIQmJDSAxj4X5/iE33jzeSgtUkVp8f9dZpJVu3bt0mJHPLn2R99///33BQCAxE697wMAAOC4UewAAOlR7AAA6VHsAADpUewAAOlR7AAA6VHsAADpUewAAOmd7vQfTkxMyLn9/f3q+OHhoVxzcHBw5DVuTu2Nd2tOndK1vq+vrzp+48YNuWZhYaE6vrW1Jde4Pf3q2N2aSI+Ajz76KDTXck20t4Fa544h8l7d7L2gjj1yXp3IdeR+M62vy25pff33wmfthWPopk4+L3d2AID0KHYAgPQodgCA9Ch2AID0KHYAgPQodgCA9DreeuC4OPJR10Tjy5GI8JkzZ+Tczs5OdfzHP/6xXDM3N1cd/+abb+Sa5eVlObe3t1cd7+/vl2siWxlax/RbixyD23JyUvXC9oeTur3A6YXz2s3X6/Xfu6oN7/qb5s4OAJAexQ4AkB7FDgCQHsUOAJAexQ4AkF7HacxIyi/S1DnS7LkU3bjZJRe3t7flnEos/eIXv5Brnj9/Xh2/ffu2XOO0Tqx2a03rxFn0OE6q1s28W77Ph6b1OYo0Lo+kJ6O/wZa/3ehnUn/3Tp9+t80D3NkBANKj2AEA0qPYAQDSo9gBANKj2AEA0qPYAQDS6zjLqaL9pegY6cHBwZFfb2BgQK5xDafVMbhmz+69NjY2quODg4NyzZUrV6rj7rh7oQltN7cydHNbQjZsL8ir9VaBaOy/5TFEqWvW1ZNOcGcHAEiPYgcASI9iBwBIj2IHAEiPYgcASK9JGlOlDd0alZIcGxs78vuU4lOSkTXDw8PV8ZmZGbnmD3/4Q3XcNZx256h1k9fWr9cLIgnTkyqSco00EQaOolt/V9zf/47WNzoOAAB6FsUOAJAexQ4AkB7FDgCQHsUOAJAexQ4AkF7HWw8iIs1It7a25Jrd3V05Nzk5WR13zZ4XFxfl3NTUVHX80aNHcs2///3v6rg77l5o2NoLzaidyPFljNyf5C0i+PBEf2fHdZ1zZwcASI9iBwBIj2IHAEiPYgcASI9iBwBIj2IHAEiv460HkY7rbs3Ozk6nb/3/jY6Oyrnf/OY31fGhoSG5ZmlpSc59/vnn1fE///nPcs3Kykp1vL+/X645PDw88pxb0614ejRW3HobQcv3Ad6XyN/X43iviF4/vv/gzg4AkB7FDgCQHsUOAJAexQ4AkB7FDgCQXsdpzNaJuL6+viOvcQnOly9fVsc//fRTueazzz6Tc6dO1f8f4BKcZ86cqY6/efNGrtne3pZzKnWpjq0Uf/5IQv5fvdAQO6KbiT2cbL1wLXerkX2nuLMDAKRHsQMApEexAwCkR7EDAKRHsQMApEexAwCk99H3HWZUp6en5ZyKyLuXVs2RXbzURe4PDg6q4+fOnZNrPvnkEzl37dq16vhvf/tbueZ3v/tddfzBgwdyzerqqpxTWy2i2wsiceTIdxvR+jP1QvQa6FQ0Vq/W9cJWFHcMkeNzazp5sAB3dgCA9Ch2AID0KHYAgPQodgCA9Ch2AID0KHYAgPSO9akHjoqGnz7d8SH9D0NDQ0de8+WXX8q5hYWF6vjVq1flGrXNQW2zKCUWz43G9FvHfSMiWwVab6cAek3r3203fxetnyJyXNsmuLMDAKRHsQMApEexAwCkR7EDAKRHsQMApNdxI+jLly/LOdWEWY07fX19R15Tim4S7V7PpSQHBgaq4y4pND4+Xh3f39+Xa+bn5+Xc+vr6kV8vonXaK5Ko3d3dlXPu85LGRHatGyq31voY1Ou5BwHQCBoAgEKxAwB8ACh2AID0KHYAgPQodgCA9Ch2AID0Os6IR5rxto6kuuipOoa9vT25xkXaVZTVvd6rV6+q42NjY3KNO0eR8+q+J3X+IvF9tTWjlFImJyflnGrYvbGxIde8fPlSzm1tbVXH2ZIAtNMLWxze9TfNnR0AID2KHQAgPYodACA9ih0AID2KHQAgvaN37D2CSILTJSRdIujw8PDIa6KpRkU1vnZNjt2cer3ocat16tyVohOcZ8+elWuuXLki51QydXt7W65RDbFLKWVpaak6vri4KNd00jQWOAm6lYSP/F3ptQbW3NkBANKj2AEA0qPYAQDSo9gBANKj2AEA0qPYAQDSa7L1wDVoVlpvFYjE6t2citq6NZE4beT1Ils6StFbGRx1fK4hdl9fn5wbHR2tjk9NTck17rjVtoQHDx7INU+fPq2Or62tyTXA+xKJ/UfWdFPk+CJ15n+sf6fVAACcABQ7AEB6FDsAQHoUOwBAehQ7AEB6FDsAQHpNth5EnhCgRJ5sED2GSATXxV/VMUTPj3qvyBYCxx2fOuebm5tyjYr2u/dST0MopZTJyUk5Nzs7Wx133+3p0/XL/v79+3LNxsaGnAPel8jfltZPMGj5Ps67/t3jzg4AkB7FDgCQHsUOAJAexQ4AkB7FDgCQXsdpzF5vHto6CdmtY1DJwFJKOXv2bHV8d3dXrtnZ2ZFzLVOcLhn75MkTOacaN7tG0O69Pv744+q4SmmWUsrw8HB1fH9/X6759ttv5Zw7PqDXRBrqd/Pvv0qhuwbzHb3uO60GAOAEoNgBANKj2AEA0qPYAQDSo9gBANKj2AEA0muy9UDNtW5S2k3dOvahoSE5Nzc3Vx13EfkXL17IuZcvX1bH37x5I9dEIsd7e3tybmVlpTruGku71xsYGKiOqy0JpZRy48aN6rj7Lpzbt29Xx91xA+9L67+xaquA+9vhthGoOdeEvxPc2QEA0qPYAQDSo9gBANKj2AEA0qPYAQDSo9gBANLreOuBi4qqbvqR7Qofmv7+fjl3/fr16vjg4KBcozr6l1LKwsJCdXxpaUmu2draqo67JyhEvlv3tIZnz57JOXUuxsbG5JqLFy9Wxy9fvizX/PSnP5VzatvEw4cP5RrgJIn8LXdbBdyc+pt47tw5uaYT3NkBANKj2AEA0qPYAQDSo9gBANKj2AEA0us4jem0bAQdpd6rm8egzoNLLrq5169fV8cvXLgg11y7dk3OTU9PV8dd2vH58+fVcdVUuhSd4CyllMPDw+q4S3u5RtUq8Tg+Pi7XjIyMVMddglMlY0spZW1tTc4prmG3+t6BFlwSMpKkVq939uxZucYlK6empqrj7jfdCe7sAADpUewAAOlR7AAA6VHsAADpUewAAOlR7AAA6R3r1oNuNnvu5e0PjovVqy0BLtLr4rmzs7PVcdcA+fHjx9Xx+fl5uebJkydyTjWd3t/fl2vcdbS9vV0dv3Pnjlxz+nT9sr9586Zcc+PGDTn3s5/9rDo+MDAg10TOkdvuoRpp7+3tyTXIy/1m1PYft85tFZicnKyOq4brpei/RaWU8qMf/ag6/oMf/ECu6QR3dgCA9Ch2AID0KHYAgPQodgCA9Ch2AID0mqQxlcij3LuZ4Gwt8plc01/VhPnMmTNyTV9fn5xTj7sfHh6Wa1QD5JmZGblmdHRUzt2+fbs67tKdkUbay8vLcs3du3er44ODg3KNO68//OEPq+MujXn+/Hk5p9KYGxsbco1qRr2+vi7XqCSrm3Op2d3dXTnnEoDw3LWnrjGX2HaN5K9cuVIddw3mVUpS/S5K8clnlQ5fWVmRazrBnR0AID2KHQAgPYodACA9ih0AID2KHQAgPYodACC9Y9160JqL8KsmzJE1rbljcLF61fj3wYMHoeNQsXG3VUBFjl1jWLc1QsWeHz16JNfcv39fzqk4sovBR87rpUuX5Jx6L9UgtxS/LUE10H316pVcs7W1VR13jaDVmlL0lgW3/WFzc1POqa0MrhG6WuO+W/d76tbv3X236rcxNDQk14yNjck59Xty16vbRqC2C3z66adyjdpG4LbXqG1Qpejv0F0rneDODgCQHsUOAJAexQ4AkB7FDgCQHsUOAJAexQ4AkF6TrQcqWt866nuSn4ignDql/7+hOsW77t8unru6ulodn5ubk2tGRkaq4y5e/ZOf/ETOqdj/+Pi4XDM7Oyvn1FMU7t27J9eoc+Si/eqpAqWUsri4WB2/ceOGXOPi5CqG7p40obYYuCcbuJi+ive713NbDyJbI9TfD7fGHYOa29nZkWtOn9Z/ItX2G7flZGJiojo+NTUl17jXm56ero6rpxeUop9SUIq+xtxTFFpzT3l4F9zZAQDSo9gBANKj2AEA0qPYAQDSo9gBANLrOI3Z60nIbiVCuynymVwaTaUQXcJOef36tZxTKdJSdJPjn//853KNS1aqc+SaUav0pLvGVZK1lFKGh4er45EUXSm6MbdLqanUoEsuRpKargmzaxIdaQQdSQC6hKn6ft1ncg3P1bXsUrMqdenSmK6ps0r1umulW3/L3d+pSIN+1ci+U9zZAQDSo9gBANKj2AEA0qPYAQDSo9gBANKj2AEA0mvSCLqlSCT1OF4vEvvvhW0O7hhUDH1hYUGuUU1y19fX5RoXJ+/v76+OX716Va757LPP5Nzly5er4y7+PT8/Xx13DYFdhF/F3d12hfPnz8s5tfXANY9WWy1cBF01Zy5Fx/FdTN810lbv5c65ey9FNVoupZQLFy5Ux9U1WYrfRqO2lnz88cdyjdqmMjg4KNe4uch2iojI3z33t9edc/V6bltJJ7izAwCkR7EDAKRHsQMApEexAwCkR7EDAKRHsQMApHesWw96/UkJTi9sI1Ci51XFqF28enl5uTruYutuTnHRfhUZL0XHv10EXW1XWFlZkWvcdgo3p7x48ULOra2tVcfdUxTU5x0aGpJr1BYHN+ci7QMDA0eec1sP1FYG9zSEubk5OaeeHuAi7eoJGaXoLQGtt4g4kdi/+72r14ts03LXyqlT+j5LnQv1ZI9OcWcHAEiPYgcASI9iBwBIj2IHAEiPYgcASK/jeItL90TSga3XRBo3t9a6eXTrY295fK9fv5ZrHj58KOf29/er4yr1WYpPY05NTVXHr1+/LtdcvHixOu4SYi5ZFrmW3eupZKpLcKoE7MjIiFzjvkPVqNo18HXJT7VuY2NDrlFNk13SdnZ2Vs6pFK5Lkar0ZCk6het+T+o8uESoS/uq35O7Jl2q0X2/ivq8kcSlm3PXcie4swMApEexAwCkR7EDAKRHsQMApEexAwCkR7EDAKTXpBF0JCIfaWDarSi+03qrQOtm2a7Ja+QYIsenGviWUsq3335bHXdbD27evCnnVCNhF+2/cuVKddw1Wt7e3pZz09PT1XEXW19fX5dzKmoeaTjtGmy7Y1DXsov9u0i7ivdHtnu4LQ6uSbRq0Oxi8DMzM3JONQ53155a45qnq+0FpejzF4n2l6KbW587d06uUd+teq1SYn9XItsi/ht3dgCA9Ch2AID0KHYAgPQodgCA9Ch2AID0mjSCVokglwyMNCXuVsPptx1Ht16rm59XUcfuPpNL5alrZWlpSa5ZW1uTc6urq9Vxl+BU16VrxuvSaCrd5hoMO6pRtUoTumNwyUBHJUldIs59XtXUOZIwjTTRLsWnEBWXqFVz7npV155Lpbrzqn7v7u+AO0cq3ey+d3Veo3+L1DqXtO0Ed3YAgPQodgCA9Ch2AID0KHYAgPQodgCA9Ch2AID0Ot564KLmKk7bunGzE9nKEJmLNE2OnLu3rYtouZXBHZuL8Kumtu7YVBy6lFIWFxer4+fPn5drVITZxczddorNzc3quGtY7JpOq+Nw0evx8fEjr3HnXDW+dg2LI9sSRkZG5Bq1XcFdD665tbr23JaEyNYId62o94r+ntT36753t3VDHUdk+0PrLVfuvHaCOzsAQHoUOwBAehQ7AEB6FDsAQHoUOwBAehQ7AEB6TbYetOTi0JFjcN3EXexfxb9d9/vBwcHOD+z/iTwhIPrEAbXOnQfl9evXcs5Fm9Wcirq/jYrcu0i7OkcbGxtHXlOKPn9qS0IpsS0s7lpWUXMV3y+llAsXLsi5mZmZ6rg7r+6aUNsFJiYm5Br1W3OfyW33UFsPok9aUX8j3DGoNZHtCm+by+Zdn4jDnR0AID2KHQAgPYodACA9ih0AID2KHQAgvWNNY0YaD0dFmpG6dJtKiV29elWuUekx12DYNVhVCTuX3HKvpxJxKqVWij5214zXNc+NpDH39vbk3OjoaHV8bm5OrlFpPtdw151zlcJ158jNqXSnS82q712dn1L897S2tlYdn52dlWtUMta9l7v21PfujttdK645uOIaKqvvw/0GP6T0ZK/hzg4AkB7FDgCQHsUOAJAexQ4AkB7FDgCQHsUOAJBex1sPIlo3j440iXbH4ObUNoLr16/LNSqe7uLGbvuDikq7rQyuUbWK97sYvGpq6z6Ti+mr43NrIk2TXTNe9b27rQf9/f1ybmxsrDrutgq4713NuTXq+3Dnzl0ras6tcbF/de2536DaltDN+L773tWxd3PLVetj6IXPdFy4swMApEexAwCkR7EDAKRHsQMApEexAwCkR7EDAKR3rE89aK31MbhouJpzsX8ViXaRbPeZ1DG4mL57PRWjdpF21dHfccenjsFFm11nfMW9nnrqgXoiQym+o7+K47969Uqucds9VKd9ddyllDI9PV0ddx343XUZWeOuFXX+3FMF1LG779Y9PSMSq3dz6nfT+u9U5PXc37bIe0W2fUXepxR9jbljcH+X/4M7OwBAehQ7AEB6FDsAQHoUOwBAehQ7AEB6TRpB93JSM5J2LKWUhYWF6viXX34p10QaQUcarLq0o6OSb6rhdCmlTE5OVsfdZ3INlVUKMdIQuxR9fO711DXx+vVrucalJ9+8eXPkY4g0VJ6fn5drXr58WR0fGRmRa1yzbLXOXXvu86o0pkuLRho+uzUq1euaPUd+n+7vijo+95uJJCvd66nrtRT9G3DXv7peI02+S9HH7o7717/+tZz7D+7sAADpUewAAOlR7AAA6VHsAADpUewAAOlR7AAA6XWcYY/EX1s3CG3NRaVVQ9m7d+8e+X3cuXOfV825+Hekca2LXk9MTFTHR0dHj/w+pcSizWp7QSml3Lx5szquGiOXoiP37rhdw+Ll5eXqeLQBuOK+dzXnroexsTE5NzMzUx2fmpqSa9xWBvX9RhqNu+0FriGw+h1GfoOOOz4Xn4+8nvo9LS4uyjUrKytybmlp6UjvU4r+bqNbDxT3d5StBwAAFIodAOADQLEDAKRHsQMApEexAwCk994aQXfr0fXR91HrXMLOpfmO+j6l6MSSW+MawEbWqAbD0QSbOkfuvKpEaCm6YbFL5e3u7lbHXXPm4eFhOafOn0saumSlej2XRlOv5z7TxYsX5ZxKwLrXc4k91fDZXXuRJswu1atSje56VansUvS17K49lcbc3NyUa9bW1uScag7+6NEjuUZd/6XEUvfqt+vex10r7py/C+7sAADpUewAAOlR7AAA6VHsAADpUewAAOlR7AAA6X30fYfZ/MuXL8u5SIPV1k1ZI3HtyHu5pqxKZDuAWxf9TJFzriLjrnm0m1Pnz23buHHjhpz75S9/WR0fHx+Xa9RnclscXORebRFx0Xl1DKXo6Lq79lSjardlwjWCVg2f3XG761ydP7V1JCqy9cDF/l+8eCHnVOTenSO1VeDZs2dyzerqqpxT2xJcfN9t89nY2KiOqy1Ibm59fV2uefXqlZzb2tqqjrvtCqqB9X/jzg4AkB7FDgCQHsUOAJAexQ4AkB7FDgCQHsUOAJBek6cetBTdetD6yQuRmL6KNrsO924ucgyOikS7WP3o6OiR17jPpOL4bqvAr371KzmnuvO7c3ThwoXquIv2u+NT3d1d13e1VaAUvSXAHV9kq4zrzq8i/Kpr/9vmVNT80qVLco3alqC2epTiz7mK3D9+/FiucXPq9dxTCtRWBhX5L0VH8UspZXl5uTq+sLAg16jtD+713FYBd85bijwF479xZwcASI9iBwBIj2IHAEiPYgcASI9iBwBIr+M05rsmYd6XaHJRrXNNXlXKTzXVLcU36lWJPZfkc59XvdfQ0JBco+ZcOtG9nmoW7JpHu3OuPq87Ryrl55rdOiqZ6hp2uzmVbnO/QdVIO/I+pejEo0tCuibM6ntyiVCV6nXv45oFP336tDr+1VdfyTX37t2Tc+q6dMnFlZWV6rhLSKrjLkWnO12CM/o38ahcc/f3UU+4swMApEexAwCkR7EDAKRHsQMApEexAwCkR7EDAKR3rFsPemG7QvQYVGTbReRnZ2er45988olc47YeqPdS0flSfPRaNU1WjYdL0efBNXt2Wy1U81wX13YNhlV83sWet7e3j3RspZSyubkp5wYHB4+8xkXDW3Lfk4v9q+0j7vp3WzfU9gy3lWFpaak67hotu+v/zp07RxovxTdUVu/15MkTuUbNqQbMpfjtI72s9XHTCBoAgLeg2AEA0qPYAQDSo9gBANKj2AEA0qPYAQDS63jrQWu9sC3BUcfnOsWvr69Xx11U2sVzXbd/xUW5NzY2jvx6KrruIu2rq6tyTn1et71APSmhFN3B3UXQHzx4UB2Pdorv6+urjrvvwlHn1sX+I9+TezqF2jahPmsp/nu6dOlSdVxtL3Dv5a7j58+fy7l//vOf1fG///3vco3bRqCuc7cFo1tPHDjJ3Lahd3rdY3lVAAB6CMUOAJAexQ4AkB7FDgCQHsUOAJBek0bQas4lj9Scex/3epFjcCIJO5UEcw2GXVNn1TzXNfB11Ou51KdK2LnknTtHKvGomjOX4r9D1YR5fHxcrlEpxGgKTF0rLj2pvotSShkdHT3S+5Siz4Nb45Ka6jt0Kdfz58/LubNnz1bHV1ZW5BqVfHbp5n/84x9yTqUxv/rqK7nGJXTVb4DE5btR5+9dzyt3dgCA9Ch2AID0KHYAgPQodgCA9Ch2AID0KHYAgPQ63nrQOk7beqvAUd8n+l5ujWpQ6+LLLv6t4uRqvBS/jcBF4VtyWw/UnNue4Zpvuy0QijoPKh5fSinDw8NyTm0fmZiYkGtmZ2flnNpaEvn+3PUV2XrguO0er169qo6771Y17L53796R15RSyl/+8pfquGrgXorfuoGThTs7AEB6FDsAQHoUOwBAehQ7AEB6FDsAQHrvrRF0r1NNgQ8PD+UaNeeShjs7O3JOpTjdMUREvqdoyrV1A/BuccenUpxTU1NyjUs7qnSnS4uqa8JdX67xtTrnFy9eDL3emzdvquMqwVxKKffv36+OP3r0SK758ssv5ZxqIO2+W/dbO6nXcq87rnPEnR0AID2KHQAgPYodACA9ih0AID2KHQAgPYodACC9jrcetNYLEdxIRNitUdHryBp3DJE1TmSrgFsT2abSWuR9IuehlFK2t7er48+ePZNr3NYD1czbNW4eGho60muV4j+T2mIwPT0t17iGyqrh8507d+Qadf5u3bol18zPz8s597tRItt8euFvG/437uwAAOlR7AAA6VHsAADpUewAAOlR7AAA6VHsAADpvbetByf1SQmtO5q3jv1H3qub57z1OWq5xol87wcHB3LNixcv5Jzq9q+2F5Sin7AwNjYm10xMTBz59Vx8X20vKKWUe/fuVce/++47uUbNqdeKav0UEfQm7uwAAOlR7AAA6VHsAADpUewAAOlR7AAA6X3QjaAjMiYXI68XTaVGUrjdah4dbQQd+UwuqfngwYPq+JkzZ+SawcHB6ng0jane6+XLl3KNa3x99+7d6rhr6nz79m05F0Hq8sPGnR0AID2KHQAgPYodACA9ih0AID2KHQAgPYodACC9Y916cFK3F0S1brTcC82ye+EYuiW6naL1udjb26uOb25uyjVqK4NrHn36tP757+/vV8dXVlbkGrVlopRSHj9+XB1/+PDhkY8BiODODgCQHsUOAJAexQ4AkB7FDgCQHsUOAJAexQ4AkN57e+qB0q2u/VlF4/Mt38eJPEWhW3rhyQullNLf318dHxgYkGvUdoW+vj65xs3961//qo6vra3JNfPz83Lum2++qY677RRAS9zZAQDSo9gBANKj2AEA0qPYAQDSo9gBANJrksbsVuKxdTPe1scdSexFjqF14rJbx30c79XNY+/WMQwODlbHx8fH5ZozZ85Ux7e3t+Wa3d1dOac+k2roXEopX3zxhZxbXFyUc0A3cGcHAEiPYgcASI9iBwBIj2IHAEiPYgcASI9iBwBIr+caQTvdbMarouGR7Q+tY/q98Hrdajj9tvfqhdeLfO+nT+uf3tTUVHV8enparpmbm6uOnzql/z+7uroq57a2tqrjf/3rX+Wahw8fyjngfePODgCQHsUOAJAexQ4AkB7FDgCQHsUOAJBez6UxXVLu8PDwyOu61aT6OJzUY++F5syt3yuSwnXn4eLFi3Lu2rVr1fGZmRm5RiU4XRpzaWlJzn399dfV8du3b8s1jjoO95uOiKSEu5XOxfvFnR0AID2KHQAgPYodACA9ih0AID2KHQAgPYodACC9jrceuAiz4iK9ka0C3Ww+jJOtF66HiYkJOXf16lU5d+XKleq4266wv79fHXcNp588eSLn1NaD3d1ducaJbDGI/I2I/P2Ibj2IbGXohevyQ8WdHQAgPYodACA9ih0AID2KHQAgPYodACA9ih0AIL2Otx7QGRwfMnf99/X1Vcfn5ubkmunpaTk3Pj5eHR8dHZVr+vv7q+PLy8tyzXfffSfnnj59Wh3v5u828qSEbh5fy+1T/D08ftzZAQDSo9gBANKj2AEA0qPYAQDSo9gBANJrksbsViopcgwnuSlrt5pln+RzFBE5ry4BeOnSpeq4S09OTk7KubGxser4wMCAXLO3t1cdv3//vlxz584dOacaPvd60+RIw/pe0Pq84n87mVcGAABHQLEDAKRHsQMApEexAwCkR7EDAKRHsQMApNfx1gOnW/HXyPtEY/otj8H50GL/Si+ch2j8WzV1VlsI3JpSSpmYmKiOu1j9kydPquN/+tOf5JrV1VU5p5pbR7dn9MJvLfI+rRvg4+2Oq1k2d3YAgPQodgCA9Ch2AID0KHYAgPQodgCA9Ch2AID0mmw9iERF1Vw0Xtqtruqto8itj6FbWyO6uS2iW09ycJ/p7Nmzcu78+fPV8eHhYblmZGREzg0ODlbH3VaBP/7xj9Xxp0+fyjVqe4HT+qkkvaCXj+1tWv8+u3UuWv+mO8GdHQAgPYodACA9ih0AID2KHQAgPYodACC9jtOYrZOVGVODva7lOYomo3rhe4+scclKldQcHR2Va3Z3d+Wcaur8+9//Xq65detWdfzg4ECucWlM1dTZnSPXqPpD+n12M915XMnFFq/XC8fw37izAwCkR7EDAKRHsQMApEexAwCkR7EDAKRHsQMApNekEXRE6+0KrSPMvRCJPqlx7W4eX8ttEy46797n9evX1fFz587JNW7rweeff14d/9vf/ibXRCLoantB9PV6uYlwVpFz3nqr0UlppM2dHQAgPYodACA9ih0AID2KHQAgPYodACC9JmnMlsnKXklVqmSeS7BFtE6WtX7cfa8fQ8vEqktj7uzsyLm1tbXq+Obmplzz9OlTOff1119XxyO/p2jCtFtNfCMpP/cb7IUmzN1srN4LIulmp/Xf2P/gzg4AkB7FDgCQHsUOAJAexQ4AkB7FDgCQHsUOAJBex1sPIk1jIxHc6FaB1pH2yOtF9ELj2tbHEImTd/N7ikTat7e35dzS0lJ1/IsvvpBrbt26JedUY2n3mSIx725+T5HXU/r6+o68ppT2zedbir5P5PxFvkN3fK23ChzXOefODgCQHsUOAJAexQ4AkB7FDgCQHsUOAJAexQ4AkF7HWw9c3FfNRTquu1isi7i2fkpBy276kfdx79V6e0brLu2tI+3uO2wZe45+JrX1YGVlRa5xT1Ho7++vjrvf4P7+vpyLiFwrrZ92EdHNJw5E/kZ0aytD9IkD6jNFt3soka0M73ruuLMDAKRHsQMApEexAwCkR7EDAKRHsQMApNdxGnNwcFDORRJxKi0UbTh6cHBQHY+m/Hq9AWzL12udnowkwdwalwRT36G6HkrR58gdg5tTx+DOkUpcutdzn0kdX+vrP5o0jKSlWyeie+G31q01rZsz7+3tNX2994E7OwBAehQ7AEB6FDsAQHoUOwBAehQ7AEB6FDsAQHoffd+tjD0AAO8Jd3YAgPQodgCA9Ch2AID0KHYAgPQodgCA9Ch2AID0KHYAgPQodgCA9Ch2AID0/g/R0rVNsYY9LAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "model.eval()\n", + "guidance_scale = 7.0\n", + "conditioning = torch.cat([-1 * torch.ones(1, 1, 1).float(), torch.ones(1, 1, 1).float()], dim=0).to(device)\n", + "\n", + "noise = torch.randn((1, 1, 64, 64))\n", + "noise = noise.to(device)\n", + "scheduler.set_timesteps(num_inference_steps=1000)\n", + "progress_bar = tqdm(scheduler.timesteps)\n", + "for t in progress_bar:\n", + " with autocast(enabled=True):\n", + " with torch.no_grad():\n", + " noise_input = torch.cat([noise] * 2)\n", + " model_output = model(noise_input, timesteps=torch.Tensor((t,)).to(noise.device), context=conditioning)\n", + " noise_pred_uncond, noise_pred_text = model_output.chunk(2)\n", + " noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)\n", + "\n", + " noise, _ = scheduler.step(noise_pred, t, noise)\n", + "\n", + "plt.style.use(\"default\")\n", + "plt.imshow(noise[0, 0].cpu(), vmin=0, vmax=1, cmap=\"gray\")\n", + "plt.tight_layout()\n", + "plt.axis(\"off\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "3483b097", + "metadata": {}, + "source": [ + "### Cleanup data directory\n", + "\n", + "Remove directory if a temporary was used." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "b00d4f9a", + "metadata": {}, + "outputs": [], + "source": [ + "if directory is None:\n", + " shutil.rmtree(root_dir)" + ] + } + ], + "metadata": { + "jupytext": { + "formats": "py:percent,ipynb" + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.py b/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.py index 16549b46..f433db53 100644 --- a/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.py +++ b/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.py @@ -1,7 +1,7 @@ # --- # jupyter: # jupytext: -# formats: ipynb,py:percent +# formats: py:percent,ipynb # text_representation: # extension: .py # format_name: percent @@ -14,12 +14,14 @@ # --- # %% [markdown] -# # Denoising Diffusion Probabilistic Models with MedNIST Dataset +# # Classifier-free Guidance +# +# This tutorial illustrates how to use MONAI for training a denoising diffusion probabilistic model (DDPM)[1] to create synthetic 2D images using the classifier-free guidance technique [2] to perform conditioning. # -# This tutorial illustrates how to use MONAI for training a denoising diffusion probabilistic model (DDPM)[1] to create -# synthetic 2D images. # # [1] - Ho et al. "Denoising Diffusion Probabilistic Models" https://arxiv.org/abs/2006.11239 +# [2] - Ho and Salimans "Classifier-Free Diffusion Guidance" https://arxiv.org/abs/2207.12598 +# # # TODO: Add Open in Colab # @@ -71,12 +73,6 @@ # %% [markdown] # ## Setup data directory -# -# You can specify a directory with the MONAI_DATA_DIRECTORY environment variable. -# -# This allows you to save results and reuse downloads. -# -# If not specified a temporary directory will be used. # %% jupyter={"outputs_hidden": false} directory = os.environ.get("MONAI_DATA_DIRECTORY") @@ -92,8 +88,8 @@ # %% [markdown] # ## Setup MedNIST Dataset and training and validation dataloaders # In this tutorial, we will train our models on the MedNIST dataset available on MONAI -# (https://docs.monai.io/en/stable/apps.html#monai.apps.MedNISTDataset). In order to train faster, we will select just -# one of the available classes ("Hand"), resulting in a training set with 7999 2D images. +# (https://docs.monai.io/en/stable/apps.html#monai.apps.MedNISTDataset). +# Here, we will use the "Hand" and "HeadCT", where our conditioning variable `class` will specify the modality. # %% jupyter={"outputs_hidden": false} train_data = MedNISTDataset(root_dir=root_dir, section="training", download=True, progress=False, seed=0) @@ -103,12 +99,21 @@ train_datalist.append({"image": item["image"], "class": 1 if item["class_name"] == "Hand" else 2}) # %% [markdown] -# Here we use transforms to augment the training dataset: +# Here we use transforms to augment the training dataset, as usual: # # 1. `LoadImaged` loads the hands images from files. # 1. `EnsureChannelFirstd` ensures the original data to construct "channel first" shape. # 1. `ScaleIntensityRanged` extracts intensity range [0, 255] and scales to [0, 1]. # 1. `RandAffined` efficiently performs rotate, scale, shear, translate, etc. together based on PyTorch affine transform. +# +# ### Classifier-free guidance during training +# +# In order to use the classifier-free guidance during training time, we need to not just have the `class` variable saying the modality of the image (`1` for Hands and `2` for HeadCTs) but we also need to train the model with an "unconditional" class. +# Here we specify the "unconditional" class with the value `-1` with a probability of training on unconditional being 15%. Specified in the following line using MONAI's RandLambdad: +# +# `transforms.RandLambdad(keys=["class"], prob=0.15, func=lambda x: -1 * torch.ones_like(x))` +# +# Finally, our conditioning variable need to have the format (batch_size, 1, cross_attention_dim) when feeding into the model. For this reason, we use Lambdad to reshape our variables in the right format. # %% jupyter={"outputs_hidden": false} train_transforms = transforms.Compose( @@ -126,7 +131,9 @@ prob=0.5, ), transforms.RandLambdad(keys=["class"], prob=0.15, func=lambda x: -1 * torch.ones_like(x)), - transforms.EnsureTyped(keys=["class"], dtype=torch.float), + transforms.Lambdad( + keys=["class"], func=lambda x: torch.tensor(x, dtype=torch.float32).unsqueeze(0).unsqueeze(0) + ), ] ) train_ds = CacheDataset(data=train_datalist, transform=train_transforms) @@ -145,7 +152,9 @@ transforms.LoadImaged(keys=["image"]), transforms.EnsureChannelFirstd(keys=["image"]), transforms.ScaleIntensityRanged(keys=["image"], a_min=0.0, a_max=255.0, b_min=0.0, b_max=1.0, clip=True), - transforms.EnsureTyped(keys=["class"], dtype=torch.float), + transforms.Lambdad( + keys=["class"], func=lambda x: torch.tensor(x, dtype=torch.float32).unsqueeze(0).unsqueeze(0) + ), ] ) val_ds = CacheDataset(data=val_datalist, transform=val_transforms) @@ -170,7 +179,14 @@ # ### Define network, scheduler, optimizer, and inferer # At this step, we instantiate the MONAI components to create a DDPM, the UNET, the noise scheduler, and the inferer used for training and sampling. We are using # the original DDPM scheduler containing 1000 timesteps in its Markov chain, and a 2D UNET with attention mechanisms -# in the 2nd and 3rd levels, each with 1 attention head. +# in the 3rd level, each with 1 attention head (`num_head_channels=64`). +# +# In order to pass conditioning variables with dimension of 1 (just specifying the modality of the image), we use: +# +# ` +# with_conditioning=True, +# cross_attention_dim=1, +# ` # %% jupyter={"outputs_hidden": false} device = torch.device("cuda") @@ -222,9 +238,7 @@ noise = torch.randn_like(images).to(device) # Get model prediction - noise_pred = inferer( - inputs=images, diffusion_model=model, noise=noise, condition=classes.unsqueeze(-1).unsqueeze(-1) - ) + noise_pred = inferer(inputs=images, diffusion_model=model, noise=noise, condition=classes) loss = F.mse_loss(noise_pred.float(), noise.float()) @@ -250,9 +264,7 @@ with torch.no_grad(): with autocast(enabled=True): noise = torch.randn_like(images).to(device) - noise_pred = inferer( - inputs=images, diffusion_model=model, noise=noise, condition=classes.unsqueeze(-1).unsqueeze(-1) - ) + noise_pred = inferer(inputs=images, diffusion_model=model, noise=noise, condition=classes) val_loss = F.mse_loss(noise_pred.float(), noise.float()) val_epoch_loss += val_loss.item() @@ -287,12 +299,14 @@ plt.show() # %% [markdown] -# ### Plotting sampling process along DDPM's Markov chain +# ### Sapling process with classifier-free guidance +# In order to sample using classifier-free guidance, for each step of the process we need to have 2 elements, one generated conditioned in the desired class (here we want to condition on Hands `=1`) and one using the unconditional class (`=-1`). +# Instead using directly the predicted class in every step, we use the unconditional plus the direction vector pointing to the condition that we want (`noise_pred_text - noise_pred_uncond`). The effect of the condition is defined by the `guidance_scale` defining the influence of our direction vector. # %% jupyter={"outputs_hidden": false} model.eval() -guidance_scale = 0.7 -conditioning = torch.cat([-1 * torch.ones(1, 1, 1, 1).float(), torch.ones(1, 1, 1, 1).float()], dim=0).to(device) +guidance_scale = 7.0 +conditioning = torch.cat([-1 * torch.ones(1, 1, 1).float(), torch.ones(1, 1, 1).float()], dim=0).to(device) noise = torch.randn((1, 1, 64, 64)) noise = noise.to(device) @@ -302,12 +316,10 @@ with autocast(enabled=True): with torch.no_grad(): noise_input = torch.cat([noise] * 2) - model_output = model(noise_input, timesteps=torch.Tensor((t,)).to(noise.device), context=conditioning) noise_pred_uncond, noise_pred_text = model_output.chunk(2) noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) - # 2. compute previous image: x_t -> x_t-1 noise, _ = scheduler.step(noise_pred, t, noise) plt.style.use("default") From 603ccdd556e8e40a2f046798939a4d5a000db699 Mon Sep 17 00:00:00 2001 From: Walter Hugo Lopez Pinaya Date: Fri, 16 Dec 2022 09:50:15 +0000 Subject: [PATCH 3/3] Fix typo Signed-off-by: Walter Hugo Lopez Pinaya --- .../2d_ddpm_classifier_free_guidance_tutorial.ipynb | 4 ++-- .../2d_ddpm_classifier_free_guidance_tutorial.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.ipynb b/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.ipynb index 7b418b8d..f8e67fbd 100644 --- a/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.ipynb +++ b/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.ipynb @@ -181,7 +181,7 @@ "source": [ "## Setup MedNIST Dataset and training and validation dataloaders\n", "In this tutorial, we will train our models on the MedNIST dataset available on MONAI\n", - "(https://docs.monai.io/en/stable/apps.html#monai.apps.MedNISTDataset). \n", + "(https://docs.monai.io/en/stable/apps.html#monai.apps.MedNISTDataset).\n", "Here, we will use the \"Hand\" and \"HeadCT\", where our conditioning variable `class` will specify the modality." ] }, @@ -670,7 +670,7 @@ "id": "0cd48c2d", "metadata": {}, "source": [ - "### Sapling process with classifier-free guidance\n", + "### Sampling process with classifier-free guidance\n", "In order to sample using classifier-free guidance, for each step of the process we need to have 2 elements, one generated conditioned in the desired class (here we want to condition on Hands `=1`) and one using the unconditional class (`=-1`).\n", "Instead using directly the predicted class in every step, we use the unconditional plus the direction vector pointing to the condition that we want (`noise_pred_text - noise_pred_uncond`). The effect of the condition is defined by the `guidance_scale` defining the influence of our direction vector." ] diff --git a/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.py b/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.py index f433db53..d59d634c 100644 --- a/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.py +++ b/tutorials/generative/classifier_free_guidance/2d_ddpm_classifier_free_guidance_tutorial.py @@ -299,7 +299,7 @@ plt.show() # %% [markdown] -# ### Sapling process with classifier-free guidance +# ### Sampling process with classifier-free guidance # In order to sample using classifier-free guidance, for each step of the process we need to have 2 elements, one generated conditioned in the desired class (here we want to condition on Hands `=1`) and one using the unconditional class (`=-1`). # Instead using directly the predicted class in every step, we use the unconditional plus the direction vector pointing to the condition that we want (`noise_pred_text - noise_pred_uncond`). The effect of the condition is defined by the `guidance_scale` defining the influence of our direction vector.