From 587f12a1f22b7bd5ee95fc9b412c94f52889eff8 Mon Sep 17 00:00:00 2001 From: Richard Brown <33289025+rijobro@users.noreply.github.com> Date: Wed, 20 Jan 2021 09:48:52 +0000 Subject: [PATCH 1/4] initial learning rate commit --- modules/learning_rate.ipynb | 1592 +++++++++++++++++++++++++++++++++++ 1 file changed, 1592 insertions(+) create mode 100644 modules/learning_rate.ipynb diff --git a/modules/learning_rate.ipynb b/modules/learning_rate.ipynb new file mode 100644 index 0000000000..7971f425be --- /dev/null +++ b/modules/learning_rate.ipynb @@ -0,0 +1,1592 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Optimal learning rates\n", + "\n", + "In this tutorial, we'll use the MedNIST dataset to explore MONAI's `LearningRateFinder` and use it to get an initial estimate of a learning rate.\n", + "\n", + "We then employ one of Pytorch's cyclical learning rate schedulers to vary the learning rate over the course of the optimisation. This has been shown to give improved results: https://arxiv.org/abs/1506.01186.\n", + "\n", + "TODO:\n", + "* make a bullet list of learning points\n", + "\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Project-MONAI/tutorials/blob/master/modules/learning_rate.ipynb)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup environment" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "!python -c \"import monai\" || pip install -q monai[pillow, tqdm]\n", + "!python -c \"import matplotlib\" || pip install -q matplotlib\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup imports" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "MONAI version: 0.4.0+66.g028a965.dirty\n", + "Numpy version: 1.19.5\n", + "Pytorch version: 1.7.1\n", + "MONAI flags: HAS_EXT = False, USE_COMPILED = False\n", + "MONAI rev id: 028a965b208e2ef4265d51ec9c7ec8bd43b83359\n", + "\n", + "Optional dependencies:\n", + "Pytorch Ignite version: 0.4.2\n", + "Nibabel version: 3.2.1\n", + "scikit-image version: 0.18.1\n", + "Pillow version: 8.1.0\n", + "Tensorboard version: 2.4.0\n", + "gdown version: 3.12.2\n", + "TorchVision version: 0.8.2\n", + "ITK version: 5.1.2\n", + "tqdm version: 4.51.0\n", + "lmdb version: 1.0.0\n", + "psutil version: 5.8.0\n", + "\n", + "For details about installing the optional dependencies, please visit:\n", + " https://docs.monai.io/en/latest/installation.html#installing-the-recommended-dependencies\n", + "\n" + ] + } + ], + "source": [ + "# Copyright 2020 MONAI Consortium\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License.\n", + "\n", + "import os\n", + "import shutil\n", + "import tempfile\n", + "import matplotlib.pyplot as plt\n", + "from math import ceil, floor, log10\n", + "import torch\n", + "import numpy as np\n", + "from sklearn.metrics import classification_report\n", + "from torch.utils.data import DataLoader\n", + "from tqdm import trange\n", + "\n", + "from monai.apps import MedNISTDataset\n", + "from monai.config import print_config\n", + "from monai.metrics import compute_roc_auc\n", + "from monai.networks.nets import densenet121\n", + "from monai.networks.utils import eval_mode\n", + "from monai.optimizers import LearningRateFinder\n", + "from monai.transforms import (\n", + " AddChanneld,\n", + " Compose,\n", + " LoadImaged,\n", + " RandFlipd,\n", + " RandRotated,\n", + " RandZoomd,\n", + " ScaleIntensityd,\n", + " ToTensord,\n", + ")\n", + "from monai.utils import set_determinism\n", + "\n", + "print_config()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup data directory\n", + "\n", + "You can specify a directory with the `MONAI_DATA_DIRECTORY` environment variable.\n", + "This allows you to save results and reuse downloads.\n", + "If not specified a temporary directory will be used." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/home/rbrown/data/MONAI\n" + ] + } + ], + "source": [ + "directory = os.environ.get(\"MONAI_DATA_DIRECTORY\")\n", + "root_dir = tempfile.mkdtemp() if directory is None else directory\n", + "print(root_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set deterministic training for reproducibility" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "set_determinism(seed=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define MONAI transforms, Dataset and Dataloader to pre-process data" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "train_transforms = Compose(\n", + " [\n", + " LoadImaged(keys=\"image\"),\n", + " AddChanneld(keys=\"image\"),\n", + " ScaleIntensityd(keys=\"image\"),\n", + " RandRotated(keys=\"image\", range_x=np.pi / 12, prob=0.5, keep_size=True),\n", + " RandFlipd(keys=\"image\", spatial_axis=0, prob=0.5),\n", + " RandZoomd(keys=\"image\", min_zoom=0.9, max_zoom=1.1, prob=0.5),\n", + " ToTensord(keys=\"image\"),\n", + " ]\n", + ")\n", + "\n", + "val_transforms = Compose(\n", + " [\n", + " LoadImaged(keys=\"image\"),\n", + " AddChanneld(keys=\"image\"),\n", + " ScaleIntensityd(keys=\"image\"),\n", + " ToTensord(keys=\"image\"),\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Verified 'MedNIST.tar.gz', md5: 0bc7306e7427e00ad1c5526a6677552d.\n", + "file /home/rbrown/data/MONAI/MedNIST.tar.gz exists, skip downloading.\n", + "extracted file /home/rbrown/data/MONAI/MedNIST exists, skip extracting.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading dataset: 100%|██████████| 249/249 [00:00<00:00, 1207.74it/s]\n", + "Loading dataset: 0%| | 0/25 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, axes = plt.subplots(3, 3, figsize=(15, 15), facecolor='white')\n", + "for i, k in enumerate(np.random.randint(len(train_ds), size=9)):\n", + " data = train_ds[k]\n", + " im, title = data[\"image\"], data[\"class_name\"]\n", + " ax = axes[i//3, i%3]\n", + " im_show = ax.imshow(im[0])\n", + " ax.set_title(title, fontsize=25)\n", + " ax.axis('off')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define network and optimizer\n", + "\n", + "1. Set learning rate for how much the model is updated per batch.\n", + "1. Set total epoch number, as we have shuffle and random transforms, so the training data of every epoch is different.\n", + "And as this is just a get start tutorial, let's just train 4 epochs.\n", + "If train 10 epochs, the model can achieve 100% accuracy on test dataset.\n", + "1. Use DenseNet from MONAI and move to GPU devide, this DenseNet can support both 2D and 3D classification tasks.\n", + "1. Use Adam optimizer." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "lr = 1e-5\n", + "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + "loss_function = torch.nn.CrossEntropyLoss()\n", + "model = densenet121(spatial_dims=2, in_channels=1,\n", + " out_channels=num_classes).to(device)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Estimate optimal learning rate\n", + "\n", + "Use MONAI's `LearningRateFinder` to get an initial estimate of a learning rate. Assume that it's in the range 1e-5, 1e-2. If that weren't the case (which we'd notice in the plot), we could just try again over a larger/different window.\n", + "\n", + "We then extract the learning rate with the steepest gradient, and set the upper and lower learning rates of a cyclical optimsation to be the nearest powers of 10 above and below this value." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Computing optimal learning rate: 95%|█████████▌| 19/20 [00:41<00:02, 2.20s/it]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Stopping early, the loss has diverged\n", + "Resetting model and optimizer\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAuM0lEQVR4nO3de1xUdd4H8M9cuA8gCLQGGhJeuOnIRbyiWHiPVkkT3cq00NrNp4d9ueW+nvXSbpfn5VOrbZu7tFY+6hOVW5muKe0mtlrJTpllumUqCmjMiAIzwAxz+T1/DDNAjAjKYWbg8369eM2cc37nnO9wdL78fuf3Oz+ZEEKAiIjoR+TuDoCIiDwTEwQREbnEBEFERC4xQRARkUtMEERE5BITBBERuaR0dwA9KSIiArGxse4Og4jIa5SXl+Py5csut/WpBBEbGwuNRuPuMIiIvEZ6evo1t7GJiYiIXGKCICIil5ggiIjIpT51D8IVs9mMyspKGI1Gd4dCEvP390dMTAx8fHzcHQpRn9DnE0RlZSWCg4MRGxsLmUzm7nBIIkII1NTUoLKyEkOHDnV3OER9Qp9vYjIajRg4cCCTQx8nk8kwcOBA1hSJelCfTxAAmBz6CV5n6o9OVNXh4+90khy7XySIbhEC+Owz4N137a8STZexadMmNDY2SnLsrqqtrcXLL7/ca+eLjY11DsiZMGHCDR/n9ddfx8WLF3sqLCKvtuOz8yh867gkx2aCaGvfPmDIECAnB1i61P46ZIh9fQ/rKwnCYrHc0H6ffPLJDZ+TCYKolVZvwi0hfpIcmwnCYd8+4J57gMpKwGAA6uvtr5WV9vU3mCQaGhowZ84cjB49GsnJyXjzzTfx4osv4uLFi8jOzkZ2djYAoKSkBOPHj0dqaioWLFgAg8EAAPj8888xZcoUpKWlYcaMGbh06RIAYOrUqfiP//gPqNVqJCcno6yszHm+ZcuWYezYsRgzZgx2794NAPjmm28wduxYqNVqjBo1CqdPn8aTTz6JM2fOQK1WY/Xq1R1i/+1vf4sRI0Zg0qRJyM/Px//8z/84z/34448jPT0dmzdvxp49e5CZmYkxY8bgzjvvRHV1NQCgpqYG06dPR1JSEh566CG0nbxQpVI532/cuBEZGRkYNWoU1q1bB8A+/D8hIQEPP/wwkpKSMH36dDQ1NWHXrl3QaDRYsmQJ1Go1mpqabui6EPUVWr0RUcHSJAiIPiQtLa3DupMnT15/R5tNiOhoIewNSq5/YmLs5bpp165d4qGHHnIu19bWCiGEuO2224ROpxNCCKHT6cTkyZOFwWAQQgjx3HPPiQ0bNojm5mYxfvx4odVqhRBCFBcXiwcffFAIIcSUKVOcxz106JBISkoSQgixZs0asX37diGEEFevXhXDhg0TBoNB/OIXvxA7duwQQghhMplEY2OjOHfunHO/HysrKxOjR48WTU1Nor6+XsTHx4uNGzc6z/3II484y165ckXYWn43r7zyiigsLBRCCPHYY4+JDRs2CCGE2Lt3rwDg/MxBQUFCCCEOHDggHn74YWGz2YTVahVz5swRhw4dEufOnRMKhUIcO3ZMCCHEggULnJ9rypQp4l//+pfLuLt0vYn6kIzffSh+9fbxG97f1femQ5/v5tolR48CdXWdl6mtBcrKgMzMbh06JSUFv/zlL/HEE09g7ty5mDx5cocyn332GU6ePImJEycCAJqbmzF+/Hh8++23OHHiBHJycgAAVqsVgwYNcu6Xn58PAMjKykJ9fT1qa2tRUlKC999/3/nXvtFoxIULFzB+/Hg8/fTTqKysxPz58zFs2LBO4z5y5Ajuvvtu+Pv7w9/fH3fddVe77ffee6/zfWVlJe69915cunQJzc3Nzm6mH3/8Md555x0AwJw5cxAWFtbhPCUlJSgpKcGYMWMAAAaDAadPn8aQIUMwdOhQqNVqAEBaWhrKy8s7jZmov7HaBC4bTIiSqImJCQIALl0C5NdpbZPLgRto9x4+fDi++OIL7Nu3D//1X/+FO+64A2vXrm1XRgiBnJwcvPHGG+3Wf/3110hKSsKnn37q8tg/7rUjk8kghMBf//pXjBgxot22hIQEZGZm4m9/+xtmz56NP//5z4iLi+v253EICgpyvn/sscdQWFiI3NxclJaWYv369V0+jhACa9aswYoVK9qtLy8vh59f6z96hULB5iSiH6kxmGATkKyJifcgAGDQIMBm67yMzQbcemu3D33x4kUEBgbiZz/7GVavXo0vvvgCABAcHAy9Xg8AGDduHI4cOYLvv/8egP0+wnfffYcRI0ZAp9M5E4TZbMY333zjPPabb74JADh8+DBCQ0MRGhqKGTNm4A9/+IOzvf/YsWMAgLNnzyIuLg6rVq3C3Xffja+++qpdDD82ceJE7NmzB0ajEQaDAXv37r3mZ6yrq0N0dDQAYNu2bc71WVlZ+L//+z8AwAcffICrV6922HfGjBl49dVXnfdcqqqqoNVqO/2ddhY3UX+i1ZsAAJHB/pIcnzUIwN5sFBpqvyl9LQMGAGPHdvvQX3/9NVavXg25XA4fHx9s2bIFAFBQUICZM2fi1ltvxcGDB/H6668jPz8fJpP9gv/ud7/D8OHDsWvXLqxatQp1dXWwWCx4/PHHkZSUBMD+aIkxY8bAbDbj1VdfBQD85je/weOPP45Ro0bBZrNh6NCh2Lt3L9566y1s374dPj4++MlPfoJf//rXCA8Px8SJE5GcnIxZs2Zh48aNzrgzMjKQm5uLUaNG4ZZbbkFKSgpCQ0Ndfsb169djwYIFCAsLw7Rp03Du3DkAwLp165Cfn4+kpCRMmDABQ4YM6bDv9OnTcerUKYwfPx6A/eb1jh07oFAorvk7Xbp0KVauXImAgAB8+umnCAgI6O5lIeoTtHr7wFCpmph4k9rhb38TIiDA9Q3qgAD7dg/S2Y3anqLX64UQQjQ0NIi0tDTx+eefS3q+nsCb1NSfvHH0vLjtib2i8mrjDR+js5vUkjUxGY1GjB07FqNHj0ZSUpKz+2JbL7zwAhITEzFq1CjccccdOH/+vHObQqGAWq2GWq1Gbm6uVGG2mj0b2LULiIkBVCogJMT+GhNjXz97tvQxeJiCggKo1WqkpqYiLy8Pqamp7g6JiNpwNjGpvOwmtZ+fHz766COoVCqYzWZMmjQJs2bNwrhx45xlxowZA41Gg8DAQGzZsgW/+tWvnO3qAQEB+PLLL6UKz7XZs4ELF+y9lS5etN9zGDsW8MBHOJSWlkp+Dsf9AyLyTFq9EWGBPvBVSvO3vmQJQiaTOQdDmc1mmM3mDr1uHIPEAPuN2h07dkgVTtfJZN3uykpE5A7V9SZESXSDGpC4F5PVaoVarUZUVBRycnKQ2ckX79atWzFr1iznstFoRHp6OsaNG4f33nvvmvsVFRUhPT0d6enp0OlcP7BKSPQ8JfIsvM7U32j10o2BACTuxaRQKPDll1+itrYW8+bNw4kTJ5CcnNyh3I4dO6DRaHDo0CHnuvPnzyM6Ohpnz57FtGnTkJKSgttvv73DvgUFBSgoKADgevJtf39/1NTU8JHffZxomQ/C31+6v6aIPI2u3ojbIwdKdvxe6eY6YMAAZGdnY//+/R0SxN///nc8/fTTOHToULuBUY5+9XFxcZg6dSqOHTvmMkFcT0xMDCorK69Zu6C+wzGjHFF/IISAziBtE5NkCUKn08HHxwcDBgxAU1MTPvzwQzzxxBPtyhw7dgwrVqzA/v37ERUV5Vx/9epVBAYGws/PD5cvX8aRI0fwq1/96obi8PHx4QxjRNTnXG00w2wVkj3JFZAwQVy6dAkPPPAArFYrbDYbFi5ciLlz52Lt2rVIT09Hbm4uVq9eDYPBgAULFgAAhgwZgvfffx+nTp3CihUrIJfLYbPZ8OSTTyIxMVGqUImIvI5zkJw31iBGjRrlfMxDW0899ZTz/d///neX+06YMAFff/21VKEREXk9bb19DISUN6n5LCYiIi9UXe+oQTBBEBFRG45R1F47DoKIiKSh05sQ7KdEgO+1H2x5s5ggiIi8kFZvlPT+A8AEQUTklbQSP2YDYIIgIvJKUj9mA2CCICLyOkIIexOThD2YACYIIiKvU2+0wGi2sYmJiIja00k91WgLJggiIi/jGEUdySYmIiJqyzFI7pYQNjEREVEbrQ/qYw2CiIja0NabEOCjgMpP2il9mCCIiLyMYwyE1LNkMkEQEXmZ6nrpx0AATBBERF5Hp5f+MRsAEwQRkdfR6k2Sd3EFmCCIiLxKY7MFBpNF8i6uABMEEZFXcU41yhoEERG15ZxJTuLHbABMEEREXqV1kJwXNzEZjUaMHTsWo0ePRlJSEtatW9ehjMlkwr333ov4+HhkZmaivLzcue3ZZ59FfHw8RowYgQMHDkgVJhGRV6nuxSYmyYbh+fn54aOPPoJKpYLZbMakSZMwa9YsjBs3zllm69atCAsLw/fff4/i4mI88cQTePPNN3Hy5EkUFxfjm2++wcWLF3HnnXfiu+++g0Ih3dyrRETeQKs3wlchx4BAH8nPJVkNQiaTQaVSAQDMZjPMZnOHUX+7d+/GAw88AAC455578I9//ANCCOzevRuLFi2Cn58fhg4divj4eJSVlUkVKhGR19DV27u4Sj2KGpD4HoTVaoVarUZUVBRycnKQmZnZbntVVRUGDx4MAFAqlQgNDUVNTU279QAQExODqqoql+coKipCeno60tPTodPppPswREQeoDemGnWQNEEoFAp8+eWXqKysRFlZGU6cONHj5ygoKIBGo4FGo0FkZGSPH5+IyJP0xlSjDr3Si2nAgAHIzs7G/v37262Pjo5GRUUFAMBisaCurg4DBw5stx4AKisrER0d3RuhEhF5NG0vPWYDkDBB6HQ61NbWAgCamprw4YcfYuTIke3K5ObmYtu2bQCAXbt2Ydq0aZDJZMjNzUVxcTFMJhPOnTuH06dPY+zYsVKFSkTkFUwWK2obzb1Wg5CsF9OlS5fwwAMPwGq1wmazYeHChZg7dy7Wrl2L9PR05ObmYvny5bjvvvsQHx+P8PBwFBcXAwCSkpKwcOFCJCYmQqlU4o9//CN7MBFRv+ccRd1L9yBkQgjRK2fqBenp6dBoNO4Og4hIEp+fv4q8LZ/gtaUZyB4Z1SPH7Ox7kyOpiYi8hK5lFHVvPMkVYIIgIvIajucw9caTXAEmCCIir6GtN0Ehl2FgkG+vnI8JgojIS2j1RkSofCGXSz+KGmCCICLyGr05BgJggiAi8hrV9aZeGwMBMEEQEXkNnd7Ya2MgACYIIiKvYLHaUNPQzCYmIiJq77KhGUL03ihqgAmCiMgr9OZUow5MEEREXkDbi1ONOjBBEBF5AccoajYxERFRO9X1RshkQISKCYKIiNrQ6k0ID/SFj6L3vraZIIiIvIB9DETv3aAGmCCIiLyC/TEbvde8BDBBEBF5BW0vP2YDYIIgIvJ4NpvAZYOpV3swAUwQREQe70pjMyw20auD5AAmCCIij1dd7xhFzRoEERG14Y5BcoCECaKiogLZ2dlITExEUlISNm/e3KHMxo0boVaroVarkZycDIVCgStXrgAAYmNjkZKSArVajfT0dKnCJCLyeDrnYzZ6t4lJKdmBlUo8//zzSE1NhV6vR1paGnJycpCYmOgss3r1aqxevRoAsGfPHvz+979HeHi4c/vBgwcREREhVYhERF7B8aC+yL7SxDRo0CCkpqYCAIKDg5GQkICqqqprln/jjTeQn58vVThERF5LqzchNMAH/j6KXj1vr9yDKC8vx7Fjx5CZmelye2NjI/bv34+8vDznOplMhunTpyMtLQ1FRUW9ESYRkUdyxxgIQMImJgeDwYC8vDxs2rQJISEhLsvs2bMHEydObNe8dPjwYURHR0Or1SInJwcjR45EVlZWh32LioqcCUSn00nzIYiI3Ejby1ONOkhagzCbzcjLy8OSJUswf/78a5YrLi7u0LwUHR0NAIiKisK8efNQVlbmct+CggJoNBpoNBpERkb2XPBERB6iut7U6zeoAQkThBACy5cvR0JCAgoLC69Zrq6uDocOHcLdd9/tXNfQ0AC9Xu98X1JSguTkZKlCJSLyWEII6NzwHCZAwiamI0eOYPv27c6uqgDwzDPP4MKFCwCAlStXAgDeffddTJ8+HUFBQc59q6urMW/ePACAxWLB4sWLMXPmTKlCJSLyWHVNZjRbbb3+JFdAwgQxadIkCCGuW27p0qVYunRpu3VxcXE4fvy4RJEREXkP5yA5N9QgOJKaiMiDuWMuagcmCCIiD+YYJOeOJiYmCCIiD8YmJiIicqm63oggXwWC/CQfttYBEwQRkQfT6k1uaV4CmCCIiDyazk2P2QCYIIiIPJr9MRusQRAR0Y9o3TSKGmCCICLyWAaTBY3NViYIIiJqT+uYi9oNT3IFmCCIiDxWtZumGnVggiAi8lDOUdRsYiIiorZ0jlHUntyLqaGhATabDQDw3Xff4f3334fZbJY0MCKi/k6rN8FPKUeIf++Poga6mCCysrJgNBpRVVWF6dOnY/v27R0e0U1ERD1LW2+falQmk7nl/F1KEEIIBAYG4p133sGjjz6Kt99+G998843UsRER9Wv2MRDuaV4CupEgPv30U+zcuRNz5swBAFitVkkDIyLq79w5SA7oYoLYtGkTnn32WcybNw9JSUk4e/YssrOzpY6NiKhfq643ujVBdOnOx5QpUzBlyhQAgM1mQ0REBF588UVJAyMi6s+MZiv0RovbejABXaxBLF68GPX19WhoaEBycjISExOxceNGqWMjIuq33DnVqEOXEsTJkycREhKC9957D7NmzcK5c+ewfft2qWMjIuq33DnVqEOXEoTZbIbZbMZ7772H3Nxc+Pj4XLfbVUVFBbKzs5GYmIikpCRs3ry5Q5nS0lKEhoZCrVZDrVbjqaeecm7bv38/RowYgfj4eDz33HPd/FhERN7NnVONOnTpHsSKFSsQGxuL0aNHIysrC+fPn0dISEjnB1Yq8fzzzyM1NRV6vR5paWnIyclBYmJiu3KTJ0/G3r17262zWq34+c9/jg8//BAxMTHIyMhAbm5uh32JiPoq54P6PL2JadWqVaiqqsK+ffsgk8lw22234eDBg53uM2jQIKSmpgIAgoODkZCQgKqqqi4FVVZWhvj4eMTFxcHX1xeLFi3C7t27u7QvEVFfoNWboJTLEBbo67YYupQg6urqUFhYiPT0dKSnp+OXv/wlGhoaunyS8vJyHDt2DJmZmR22ffrppxg9ejRmzZrlHHxXVVWFwYMHO8vExMR0ObkQEfUF1fUmRAb7QS53zyhqoIsJYtmyZQgODsZbb72Ft956CyEhIXjwwQe7dAKDwYC8vDxs2rSpQ7NUamoqzp8/j+PHj+Oxxx7DT3/6025/gKKiImfi0ul03d6fiMgTuXOqUYcuJYgzZ85gw4YNiIuLQ1xcHNatW4ezZ89edz+z2Yy8vDwsWbIE8+fP77A9JCQEKpUKADB79myYzWZcvnwZ0dHRqKiocJarrKxEdHS0y3MUFBRAo9FAo9EgMjKyKx+HiMjj6dw8ihroYoIICAjA4cOHnctHjhxBQEBAp/sIIbB8+XIkJCSgsLDQZZkffvgBQggA9vsONpsNAwcOREZGBk6fPo1z586hubkZxcXFyM3N7epnIiLyeu5+zAbQxV5Mf/rTn3D//fejrq4OABAWFoZt27Z1us+RI0ewfft2pKSkQK1WAwCeeeYZXLhwAQCwcuVK7Nq1C1u2bIFSqURAQACKi4shk8mgVCrx0ksvYcaMGbBarVi2bBmSkpJu4mMSEXmPZosNVxqa3fqgPgCQCcef8F1QX18PwN40tGnTJjz++ONSxXVD0tPTodFo3B0GEdFNuVjbhAnPfYRn56cgf+wQSc/V2fdmt2aUCwkJcd5ofuGFF24+MiIi6sATBskBNzHlaDcqHkRE1A3VzkFyXtCLyRV3zXBERNTXOWoQt4R48E3q4OBgl4lACIGmpibJgiIi6s909UbIZcBAlQcnCL1e31txEBFRC63ehIEqPyjcOIoauIkmJiIikoYnjIEAmCCIiDyOVu/eqUYdmCCIiDyMtt7k9h5MABMEEZFHsdoELhtMiHJzDyaACYKIyKPUGEywCfdONerABEFE5EE8ZRQ1wARBRORRtHr3TzXqwARBRORBtPUtNQg2MRERUVuOJqZIN4+iBpggiIg8ilZvRFigD3yV7v96dn8ERETkVO0hYyAAJggiIo+i1XvGGAiACYKIyKPo6o2sQRARUXtCCOg8ZBQ1wARBROQxrjaaYbYKjxgDATBBEBF5jNZBcn28iamiogLZ2dlITExEUlISNm/e3KHMzp07MWrUKKSkpGDChAk4fvy4c1tsbCxSUlKgVquRnp4uVZhERB6jdZCcZ9QgOp1R7qYOrFTi+eefR2pqKvR6PdLS0pCTk4PExERnmaFDh+LQoUMICwvDBx98gIKCAhw9etS5/eDBg4iIiJAqRCIij1Jd7zmP2QAkrEEMGjQIqampAOxzWyckJKCqqqpdmQkTJiAsLAwAMG7cOFRWVkoVDhGRx2t9UF8fb2Jqq7y8HMeOHUNmZuY1y2zduhWzZs1yLstkMkyfPh1paWkoKirqjTCJiNxKpzch2F+JAF+Fu0MBIGETk4PBYEBeXh42bdqEkJAQl2UOHjyIrVu34vDhw851hw8fRnR0NLRaLXJycjBy5EhkZWV12LeoqMiZQHQ6nTQfgoioF3jKVKMOktYgzGYz8vLysGTJEsyfP99lma+++goPPfQQdu/ejYEDBzrXR0dHAwCioqIwb948lJWVudy/oKAAGo0GGo0GkZGRPf8hiIh6iadMNeogWYIQQmD58uVISEhAYWGhyzIXLlzA/PnzsX37dgwfPty5vqGhAXq93vm+pKQEycnJUoVKROQRPOkxG4CETUxHjhzB9u3bnV1VAeCZZ57BhQsXAAArV67EU089hZqaGjz66KP2YJRKaDQaVFdXY968eQAAi8WCxYsXY+bMmVKFSkTkdkIIj2tikixBTJo0CUKITsv85S9/wV/+8pcO6+Pi4tqNiSAi6uvqjRYYzTbc4gETBTlwJDURkQfQtYyijvSgGgQTBBGRB3COou4PN6mJiKjrnIPkPOgmNRMEEZEHaH1QHxMEERG1oa03IcBHAZWf5OOXu4wJgojIAzjGQMhkMneH4sQEQUTkAarrjbjFg25QA0wQREQeQac3IdKDblADTBBERB5Bqzd51A1qgAmCiMjtGpstMJgsHjUGAmCCICJyu9ZBcqxBEBFRG544SA5ggiAicrvWQXJsYiIiojaqW5qYbmENgoiI2tLqjfBVyhEa4OPuUNphgiAicjNdvQmRKs8aRQ0wQRARuZ2nTTXqwARBRORmnjbVqAMTBBGRm9lHUXtWDyaACYKIyK1MFitqG82sQRARUXtaZxdX1iCIiKgNxyhqT3uSKyBhgqioqEB2djYSExORlJSEzZs3dygjhMCqVasQHx+PUaNG4YsvvnBu27ZtG4YNG4Zhw4Zh27ZtUoVJRORWOg+catRBsrntlEolnn/+eaSmpkKv1yMtLQ05OTlITEx0lvnggw9w+vRpnD59GkePHsUjjzyCo0eP4sqVK9iwYQM0Gg1kMhnS0tKQm5uLsLAwqcIlInIL53OY+tNN6kGDBiE1NRUAEBwcjISEBFRVVbUrs3v3btx///2QyWQYN24camtrcenSJRw4cAA5OTkIDw9HWFgYcnJysH//fqlCJSJyG229CQq5DAODfN0dSge9Mjt2eXk5jh07hszMzHbrq6qqMHjwYOdyTEwMqqqqrrnelaKiIhQVFQEAdDqdBNETEUlHqzciQuULudyzRlEDvXCT2mAwIC8vD5s2bUJISEiPH7+goAAajQYajQaRkZE9fnwiIil56hgIQOIEYTabkZeXhyVLlmD+/PkdtkdHR6OiosK5XFlZiejo6GuuJyLqa6rrTR73FFcHyRKEEALLly9HQkICCgsLXZbJzc3F//7v/0IIgc8++wyhoaEYNGgQZsyYgZKSEly9ehVXr15FSUkJZsyYIVWoRERuo9MbEemhNQjJ7kEcOXIE27dvR0pKCtRqNQDgmWeewYULFwAAK1euxOzZs7Fv3z7Ex8cjMDAQr732GgAgPDwcv/nNb5CRkQEAWLt2LcLDw6UKlYjILSxWG2oamj2yiysgYYKYNGkShBCdlpHJZPjjH//octuyZcuwbNkyKUIjIvIIlw3NEMLzphp14EhqIiI38dSpRh2YIIiI3MTxHCZPbWJigiAichPHKGpPfFAfwARBROQWQgh8cuYyfBQyDFR53ihqgAmCiMgtiv9Vgb1fXcKjU+Pho/DMr2LPjIqIqA/7urIO697/BlnDI7HqjmHuDueamCCIiHpRbWMzHtn5OSKCfLHpXjUUHvgMJodeeVgfEREBNptA4VvHUV1vxNsrJyDcA5/g2hZrEEREvWTLoTP46N9a/GZuItSDB7g7nOtigiAi6gVHvr+M50u+xd3qW3HfuNvcHU6XMEEQEUnshzojVr1xDLdHqvDs/BTIZJ5736EtJggiIgk1W2x4dOfnMJqt2PKzNAT6es+tX++JlIjICz37wSl8caEWLy0eg/golbvD6RbWIIiIJLL3q4t47Ug5HpwYi7mjbnV3ON3GBEFEJIHvtQY8sesrpA4ZgDWzEtwdzg1hgiAi6mENJgse2fE5/HwU+OOSVPgqvfOr1jujJiLyUEII/Prdr3FGZ8Af8sdgUGiAu0O6YUwQREQ9aMdn57H7y4sozBmOifER7g7nprAXE+xthTYhIAQgYH91LANoXYb9rwPnq0DLe/t2R1nAfpyWN21fnOdoX7ZnyQDIZIBcJmt5L2tdlgH2R7/I2pVxbHOsk8tkUMjt+ypkMijk9u0KuX1Z1rJOIZNBJkf7Ms733tHXm6inHLtwFU/tPYlpI6Pw6NR4d4dz05ggAMz9wz9hNNvcHUafI5cBSrkcCrkMSoUMSrkMCrm85dW+TiGXwadNGYVc5tzuo5C3/Njf+yrkUCpa1/sqW7c5tvsoZPBRtu7np1TA30cOfx8FAnwUCPC1v/q3/AT4KOCjYDKjm3eloRk/3/kFbgnxxwsLR0PuwQ/h6yomCAAvLFTDJgRk6PgX9o//unaUkTn/Om/9Kxwt/x5aliBzLre8ylrXt65zRNGT/5gEbALtakKOmo9j2VEjgui4zmaz728TAjYhYHUs2wSsLetstjbrRcf39mUBs1XAarPB0rLOYhOwWu2vFsf6luV25awCBosFFquA2WpDs9UGs9UGs6V12bHNYru5OphCLmtJGu0TieN9oK8CQX5KqPyUCPZXtn/vq4TK376s8mt976eUe1bSEQI4ehS4dAkYNAjIzGz7j49uktUm8B/Fx3DZ0Iy/PjIBAwI9+yF8XSVZgli2bBn27t2LqKgonDhxosP2jRs3YufOnQAAi8WCU6dOQafTITw8HLGxsQgODoZCoYBSqYRGo5EqTADA7JRBkh6fpGWzCZhtNpitAmaLrU1CETBZrDCabWhqtsJotv80OX6aW5eNZpv9tbn99tomM6pqm9BgssDQ8iO6kI+Uclm7xBES4IPwQF+EBfkiPMgHYYG+CA9qWW7zPshX0fOJZd8+YMUKoLYWkMsBmw0YMAD485+B2bN79lz91Iv/OI1/nr6MZ+alICUm1N3h9BiZEF355959H3/8MVQqFe6//36XCaKtPXv24Pe//z0++ugjAEBsbCw0Gg0iIrp3gyc9PV3yZEL9mxACjc1WNJgs0Jss9sRhbPPe8WNsfa83WlDfZMbVxmZcabC/Wq9R6/FVyBHmIoGEBfkiQuWLCJUfIoP9ENnyGuR3nb/x9u0D7rkHaGrquC0gANi1i0niJpV+q8WDr/8L88ZE4/kFoz2r5tgFnX1vSlaDyMrKQnl5eZfKvvHGG8jPz5cqFKIeI5PJEORnb2aKusFjCCFQb7TgakMzrjQ2218bmlsTSJv1py7V42pDM2qbzC5rLoG+inYJw5lAgv0QGeSLrOUPw9dVcgDsSWPFCuDCBTY33aCq2iY8/uaXGHFLMJ7+qfc8hK+r3H4PorGxEfv378dLL73kXCeTyTB9+nTIZDKsWLECBQUF19y/qKgIRUVFAACdTid5vEQ3SyaTITTAB6EBPohFUJf2sdoErjQ0Q6c3QWcw4XLLq07f+nNaa8AnZ2pQ12QGAIyp+jfG1VxBZ63hlitX8cOBUkTckQV/H0UPfLr+40pDMx7d+QWsVoEtP0tDgG/f+/25PUHs2bMHEydORHh4uHPd4cOHER0dDa1Wi5ycHIwcORJZWVku9y8oKHAmkPT09F6Jmai3KeQyZ83gekwWK2oMzTC9fRV+7/oAZuM1yzZZBH5b9HccKG3ET0L8MSQ8EDHhARgSHogh4YEY3PIaqfLrE71yboTVJnDucgNOXapv86PHD/X23+uffpaKoRFdS/Texu0Jori4uEPzUnR0NAAgKioK8+bNQ1lZ2TUTBBG156dU4NYBAUBKPCDr/BZjoI8M+T8dh8Rbh+PClUZUXG3Ep2dq8O6xqnZNWn5KOWLCAjA4PBADg/wQGuCDkAClsyYU4u+D0MCW15Z1/j4e1pOrC+qNZvz7kr5dMvi2Wu/sBq+UyxAfpcL42wciYVAwMmLDMWZImJujlo5bE0RdXR0OHTqEHTt2ONc1NDTAZrMhODgYDQ0NKCkpwdq1a90YJZGXyswEQkMBg+GaRRRhYZh631xM/dEXucliRdXVJnvSuNKIiqtNuFBjTyCnqw2oazLDYLJ0enofhcyZPEIC7D/25KGEys8Hwf72rsL2LsM+zq7Dzld/JfyUPddsI4RAs9WGZov9R2+04N8/tEkGP9Sj4krr/ZoBgT5I+EkIlmTehoRBIUgYFIz4KFWPxuTpJEsQ+fn5KC0txeXLlxETE4MNGzbAbLa3ja5cuRIA8O6772L69OkICmqtnlVXV2PevHkA7N1fFy9ejJkzZ0oVJlHfJZMBRUWd92L6859d3qD2UyoQF6lCXOS15y+wWO1fsnVNZtQbzfbXJvty+3X217rGZlRcaURdkxl6oxlm6/U7UPoq5M5kofJrTSaBvgpYbQImi63lS9+KZovNvuxc1/pjalm+1q9p6MAgjIoZgEUZQ5AwKBgJg0LwkxB/r6sB9TTJurm6A7u5ErngoeMgTBYr9MbWLsF6owV6o7ld92C90QKDyWzvStzSndhgtKCx2QKFXAZfpQK+Sjn8WkbW+yrl8Gt59W2z7sdlfBVyBPopMSxKhRE/CfaqWd56mlu6uRKRh5g9296VtawMuHgRuPVWYOxYt3dt9VMq4KdSIEJ1/Rvv5B5MEET9gUxmvydB1A183DcREbnEBEFERC4xQRARkUtMEERE5BITBBERucQEQURELjFBEBGRS31qJHVERARiY2Ody3V1dQgN7Ti7k6v1Op0OkZGRUod4TdeKtbeO1Z19rle2s+3d3cZrdXP78FrxWl3vWpWXl+Py5cuuTyr6sIcffrjL69PS0qQOp1PXirW3jtWdfa5XtrPt3d3Ga3Vz+/Ba8Vp19Vq50qebmO66665urXennozpRo7VnX2uV7az7d3dxmt1c/vwWvFa3cy16lNNTDeDD/rzHrxW3oPXyrv16RpEd3Q2rSl5Fl4r78Fr5d1YgyAiIpdYgyAiIpeYIIiIyCUmCCIicokJogtKS0sxefJkrFy5EqWlpe4Oh66joaEB6enp2Lt3r7tDoU6cOnUKK1euxD333IMtW7a4Oxxyoc8niGXLliEqKgrJycnt1u/fvx8jRoxAfHw8nnvuuU6PIZPJoFKpYDQaERMTI2W4/VpPXCsA+O///m8sXLhQqjAJPXOtEhIS8Kc//QlvvfUWjhw5ImW4dIP6fC+mjz/+GCqVCvfffz9OnDgBALBarRg+fDg+/PBDxMTEICMjA2+88QasVivWrFnTbv9XX30VERERkMvlqK6uRmFhIXbu3OmOj9Ln9cS1On78OGpqamA0GhEREYG5c+e646P0eT1xraKiovD+++9jy5YtuO+++7B48WJ3fBTqRJ+fkzorKwvl5eXt1pWVlSE+Ph5xcXEAgEWLFmH37t1Ys2ZNp80SYWFhMJlMUobbr/XEtSotLUVDQwNOnjyJgIAAzJ49G3J5n68o97qe+n+Vm5uL3NxczJkzhwnCA/X5BOFKVVUVBg8e7FyOiYnB0aNHr1n+nXfewYEDB1BbW4tf/OIXvREitejutXr66acBAK+//rqz5ke9o7vXqrS0FO+88w5MJhNmz57dGyFSN/XLBNFd8+fPx/z5890dBnXD0qVL3R0CXcfUqVMxdepUd4dBneiXf15FR0ejoqLCuVxZWYno6Gg3RkTXwmvlPXit+p5+mSAyMjJw+vRpnDt3Ds3NzSguLkZubq67wyIXeK28B69V39PnE0R+fj7Gjx+Pb7/9FjExMdi6dSuUSiVeeuklzJgxAwkJCVi4cCGSkpLcHWq/x2vlPXit+oc+382ViIhuTJ+vQRAR0Y1hgiAiIpeYIIiIyCUmCCIicokJgoiIXGKCICIil5ggqM9TqVS9er4JEyb06vlqa2vx8ssv9+o5qX9ggiDqJovF0un2Tz75pFfPyQRBUmGCoH7pzJkzmDlzJtLS0jB58mT8+9//BgDs2bMHmZmZGDNmDO68805UV1cDANavX4/77rsPEydOxH333Yf169dj2bJlmDp1KuLi4vDiiy86j+2osZSWlmLq1Km45557MHLkSCxZsgSOcan79u3DyJEjkZaWhlWrVrmct+L1119Hbm4upk2bhjvuuAMGgwF33HEHUlNTkZKSgt27dwMAnnzySZw5cwZqtRqrV68GAGzcuBEZGRkYNWoU1q1bJ90vkvo2QdTHBQUFdVg3bdo08d133wkhhPjss89Edna2EEKIK1euCJvNJoQQ4pVXXhGFhYVCCCHWrVsnUlNTRWNjo3N5/Pjxwmg0Cp1OJ8LDw0Vzc3O78x08eFCEhISIiooKYbVaxbhx48Q///lP0dTUJGJiYsTZs2eFEEIsWrRIzJkzp0OMr732moiOjhY1NTVCCCHMZrOoq6sTQgih0+nE7bffLmw2mzh37pxISkpy7nfgwAHx8MMPC5vNJqxWq5gzZ444dOjQTf4WqT/i476p3zEYDPjkk0+wYMEC5zrHRFCVlZW49957cenSJTQ3N2Po0KHOMrm5uQgICHAuz5kzB35+fvDz80NUVBSqq6s7TEk7duxY5zq1Wo3y8nKoVCrExcU5j52fn4+ioiKXsebk5CA8PBwAIITAr3/9a3z88ceQy+Woqqpy1nDaKikpQUlJCcaMGeP8vKdPn0ZWVla3f1fUvzFBUL9js9kwYMAAfPnllx22PfbYYygsLERubi5KS0uxfv1657agoKB2Zf38/JzvFQqFy/sEXSnTmbbn3LlzJ3Q6HT7//HP4+PggNjYWRqOxwz5CCKxZswYrVqzo1rmIfoz3IKjfCQkJwdChQ/H2228DsH+hHj9+HABQV1fnnMNg27Ztkpx/xIgROHv2rHPKzjfffLNL+9XV1SEqKgo+Pj44ePAgzp8/DwAIDg6GXq93lpsxYwZeffVVGAwGAPaZ3rRabc9+COoXWIOgPq+xsbFd009hYSF27tyJRx55BL/73e9gNpuxaNEijB49GuvXr8eCBQsQFhaGadOm4dy5cz0eT0BAAF5++WXMnDkTQUFByMjI6NJ+S5YswV133YWUlBSkp6dj5MiRAICBAwdi4sSJSE5OxqxZs7Bx40acOnUK48ePB2C/ab5jxw5ERUX1+Gehvo2P+yZyA4PBAJVKBSEEfv7zn2PYsGH4z//8T3eHRdQOm5iI3OCVV16BWq1GUlIS6urqeL+APBJrEERE5BJrEERE5BITBBERucQEQURELjFBEBGRS0wQRETkEhMEERG59P9Rcra4KPOdsQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "lower_lr, upper_lr = 1e-5, 1e-2\n", + "optimizer = torch.optim.Adam(model.parameters(), lower_lr)\n", + "lr_finder = LearningRateFinder(model, optimizer, loss_function, device=device)\n", + "lr_finder.range_test(train_loader, val_loader, end_lr=upper_lr, num_iter=20)\n", + "ax=plt.subplots(1, 1, facecolor='white')[1]\n", + "_ = lr_finder.plot(ax=ax)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "lr w/ steepest gradient: 5.455595e-04\n", + "lower: 1e-04, upper: 1e-03\n" + ] + } + ], + "source": [ + "steepest_lr = lr_finder.get_steepest_gradient()[0]\n", + "lower_lr = 10 ** floor(log10(steepest_lr))\n", + "upper_lr = 10 ** ceil(log10(steepest_lr))\n", + "print(f\"lr w/ steepest gradient: {steepest_lr:e}\")\n", + "print(f\"lower: {lower_lr:1.0e}, upper: {upper_lr:1.0e}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Model training\n", + "\n", + "Execute a typical PyTorch training that run epoch loop and step loop, and do validation after every epoch.\n", + "Will save the model weights to file if got best validation accuracy." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib notebook\n", + "# def get_colour(q):\n", + "# return plt.rcParams['axes.prop_cycle'].by_key()['color'][q]\n", + "\n", + "def plot_range(data, wrapped_generator):\n", + " plt.ion()\n", + " for q in data.values():\n", + " for d in q.values():\n", + " if isinstance(d, dict):\n", + " ax = d[\"line\"].axes\n", + " ax.legend()\n", + " fig = ax.get_figure()\n", + " fig.show()\n", + " \n", + " for i in wrapped_generator:\n", + " for q in data.values():\n", + " for d in q.values():\n", + " if isinstance(d, dict):\n", + " d[\"line\"].set_data(d[\"x\"], d[\"y\"])\n", + " ax = d[\"line\"].axes\n", + " ax.legend()\n", + " ax.relim()\n", + " ax.autoscale_view()\n", + " fig.canvas.draw()\n", + " yield i" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "def train(max_epochs, axes, data):\n", + " for z, d in enumerate(data.keys()):\n", + " data[d][\"model\"] = densenet121(\n", + " spatial_dims=2, in_channels=1,\n", + " out_channels=num_classes).to(device)\n", + "\n", + " if \"lr_lims\" in data[d]:\n", + " data[d][\"optimizer\"] = torch.optim.Adam(\n", + " data[d][\"model\"].parameters(), data[d][\"lr_lims\"][0])\n", + " # In the paper referenced at the top of this notebook, a step\n", + " # size of 8 times the number of iterations per epoch is suggested.\n", + " step_size = 8 * len(train_loader)\n", + " data[d][\"scheduler\"] = torch.optim.lr_scheduler.CyclicLR(\n", + " data[d][\"optimizer\"], base_lr=data[d][\"lr_lims\"][0], \n", + " max_lr=data[d][\"lr_lims\"][1], step_size_up=step_size,\n", + " cycle_momentum=False,\n", + " )\n", + " else:\n", + " data[d][\"optimizer\"] = torch.optim.Adam(\n", + " data[d][\"model\"].parameters(), data[d][\"lr_lim\"])\n", + "\n", + " for q, i in enumerate([\"train\", \"auc\", \"acc\"]):\n", + " data[d][i] = {\"x\":[], \"y\":[]}\n", + " data[d][i][\"line\"], = axes[q].plot(\n", + " data[d][i][\"x\"], data[d][i][\"y\"], label=d)\n", + "# get_colour(z), label=d)\n", + "\n", + " val_interval = 1\n", + " \n", + " for epoch in plot_range(data, trange(max_epochs)):\n", + " \n", + " for d in data.keys():\n", + " data[d][\"epoch_loss\"] = 0\n", + " for batch_data in train_loader:\n", + " inputs = batch_data[\"image\"].to(device)\n", + " labels = batch_data[\"label\"].to(device)\n", + " \n", + " for d in data.keys():\n", + " data[d][\"optimizer\"].zero_grad()\n", + " outputs = data[d][\"model\"](inputs)\n", + " loss = loss_function(outputs, labels)\n", + " loss.backward()\n", + " data[d][\"optimizer\"].step()\n", + " if \"scheduler\" in data[d]:\n", + " data[d][\"scheduler\"].step()\n", + " data[d][\"epoch_loss\"] += loss.item()\n", + " for d in data.keys():\n", + " data[d][\"epoch_loss\"] /= len(train_loader)\n", + " data[d][\"train\"][\"x\"].append(epoch+1)\n", + " data[d][\"train\"][\"y\"].append(data[d][\"epoch_loss\"])\n", + "\n", + " if (epoch + 1) % val_interval == 0:\n", + " with eval_mode(*[data[d][\"model\"] for d in data.keys()]):\n", + " for d in data:\n", + " data[d][\"y_pred\"] = torch.tensor([], dtype=torch.float32, device=device)\n", + " y = torch.tensor([], dtype=torch.long, device=device)\n", + " for val_data in val_loader:\n", + " val_images = val_data[\"image\"].to(device)\n", + " val_labels = val_data[\"label\"].to(device)\n", + " for d in data:\n", + " data[d][\"y_pred\"] = torch.cat(\n", + " [data[d][\"y_pred\"], data[d][\"model\"](val_images)], dim=0)\n", + " y = torch.cat([y, val_labels], dim=0)\n", + " \n", + " for d in data:\n", + " auc_metric = compute_roc_auc(\n", + " data[d][\"y_pred\"], y, to_onehot_y=True, softmax=True)\n", + " data[d][\"auc\"][\"x\"].append(epoch+1)\n", + " data[d][\"auc\"][\"y\"].append(auc_metric)\n", + " \n", + " acc_value = torch.eq(data[d][\"y_pred\"].argmax(dim=1), y)\n", + " acc_metric = acc_value.sum().item() / len(acc_value)\n", + " data[d][\"acc\"][\"x\"].append(epoch+1)\n", + " data[d][\"acc\"][\"y\"].append(acc_metric)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "application/javascript": [ + "/* Put everything inside the global mpl namespace */\n", + "/* global mpl */\n", + "window.mpl = {};\n", + "\n", + "mpl.get_websocket_type = function () {\n", + " if (typeof WebSocket !== 'undefined') {\n", + " return WebSocket;\n", + " } else if (typeof MozWebSocket !== 'undefined') {\n", + " return MozWebSocket;\n", + " } else {\n", + " alert(\n", + " 'Your browser does not have WebSocket support. ' +\n", + " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", + " 'Firefox 4 and 5 are also supported but you ' +\n", + " 'have to enable WebSockets in about:config.'\n", + " );\n", + " }\n", + "};\n", + "\n", + "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n", + " this.id = figure_id;\n", + "\n", + " this.ws = websocket;\n", + "\n", + " this.supports_binary = this.ws.binaryType !== undefined;\n", + "\n", + " if (!this.supports_binary) {\n", + " var warnings = document.getElementById('mpl-warnings');\n", + " if (warnings) {\n", + " warnings.style.display = 'block';\n", + " warnings.textContent =\n", + " 'This browser does not support binary websocket messages. ' +\n", + " 'Performance may be slow.';\n", + " }\n", + " }\n", + "\n", + " this.imageObj = new Image();\n", + "\n", + " this.context = undefined;\n", + " this.message = undefined;\n", + " this.canvas = undefined;\n", + " this.rubberband_canvas = undefined;\n", + " this.rubberband_context = undefined;\n", + " this.format_dropdown = undefined;\n", + "\n", + " this.image_mode = 'full';\n", + "\n", + " this.root = document.createElement('div');\n", + " this.root.setAttribute('style', 'display: inline-block');\n", + " this._root_extra_style(this.root);\n", + "\n", + " parent_element.appendChild(this.root);\n", + "\n", + " this._init_header(this);\n", + " this._init_canvas(this);\n", + " this._init_toolbar(this);\n", + "\n", + " var fig = this;\n", + "\n", + " this.waiting = false;\n", + "\n", + " this.ws.onopen = function () {\n", + " fig.send_message('supports_binary', { value: fig.supports_binary });\n", + " fig.send_message('send_image_mode', {});\n", + " if (fig.ratio !== 1) {\n", + " fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n", + " }\n", + " fig.send_message('refresh', {});\n", + " };\n", + "\n", + " this.imageObj.onload = function () {\n", + " if (fig.image_mode === 'full') {\n", + " // Full images could contain transparency (where diff images\n", + " // almost always do), so we need to clear the canvas so that\n", + " // there is no ghosting.\n", + " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", + " }\n", + " fig.context.drawImage(fig.imageObj, 0, 0);\n", + " };\n", + "\n", + " this.imageObj.onunload = function () {\n", + " fig.ws.close();\n", + " };\n", + "\n", + " this.ws.onmessage = this._make_on_message_function(this);\n", + "\n", + " this.ondownload = ondownload;\n", + "};\n", + "\n", + "mpl.figure.prototype._init_header = function () {\n", + " var titlebar = document.createElement('div');\n", + " titlebar.classList =\n", + " 'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n", + " var titletext = document.createElement('div');\n", + " titletext.classList = 'ui-dialog-title';\n", + " titletext.setAttribute(\n", + " 'style',\n", + " 'width: 100%; text-align: center; padding: 3px;'\n", + " );\n", + " titlebar.appendChild(titletext);\n", + " this.root.appendChild(titlebar);\n", + " this.header = titletext;\n", + "};\n", + "\n", + "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n", + "\n", + "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n", + "\n", + "mpl.figure.prototype._init_canvas = function () {\n", + " var fig = this;\n", + "\n", + " var canvas_div = (this.canvas_div = document.createElement('div'));\n", + " canvas_div.setAttribute(\n", + " 'style',\n", + " 'border: 1px solid #ddd;' +\n", + " 'box-sizing: content-box;' +\n", + " 'clear: both;' +\n", + " 'min-height: 1px;' +\n", + " 'min-width: 1px;' +\n", + " 'outline: 0;' +\n", + " 'overflow: hidden;' +\n", + " 'position: relative;' +\n", + " 'resize: both;'\n", + " );\n", + "\n", + " function on_keyboard_event_closure(name) {\n", + " return function (event) {\n", + " return fig.key_event(event, name);\n", + " };\n", + " }\n", + "\n", + " canvas_div.addEventListener(\n", + " 'keydown',\n", + " on_keyboard_event_closure('key_press')\n", + " );\n", + " canvas_div.addEventListener(\n", + " 'keyup',\n", + " on_keyboard_event_closure('key_release')\n", + " );\n", + "\n", + " this._canvas_extra_style(canvas_div);\n", + " this.root.appendChild(canvas_div);\n", + "\n", + " var canvas = (this.canvas = document.createElement('canvas'));\n", + " canvas.classList.add('mpl-canvas');\n", + " canvas.setAttribute('style', 'box-sizing: content-box;');\n", + "\n", + " this.context = canvas.getContext('2d');\n", + "\n", + " var backingStore =\n", + " this.context.backingStorePixelRatio ||\n", + " this.context.webkitBackingStorePixelRatio ||\n", + " this.context.mozBackingStorePixelRatio ||\n", + " this.context.msBackingStorePixelRatio ||\n", + " this.context.oBackingStorePixelRatio ||\n", + " this.context.backingStorePixelRatio ||\n", + " 1;\n", + "\n", + " this.ratio = (window.devicePixelRatio || 1) / backingStore;\n", + " if (this.ratio !== 1) {\n", + " fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n", + " }\n", + "\n", + " var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n", + " 'canvas'\n", + " ));\n", + " rubberband_canvas.setAttribute(\n", + " 'style',\n", + " 'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n", + " );\n", + "\n", + " // Apply a ponyfill if ResizeObserver is not implemented by browser.\n", + " if (this.ResizeObserver === undefined) {\n", + " if (window.ResizeObserver !== undefined) {\n", + " this.ResizeObserver = window.ResizeObserver;\n", + " } else {\n", + " var obs = _JSXTOOLS_RESIZE_OBSERVER({});\n", + " this.ResizeObserver = obs.ResizeObserver;\n", + " }\n", + " }\n", + "\n", + " this.resizeObserverInstance = new this.ResizeObserver(function (entries) {\n", + " var nentries = entries.length;\n", + " for (var i = 0; i < nentries; i++) {\n", + " var entry = entries[i];\n", + " var width, height;\n", + " if (entry.contentBoxSize) {\n", + " if (entry.contentBoxSize instanceof Array) {\n", + " // Chrome 84 implements new version of spec.\n", + " width = entry.contentBoxSize[0].inlineSize;\n", + " height = entry.contentBoxSize[0].blockSize;\n", + " } else {\n", + " // Firefox implements old version of spec.\n", + " width = entry.contentBoxSize.inlineSize;\n", + " height = entry.contentBoxSize.blockSize;\n", + " }\n", + " } else {\n", + " // Chrome <84 implements even older version of spec.\n", + " width = entry.contentRect.width;\n", + " height = entry.contentRect.height;\n", + " }\n", + "\n", + " // Keep the size of the canvas and rubber band canvas in sync with\n", + " // the canvas container.\n", + " if (entry.devicePixelContentBoxSize) {\n", + " // Chrome 84 implements new version of spec.\n", + " canvas.setAttribute(\n", + " 'width',\n", + " entry.devicePixelContentBoxSize[0].inlineSize\n", + " );\n", + " canvas.setAttribute(\n", + " 'height',\n", + " entry.devicePixelContentBoxSize[0].blockSize\n", + " );\n", + " } else {\n", + " canvas.setAttribute('width', width * fig.ratio);\n", + " canvas.setAttribute('height', height * fig.ratio);\n", + " }\n", + " canvas.setAttribute(\n", + " 'style',\n", + " 'width: ' + width + 'px; height: ' + height + 'px;'\n", + " );\n", + "\n", + " rubberband_canvas.setAttribute('width', width);\n", + " rubberband_canvas.setAttribute('height', height);\n", + "\n", + " // And update the size in Python. We ignore the initial 0/0 size\n", + " // that occurs as the element is placed into the DOM, which should\n", + " // otherwise not happen due to the minimum size styling.\n", + " if (width != 0 && height != 0) {\n", + " fig.request_resize(width, height);\n", + " }\n", + " }\n", + " });\n", + " this.resizeObserverInstance.observe(canvas_div);\n", + "\n", + " function on_mouse_event_closure(name) {\n", + " return function (event) {\n", + " return fig.mouse_event(event, name);\n", + " };\n", + " }\n", + "\n", + " rubberband_canvas.addEventListener(\n", + " 'mousedown',\n", + " on_mouse_event_closure('button_press')\n", + " );\n", + " rubberband_canvas.addEventListener(\n", + " 'mouseup',\n", + " on_mouse_event_closure('button_release')\n", + " );\n", + " // Throttle sequential mouse events to 1 every 20ms.\n", + " rubberband_canvas.addEventListener(\n", + " 'mousemove',\n", + " on_mouse_event_closure('motion_notify')\n", + " );\n", + "\n", + " rubberband_canvas.addEventListener(\n", + " 'mouseenter',\n", + " on_mouse_event_closure('figure_enter')\n", + " );\n", + " rubberband_canvas.addEventListener(\n", + " 'mouseleave',\n", + " on_mouse_event_closure('figure_leave')\n", + " );\n", + "\n", + " canvas_div.addEventListener('wheel', function (event) {\n", + " if (event.deltaY < 0) {\n", + " event.step = 1;\n", + " } else {\n", + " event.step = -1;\n", + " }\n", + " on_mouse_event_closure('scroll')(event);\n", + " });\n", + "\n", + " canvas_div.appendChild(canvas);\n", + " canvas_div.appendChild(rubberband_canvas);\n", + "\n", + " this.rubberband_context = rubberband_canvas.getContext('2d');\n", + " this.rubberband_context.strokeStyle = '#000000';\n", + "\n", + " this._resize_canvas = function (width, height, forward) {\n", + " if (forward) {\n", + " canvas_div.style.width = width + 'px';\n", + " canvas_div.style.height = height + 'px';\n", + " }\n", + " };\n", + "\n", + " // Disable right mouse context menu.\n", + " this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n", + " event.preventDefault();\n", + " return false;\n", + " });\n", + "\n", + " function set_focus() {\n", + " canvas.focus();\n", + " canvas_div.focus();\n", + " }\n", + "\n", + " window.setTimeout(set_focus, 100);\n", + "};\n", + "\n", + "mpl.figure.prototype._init_toolbar = function () {\n", + " var fig = this;\n", + "\n", + " var toolbar = document.createElement('div');\n", + " toolbar.classList = 'mpl-toolbar';\n", + " this.root.appendChild(toolbar);\n", + "\n", + " function on_click_closure(name) {\n", + " return function (_event) {\n", + " return fig.toolbar_button_onclick(name);\n", + " };\n", + " }\n", + "\n", + " function on_mouseover_closure(tooltip) {\n", + " return function (event) {\n", + " if (!event.currentTarget.disabled) {\n", + " return fig.toolbar_button_onmouseover(tooltip);\n", + " }\n", + " };\n", + " }\n", + "\n", + " fig.buttons = {};\n", + " var buttonGroup = document.createElement('div');\n", + " buttonGroup.classList = 'mpl-button-group';\n", + " for (var toolbar_ind in mpl.toolbar_items) {\n", + " var name = mpl.toolbar_items[toolbar_ind][0];\n", + " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", + " var image = mpl.toolbar_items[toolbar_ind][2];\n", + " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", + "\n", + " if (!name) {\n", + " /* Instead of a spacer, we start a new button group. */\n", + " if (buttonGroup.hasChildNodes()) {\n", + " toolbar.appendChild(buttonGroup);\n", + " }\n", + " buttonGroup = document.createElement('div');\n", + " buttonGroup.classList = 'mpl-button-group';\n", + " continue;\n", + " }\n", + "\n", + " var button = (fig.buttons[name] = document.createElement('button'));\n", + " button.classList = 'mpl-widget';\n", + " button.setAttribute('role', 'button');\n", + " button.setAttribute('aria-disabled', 'false');\n", + " button.addEventListener('click', on_click_closure(method_name));\n", + " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", + "\n", + " var icon_img = document.createElement('img');\n", + " icon_img.src = '_images/' + image + '.png';\n", + " icon_img.srcset = '_images/' + image + '_large.png 2x';\n", + " icon_img.alt = tooltip;\n", + " button.appendChild(icon_img);\n", + "\n", + " buttonGroup.appendChild(button);\n", + " }\n", + "\n", + " if (buttonGroup.hasChildNodes()) {\n", + " toolbar.appendChild(buttonGroup);\n", + " }\n", + "\n", + " var fmt_picker = document.createElement('select');\n", + " fmt_picker.classList = 'mpl-widget';\n", + " toolbar.appendChild(fmt_picker);\n", + " this.format_dropdown = fmt_picker;\n", + "\n", + " for (var ind in mpl.extensions) {\n", + " var fmt = mpl.extensions[ind];\n", + " var option = document.createElement('option');\n", + " option.selected = fmt === mpl.default_extension;\n", + " option.innerHTML = fmt;\n", + " fmt_picker.appendChild(option);\n", + " }\n", + "\n", + " var status_bar = document.createElement('span');\n", + " status_bar.classList = 'mpl-message';\n", + " toolbar.appendChild(status_bar);\n", + " this.message = status_bar;\n", + "};\n", + "\n", + "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n", + " // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n", + " // which will in turn request a refresh of the image.\n", + " this.send_message('resize', { width: x_pixels, height: y_pixels });\n", + "};\n", + "\n", + "mpl.figure.prototype.send_message = function (type, properties) {\n", + " properties['type'] = type;\n", + " properties['figure_id'] = this.id;\n", + " this.ws.send(JSON.stringify(properties));\n", + "};\n", + "\n", + "mpl.figure.prototype.send_draw_message = function () {\n", + " if (!this.waiting) {\n", + " this.waiting = true;\n", + " this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n", + " }\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", + " var format_dropdown = fig.format_dropdown;\n", + " var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n", + " fig.ondownload(fig, format);\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_resize = function (fig, msg) {\n", + " var size = msg['size'];\n", + " if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n", + " fig._resize_canvas(size[0], size[1], msg['forward']);\n", + " fig.send_message('refresh', {});\n", + " }\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n", + " var x0 = msg['x0'] / fig.ratio;\n", + " var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n", + " var x1 = msg['x1'] / fig.ratio;\n", + " var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n", + " x0 = Math.floor(x0) + 0.5;\n", + " y0 = Math.floor(y0) + 0.5;\n", + " x1 = Math.floor(x1) + 0.5;\n", + " y1 = Math.floor(y1) + 0.5;\n", + " var min_x = Math.min(x0, x1);\n", + " var min_y = Math.min(y0, y1);\n", + " var width = Math.abs(x1 - x0);\n", + " var height = Math.abs(y1 - y0);\n", + "\n", + " fig.rubberband_context.clearRect(\n", + " 0,\n", + " 0,\n", + " fig.canvas.width / fig.ratio,\n", + " fig.canvas.height / fig.ratio\n", + " );\n", + "\n", + " fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n", + " // Updates the figure title.\n", + " fig.header.textContent = msg['label'];\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n", + " var cursor = msg['cursor'];\n", + " switch (cursor) {\n", + " case 0:\n", + " cursor = 'pointer';\n", + " break;\n", + " case 1:\n", + " cursor = 'default';\n", + " break;\n", + " case 2:\n", + " cursor = 'crosshair';\n", + " break;\n", + " case 3:\n", + " cursor = 'move';\n", + " break;\n", + " }\n", + " fig.rubberband_canvas.style.cursor = cursor;\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_message = function (fig, msg) {\n", + " fig.message.textContent = msg['message'];\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n", + " // Request the server to send over a new figure.\n", + " fig.send_draw_message();\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n", + " fig.image_mode = msg['mode'];\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n", + " for (var key in msg) {\n", + " if (!(key in fig.buttons)) {\n", + " continue;\n", + " }\n", + " fig.buttons[key].disabled = !msg[key];\n", + " fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n", + " }\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n", + " if (msg['mode'] === 'PAN') {\n", + " fig.buttons['Pan'].classList.add('active');\n", + " fig.buttons['Zoom'].classList.remove('active');\n", + " } else if (msg['mode'] === 'ZOOM') {\n", + " fig.buttons['Pan'].classList.remove('active');\n", + " fig.buttons['Zoom'].classList.add('active');\n", + " } else {\n", + " fig.buttons['Pan'].classList.remove('active');\n", + " fig.buttons['Zoom'].classList.remove('active');\n", + " }\n", + "};\n", + "\n", + "mpl.figure.prototype.updated_canvas_event = function () {\n", + " // Called whenever the canvas gets updated.\n", + " this.send_message('ack', {});\n", + "};\n", + "\n", + "// A function to construct a web socket function for onmessage handling.\n", + "// Called in the figure constructor.\n", + "mpl.figure.prototype._make_on_message_function = function (fig) {\n", + " return function socket_on_message(evt) {\n", + " if (evt.data instanceof Blob) {\n", + " /* FIXME: We get \"Resource interpreted as Image but\n", + " * transferred with MIME type text/plain:\" errors on\n", + " * Chrome. But how to set the MIME type? It doesn't seem\n", + " * to be part of the websocket stream */\n", + " evt.data.type = 'image/png';\n", + "\n", + " /* Free the memory for the previous frames */\n", + " if (fig.imageObj.src) {\n", + " (window.URL || window.webkitURL).revokeObjectURL(\n", + " fig.imageObj.src\n", + " );\n", + " }\n", + "\n", + " fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n", + " evt.data\n", + " );\n", + " fig.updated_canvas_event();\n", + " fig.waiting = false;\n", + " return;\n", + " } else if (\n", + " typeof evt.data === 'string' &&\n", + " evt.data.slice(0, 21) === 'data:image/png;base64'\n", + " ) {\n", + " fig.imageObj.src = evt.data;\n", + " fig.updated_canvas_event();\n", + " fig.waiting = false;\n", + " return;\n", + " }\n", + "\n", + " var msg = JSON.parse(evt.data);\n", + " var msg_type = msg['type'];\n", + "\n", + " // Call the \"handle_{type}\" callback, which takes\n", + " // the figure and JSON message as its only arguments.\n", + " try {\n", + " var callback = fig['handle_' + msg_type];\n", + " } catch (e) {\n", + " console.log(\n", + " \"No handler for the '\" + msg_type + \"' message type: \",\n", + " msg\n", + " );\n", + " return;\n", + " }\n", + "\n", + " if (callback) {\n", + " try {\n", + " // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n", + " callback(fig, msg);\n", + " } catch (e) {\n", + " console.log(\n", + " \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n", + " e,\n", + " e.stack,\n", + " msg\n", + " );\n", + " }\n", + " }\n", + " };\n", + "};\n", + "\n", + "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", + "mpl.findpos = function (e) {\n", + " //this section is from http://www.quirksmode.org/js/events_properties.html\n", + " var targ;\n", + " if (!e) {\n", + " e = window.event;\n", + " }\n", + " if (e.target) {\n", + " targ = e.target;\n", + " } else if (e.srcElement) {\n", + " targ = e.srcElement;\n", + " }\n", + " if (targ.nodeType === 3) {\n", + " // defeat Safari bug\n", + " targ = targ.parentNode;\n", + " }\n", + "\n", + " // pageX,Y are the mouse positions relative to the document\n", + " var boundingRect = targ.getBoundingClientRect();\n", + " var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n", + " var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n", + "\n", + " return { x: x, y: y };\n", + "};\n", + "\n", + "/*\n", + " * return a copy of an object with only non-object keys\n", + " * we need this to avoid circular references\n", + " * http://stackoverflow.com/a/24161582/3208463\n", + " */\n", + "function simpleKeys(original) {\n", + " return Object.keys(original).reduce(function (obj, key) {\n", + " if (typeof original[key] !== 'object') {\n", + " obj[key] = original[key];\n", + " }\n", + " return obj;\n", + " }, {});\n", + "}\n", + "\n", + "mpl.figure.prototype.mouse_event = function (event, name) {\n", + " var canvas_pos = mpl.findpos(event);\n", + "\n", + " if (name === 'button_press') {\n", + " this.canvas.focus();\n", + " this.canvas_div.focus();\n", + " }\n", + "\n", + " var x = canvas_pos.x * this.ratio;\n", + " var y = canvas_pos.y * this.ratio;\n", + "\n", + " this.send_message(name, {\n", + " x: x,\n", + " y: y,\n", + " button: event.button,\n", + " step: event.step,\n", + " guiEvent: simpleKeys(event),\n", + " });\n", + "\n", + " /* This prevents the web browser from automatically changing to\n", + " * the text insertion cursor when the button is pressed. We want\n", + " * to control all of the cursor setting manually through the\n", + " * 'cursor' event from matplotlib */\n", + " event.preventDefault();\n", + " return false;\n", + "};\n", + "\n", + "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n", + " // Handle any extra behaviour associated with a key event\n", + "};\n", + "\n", + "mpl.figure.prototype.key_event = function (event, name) {\n", + " // Prevent repeat events\n", + " if (name === 'key_press') {\n", + " if (event.which === this._key) {\n", + " return;\n", + " } else {\n", + " this._key = event.which;\n", + " }\n", + " }\n", + " if (name === 'key_release') {\n", + " this._key = null;\n", + " }\n", + "\n", + " var value = '';\n", + " if (event.ctrlKey && event.which !== 17) {\n", + " value += 'ctrl+';\n", + " }\n", + " if (event.altKey && event.which !== 18) {\n", + " value += 'alt+';\n", + " }\n", + " if (event.shiftKey && event.which !== 16) {\n", + " value += 'shift+';\n", + " }\n", + "\n", + " value += 'k';\n", + " value += event.which.toString();\n", + "\n", + " this._key_event_extra(event, name);\n", + "\n", + " this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n", + " return false;\n", + "};\n", + "\n", + "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n", + " if (name === 'download') {\n", + " this.handle_save(this, null);\n", + " } else {\n", + " this.send_message('toolbar_button', { name: name });\n", + " }\n", + "};\n", + "\n", + "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n", + " this.message.textContent = tooltip;\n", + "};\n", + "\n", + "///////////////// REMAINING CONTENT GENERATED BY embed_js.py /////////////////\n", + "// prettier-ignore\n", + "var _JSXTOOLS_RESIZE_OBSERVER=function(A){var t,i=new WeakMap,n=new WeakMap,a=new WeakMap,r=new WeakMap,o=new Set;function s(e){if(!(this instanceof s))throw new TypeError(\"Constructor requires 'new' operator\");i.set(this,e)}function h(){throw new TypeError(\"Function is not a constructor\")}function c(e,t,i,n){e=0 in arguments?Number(arguments[0]):0,t=1 in arguments?Number(arguments[1]):0,i=2 in arguments?Number(arguments[2]):0,n=3 in arguments?Number(arguments[3]):0,this.right=(this.x=this.left=e)+(this.width=i),this.bottom=(this.y=this.top=t)+(this.height=n),Object.freeze(this)}function d(){t=requestAnimationFrame(d);var s=new WeakMap,p=new Set;o.forEach((function(t){r.get(t).forEach((function(i){var r=t instanceof window.SVGElement,o=a.get(t),d=r?0:parseFloat(o.paddingTop),f=r?0:parseFloat(o.paddingRight),l=r?0:parseFloat(o.paddingBottom),u=r?0:parseFloat(o.paddingLeft),g=r?0:parseFloat(o.borderTopWidth),m=r?0:parseFloat(o.borderRightWidth),w=r?0:parseFloat(o.borderBottomWidth),b=u+f,F=d+l,v=(r?0:parseFloat(o.borderLeftWidth))+m,W=g+w,y=r?0:t.offsetHeight-W-t.clientHeight,E=r?0:t.offsetWidth-v-t.clientWidth,R=b+v,z=F+W,M=r?t.width:parseFloat(o.width)-R-E,O=r?t.height:parseFloat(o.height)-z-y;if(n.has(t)){var k=n.get(t);if(k[0]===M&&k[1]===O)return}n.set(t,[M,O]);var S=Object.create(h.prototype);S.target=t,S.contentRect=new c(u,d,M,O),s.has(i)||(s.set(i,[]),p.add(i)),s.get(i).push(S)}))})),p.forEach((function(e){i.get(e).call(e,s.get(e),e)}))}return s.prototype.observe=function(i){if(i instanceof window.Element){r.has(i)||(r.set(i,new Set),o.add(i),a.set(i,window.getComputedStyle(i)));var n=r.get(i);n.has(this)||n.add(this),cancelAnimationFrame(t),t=requestAnimationFrame(d)}},s.prototype.unobserve=function(i){if(i instanceof window.Element&&r.has(i)){var n=r.get(i);n.has(this)&&(n.delete(this),n.size||(r.delete(i),o.delete(i))),n.size||r.delete(i),o.size||cancelAnimationFrame(t)}},A.DOMRectReadOnly=c,A.ResizeObserver=s,A.ResizeObserverEntry=h,A}; // eslint-disable-line\n", + "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", + "\n", + "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", + "\n", + "mpl.default_extension = \"png\";/* global mpl */\n", + "\n", + "var comm_websocket_adapter = function (comm) {\n", + " // Create a \"websocket\"-like object which calls the given IPython comm\n", + " // object with the appropriate methods. Currently this is a non binary\n", + " // socket, so there is still some room for performance tuning.\n", + " var ws = {};\n", + "\n", + " ws.close = function () {\n", + " comm.close();\n", + " };\n", + " ws.send = function (m) {\n", + " //console.log('sending', m);\n", + " comm.send(m);\n", + " };\n", + " // Register the callback with on_msg.\n", + " comm.on_msg(function (msg) {\n", + " //console.log('receiving', msg['content']['data'], msg);\n", + " // Pass the mpl event to the overridden (by mpl) onmessage function.\n", + " ws.onmessage(msg['content']['data']);\n", + " });\n", + " return ws;\n", + "};\n", + "\n", + "mpl.mpl_figure_comm = function (comm, msg) {\n", + " // This is the function which gets called when the mpl process\n", + " // starts-up an IPython Comm through the \"matplotlib\" channel.\n", + "\n", + " var id = msg.content.data.id;\n", + " // Get hold of the div created by the display call when the Comm\n", + " // socket was opened in Python.\n", + " var element = document.getElementById(id);\n", + " var ws_proxy = comm_websocket_adapter(comm);\n", + "\n", + " function ondownload(figure, _format) {\n", + " window.open(figure.canvas.toDataURL());\n", + " }\n", + "\n", + " var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n", + "\n", + " // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n", + " // web socket which is closed, not our websocket->open comm proxy.\n", + " ws_proxy.onopen();\n", + "\n", + " fig.parent_element = element;\n", + " fig.cell_info = mpl.find_output_cell(\"
\");\n", + " if (!fig.cell_info) {\n", + " console.error('Failed to find cell for figure', id, fig);\n", + " return;\n", + " }\n", + " fig.cell_info[0].output_area.element.on(\n", + " 'cleared',\n", + " { fig: fig },\n", + " fig._remove_fig_handler\n", + " );\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_close = function (fig, msg) {\n", + " var width = fig.canvas.width / fig.ratio;\n", + " fig.cell_info[0].output_area.element.off(\n", + " 'cleared',\n", + " fig._remove_fig_handler\n", + " );\n", + " fig.resizeObserverInstance.unobserve(fig.canvas_div);\n", + "\n", + " // Update the output cell to use the data from the current canvas.\n", + " fig.push_to_output();\n", + " var dataURL = fig.canvas.toDataURL();\n", + " // Re-enable the keyboard manager in IPython - without this line, in FF,\n", + " // the notebook keyboard shortcuts fail.\n", + " IPython.keyboard_manager.enable();\n", + " fig.parent_element.innerHTML =\n", + " '';\n", + " fig.close_ws(fig, msg);\n", + "};\n", + "\n", + "mpl.figure.prototype.close_ws = function (fig, msg) {\n", + " fig.send_message('closing', msg);\n", + " // fig.ws.close()\n", + "};\n", + "\n", + "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n", + " // Turn the data on the canvas into data in the output cell.\n", + " var width = this.canvas.width / this.ratio;\n", + " var dataURL = this.canvas.toDataURL();\n", + " this.cell_info[1]['text/html'] =\n", + " '';\n", + "};\n", + "\n", + "mpl.figure.prototype.updated_canvas_event = function () {\n", + " // Tell IPython that the notebook contents must change.\n", + " IPython.notebook.set_dirty(true);\n", + " this.send_message('ack', {});\n", + " var fig = this;\n", + " // Wait a second, then push the new image to the DOM so\n", + " // that it is saved nicely (might be nice to debounce this).\n", + " setTimeout(function () {\n", + " fig.push_to_output();\n", + " }, 1000);\n", + "};\n", + "\n", + "mpl.figure.prototype._init_toolbar = function () {\n", + " var fig = this;\n", + "\n", + " var toolbar = document.createElement('div');\n", + " toolbar.classList = 'btn-toolbar';\n", + " this.root.appendChild(toolbar);\n", + "\n", + " function on_click_closure(name) {\n", + " return function (_event) {\n", + " return fig.toolbar_button_onclick(name);\n", + " };\n", + " }\n", + "\n", + " function on_mouseover_closure(tooltip) {\n", + " return function (event) {\n", + " if (!event.currentTarget.disabled) {\n", + " return fig.toolbar_button_onmouseover(tooltip);\n", + " }\n", + " };\n", + " }\n", + "\n", + " fig.buttons = {};\n", + " var buttonGroup = document.createElement('div');\n", + " buttonGroup.classList = 'btn-group';\n", + " var button;\n", + " for (var toolbar_ind in mpl.toolbar_items) {\n", + " var name = mpl.toolbar_items[toolbar_ind][0];\n", + " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", + " var image = mpl.toolbar_items[toolbar_ind][2];\n", + " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", + "\n", + " if (!name) {\n", + " /* Instead of a spacer, we start a new button group. */\n", + " if (buttonGroup.hasChildNodes()) {\n", + " toolbar.appendChild(buttonGroup);\n", + " }\n", + " buttonGroup = document.createElement('div');\n", + " buttonGroup.classList = 'btn-group';\n", + " continue;\n", + " }\n", + "\n", + " button = fig.buttons[name] = document.createElement('button');\n", + " button.classList = 'btn btn-default';\n", + " button.href = '#';\n", + " button.title = name;\n", + " button.innerHTML = '';\n", + " button.addEventListener('click', on_click_closure(method_name));\n", + " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", + " buttonGroup.appendChild(button);\n", + " }\n", + "\n", + " if (buttonGroup.hasChildNodes()) {\n", + " toolbar.appendChild(buttonGroup);\n", + " }\n", + "\n", + " // Add the status bar.\n", + " var status_bar = document.createElement('span');\n", + " status_bar.classList = 'mpl-message pull-right';\n", + " toolbar.appendChild(status_bar);\n", + " this.message = status_bar;\n", + "\n", + " // Add the close button to the window.\n", + " var buttongrp = document.createElement('div');\n", + " buttongrp.classList = 'btn-group inline pull-right';\n", + " button = document.createElement('button');\n", + " button.classList = 'btn btn-mini btn-primary';\n", + " button.href = '#';\n", + " button.title = 'Stop Interaction';\n", + " button.innerHTML = '';\n", + " button.addEventListener('click', function (_evt) {\n", + " fig.handle_close(fig, {});\n", + " });\n", + " button.addEventListener(\n", + " 'mouseover',\n", + " on_mouseover_closure('Stop Interaction')\n", + " );\n", + " buttongrp.appendChild(button);\n", + " var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n", + " titlebar.insertBefore(buttongrp, titlebar.firstChild);\n", + "};\n", + "\n", + "mpl.figure.prototype._remove_fig_handler = function (event) {\n", + " var fig = event.data.fig;\n", + " if (event.target !== this) {\n", + " // Ignore bubbled events from children.\n", + " return;\n", + " }\n", + " fig.close_ws(fig, {});\n", + "};\n", + "\n", + "mpl.figure.prototype._root_extra_style = function (el) {\n", + " el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n", + "};\n", + "\n", + "mpl.figure.prototype._canvas_extra_style = function (el) {\n", + " // this is important to make the div 'focusable\n", + " el.setAttribute('tabindex', 0);\n", + " // reach out to IPython and tell the keyboard manager to turn it's self\n", + " // off when our div gets focus\n", + "\n", + " // location in version 3\n", + " if (IPython.notebook.keyboard_manager) {\n", + " IPython.notebook.keyboard_manager.register_events(el);\n", + " } else {\n", + " // location in version 2\n", + " IPython.keyboard_manager.register_events(el);\n", + " }\n", + "};\n", + "\n", + "mpl.figure.prototype._key_event_extra = function (event, _name) {\n", + " var manager = IPython.notebook.keyboard_manager;\n", + " if (!manager) {\n", + " manager = IPython.keyboard_manager;\n", + " }\n", + "\n", + " // Check for shift+enter\n", + " if (event.shiftKey && event.which === 13) {\n", + " this.canvas_div.blur();\n", + " // select the cell after this one\n", + " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", + " IPython.notebook.select(index + 1);\n", + " }\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", + " fig.ondownload(fig, null);\n", + "};\n", + "\n", + "mpl.find_output_cell = function (html_output) {\n", + " // Return the cell and output element which can be found *uniquely* in the notebook.\n", + " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", + " // IPython event is triggered only after the cells have been serialised, which for\n", + " // our purposes (turning an active figure into a static one), is too late.\n", + " var cells = IPython.notebook.get_cells();\n", + " var ncells = cells.length;\n", + " for (var i = 0; i < ncells; i++) {\n", + " var cell = cells[i];\n", + " if (cell.cell_type === 'code') {\n", + " for (var j = 0; j < cell.output_area.outputs.length; j++) {\n", + " var data = cell.output_area.outputs[j];\n", + " if (data.data) {\n", + " // IPython >= 3 moved mimebundle to data attribute of output\n", + " data = data.data;\n", + " }\n", + " if (data['text/html'] === html_output) {\n", + " return [cell, data, j];\n", + " }\n", + " }\n", + " }\n", + " }\n", + "};\n", + "\n", + "// Register the function which deals with the matplotlib target/channel.\n", + "// The kernel may be null if the page has been refreshed.\n", + "if (IPython.notebook.kernel !== null) {\n", + " IPython.notebook.kernel.comm_manager.register_target(\n", + " 'matplotlib',\n", + " mpl.mpl_figure_comm\n", + " );\n", + "}\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [04:36<00:00, 2.77s/it]\n" + ] + } + ], + "source": [ + "fig, axes=plt.subplots(3, 1, figsize=(10, 10), facecolor='white')\n", + "for ax in axes:\n", + " ax.set_xlabel('Epoch')\n", + "axes[0].set_ylabel('Train loss')\n", + "axes[1].set_ylabel('AUC')\n", + "axes[2].set_ylabel('ACC')\n", + "\n", + "max_epochs = 100\n", + "data = {}\n", + "data[\"Default LR\"] = {\"lr_lim\": 1e-5}\n", + "data[\"Steepest LR\"] = {\"lr_lim\": steepest_lr}\n", + "data[\"Cyclical LR\"] = {\"lr_lims\": (0.8*steepest_lr, 1.2*steepest_lr)}\n", + "\n", + "train(max_epochs, axes, data)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cleanup data directory\n", + "\n", + "Remove directory if a temporary was used." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "if directory is None:\n", + " shutil.rmtree(root_dir)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 4c1309d67f043462471e133648e17fe902acf01e Mon Sep 17 00:00:00 2001 From: Richard Brown <33289025+rijobro@users.noreply.github.com> Date: Wed, 20 Jan 2021 12:43:40 +0000 Subject: [PATCH 2/4] finished --- modules/._learning_rate.ipynb | Bin 0 -> 4096 bytes modules/learning_rate.ipynb | 225 +++++++++++++++------------------- 2 files changed, 101 insertions(+), 124 deletions(-) create mode 100644 modules/._learning_rate.ipynb diff --git a/modules/._learning_rate.ipynb b/modules/._learning_rate.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..d489ed98b676fe49e4b919e63f73d6dc10171041 GIT binary patch literal 4096 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDJkFz{^v(m+1nBL)UWIUt(=a103v0xIE$ z=wR3bWJjQB0htaG7hqtJO3u&KODrhJN!80qEG{W6PEAQkEJ;-k2!`r81Ef8G#v#<@ zCMM?q1)Wqm5Bh0E*Q%TmS$7 literal 0 HcmV?d00001 diff --git a/modules/learning_rate.ipynb b/modules/learning_rate.ipynb index 7971f425be..8082b3e4d1 100644 --- a/modules/learning_rate.ipynb +++ b/modules/learning_rate.ipynb @@ -8,10 +8,9 @@ "\n", "In this tutorial, we'll use the MedNIST dataset to explore MONAI's `LearningRateFinder` and use it to get an initial estimate of a learning rate.\n", "\n", - "We then employ one of Pytorch's cyclical learning rate schedulers to vary the learning rate over the course of the optimisation. This has been shown to give improved results: https://arxiv.org/abs/1506.01186.\n", + "We then employ one of Pytorch's cyclical learning rate schedulers to vary the learning rate over the course of the optimisation. This has been shown to give improved results: https://arxiv.org/abs/1506.01186. We'll compare this to the optimiser's (ADAM) default learning rate and the learning rate suggested by `LearningRateFinder`.\n", "\n", - "TODO:\n", - "* make a bullet list of learning points\n", + "This 2D classification is fairly easy, so to make it a little harder (and faster), we'll use a small network, only a subset of the images (~250 and ~25 for training and validation, respectively), we'll crop the images (from 64x64 to 20x20) and we won't use any random transformations. In a more difficult scenario, we probably wouldn't want to do any of these things.\n", "\n", "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Project-MONAI/tutorials/blob/master/modules/learning_rate.ipynb)" ] @@ -30,8 +29,7 @@ "outputs": [], "source": [ "!python -c \"import monai\" || pip install -q monai[pillow, tqdm]\n", - "!python -c \"import matplotlib\" || pip install -q matplotlib\n", - "%matplotlib inline" + "!python -c \"import matplotlib\" || pip install -q matplotlib" ] }, { @@ -103,11 +101,12 @@ "from monai.apps import MedNISTDataset\n", "from monai.config import print_config\n", "from monai.metrics import compute_roc_auc\n", - "from monai.networks.nets import densenet121\n", + "from monai.networks.nets import DenseNet\n", "from monai.networks.utils import eval_mode\n", "from monai.optimizers import LearningRateFinder\n", "from monai.transforms import (\n", " AddChanneld,\n", + " CenterSpatialCropd,\n", " Compose,\n", " LoadImaged,\n", " RandFlipd,\n", @@ -173,7 +172,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Define MONAI transforms, Dataset and Dataloader to pre-process data" + "## Define MONAI transforms and get dataset and data loader" ] }, { @@ -182,23 +181,12 @@ "metadata": {}, "outputs": [], "source": [ - "train_transforms = Compose(\n", - " [\n", - " LoadImaged(keys=\"image\"),\n", - " AddChanneld(keys=\"image\"),\n", - " ScaleIntensityd(keys=\"image\"),\n", - " RandRotated(keys=\"image\", range_x=np.pi / 12, prob=0.5, keep_size=True),\n", - " RandFlipd(keys=\"image\", spatial_axis=0, prob=0.5),\n", - " RandZoomd(keys=\"image\", min_zoom=0.9, max_zoom=1.1, prob=0.5),\n", - " ToTensord(keys=\"image\"),\n", - " ]\n", - ")\n", - "\n", - "val_transforms = Compose(\n", + "transforms = Compose(\n", " [\n", " LoadImaged(keys=\"image\"),\n", " AddChanneld(keys=\"image\"),\n", " ScaleIntensityd(keys=\"image\"),\n", + " CenterSpatialCropd(keys=\"image\", roi_size=(20, 20)),\n", " ToTensord(keys=\"image\"),\n", " ]\n", ")" @@ -222,8 +210,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "Loading dataset: 100%|██████████| 249/249 [00:00<00:00, 1207.74it/s]\n", - "Loading dataset: 0%| | 0/25 [00:00" ] @@ -308,6 +283,7 @@ } ], "source": [ + "%matplotlib inline\n", "fig, axes = plt.subplots(3, 3, figsize=(15, 15), facecolor='white')\n", "for i, k in enumerate(np.random.randint(len(train_ds), size=9)):\n", " data = train_ds[k]\n", @@ -322,14 +298,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Define network and optimizer\n", - "\n", - "1. Set learning rate for how much the model is updated per batch.\n", - "1. Set total epoch number, as we have shuffle and random transforms, so the training data of every epoch is different.\n", - "And as this is just a get start tutorial, let's just train 4 epochs.\n", - "If train 10 epochs, the model can achieve 100% accuracy on test dataset.\n", - "1. Use DenseNet from MONAI and move to GPU devide, this DenseNet can support both 2D and 3D classification tasks.\n", - "1. Use Adam optimizer." + "## Define loss function and network" ] }, { @@ -338,11 +307,18 @@ "metadata": {}, "outputs": [], "source": [ - "lr = 1e-5\n", "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", "loss_function = torch.nn.CrossEntropyLoss()\n", - "model = densenet121(spatial_dims=2, in_channels=1,\n", - " out_channels=num_classes).to(device)" + "def get_new_net():\n", + " return DenseNet(\n", + " spatial_dims=2, \n", + " in_channels=1, \n", + " out_channels=num_classes, \n", + " init_features=2, \n", + " growth_rate=2, \n", + " block_config=(2,)\n", + " ).to(device)\n", + "model = get_new_net()" ] }, { @@ -351,9 +327,9 @@ "source": [ "# Estimate optimal learning rate\n", "\n", - "Use MONAI's `LearningRateFinder` to get an initial estimate of a learning rate. Assume that it's in the range 1e-5, 1e-2. If that weren't the case (which we'd notice in the plot), we could just try again over a larger/different window.\n", + "Use MONAI's `LearningRateFinder` to get an initial estimate of a learning rate. Assume that it's in the range 1e-5, 1e0. If that weren't the case (which we'd notice in the plot), we could just try again over a larger/different window. \n", "\n", - "We then extract the learning rate with the steepest gradient, and set the upper and lower learning rates of a cyclical optimsation to be the nearest powers of 10 above and below this value." + "We can plot the results and extract the learning rate with the steepest gradient." ] }, { @@ -365,7 +341,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "Computing optimal learning rate: 95%|█████████▌| 19/20 [00:41<00:02, 2.20s/it]\n" + "Computing optimal learning rate: 90%|█████████ | 18/20 [00:14<00:01, 1.26it/s]\n" ] }, { @@ -378,9 +354,9 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAuM0lEQVR4nO3de1xUdd4H8M9cuA8gCLQGGhJeuOnIRbyiWHiPVkkT3cq00NrNp4d9ueW+nvXSbpfn5VOrbZu7tFY+6hOVW5muKe0mtlrJTpllumUqCmjMiAIzwAxz+T1/DDNAjAjKYWbg8369eM2cc37nnO9wdL78fuf3Oz+ZEEKAiIjoR+TuDoCIiDwTEwQREbnEBEFERC4xQRARkUtMEERE5BITBBERuaR0dwA9KSIiArGxse4Og4jIa5SXl+Py5csut/WpBBEbGwuNRuPuMIiIvEZ6evo1t7GJiYiIXGKCICIil5ggiIjIpT51D8IVs9mMyspKGI1Gd4dCEvP390dMTAx8fHzcHQpRn9DnE0RlZSWCg4MRGxsLmUzm7nBIIkII1NTUoLKyEkOHDnV3OER9Qp9vYjIajRg4cCCTQx8nk8kwcOBA1hSJelCfTxAAmBz6CV5n6o9OVNXh4+90khy7XySIbhEC+Owz4N137a8STZexadMmNDY2SnLsrqqtrcXLL7/ca+eLjY11DsiZMGHCDR/n9ddfx8WLF3sqLCKvtuOz8yh867gkx2aCaGvfPmDIECAnB1i61P46ZIh9fQ/rKwnCYrHc0H6ffPLJDZ+TCYKolVZvwi0hfpIcmwnCYd8+4J57gMpKwGAA6uvtr5WV9vU3mCQaGhowZ84cjB49GsnJyXjzzTfx4osv4uLFi8jOzkZ2djYAoKSkBOPHj0dqaioWLFgAg8EAAPj8888xZcoUpKWlYcaMGbh06RIAYOrUqfiP//gPqNVqJCcno6yszHm+ZcuWYezYsRgzZgx2794NAPjmm28wduxYqNVqjBo1CqdPn8aTTz6JM2fOQK1WY/Xq1R1i/+1vf4sRI0Zg0qRJyM/Px//8z/84z/34448jPT0dmzdvxp49e5CZmYkxY8bgzjvvRHV1NQCgpqYG06dPR1JSEh566CG0nbxQpVI532/cuBEZGRkYNWoU1q1bB8A+/D8hIQEPP/wwkpKSMH36dDQ1NWHXrl3QaDRYsmQJ1Go1mpqabui6EPUVWr0RUcHSJAiIPiQtLa3DupMnT15/R5tNiOhoIewNSq5/YmLs5bpp165d4qGHHnIu19bWCiGEuO2224ROpxNCCKHT6cTkyZOFwWAQQgjx3HPPiQ0bNojm5mYxfvx4odVqhRBCFBcXiwcffFAIIcSUKVOcxz106JBISkoSQgixZs0asX37diGEEFevXhXDhg0TBoNB/OIXvxA7duwQQghhMplEY2OjOHfunHO/HysrKxOjR48WTU1Nor6+XsTHx4uNGzc6z/3II484y165ckXYWn43r7zyiigsLBRCCPHYY4+JDRs2CCGE2Lt3rwDg/MxBQUFCCCEOHDggHn74YWGz2YTVahVz5swRhw4dEufOnRMKhUIcO3ZMCCHEggULnJ9rypQp4l//+pfLuLt0vYn6kIzffSh+9fbxG97f1femQ5/v5tolR48CdXWdl6mtBcrKgMzMbh06JSUFv/zlL/HEE09g7ty5mDx5cocyn332GU6ePImJEycCAJqbmzF+/Hh8++23OHHiBHJycgAAVqsVgwYNcu6Xn58PAMjKykJ9fT1qa2tRUlKC999/3/nXvtFoxIULFzB+/Hg8/fTTqKysxPz58zFs2LBO4z5y5Ajuvvtu+Pv7w9/fH3fddVe77ffee6/zfWVlJe69915cunQJzc3Nzm6mH3/8Md555x0AwJw5cxAWFtbhPCUlJSgpKcGYMWMAAAaDAadPn8aQIUMwdOhQqNVqAEBaWhrKy8s7jZmov7HaBC4bTIiSqImJCQIALl0C5NdpbZPLgRto9x4+fDi++OIL7Nu3D//1X/+FO+64A2vXrm1XRgiBnJwcvPHGG+3Wf/3110hKSsKnn37q8tg/7rUjk8kghMBf//pXjBgxot22hIQEZGZm4m9/+xtmz56NP//5z4iLi+v253EICgpyvn/sscdQWFiI3NxclJaWYv369V0+jhACa9aswYoVK9qtLy8vh59f6z96hULB5iSiH6kxmGATkKyJifcgAGDQIMBm67yMzQbcemu3D33x4kUEBgbiZz/7GVavXo0vvvgCABAcHAy9Xg8AGDduHI4cOYLvv/8egP0+wnfffYcRI0ZAp9M5E4TZbMY333zjPPabb74JADh8+DBCQ0MRGhqKGTNm4A9/+IOzvf/YsWMAgLNnzyIuLg6rVq3C3Xffja+++qpdDD82ceJE7NmzB0ajEQaDAXv37r3mZ6yrq0N0dDQAYNu2bc71WVlZ+L//+z8AwAcffICrV6922HfGjBl49dVXnfdcqqqqoNVqO/2ddhY3UX+i1ZsAAJHB/pIcnzUIwN5sFBpqvyl9LQMGAGPHdvvQX3/9NVavXg25XA4fHx9s2bIFAFBQUICZM2fi1ltvxcGDB/H6668jPz8fJpP9gv/ud7/D8OHDsWvXLqxatQp1dXWwWCx4/PHHkZSUBMD+aIkxY8bAbDbj1VdfBQD85je/weOPP45Ro0bBZrNh6NCh2Lt3L9566y1s374dPj4++MlPfoJf//rXCA8Px8SJE5GcnIxZs2Zh48aNzrgzMjKQm5uLUaNG4ZZbbkFKSgpCQ0Ndfsb169djwYIFCAsLw7Rp03Du3DkAwLp165Cfn4+kpCRMmDABQ4YM6bDv9OnTcerUKYwfPx6A/eb1jh07oFAorvk7Xbp0KVauXImAgAB8+umnCAgI6O5lIeoTtHr7wFCpmph4k9rhb38TIiDA9Q3qgAD7dg/S2Y3anqLX64UQQjQ0NIi0tDTx+eefS3q+nsCb1NSfvHH0vLjtib2i8mrjDR+js5vUkjUxGY1GjB07FqNHj0ZSUpKz+2JbL7zwAhITEzFq1CjccccdOH/+vHObQqGAWq2GWq1Gbm6uVGG2mj0b2LULiIkBVCogJMT+GhNjXz97tvQxeJiCggKo1WqkpqYiLy8Pqamp7g6JiNpwNjGpvOwmtZ+fHz766COoVCqYzWZMmjQJs2bNwrhx45xlxowZA41Gg8DAQGzZsgW/+tWvnO3qAQEB+PLLL6UKz7XZs4ELF+y9lS5etN9zGDsW8MBHOJSWlkp+Dsf9AyLyTFq9EWGBPvBVSvO3vmQJQiaTOQdDmc1mmM3mDr1uHIPEAPuN2h07dkgVTtfJZN3uykpE5A7V9SZESXSDGpC4F5PVaoVarUZUVBRycnKQ2ckX79atWzFr1iznstFoRHp6OsaNG4f33nvvmvsVFRUhPT0d6enp0OlcP7BKSPQ8JfIsvM7U32j10o2BACTuxaRQKPDll1+itrYW8+bNw4kTJ5CcnNyh3I4dO6DRaHDo0CHnuvPnzyM6Ohpnz57FtGnTkJKSgttvv73DvgUFBSgoKADgevJtf39/1NTU8JHffZxomQ/C31+6v6aIPI2u3ojbIwdKdvxe6eY6YMAAZGdnY//+/R0SxN///nc8/fTTOHToULuBUY5+9XFxcZg6dSqOHTvmMkFcT0xMDCorK69Zu6C+wzGjHFF/IISAziBtE5NkCUKn08HHxwcDBgxAU1MTPvzwQzzxxBPtyhw7dgwrVqzA/v37ERUV5Vx/9epVBAYGws/PD5cvX8aRI0fwq1/96obi8PHx4QxjRNTnXG00w2wVkj3JFZAwQVy6dAkPPPAArFYrbDYbFi5ciLlz52Lt2rVIT09Hbm4uVq9eDYPBgAULFgAAhgwZgvfffx+nTp3CihUrIJfLYbPZ8OSTTyIxMVGqUImIvI5zkJw31iBGjRrlfMxDW0899ZTz/d///neX+06YMAFff/21VKEREXk9bb19DISUN6n5LCYiIi9UXe+oQTBBEBFRG45R1F47DoKIiKSh05sQ7KdEgO+1H2x5s5ggiIi8kFZvlPT+A8AEQUTklbQSP2YDYIIgIvJKUj9mA2CCICLyOkIIexOThD2YACYIIiKvU2+0wGi2sYmJiIja00k91WgLJggiIi/jGEUdySYmIiJqyzFI7pYQNjEREVEbrQ/qYw2CiIja0NabEOCjgMpP2il9mCCIiLyMYwyE1LNkMkEQEXmZ6nrpx0AATBBERF5Hp5f+MRsAEwQRkdfR6k2Sd3EFmCCIiLxKY7MFBpNF8i6uABMEEZFXcU41yhoEERG15ZxJTuLHbABMEEREXqV1kJwXNzEZjUaMHTsWo0ePRlJSEtatW9ehjMlkwr333ov4+HhkZmaivLzcue3ZZ59FfHw8RowYgQMHDkgVJhGRV6nuxSYmyYbh+fn54aOPPoJKpYLZbMakSZMwa9YsjBs3zllm69atCAsLw/fff4/i4mI88cQTePPNN3Hy5EkUFxfjm2++wcWLF3HnnXfiu+++g0Ih3dyrRETeQKs3wlchx4BAH8nPJVkNQiaTQaVSAQDMZjPMZnOHUX+7d+/GAw88AAC455578I9//ANCCOzevRuLFi2Cn58fhg4divj4eJSVlUkVKhGR19DV27u4Sj2KGpD4HoTVaoVarUZUVBRycnKQmZnZbntVVRUGDx4MAFAqlQgNDUVNTU279QAQExODqqoql+coKipCeno60tPTodPppPswREQeoDemGnWQNEEoFAp8+eWXqKysRFlZGU6cONHj5ygoKIBGo4FGo0FkZGSPH5+IyJP0xlSjDr3Si2nAgAHIzs7G/v37262Pjo5GRUUFAMBisaCurg4DBw5stx4AKisrER0d3RuhEhF5NG0vPWYDkDBB6HQ61NbWAgCamprw4YcfYuTIke3K5ObmYtu2bQCAXbt2Ydq0aZDJZMjNzUVxcTFMJhPOnTuH06dPY+zYsVKFSkTkFUwWK2obzb1Wg5CsF9OlS5fwwAMPwGq1wmazYeHChZg7dy7Wrl2L9PR05ObmYvny5bjvvvsQHx+P8PBwFBcXAwCSkpKwcOFCJCYmQqlU4o9//CN7MBFRv+ccRd1L9yBkQgjRK2fqBenp6dBoNO4Og4hIEp+fv4q8LZ/gtaUZyB4Z1SPH7Ox7kyOpiYi8hK5lFHVvPMkVYIIgIvIajucw9caTXAEmCCIir6GtN0Ehl2FgkG+vnI8JgojIS2j1RkSofCGXSz+KGmCCICLyGr05BgJggiAi8hrV9aZeGwMBMEEQEXkNnd7Ya2MgACYIIiKvYLHaUNPQzCYmIiJq77KhGUL03ihqgAmCiMgr9OZUow5MEEREXkDbi1ONOjBBEBF5AccoajYxERFRO9X1RshkQISKCYKIiNrQ6k0ID/SFj6L3vraZIIiIvIB9DETv3aAGmCCIiLyC/TEbvde8BDBBEBF5BW0vP2YDYIIgIvJ4NpvAZYOpV3swAUwQREQe70pjMyw20auD5AAmCCIij1dd7xhFzRoEERG14Y5BcoCECaKiogLZ2dlITExEUlISNm/e3KHMxo0boVaroVarkZycDIVCgStXrgAAYmNjkZKSArVajfT0dKnCJCLyeDrnYzZ6t4lJKdmBlUo8//zzSE1NhV6vR1paGnJycpCYmOgss3r1aqxevRoAsGfPHvz+979HeHi4c/vBgwcREREhVYhERF7B8aC+yL7SxDRo0CCkpqYCAIKDg5GQkICqqqprln/jjTeQn58vVThERF5LqzchNMAH/j6KXj1vr9yDKC8vx7Fjx5CZmelye2NjI/bv34+8vDznOplMhunTpyMtLQ1FRUW9ESYRkUdyxxgIQMImJgeDwYC8vDxs2rQJISEhLsvs2bMHEydObNe8dPjwYURHR0Or1SInJwcjR45EVlZWh32LioqcCUSn00nzIYiI3Ejby1ONOkhagzCbzcjLy8OSJUswf/78a5YrLi7u0LwUHR0NAIiKisK8efNQVlbmct+CggJoNBpoNBpERkb2XPBERB6iut7U6zeoAQkThBACy5cvR0JCAgoLC69Zrq6uDocOHcLdd9/tXNfQ0AC9Xu98X1JSguTkZKlCJSLyWEII6NzwHCZAwiamI0eOYPv27c6uqgDwzDPP4MKFCwCAlStXAgDeffddTJ8+HUFBQc59q6urMW/ePACAxWLB4sWLMXPmTKlCJSLyWHVNZjRbbb3+JFdAwgQxadIkCCGuW27p0qVYunRpu3VxcXE4fvy4RJEREXkP5yA5N9QgOJKaiMiDuWMuagcmCCIiD+YYJOeOJiYmCCIiD8YmJiIicqm63oggXwWC/CQfttYBEwQRkQfT6k1uaV4CmCCIiDyazk2P2QCYIIiIPJr9MRusQRAR0Y9o3TSKGmCCICLyWAaTBY3NViYIIiJqT+uYi9oNT3IFmCCIiDxWtZumGnVggiAi8lDOUdRsYiIiorZ0jlHUntyLqaGhATabDQDw3Xff4f3334fZbJY0MCKi/k6rN8FPKUeIf++Poga6mCCysrJgNBpRVVWF6dOnY/v27R0e0U1ERD1LW2+falQmk7nl/F1KEEIIBAYG4p133sGjjz6Kt99+G998843UsRER9Wv2MRDuaV4CupEgPv30U+zcuRNz5swBAFitVkkDIyLq79w5SA7oYoLYtGkTnn32WcybNw9JSUk4e/YssrOzpY6NiKhfq643ujVBdOnOx5QpUzBlyhQAgM1mQ0REBF588UVJAyMi6s+MZiv0RovbejABXaxBLF68GPX19WhoaEBycjISExOxceNGqWMjIuq33DnVqEOXEsTJkycREhKC9957D7NmzcK5c+ewfft2qWMjIuq33DnVqEOXEoTZbIbZbMZ7772H3Nxc+Pj4XLfbVUVFBbKzs5GYmIikpCRs3ry5Q5nS0lKEhoZCrVZDrVbjqaeecm7bv38/RowYgfj4eDz33HPd/FhERN7NnVONOnTpHsSKFSsQGxuL0aNHIysrC+fPn0dISEjnB1Yq8fzzzyM1NRV6vR5paWnIyclBYmJiu3KTJ0/G3r17262zWq34+c9/jg8//BAxMTHIyMhAbm5uh32JiPoq54P6PL2JadWqVaiqqsK+ffsgk8lw22234eDBg53uM2jQIKSmpgIAgoODkZCQgKqqqi4FVVZWhvj4eMTFxcHX1xeLFi3C7t27u7QvEVFfoNWboJTLEBbo67YYupQg6urqUFhYiPT0dKSnp+OXv/wlGhoaunyS8vJyHDt2DJmZmR22ffrppxg9ejRmzZrlHHxXVVWFwYMHO8vExMR0ObkQEfUF1fUmRAb7QS53zyhqoIsJYtmyZQgODsZbb72Ft956CyEhIXjwwQe7dAKDwYC8vDxs2rSpQ7NUamoqzp8/j+PHj+Oxxx7DT3/6025/gKKiImfi0ul03d6fiMgTuXOqUYcuJYgzZ85gw4YNiIuLQ1xcHNatW4ezZ89edz+z2Yy8vDwsWbIE8+fP77A9JCQEKpUKADB79myYzWZcvnwZ0dHRqKiocJarrKxEdHS0y3MUFBRAo9FAo9EgMjKyKx+HiMjj6dw8ihroYoIICAjA4cOHnctHjhxBQEBAp/sIIbB8+XIkJCSgsLDQZZkffvgBQggA9vsONpsNAwcOREZGBk6fPo1z586hubkZxcXFyM3N7epnIiLyeu5+zAbQxV5Mf/rTn3D//fejrq4OABAWFoZt27Z1us+RI0ewfft2pKSkQK1WAwCeeeYZXLhwAQCwcuVK7Nq1C1u2bIFSqURAQACKi4shk8mgVCrx0ksvYcaMGbBarVi2bBmSkpJu4mMSEXmPZosNVxqa3fqgPgCQCcef8F1QX18PwN40tGnTJjz++ONSxXVD0tPTodFo3B0GEdFNuVjbhAnPfYRn56cgf+wQSc/V2fdmt2aUCwkJcd5ofuGFF24+MiIi6sATBskBNzHlaDcqHkRE1A3VzkFyXtCLyRV3zXBERNTXOWoQt4R48E3q4OBgl4lACIGmpibJgiIi6s909UbIZcBAlQcnCL1e31txEBFRC63ehIEqPyjcOIoauIkmJiIikoYnjIEAmCCIiDyOVu/eqUYdmCCIiDyMtt7k9h5MABMEEZFHsdoELhtMiHJzDyaACYKIyKPUGEywCfdONerABEFE5EE8ZRQ1wARBRORRtHr3TzXqwARBRORBtPUtNQg2MRERUVuOJqZIN4+iBpggiIg8ilZvRFigD3yV7v96dn8ERETkVO0hYyAAJggiIo+i1XvGGAiACYKIyKPo6o2sQRARUXtCCOg8ZBQ1wARBROQxrjaaYbYKjxgDATBBEBF5jNZBcn28iamiogLZ2dlITExEUlISNm/e3KHMzp07MWrUKKSkpGDChAk4fvy4c1tsbCxSUlKgVquRnp4uVZhERB6jdZCcZ9QgOp1R7qYOrFTi+eefR2pqKvR6PdLS0pCTk4PExERnmaFDh+LQoUMICwvDBx98gIKCAhw9etS5/eDBg4iIiJAqRCIij1Jd7zmP2QAkrEEMGjQIqampAOxzWyckJKCqqqpdmQkTJiAsLAwAMG7cOFRWVkoVDhGRx2t9UF8fb2Jqq7y8HMeOHUNmZuY1y2zduhWzZs1yLstkMkyfPh1paWkoKirqjTCJiNxKpzch2F+JAF+Fu0MBIGETk4PBYEBeXh42bdqEkJAQl2UOHjyIrVu34vDhw851hw8fRnR0NLRaLXJycjBy5EhkZWV12LeoqMiZQHQ6nTQfgoioF3jKVKMOktYgzGYz8vLysGTJEsyfP99lma+++goPPfQQdu/ejYEDBzrXR0dHAwCioqIwb948lJWVudy/oKAAGo0GGo0GkZGRPf8hiIh6iadMNeogWYIQQmD58uVISEhAYWGhyzIXLlzA/PnzsX37dgwfPty5vqGhAXq93vm+pKQEycnJUoVKROQRPOkxG4CETUxHjhzB9u3bnV1VAeCZZ57BhQsXAAArV67EU089hZqaGjz66KP2YJRKaDQaVFdXY968eQAAi8WCxYsXY+bMmVKFSkTkdkIIj2tikixBTJo0CUKITsv85S9/wV/+8pcO6+Pi4tqNiSAi6uvqjRYYzTbc4gETBTlwJDURkQfQtYyijvSgGgQTBBGRB3COou4PN6mJiKjrnIPkPOgmNRMEEZEHaH1QHxMEERG1oa03IcBHAZWf5OOXu4wJgojIAzjGQMhkMneH4sQEQUTkAarrjbjFg25QA0wQREQeQac3IdKDblADTBBERB5Bqzd51A1qgAmCiMjtGpstMJgsHjUGAmCCICJyu9ZBcqxBEBFRG544SA5ggiAicrvWQXJsYiIiojaqW5qYbmENgoiI2tLqjfBVyhEa4OPuUNphgiAicjNdvQmRKs8aRQ0wQRARuZ2nTTXqwARBRORmnjbVqAMTBBGRm9lHUXtWDyaACYKIyK1MFitqG82sQRARUXtaZxdX1iCIiKgNxyhqT3uSKyBhgqioqEB2djYSExORlJSEzZs3dygjhMCqVasQHx+PUaNG4YsvvnBu27ZtG4YNG4Zhw4Zh27ZtUoVJRORWOg+catRBsrntlEolnn/+eaSmpkKv1yMtLQ05OTlITEx0lvnggw9w+vRpnD59GkePHsUjjzyCo0eP4sqVK9iwYQM0Gg1kMhnS0tKQm5uLsLAwqcIlInIL53OY+tNN6kGDBiE1NRUAEBwcjISEBFRVVbUrs3v3btx///2QyWQYN24camtrcenSJRw4cAA5OTkIDw9HWFgYcnJysH//fqlCJSJyG229CQq5DAODfN0dSge9Mjt2eXk5jh07hszMzHbrq6qqMHjwYOdyTEwMqqqqrrnelaKiIhQVFQEAdDqdBNETEUlHqzciQuULudyzRlEDvXCT2mAwIC8vD5s2bUJISEiPH7+goAAajQYajQaRkZE9fnwiIil56hgIQOIEYTabkZeXhyVLlmD+/PkdtkdHR6OiosK5XFlZiejo6GuuJyLqa6rrTR73FFcHyRKEEALLly9HQkICCgsLXZbJzc3F//7v/0IIgc8++wyhoaEYNGgQZsyYgZKSEly9ehVXr15FSUkJZsyYIVWoRERuo9MbEemhNQjJ7kEcOXIE27dvR0pKCtRqNQDgmWeewYULFwAAK1euxOzZs7Fv3z7Ex8cjMDAQr732GgAgPDwcv/nNb5CRkQEAWLt2LcLDw6UKlYjILSxWG2oamj2yiysgYYKYNGkShBCdlpHJZPjjH//octuyZcuwbNkyKUIjIvIIlw3NEMLzphp14EhqIiI38dSpRh2YIIiI3MTxHCZPbWJigiAichPHKGpPfFAfwARBROQWQgh8cuYyfBQyDFR53ihqgAmCiMgtiv9Vgb1fXcKjU+Pho/DMr2LPjIqIqA/7urIO697/BlnDI7HqjmHuDueamCCIiHpRbWMzHtn5OSKCfLHpXjUUHvgMJodeeVgfEREBNptA4VvHUV1vxNsrJyDcA5/g2hZrEEREvWTLoTP46N9a/GZuItSDB7g7nOtigiAi6gVHvr+M50u+xd3qW3HfuNvcHU6XMEEQEUnshzojVr1xDLdHqvDs/BTIZJ5736EtJggiIgk1W2x4dOfnMJqt2PKzNAT6es+tX++JlIjICz37wSl8caEWLy0eg/golbvD6RbWIIiIJLL3q4t47Ug5HpwYi7mjbnV3ON3GBEFEJIHvtQY8sesrpA4ZgDWzEtwdzg1hgiAi6mENJgse2fE5/HwU+OOSVPgqvfOr1jujJiLyUEII/Prdr3FGZ8Af8sdgUGiAu0O6YUwQREQ9aMdn57H7y4sozBmOifER7g7nprAXE+xthTYhIAQgYH91LANoXYb9rwPnq0DLe/t2R1nAfpyWN21fnOdoX7ZnyQDIZIBcJmt5L2tdlgH2R7/I2pVxbHOsk8tkUMjt+ypkMijk9u0KuX1Z1rJOIZNBJkf7Ms733tHXm6inHLtwFU/tPYlpI6Pw6NR4d4dz05ggAMz9wz9hNNvcHUafI5cBSrkcCrkMSoUMSrkMCrm85dW+TiGXwadNGYVc5tzuo5C3/Njf+yrkUCpa1/sqW7c5tvsoZPBRtu7np1TA30cOfx8FAnwUCPC1v/q3/AT4KOCjYDKjm3eloRk/3/kFbgnxxwsLR0PuwQ/h6yomCAAvLFTDJgRk6PgX9o//unaUkTn/Om/9Kxwt/x5aliBzLre8ylrXt65zRNGT/5gEbALtakKOmo9j2VEjgui4zmaz728TAjYhYHUs2wSsLetstjbrRcf39mUBs1XAarPB0rLOYhOwWu2vFsf6luV25awCBosFFquA2WpDs9UGs9UGs6V12bHNYru5OphCLmtJGu0TieN9oK8CQX5KqPyUCPZXtn/vq4TK376s8mt976eUe1bSEQI4ehS4dAkYNAjIzGz7j49uktUm8B/Fx3DZ0Iy/PjIBAwI9+yF8XSVZgli2bBn27t2LqKgonDhxosP2jRs3YufOnQAAi8WCU6dOQafTITw8HLGxsQgODoZCoYBSqYRGo5EqTADA7JRBkh6fpGWzCZhtNpitAmaLrU1CETBZrDCabWhqtsJotv80OX6aW5eNZpv9tbn99tomM6pqm9BgssDQ8iO6kI+Uclm7xBES4IPwQF+EBfkiPMgHYYG+CA9qWW7zPshX0fOJZd8+YMUKoLYWkMsBmw0YMAD485+B2bN79lz91Iv/OI1/nr6MZ+alICUm1N3h9BiZEF355959H3/8MVQqFe6//36XCaKtPXv24Pe//z0++ugjAEBsbCw0Gg0iIrp3gyc9PV3yZEL9mxACjc1WNJgs0Jss9sRhbPPe8WNsfa83WlDfZMbVxmZcabC/Wq9R6/FVyBHmIoGEBfkiQuWLCJUfIoP9ENnyGuR3nb/x9u0D7rkHaGrquC0gANi1i0niJpV+q8WDr/8L88ZE4/kFoz2r5tgFnX1vSlaDyMrKQnl5eZfKvvHGG8jPz5cqFKIeI5PJEORnb2aKusFjCCFQb7TgakMzrjQ2218bmlsTSJv1py7V42pDM2qbzC5rLoG+inYJw5lAgv0QGeSLrOUPw9dVcgDsSWPFCuDCBTY33aCq2iY8/uaXGHFLMJ7+qfc8hK+r3H4PorGxEfv378dLL73kXCeTyTB9+nTIZDKsWLECBQUF19y/qKgIRUVFAACdTid5vEQ3SyaTITTAB6EBPohFUJf2sdoErjQ0Q6c3QWcw4XLLq07f+nNaa8AnZ2pQ12QGAIyp+jfG1VxBZ63hlitX8cOBUkTckQV/H0UPfLr+40pDMx7d+QWsVoEtP0tDgG/f+/25PUHs2bMHEydORHh4uHPd4cOHER0dDa1Wi5ycHIwcORJZWVku9y8oKHAmkPT09F6Jmai3KeQyZ83gekwWK2oMzTC9fRV+7/oAZuM1yzZZBH5b9HccKG3ET0L8MSQ8EDHhARgSHogh4YEY3PIaqfLrE71yboTVJnDucgNOXapv86PHD/X23+uffpaKoRFdS/Texu0Jori4uEPzUnR0NAAgKioK8+bNQ1lZ2TUTBBG156dU4NYBAUBKPCDr/BZjoI8M+T8dh8Rbh+PClUZUXG3Ep2dq8O6xqnZNWn5KOWLCAjA4PBADg/wQGuCDkAClsyYU4u+D0MCW15Z1/j4e1pOrC+qNZvz7kr5dMvi2Wu/sBq+UyxAfpcL42wciYVAwMmLDMWZImJujlo5bE0RdXR0OHTqEHTt2ONc1NDTAZrMhODgYDQ0NKCkpwdq1a90YJZGXyswEQkMBg+GaRRRhYZh631xM/dEXucliRdXVJnvSuNKIiqtNuFBjTyCnqw2oazLDYLJ0enofhcyZPEIC7D/25KGEys8Hwf72rsL2LsM+zq7Dzld/JfyUPddsI4RAs9WGZov9R2+04N8/tEkGP9Sj4krr/ZoBgT5I+EkIlmTehoRBIUgYFIz4KFWPxuTpJEsQ+fn5KC0txeXLlxETE4MNGzbAbLa3ja5cuRIA8O6772L69OkICmqtnlVXV2PevHkA7N1fFy9ejJkzZ0oVJlHfJZMBRUWd92L6859d3qD2UyoQF6lCXOS15y+wWO1fsnVNZtQbzfbXJvty+3X217rGZlRcaURdkxl6oxlm6/U7UPoq5M5kofJrTSaBvgpYbQImi63lS9+KZovNvuxc1/pjalm+1q9p6MAgjIoZgEUZQ5AwKBgJg0LwkxB/r6sB9TTJurm6A7u5ErngoeMgTBYr9MbWLsF6owV6o7ld92C90QKDyWzvStzSndhgtKCx2QKFXAZfpQK+Sjn8WkbW+yrl8Gt59W2z7sdlfBVyBPopMSxKhRE/CfaqWd56mlu6uRKRh5g9296VtawMuHgRuPVWYOxYt3dt9VMq4KdSIEJ1/Rvv5B5MEET9gUxmvydB1A183DcREbnEBEFERC4xQRARkUtMEERE5BITBBERucQEQURELjFBEBGRS31qJHVERARiY2Ody3V1dQgN7Ti7k6v1Op0OkZGRUod4TdeKtbeO1Z19rle2s+3d3cZrdXP78FrxWl3vWpWXl+Py5cuuTyr6sIcffrjL69PS0qQOp1PXirW3jtWdfa5XtrPt3d3Ga3Vz+/Ba8Vp19Vq50qebmO66665urXennozpRo7VnX2uV7az7d3dxmt1c/vwWvFa3cy16lNNTDeDD/rzHrxW3oPXyrv16RpEd3Q2rSl5Fl4r78Fr5d1YgyAiIpdYgyAiIpeYIIiIyCUmCCIicokJogtKS0sxefJkrFy5EqWlpe4Oh66joaEB6enp2Lt3r7tDoU6cOnUKK1euxD333IMtW7a4Oxxyoc8niGXLliEqKgrJycnt1u/fvx8jRoxAfHw8nnvuuU6PIZPJoFKpYDQaERMTI2W4/VpPXCsA+O///m8sXLhQqjAJPXOtEhIS8Kc//QlvvfUWjhw5ImW4dIP6fC+mjz/+GCqVCvfffz9OnDgBALBarRg+fDg+/PBDxMTEICMjA2+88QasVivWrFnTbv9XX30VERERkMvlqK6uRmFhIXbu3OmOj9Ln9cS1On78OGpqamA0GhEREYG5c+e646P0eT1xraKiovD+++9jy5YtuO+++7B48WJ3fBTqRJ+fkzorKwvl5eXt1pWVlSE+Ph5xcXEAgEWLFmH37t1Ys2ZNp80SYWFhMJlMUobbr/XEtSotLUVDQwNOnjyJgIAAzJ49G3J5n68o97qe+n+Vm5uL3NxczJkzhwnCA/X5BOFKVVUVBg8e7FyOiYnB0aNHr1n+nXfewYEDB1BbW4tf/OIXvREitejutXr66acBAK+//rqz5ke9o7vXqrS0FO+88w5MJhNmz57dGyFSN/XLBNFd8+fPx/z5890dBnXD0qVL3R0CXcfUqVMxdepUd4dBneiXf15FR0ejoqLCuVxZWYno6Gg3RkTXwmvlPXit+p5+mSAyMjJw+vRpnDt3Ds3NzSguLkZubq67wyIXeK28B69V39PnE0R+fj7Gjx+Pb7/9FjExMdi6dSuUSiVeeuklzJgxAwkJCVi4cCGSkpLcHWq/x2vlPXit+oc+382ViIhuTJ+vQRAR0Y1hgiAiIpeYIIiIyCUmCCIicokJgoiIXGKCICIil5ggqM9TqVS9er4JEyb06vlqa2vx8ssv9+o5qX9ggiDqJovF0un2Tz75pFfPyQRBUmGCoH7pzJkzmDlzJtLS0jB58mT8+9//BgDs2bMHmZmZGDNmDO68805UV1cDANavX4/77rsPEydOxH333Yf169dj2bJlmDp1KuLi4vDiiy86j+2osZSWlmLq1Km45557MHLkSCxZsgSOcan79u3DyJEjkZaWhlWrVrmct+L1119Hbm4upk2bhjvuuAMGgwF33HEHUlNTkZKSgt27dwMAnnzySZw5cwZqtRqrV68GAGzcuBEZGRkYNWoU1q1bJ90vkvo2QdTHBQUFdVg3bdo08d133wkhhPjss89Edna2EEKIK1euCJvNJoQQ4pVXXhGFhYVCCCHWrVsnUlNTRWNjo3N5/Pjxwmg0Cp1OJ8LDw0Vzc3O78x08eFCEhISIiooKYbVaxbhx48Q///lP0dTUJGJiYsTZs2eFEEIsWrRIzJkzp0OMr732moiOjhY1NTVCCCHMZrOoq6sTQgih0+nE7bffLmw2mzh37pxISkpy7nfgwAHx8MMPC5vNJqxWq5gzZ444dOjQTf4WqT/i476p3zEYDPjkk0+wYMEC5zrHRFCVlZW49957cenSJTQ3N2Po0KHOMrm5uQgICHAuz5kzB35+fvDz80NUVBSqq6s7TEk7duxY5zq1Wo3y8nKoVCrExcU5j52fn4+ioiKXsebk5CA8PBwAIITAr3/9a3z88ceQy+Woqqpy1nDaKikpQUlJCcaMGeP8vKdPn0ZWVla3f1fUvzFBUL9js9kwYMAAfPnllx22PfbYYygsLERubi5KS0uxfv1657agoKB2Zf38/JzvFQqFy/sEXSnTmbbn3LlzJ3Q6HT7//HP4+PggNjYWRqOxwz5CCKxZswYrVqzo1rmIfoz3IKjfCQkJwdChQ/H2228DsH+hHj9+HABQV1fnnMNg27Ztkpx/xIgROHv2rHPKzjfffLNL+9XV1SEqKgo+Pj44ePAgzp8/DwAIDg6GXq93lpsxYwZeffVVGAwGAPaZ3rRabc9+COoXWIOgPq+xsbFd009hYSF27tyJRx55BL/73e9gNpuxaNEijB49GuvXr8eCBQsQFhaGadOm4dy5cz0eT0BAAF5++WXMnDkTQUFByMjI6NJ+S5YswV133YWUlBSkp6dj5MiRAICBAwdi4sSJSE5OxqxZs7Bx40acOnUK48ePB2C/ab5jxw5ERUX1+Gehvo2P+yZyA4PBAJVKBSEEfv7zn2PYsGH4z//8T3eHRdQOm5iI3OCVV16BWq1GUlIS6urqeL+APBJrEERE5BJrEERE5BITBBERucQEQURELjFBEBGRS0wQRETkEhMEERG59P9Rcra4KPOdsQAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA24AAANgCAYAAABUWzsXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABTAUlEQVR4nO39eZTkdX0v/j+ru2djpmdh6RlgWAOyDMvMMIC4IGoQBUPiDUSJMVeNoiZXYvB6o/kmkXyTqDfE+1Vzf9HgjTFXvXHBxIV4E7JouKIJd+wRRBCJMNMM0DUDdHfNVj291O+P7umZYQaYpavrU1WPxzme7qr6LK/qhrKfvJdXqVar1QIAAEBhdTS6AAAAAJ6d4AYAAFBwghsAAEDBCW4AAAAFJ7gBAAAUnOAGAABQcF2NLmBPRx99dE4++eRGlwEAANAQ69evzxNPPLHP84UKbieffHLWrl3b6DIAAAAaYs2aNft93lRJAACAghPcAAAACk5wAwAAKLhCrXEDAIB2NTIyko0bN6ZarTa6FGbA3Llzs3z58syaNeuAjhfcAACgADZu3Jju7u6cfPLJKZVKjS6HOqrVannyySezcePGnHLKKQd0jqmSAABQANVqNUcddZTQ1gZKpVKOOuqogxpdFdwAAKAghLb2cbC/a8ENAACaUa2W/Ou/Jn/zNxNfa7W63OYjH/lItm/fXpdrH6jBwcH86Z/+6Yzd7+STT55qgv2CF7zgkK/z6U9/Oo899ti01CS4AQBAs/nGN5ITT0wuvzx54xsnvp544sTz06xVgtvo6Oghnfed73znkO8puAEAQLv6xjeSa65JNm5Mtm5NKpWJrxs3Tjx/iOFt27Ztueqqq3L++efnnHPOyRe+8IV87GMfy2OPPZaXvvSleelLX5okuf3223PJJZdk9erVufbaa7N169Ykyfe+97285CUvyQUXXJArrrgijz/+eJLksssuy6//+q9n5cqVOeecc3LXXXdN3e/Nb35zLrrooqxatSpf/epXkyQ//OEPc9FFF2XlypU577zz8uCDD+a9731vfvKTn2TlypV5z3ves0/tv//7v58zzjgjL3rRi3Ldddflj//4j6fu/a53vStr1qzJRz/60Xz961/PxRdfnFWrVuWnf/qnUy6XkyRPPvlkXvGKV2TFihV5y1vektoeo5cLFiyY+v7mm2/OhRdemPPOOy/vf//7kyTr16/PWWedlbe+9a1ZsWJFXvGKV2THjh259dZbs3bt2rz+9a/PypUrs2PHjkP6vUypFcgFF1zQ6BIAAKAh7rvvvuc+aHy8Vjv++FptYmLk/v+3fPnEcQfp1ltvrb3lLW+Zejw4OFir1Wq1k046qbZ58+ZarVarbd68ufbiF7+4tnXr1lqtVqt96EMfqv3e7/1ebefOnbVLLrmktmnTplqtVqt9/vOfr73pTW+q1Wq12kte8pKp6/7Lv/xLbcWKFbVarVZ73/veV/vMZz5Tq9VqtYGBgdrpp59e27p1a+0//af/VPvsZz9bq9VqteHh4dr27dtrDz/88NR5T3fXXXfVzj///NqOHTtqlUqldtppp9VuvvnmqXu/4x3vmDr2qaeeqo1P/mw++clP1m688cZarVarvfOd76z93u/9Xq1Wq9Vuu+22WpKp9zx//vxarVar/f3f/33trW99a218fLw2NjZWu+qqq2r/8i//Unv44YdrnZ2dtXXr1tVqtVrt2muvnXpfL3nJS2r/9//+32f8me/vd/5MmUg7AAAAaBb/9m/J0NCzHzM4mNx1V3LxxQd16XPPPTfvfve785u/+Zt59atfnRe/+MX7HPOv//qvue+++/LCF74wSbJz585ccskleeCBB3Lvvffm8ssvT5KMjY3l2GOPnTrvuuuuS5JceumlqVQqGRwczO23356vfe1rU6Nj1Wo1fX19ueSSS/KHf/iH2bhxY/7Df/gPOf3005+17jvvvDM/+7M/m7lz52bu3Ln5mZ/5mb1ef+1rXzv1/caNG/Pa1742jz/+eHbu3Dm1Ff8dd9yRv/7rv06SXHXVVVmyZMk+97n99ttz++23Z9WqVUmSrVu35sEHH8yJJ56YU045JStXrkySXHDBBVm/fv2z1nwoBDcAAGgWjz+edDzHaqeOjuQQ1lU973nPS29vb77xjW/kt3/7t/Pyl788v/u7v7vXMbVaLZdffnn+6q/+aq/nf/CDH2TFihX57ne/u99rP30HxVKplFqtli9/+cs544wz9nrtrLPOysUXX5y//du/zZVXXpk/+7M/y6mnnnrQ72eX+fPnT33/zne+MzfeeGOuvvrqfOtb38pNN910wNep1Wp53/vel7e97W17Pb9+/frMmTNn6nFnZ+fhT4vcD2vcAACgWRx7bDI+/uzHjI8nxx130Jd+7LHHcsQRR+SXfumX8p73vCe9vb1Jku7u7mzZsiVJ8vznPz933nln/v3f/z3JxDq1H//4xznjjDOyefPmqeA2MjKSH/7wh1PX/sIXvpAk+fa3v51FixZl0aJFueKKK/Inf/InU+vJ1q1blyR56KGHcuqpp+aGG27Iz/7sz+aee+7Zq4ane+ELX5ivf/3rqVar2bp1a2677bZnfI9DQ0M5/vjjkyR/+Zd/OfX8pZdemv/1v/5XkuR//+//nYGBgX3OveKKK/KpT31qak3fo48+mk2bNj3rz/TZ6j5YRtwAAKBZXHxxsmjRxGYkz2Tx4uSiiw760j/4wQ/ynve8Jx0dHZk1a1Y+/vGPJ0muv/76vPKVr8xxxx2Xb37zm/n0pz+d6667LsPDw0mSP/iDP8jznve83HrrrbnhhhsyNDSU0dHRvOtd78qKFSuSJHPnzs2qVasyMjKST33qU0mS3/md38m73vWunHfeeRkfH88pp5yS2267LV/84hfzmc98JrNmzcqyZcvyW7/1WznyyCPzwhe+MOecc05e9apX5eabb56q+8ILL8zVV1+d8847L0uXLs25556bRYsW7fc93nTTTbn22muzZMmSvOxlL8vDDz+cJHn/+9+f6667LitWrMgLXvCCnHjiifuc+4pXvCL3339/LrnkkiQTm5Z89rOfTWdn5zP+TN/4xjfm7W9/e+bNm5fvfve7mTdv3sH+WqaUarU6NXw4BGvWrMnatWsbXQYAAMy4+++/P2edddZzH7hrV8n9TcebNy+59dbkyiunv8BDdNlll+WP//iPs2bNmrrdY+vWrVmwYEG2b9+eSy+9NLfccktWr15dt/tNl/39zp8pE5kqCQAAzeTKKyfC2fLlyYIFycKFE1+XLy9caJsp119/fVauXJnVq1fn53/+55sitB0sUyUBAKDZXHll0tc3sXvkY49NrGm76KLkaZuAFMG3vvWtut9j1/q0Via4AQBAMyqVDnrLf5qXqZIAAFAQBdp+gjo72N+14AYAAAUwd+7cPPnkk8JbG6jVannyySczd+7cAz7HVEkAACiA5cuXZ+PGjdm8eXOjS2EGzJ07N8uXLz/g4wU3AAAogFmzZuWUU05pdBkUlKmSAAAABSe4AQAAFJzgBgAAUHCCGwAAQMEJbgAAAAUnuAEAABSc4AYAAFBwghsAAEDBCW4AAAAFJ7gBAAAUnOAGAAC0jZ2j49lUqWZsvNboUg6K4AYAALSN+x6v5KIP/FO+9cCmRpdyUAQ3AACgbfQPVZMkSxfObXAlB0dwAwAA2ka5IrgBAAAUWrlSTVdHKUfNn93oUg6K4AYAALSN/ko1Pd1z0tFRanQpB0VwAwAA2samynCWLmquaZKJ4AYAALSR/ko1S7sFNwAAgMIqD1WzzIgbAABAMW0bHs2W4dH0LJzT6FIOmuAGAAC0hV2tAJY1WSuARHADAADaRLkynERwAwAAKKxdI249ghsAAEAx9e+aKmlzEgAAgGIqV6qZP7szC+Z0NbqUgya4AQAAbaFcqTZl8+1EcAMAANpEuTLclBuTJIIbAADQJvqHqlkquAEAABTT+Hgtm7YIbgAAAIU1sH1nRsZqWbpwTqNLOSSCGwAA0PKmWgEYcQMAACimTZXhJLGrJAAAQFHtGnGzxg0AAKCg+ocmgltPtzVuAAAAhbRpSzVHL5idWZ3NGYGas2oAAICD0Mw93BLBDQAAaAPlynDT7iiZCG4AAEAbKFeq6RHcAAAAiml4dCxPbttpxA0AAKCoNm+Z7OG2sDl3lEwENwAAoMWVd/Vwa9Lm24ngBgAAtLhyZWLEzVRJAACAgtrVfFs7AAAAgIIqV6qZ3dmRJUfManQph0xwAwAAWtpEK4A5KZVKjS7lkAluAABAS+uvVJt6fVsiuAEAAC1uU2W4qXeUTAQ3AACghdVqtfRXqlnaLbgBAAAU0pbh0WzfOZZli5q3+XYiuAEAAC1sU6X5WwEkghsAANDC+ocmmm8LbgAAAAVVnhxxs6skAABAQfWbKgkAAFBs5Uo1C+d2Zd7szkaXclgENwAAoGWVK9Usa/IebongBgAAtLD+ynDTT5NMBDcAAKCFbapUBTcAAICiGhuvZdOW4Sxd2NzNtxPBDQAAaFFPbh3O2Hit6VsBJIIbAADQosqV1mi+nQhuAABAi2qVHm6J4AYAALSo8mRw0w4AAACgoMqVajpKyVHzZze6lMMmuAEAAC2pf6iaY7rnpKuz+WNP878DAACA/ShvGW6JHSUTwQ0AAGhR5aFqegQ3AACA4uqvVI24AQAAFFV1ZCxDO0aydOGcRpcyLQQ3AACg5ZRbqIdbIrgBAAAtqFwZTtIaPdwSwQ0AAGhB/UbcAAAAiq08JLgBAAAUWrlSzdxZHVk4t6vRpUwLwQ0AAGg5u1oBlEqlRpcyLQQ3AACg5WyqDLfMNMlEcAMAAFpQf6UquAEAABRVrVabmCrZIq0AEsENAABoMUM7RrJzdDw93XMaXcq0EdwAAICWsquHmxE3AACAgipXhpMky6xxOzCDg4O55pprcuaZZ+ass87Kd7/73XreDgAAoOWabydJXbvR/fqv/3pe+cpX5tZbb83OnTuzffv2et4OAABgaqpkz8LWWeNWt+A2NDSUO+64I5/+9KeTJLNnz87s2bPrdTsAAIAkSblSzZIjZmVOV2ejS5k2dZsq+fDDD+eYY47Jm970pqxatSpvectbsm3btnrdDgAAIMlEcGulaZJJHYPb6Ohoent78453vCPr1q3L/Pnz86EPfWif42655ZasWbMma9asyebNm+tVDgAA0CbKleGW2lEyqWNwW758eZYvX56LL744SXLNNdekt7d3n+Ouv/76rF27NmvXrs0xxxxTr3IAAIA20V+pZmm34HZAli1blhNOOCEPPPBAkuSf/umfcvbZZ9frdgAAABkZG88TW4eztMVG3Oq6q+Sf/Mmf5PWvf3127tyZU089NX/xF39Rz9sBAABt7omtw6nVkqUttKNkUufgtnLlyqxdu7aetwAAAJjSP9nDrZWabyd1bsANAAAwk8qV4SSt1Xw7EdwAAIAWUp5svi24AQAAFFR/pZqujlKOmj+70aVMK8ENAABoGeVKNT3dc9LRUWp0KdNKcAMAAFpGuVJtuVYAieAGAAC0kHJluOV2lEwENwAAoIWUh6ottzFJIrgBAAAtYtvwaLYMjwpuAAAARbW7FcCcBlcy/QQ3AACgJfRPBjdr3AAAAApqU2U4SewqCQAAUFT9U1MlBTcAAIBC6h+qZsGcriyY09XoUqad4AYAALSETVuq6WnBjUkSwQ0AAGgR/UPVltyYJBHcAACAFlGuDAtuAAAARTU+XpucKim4AQAAFNJT23dmZKyWZda4AQAAFFO5hVsBJIIbAADQAqaCWws2304ENwAAoAWUK8NJYnMSAACAouofqqZUSo7ptsYNAACgkMqVao6aPyezOlsz4rTmuwIAANpKuVLN0hbdUTIR3AAAgBbQ38LNtxPBDQAAaAGbKtWW3VEyEdwAAIAmNzw6lie37czSbsENAACgkDbtagWwyBo3AACAQtq0ZaL5do81bgAAAMXUP9TazbcTwQ0AAGhy5crEiJvgBgAAUFDlSjWzuzqy+IhZjS6lbgQ3AACgqfVPNt8ulUqNLqVuBDcAAKCplSvVlm4FkAhuAABAkytXhlu6+XYiuAEAAE2sVqulXKm29MYkieAGAAA0sS3Do9m+cyxLF7Zu8+1EcAMAAJpYeWiiFcBSI24AAADFVK5MNN8W3AAAAAqqvw2abyeCGwAA0MTKFVMlAQAACq1cqWbh3K7Mm93Z6FLqSnADAACaVv9QNctavIdbIrgBAABNrLxluOWnSSaCGwAA0MTKQ1XBDQAAoKjGxmvZvHW45XeUTAQ3AACgST25dThj47UsXTin0aXUneAGAAA0pf42aQWQCG4AAECTKleGk8SukgAAAEVlxA0AAKDgykPVdJSSoxdY4wYAAFBI5Uo1x3TPSWdHqdGl1J3gBgAANKX+SrUtWgEkghsAANCkNlWG22J9WyK4AQAATaq/UhXcAAAAiqo6MpahHSNt0QogEdwAAIAmVJ5sBdDT3fo7SiaCGwAA0IT6hyaCmxE3AACAgipvGU4Su0oCAAAUVXlyxK1HcAMAACim/ko182Z1ZuHcrkaXMiMENwAAoOmUK9UsXTgnpVKp0aXMCMENAABoOuU26uGWCG4AAEATKleG22ZHyURwAwAAmkytVku/ETcAAIDiGtw+kp2j44IbAABAUZW3TLQCWLpwToMrmTmCGwAA0FT6J3u4tUvz7URwAwAAmsymynCSmCoJAABQVP2ViRG3HlMlAQAAiqm/Us2R82dnTldno0uZMYIbAADQVDZVqunpbp/RtkRwAwAAmkx/pdpWzbcTwQ0AAGgy5cpwW+0omQhuAABAExkZG88TW4fTI7gBAAAU0+Ytw6nV2quHWyK4AQAATaQ82QpgaRu1AkgENwAAoInsDm5G3AAAAAqpXBlOErtKAgAAFFV/pZpZnaUcecTsRpcyowQ3AACgaZSHqunpnpuOjlKjS5lRghsAANA0yluq6WmzjUkSwQ0AAGgi/UPVtmsFkAhuAABAE9lUGW67HSUTwQ0AAGgS24ZHs2V4VHADAAAoqv7JHm7LFlnjBgAAUEhTzbe7jbgBAAAU0lRwa7Pm24ngBgAANIlyZThJrHEDAAAoqv6hahbM6cqCOV2NLmXGCW4AAEBTKFeqWdqGzbcTwQ0AAGgSE8Gt/aZJJoIbAADQJMqV4SwT3AAAAIppfLyWTVuqbbmjZCK4AQAATeCp7TszMlbL0m5r3AAAAAqpf2iih9syI24AAADFtGnLRHDrscYNAACgmPqHJppv25wEAACgoMqVakql5Bhr3AAAAIqpXKnmqPlzMquzPSNMe75rAACgqfRXqlm2qD1H2xLBDQAAaALlynCWdrfn+rZEcAMAAJpAudK+zbcTwQ0AACi44dGxPLVtZ9vuKJkIbgAAQMFtqky0Ali60Bo3AACAQipXJppvLzXiBgAAUEzlqRE3wQ0AAKCQ+idH3KxxAwAAKKhNlWpmd3Vk8RGzGl1KwwhuAABAofVXqlm6cE5KpVKjS2kYwQ0AACi0/qFqW0+TTAQ3AACg4DZtGU6P4AYAAFBMtVrNiFsENwAAoMC2DI9mx8iY4NboAgAAAJ5JeWiiFUDPwjkNrqSxBDcAAKCw9HCbILgBAACFVa4MJ0mWCm4AAADFVN414rZIcAMAACikcqWaRfNmZe6szkaX0lCCGwAAUFj9Q9UsbfONSRLBDQAAKLBypdr269sSwQ0AACiwcmVYcIvgBgAAFNTYeC2btw63fSuARHADAAAK6omtwxkbr2Vpm+8omQhuAABAQe1qBbC02+YkghsAAFBI/UN6uO0iuAEAAIVU3jKcJDYnieAGAAAUVHmoms6OUo5eYKqk4AYAABRSf6WaYxbMSWdHqdGlNJzgBgAAFNJE822jbYngBgAAFNREcLO+LRHcAACAgipXhu0oOUlwAwAACqc6MpahHSNG3CYJbgAAQOHs6uEmuE0Q3AAAgMIpV3YFN5uTJIIbAABQQP2TwW2ZEbckghsAAFBAmyrDSZKlNidJIrgBAAAF1F+pZt6sznTP6Wp0KYUguAEAAIXTX6lm2aK5KZVKjS6lEAQ3AACgcDZVqunptjHJLoIbAABQOLtG3JgguAEAAIVSq9VSrgzbUXIPghsAAFAog9tHsnN0PD2C2xTBDQAAKBQ93PYluAEAAIVSngxuSxfanGQXwQ0AACiU3cHNiNsughsAAFAo5cpwkqTHiNsUwQ0AACiU/ko1R86fnTldnY0upTAENwAAoFDKQ1XTJJ9GcAMAAAqlvKVqY5KnEdwAAIBC6R/SfPvpBDcAAKAwRsbG8+S2YVMln0ZwAwAACmPzluHUaloBPF1XPS9+8sknp7u7O52dnenq6sratWvreTsAAKDJ9U/2cFu2yBq3PdU1uCXJN7/5zRx99NH1vg0AANACNk0Gt55uI257MlUSAAAojP6hXSNugtue6hrcSqVSXvGKV+SCCy7ILbfcst9jbrnllqxZsyZr1qzJ5s2b61kOAABQcOUtw5nVWcqRR8xudCmFUtepkt/+9rdz/PHHZ9OmTbn88stz5pln5tJLL93rmOuvvz7XX399kmTNmjX1LAcAACi48lA1Pd1z09FRanQphVLXEbfjjz8+SdLT05PXvOY1ueuuu+p5OwAAoMn1VzTf3p+6Bbdt27Zly5YtU9/ffvvtOeecc+p1OwAAoAWUK1WtAPajblMly+VyXvOa1yRJRkdH84u/+It55StfWa/bAQAALaBcGc6LTz+m0WUUTt2C26mnnpq77767XpcHAABazNbh0WwdHrWj5H5oBwAAABRCebKHmzVu+xLcAACAQigP7QpuRtyeTnADAAAKobxFcHsmghsAAFAI/UPDSZJlgts+BDcAAKAQypVquud0Zf6cuu2h2LQENwAAoBDKlWp6bEyyX4IbAABQCP2VqlYAz0BwAwAACmFTZThLuwW3/RHcAACAhhsfr6VcqWapEbf9EtwAAICGe2r7zoyO1+wo+QwENwAAoOH6p5pv25xkfwQ3AACg4coVzbefjeAGAAA0XLky0XxbcNs/wQ0AAGi4/ko1pVJyTLepkvsjuAEAAA23qVLN0QvmZFaniLI/fioAAEDD9VeqNiZ5FoIbAADQcP1DVa0AnoXgBgAANNymLcPpEdyekeAGAAA01PDoWJ7attOI27MQ3AAAgIbaNNkKQHB7ZoIbAADQULuab/fYnOQZCW4AAEBD9U8Gt2WLjLg9E8ENAABoqPLkVMml3YLbMxHcAACAhipXqpnd1ZHFR8xqdCmFJbgBAAANVa5M9HArlUqNLqWwBDcAAKCh+oeqWWpjkmcluAEAAA1VrlSzVCuAZyW4AQAADVOr1VKuDAtuz0FwAwAAGqZSHc2OkTHNt5+D4AYAADTMpskebkv1cHtWghsAANAwu5pvL+22OcmzEdwAAICG6R+aCG7LjLg9K8ENAABomE1bhpPE5iTPQXADAAAapn+omkXzZmXurM5Gl1JoghsAANAw/ZWqHSUPgOAGAAA0zKZKNT0LbUzyXAQ3AACgYYy4HRjBDQAAaIix8Vo2bxm2MckBENwAAICGeGLrcMZrmm8fCMENAABoiKkebkbcnpPgBgAANES5MhHcltqc5DkJbgAAQEPsCm5G3J6b4AYAADREuTKczo5SjlpgxO25CG4AAEBD9FeqOWbBnHR2lBpdSuEJbgAAQEOUK1U7Sh4gwQ0AAGiIcqWapd2mSR4IwQ0AAGiI/qFqlhlxOyCCGwAAMON27BxLpTqapXaUPCCCGwAAMON293AT3A6E4AYAAMy4fj3cDorgBgAAzLjdI242JzkQghsAADDjpoKbzUkOiOAGAADMuHJlOEfM7kz3nK5Gl9IUBDcAAGDG9VeqWbpwbkqlUqNLaQqCGwAAMOPKQ1Xr2w6C4AYAAMy48paqVgAHQXADAABmVK1WS7kyrBXAQRDcAACAGTW4fSQ7R8eNuB0EwQ0AAJhR/VM93AS3AyW4AQAAM2pXcFu2yOYkB0pwAwAAZtSmyeDW023E7UAJbgAAwIzqHxpOYqrkwRDcAACAGVXeUs1R82dndpc4cqD8pAAAgBlVHqqmx2jbQRHcAACAGdVfqWbZQhuTHAzBDQAAmFHlyrD1bQdJcAMAAGbMyNh4ntwmuB0swQ0AAJgxm7cMp1ZLli0S3A6G4AYAAMyYXc23l1rjdlAENwAAYMaUh3YFNyNuB0NwAwAAZky5IrgdCsENAACYMf2V4czqLOXII2Y3upSmIrgBAAAzZlOlmp7uuenoKDW6lKYiuAEAADOmv1K1MckhENwAAIAZ01+pagVwCAQ3AABgxmyqDKenW3A7WIIbAAAwI7YOj2br8KgRt0MguAEAADNiVyuAZVoBHDTBDQAAmBG7mm/32JzkoAluAADAjOg34nbIBDcAAGBGlCvDSZKlgttBE9wAAIAZUa5U0z2nK/PndDW6lKYjuAEAADOiXKlmqR0lD4ngBgAAzIj+SjVLbUxySAQ3AABgRpSHqta3HSLBDQAAqLvx8Vo2bRkW3A6R4AYAANTdk9t2ZnS8phXAIRLcAACAuitP9nAz4nZoBDcAAKDudgc3m5McCsENAACou/7J4LZMO4BDIrgBAAB1V64Mp1RKjl5gxO1QCG4AAEDdlYeqOXrBnMzqFEEOhZ8aAABQd+UtVTtKHgbBDQAAqLv+oaqNSQ6D4AYAANRduVLVCuAwCG4AAEBdDY+OZWD7iOB2GAQ3AACgrjZVhpPEGrfDILgBAAB1NdV8Ww+3Qya4AQAAdbWr+bbNSQ6d4AYAANRV/9BEcDNV8tAJbgAAQF1t2jKc2V0dWTRvVqNLaVqCGwAAUFf9QxPNt0ulUqNLaVqCGwAAUFflStU0ycMkuAEAAHVVrlTTY2OSwyK4AQAAdVOr1dJvxO2wCW4AAEDdVKqjqY6MZ6ngdlgENwAAoG40354eghsAAFA3erhND8ENAACom6kRN5uTHBbBDQAAqJvdwc2I2+EQ3AAAgLopV4azaN6szJ3V2ehSmprgBgAA1I1WANNDcAMAAOqmXKnaUXIaCG4AAEDdlCvVLO22McnhEtwAAIC6GB0bz+Ytw1lmxO2wCW4AAEBdPLltZ8ZrSY81bodNcAMAAOpC8+3pI7gBAAB10V8R3KaL4AYAANTFpqnm2zYnOVyCGwAAUBf9lWo6O0o5aoHgdrgENwAAoC7KleEcs2BOOjtKjS6l6QluAABAXWi+PX0ENwAAoC76h6pZZn3btDig4LZt27aMj48nSX784x/na1/7WkZGRupaGAAA0NzKlWqW2lFyWhxQcLv00ktTrVbz6KOP5hWveEU+85nP5I1vfGOdSwMAAJrVjp1jqVRHBbdpckDBrVar5Ygjjshf//Vf51d/9VfzpS99KT/84Q/rXRsAANCkylOtAAS36XDAwe273/1uPve5z+Wqq65KkoyNjdW1MAAAoHlpvj29Dii4feQjH8kHP/jBvOY1r8mKFSvy0EMP5aUvfWm9awMAAJrUrhG3ZYtsTjIdug7koJe85CV5yUtekiQZHx/P0UcfnY997GN1LQwAAGheu4JbjxG3aXFAI26/+Iu/mEqlkm3btuWcc87J2WefnZtvvrnetQEAAE2qf2g4R8zuTPecAxor4jkcUHC77777snDhwnzlK1/Jq171qjz88MP5zGc+U+/aAACAJlXeMtEKoFQqNbqUlnBAwW1kZCQjIyP5yle+kquvvjqzZs3yCwAAAJ5ReaiapZpvT5sDCm5ve9vbcvLJJ2fbtm259NJLs2HDhixcuLDetQEAAE2qv1K1o+Q0OqDgdsMNN+TRRx/NN77xjZRKpZx00kn55je/We/aAACAJlSr1bKpMqyH2zQ6oOA2NDSUG2+8MWvWrMmaNWvy7ne/O9u2bat3bQAAQBMa2D6SnWPjgts0OqDg9uY3vznd3d354he/mC9+8YtZuHBh3vSmN9W7NgAAoAnt7uEmuE2XA9qb8yc/+Um+/OUvTz1+//vfn5UrV9arJgAAoIn1TwY3m5NMnwMacZs3b16+/e1vTz2+8847M2/evLoVBQAANK/y0K7gZsRtuhzQiNsnPvGJ/PIv/3KGhoaSJEuWLMlf/uVf1rUwAACgOZUrw0mSnm7BbbocUHA7//zzc/fdd6dSqSRJFi5cmI985CM577zz6locAADQfPor1Rw1f3Zmdx3QBD8OwEH9JBcuXDjVv+2//bf/VpeCAACA5rapUjVNcpodcgSu1WrTWQcAANAi+itVG5NMs0MObqVSaTrrAAAAWkS5UtUKYJo96xq37u7u/Qa0Wq2WHTt21K0oAACgOY2MjeeJrTttTDLNnjW4bdmyZabqAAAAWsCmLRM7Shpxm162eQEAAKZNebL59jKbk0wrwQ0AAJg2u5pv99icZFoJbgAAwLTpN+JWF4IbAAAwbcqV4czqLGXJEbMbXUpLEdwAAIBpU65U09M9Nx0d2odNJ8ENAACYNnq41YfgBgAATJv+SjVLbUwy7eoe3MbGxrJq1aq8+tWvrvetAACABisPVbPUxiTTru7B7aMf/WjOOuuset8GAABosK3Do9m2c0xwq4O6BreNGzfmb//2b/OWt7ylnrcBAAAKoH9IK4B6qWtwe9e73pU/+qM/SkeHpXQAANDqNk32cDPiNv3qlqhuu+229PT05IILLnjW42655ZasWbMma9asyebNm+tVDgAAUGf9U8HN5iTTrW7B7c4778zXvva1nHzyyXnd616Xf/7nf84v/dIv7XPc9ddfn7Vr12bt2rU55phj6lUOAABQZ/1G3OqmbsHtgx/8YDZu3Jj169fn85//fF72spfls5/9bL1uBwAANNimynC653Rl/pyuRpfSciw+AwAApkX/UDVLNd+uixmJwpdddlkuu+yymbgVAADQIOUtVTtK1okRNwAAYFqUh6rpsTFJXQhuAADAYRsfr2XTlmEjbnUiuAEAAIftyW07Mzpes6NknQhuAADAYStrBVBXghsAAHDYdgW3ZXaVrAvBDQAAOGy7m2/bnKQeBDcAAOCwlYeqKZWSYxYIbvUguAEAAIetXBnO0QvmpKtTxKgHP1UAAOCw9Vc0364nwQ0AADhs5UrVjpJ1JLgBAACHbSK4Wd9WL4IbAABwWKojYxnYPmKqZB0JbgAAwGHZvGU4iebb9SS4AQAAh2Wqh5vm23UjuAEAAIelPBncTJWsH8ENAAA4LP1DkyNuNiepG8ENAAA4LOVKNXO6OrJo3qxGl9KyBDcAAOCwlCvDWbpwbkqlUqNLaVmCGwAAcFj6K1Xr2+pMcAMAAA5LuVK1o2SdCW4AAMAhq9VqE8Gt28Yk9SS4AQAAh6yyYzTVkfEsM+JWV4IbAABwyMpbJloB9FjjVleCGwAAcMh29XCzOUl9CW4AAMAh668IbjNBcAMAAA7ZpsquqZI2J6knwQ0AADhk/ZVqFh8xK3NndTa6lJYmuAEAAIesXBnO0m7TJOtNcAMAAA6Z5tszQ3ADAAAOWf9QNcusb6s7wQ0AADgk24ZH88TW4Ry7aF6jS2l5ghsAAHBI7t44mPFacv4JixpdSssT3AAAgEOyrm8wSbLqhCWNLaQNCG4AAMAh6d0wkFOPmZ8l82c3upSWJ7gBAAAHrVarZd0jg1l9otG2mSC4AQAAB239k9vz1LadgtsMEdwAAICD1rthIEmy+qTFjS2kTQhuAADAQevtG8iCOV05vae70aW0BcENAAA4aL19g1l5wuJ0dpQaXUpbENwAAICDsnV4NA/0V7L6xMWNLqVtCG4AAMBBueeRicbbq06yMclMEdwAAICD8r1dG5NovD1jBDcAAOCg9PYN5LSeBVl0xKxGl9I2BDcAAOCA7W68vbjRpbQVwQ0AADhgDz2xLYPbRzTenmGCGwAAcMB2N94W3GaS4AYAAByw3r7BdM/tymnHLGh0KW1FcAMAAA7Yur6BrDxhcTo03p5RghsAAHBAtlRH8kB5i/VtDSC4AQAAB+TuR4ZSq1nf1giCGwAAcEB6+yY2Jll5wuLGFtKGBDcAAOCA9PYN5PSeBVk0T+PtmSa4AQAAz2l8vJbeDQPWtzWI4AYAADynh57Ymkp1NBdY39YQghsAAPCcejcMJklWn7S4oXW0K8ENAAB4Tr19A1k4tyunHq3xdiMIbgAAwHPq7RvIqhOXaLzdIIIbAADwrCrVkTy4aauNSRpIcAMAAJ7V9/sGJxtvL250KW1LcAMAAJ5Vb99ASiWNtxtJcAMAAJ5Vb99gntfTne65Gm83iuAGAAA8o/HxWtb1DZgm2WCCGwAA8Ix+snlrtlRHs8rGJA0luAEAAM/oexsGkiQXnCS4NZLgBgAAPKPevoEsPmJWTj16fqNLaWuCGwAA8Ix6+waz6oTFKZU03m4kwQ0AANivoe0j+XeNtwtBcAMAAPZr3SMT69tWW9/WcIIbAACwX719g+koJedrvN1wghsAALBf6/oG8ryl3Vkwp6vRpbQ9wQ0AANjH+Hgt3+8bNE2yIAQ3AABgHw9u2potw6M2JikIwQ0AANhHb9/kxiQnLm5sISQR3AAAgP343oaBLDliVk7ReLsQBDcAAGAfvX0DWX3iEo23C0JwAwAA9jK4fWce2rzNxiQFIrgBAAB7Wdc3mCRZZX1bYQhuAADAXnr7BiYaby9f3OhSmCS4AQAAe+ntG8iZyxZmvsbbhSG4AQAAU8amGm8vbnQp7EFwAwAApvy4vCXbdo5pvF0wghsAADBld+Ntwa1IBDcAAGBK74bBHDl/dk466ohGl8IeBDcAAGDKur6BrD5xscbbBSO4AQAASZKntu3MQ09ovF1EghsAAJBkYrQtsb6tiAQ3AAAgycTGJJ0dpZy3fFGjS+FpBDcAACDJxMYkZx3bnSNma7xdNIIbAACQ0bHx3L1x0DTJghLcAACAPFDeku0abxeW4AYAAKS3bzCJjUmKSnADAACybsNAjl4wOyccOa/RpbAfghsAAJDevoGsOnGJxtsFJbgBAECbe3LrcNY/ud00yQIT3AAAoM2tm1rftrihdfDMBDcAAGhz3+sbSFdHKectX9zoUngGghsAALS53g0DOfu4hZk3u7PRpfAMBDcAAGhjo2PjuWfjkPVtBSe4AQBAG/tR/5bsGBnLKuvbCk1wAwCANtbbN5BE4+2iE9wAAKCN9W4YyDHdc7J8icbbRSa4AQBAG+vtG8zqExdrvF1wghsAALSpJ7YOp+8pjbebgeAGAABtqnfD5Pq2kwS3ohPcAACgTfX2Daaro5Rzj1/U6FJ4DoIbAAC0qd6+gaw4flHmztJ4u+gENwAAaEMjY+O5Z+PExiQUn+AGAABt6P7HK6mOjNuYpEkIbgAA0IZsTNJcBDcAAGhDvX2DWbpwTo5bNLfRpXAABDcAAGhDvX0DWX3iEo23m4TgBgAAbWbTlmo2Duywvq2JCG4AANBmejcMJklWn7S4oXVw4AQ3AABoM+v6BjKrs5QVx2m83SwENwAAaDO9fQNZcZzG281EcAMAgDayc3Q892wcsr6tyQhuAADQRu5/vJLh0fFcoH9bUxHcAACgjXxvqvH24sYWwkER3AAAoI309g3k2EVzc+yieY0uhYMguAEAQBtZ1zdofVsTEtwAAKBNlCvVPDq4I6tOXNzoUjhIghsAALSJ3qn1bUbcmo3gBgAAbaK3byCzOzuy4riFjS6FgyS4AQBAm+jtG8w5xy/MnC6Nt5uN4AYAAG1g5+h4fvCoxtvNSnADAIA28MPHhrJzdNz6tiYluAEAQBvo7RtMklwguDUlwQ0AANpAb99Ajl88L0sXzm10KRwCwQ0AANpA74YB/duamOAGAAAt7vGhHXl8qGpjkiYmuAEAQIvr3TCYROPtZia4AQBAi+vtG8icro6cfazG281KcAMAgBbX2zeQc49flNld/vxvVn5zAADQwoZHx/LDRyumSTY5wQ0AAFrYvY9WsnNsPKvtKNnUBDcAAGhh6/oGksSOkk1OcAMAgBa2q/F2j8bbTU1wAwCAFta7YTAXWN/W9AQ3AABoUY8N7kh/pWp9WwsQ3AAAoEV9b8Pk+jYjbk1PcAMAgBbV2zeQubM6cpbG201PcAMAgBbV2zeY845fnFmd/uxvdn6DAADQgqojY7nvsaGsOmlxo0thGghuAADQgu59dCgjYzX921qE4AYAAC2oV+PtliK4AQBAC+rdMJgTjpyXY7rnNLoUpoHgBgAALaZWq6W3b8BoWwsR3AAAoMU8Orgjm7YM5wL921qG4AYAAC2mt28wifVtrURwAwCAFtO7YSDzZnXmzGXdjS6FaSK4AQBAi+ntG8h5yxelS+PtluE3CQAALWSi8XYlq61vaymCGwAAtJB7Ng5ldFzj7VZTt+BWrVZz0UUX5fzzz8+KFSvy/ve/v163AgAAJu1qvL3qxMWNLYRp1VWvC8+ZMyf//M//nAULFmRkZCQvetGL8qpXvSrPf/7z63VLAABoe70bBnLSUUfk6AUab7eSuo24lUqlLFiwIEkyMjKSkZGRlEqlet0OAADa3kTj7UHTJFtQXde4jY2NZeXKlenp6cnll1+eiy++eJ9jbrnllqxZsyZr1qzJ5s2b61kOAAC0tI0DO/LE1uGsNk2y5dQ1uHV2dub73/9+Nm7cmLvuuiv33nvvPsdcf/31Wbt2bdauXZtjjjmmnuUAAEBL272+zYhbq5mRXSUXL16cl770pfm7v/u7mbgdAAC0pd4NAzlitsbbrahuwW3z5s0ZHBxMkuzYsSP/8A//kDPPPLNetwMAgLbX2zeY85cv1ni7BdVtV8nHH388//E//seMjY1lfHw8v/ALv5BXv/rV9bodAAC0te07R3Pf45W8/SWnNroU6qBuwe28887LunXr6nV5AABgD/dsHMqYxtstyxgqAAC0ABuTtDbBDQAAWkDvhsGccvT8HDl/dqNLoQ4ENwAAaHK1Wi3r+gaySv+2liW4AQBAk+t7anue3LbT+rYWJrgBAECT27W+TXBrXYIbAAA0ud4Ng5k/uzNnaLzdsgQ3AABocr19A1l54uJ0dpQaXQp1IrgBAEAT275zND/q32KaZIsT3AAAoInd/YjG2+1AcAMAgCa2u/H24sYWQl0JbgAA0MR6Nwzk1GPmZ/ERGm+3MsENAACaVK1Wy7pHBk2TbAOCGwAANKn1T27PUxpvtwXBDQAAmlTvhsnG2yctbmwh1J3gBgAATaq3byAL5nTl9B6Nt1ud4AYAAE2qt28wK0/QeLsdCG4AANCEtg6P5oH+SlZrA9AWBDcAAGhC9zwymPFasvokG5O0A8ENAACa0FTj7RMEt3YguAEAQBPq7RvMaT0LsuiIWY0uhRkguAEAQJOp1Wrp7Ruwvq2NCG4AANBkHnpiWwa3j2i83UYENwAAaDK7G28Lbu1CcAMAgCbT2zeY7rldOe2YBY0uhRkiuAEAQJNZ1zeQlScsTofG221DcAMAgCaypTqSB8pbrG9rM4IbAAA0kbsfGUpN4+22I7gBAEAT6e0bSKmUrDxhcaNLYQYJbgAA0ER6+wZyes+CLJqn8XY7EdwAAKBJjI/Xsq5v0Pq2NiS4AQBAk3joiW0Z2qHxdjsS3AAAoEnsbry9uLGFMOMENwAAaBK9fQNZOLcrpx6t8Xa7EdwAAKBJ9PYNZNWJSzTebkOCGwAANIFKdSQPbtpqfVubEtwAAKAJfL9vcLLx9uJGl0IDCG4AANAENN5ub4IbAAA0gd6+wTyvpzvdczXebkeCGwAAFNxE4+2BrD7J+rZ2JbgBAEDB/WTz1mypjmb1iYsbXQoNIrgBAEDB9fbtarxtxK1dCW4AAFBwvRsGs/iIWTn16PmNLoUGEdwAAKDgvtc3kFUnLE6ppPF2uxLcAACgwIa2j+TfNd5ue4IbAAAU2LpHrG9DcAMAgELr7RtMRyk5X+Pttia4AQBAga3rG8jzlnZnwZyuRpdCAwluAABQUOPjtXy/b9A0SQQ3AAAoqgc3bc2W4dFcYGOStie4AQBAQWm8zS6CGwAAFFTvhoEcOX92Tj7qiEaXQoMJbgAAUFC9Gm8zSXADAIACGty+Mz/ZvM00SZIIbgAAUEjr+gaTJKtOXNzQOigGwQ0AAAqot29govH28sWNLoUCENwAAKCAevsGcuayhZmv8TYR3AAAoHDGphpvL250KRSE4AYAAAXz4/KWbNs5ltUabzNJcAMAgIKZarwtuDFJcAMAgILp3TCYo+bPzkkabzNJcAMAgIJZ1zeQVScu0XibKYIbAAAUyMC2nXnoiW02JmEvghsAABTIukesb2NfghsAABRI74bBdHaUct7yRY0uhQIR3AAAoEC+t2EgZx3bnSNma7zNboIbAAAUxOjYeO7eOGiaJPsQ3AAAoCAeKG/Jdo232Q/BDQAACqK3bzCJjUnYl+AGAAAF8b31T+XoBbNzwpHzGl0KBSO4AQBAAdz76FC+fs/jufzsZRpvsw/BDQAAGmx4dCzv/uLdOWr+7Lz3lWc2uhwKyB6jAADQYB/7pwfzQHlLPvXGNVl0xKxGl0MBGXEDAIAGuvuRwXz8Wz/JtRcsz8vOXNrocigowQ0AABqkOjKWd3/p7ixdODe//eqzG10OBWaqJAAANMj/948/zr9v2pq/fPNFWTTPFEmemRE3AABogO9tGMgn73go1110Ql7yvGMaXQ4FJ7gBAMAMq46M5T1fujvHLpqX37ryrEaXQxMwVRIAAGbYH//9A3noiW357K9cnO65pkjy3Iy4AQDADFq7/qn8+Z0P55eef2JedPrRjS6HJiG4AQDADNmxcyz/+Ut35/jF8/K+V5kiyYEzVRIAAGbIH/39j7L+ye35q7c+P/Pn+FOcA2fEDQAAZsC/PvRk/uLO9fmPl5yUS37qqEaXQ5MR3AAAoM62DY/mv9x6T0488oj85qvObHQ5NCHjswAAUGf/9e9+lEcGtucL11+SI2b7E5yDZ8QNAADq6Dv//kT+53c35E0vOCUXnXJko8uhSQluAABQJ1uHR/OeW+/JKUfPz3uuOKPR5dDEjNMCAECdfOAb9+exoR259e2XZN7szkaXQxMz4gYAAHVwx48353/9W1/e+uJTc8FJpkhyeAQ3AACYZpXqSN775XvyU8fMz42XP6/R5dACTJUEAIBp9oe33Z/+SjVffscLMneWKZIcPiNuAAAwjb75wKZ8Ye0jedtLfiqrTlzS6HJoEYIbAABMk6EdI3nfl3+Q5y1dkHf99OmNLocWYqokAABMk9+/7b5s3jqcW375gszpMkWS6WPEDQAApsE/3V/Ord/bmHe85Kdy3vLFjS6HFiO4AQDAYRrcvjPv++sf5Mxl3Xnny09rdDm0IFMlAQDgMP3e1+/LU9t25lNvvNAUSerCiBsAAByGv/9hf/5m3aP5tZeelnOOX9TocmhRghsAAByip7btzP/zNz/I2ccuzK+91BRJ6sdUSQAAOETv/9oPM7RjJP/zzRdndpcxEerHP10AAHAIvvGDx/P1ux/LDS87PWcft7DR5dDiBDcAADhIT2wdzm9/5d6ce/yivP2yn2p0ObQBwQ0AAA5CrVbL73zl3mytjuaPrz0/szr9SU39+acMAAAOwm33PJ7/fW9/3nX56TljWXejy6FNCG4AAHCANm2p5ne+em/OP2Fxrn/xqY0uhzYiuAEAwAGo1Wr57b+5N9t3juXD156XLlMkmUH+aQMAgAPwtbsfy+33lfPuy5+X03pMkWRmCW4AAPAcNlWq+d2v/jCrTlyct5giSQMIbgAA8CxqtVp+629+kOrIWP742vPT2VFqdEm0IcENAACexV/3Ppp/vH9T3nPFGfmpYxY0uhzalOAGAADPoH+ompu+/sNcePKSvOmFpzS6HNqY4AYAAPtRq9Xy3r++JyNj47n5GlMkaSzBDQAA9uNLazfmWw9szntfeWZOPnp+o8uhzQluAADwNI8O7sjv33ZfLj7lyPzyJSc3uhwQ3AAAYE+1Wi3v/fI9GavVcvM156fDFEkKQHADAIA9/NVdj+T/PPhE3nflWTnxqCMaXQ4kEdwAAGDKI09tzx/+7X154WlH5fUXndjocmCK4AYAAEnGx2v5zS/fkyT5rz9/nimSFIrgBgAAST73bxvynZ88mf/nqrOzfIkpkhSL4AYAQNvre3J7PvCNH+XFpx+d6y46odHlwD4ENwAA2tr4eC3vufXudHWU8l9//ryUSqZIUjyCGwAAbe1/fnd9/u3hp/I7rz47xy2e1+hyYL8ENwAA2tb6J7blQ3/3o1x2xjG5ds3yRpcDz0hwAwCgLY2N1/Kfv3R3ZnV25EP/wRRJiq2r0QUAAEAj/MWdD2fthoF8+Nrzs2zR3EaXA8/KiBsAAG3nJ5u35ua/fyA/fVZP/sPq4xtdDjwnwQ0AgLaya4rk3Fmd+cBrzjVFkqZgqiQAAG3lf/yfh7KubzAffd3K9Cw0RZLmYMQNAIC28WB5Sz78Dz/OFSuW5urzj2t0OXDABDcAANrC6Nh4/vOX7s782Z35g58zRZLmYqokAABt4c/ueCh3bxzKn1y3Ksd0z2l0OXBQjLgBANDyftRfyUf+8ce58txlefV5xza6HDhoghsAAC1tZHKK5MK5s/L7P3uOKZI0JVMlAQBoaZ/41k9y76OVfPz1q3PUAlMkaU5G3AAAaFn3PVbJx/75wfzM+cflVeeaIknzEtwAAGhJO0cnpkgumjc7/+/VKxpdDhwWwQ0AgJYztGMkv/75dbnv8Uo+8JpzsmT+7EaXBIfFGjcAAFrKXQ8/ld/4wvdTrlTzvledmVesWNbokuCwCW4AALSE0bHxfOyfHsx//+a/54Qjj8it73hBVp6wuNFlwbQQ3AAAaHqPPLU9v/75dentG8zPr16e3/vZFVkwx5+6tA7/NAMA0NS+su7R/PZX7k2plHzsulW5+vzjGl0STDvBDQCAprSlOpLf/eoP8zfrHs2ak5bkI69bmeVLjmh0WVAXghsAAE3nexsG8q4vrMtjg9X8xk8/L7/20p9KV6cN02ldghsAAE1jbLyW/983/z0f/acHc+yiufni256fC046stFlQd0JbgAANIVHB3fkNz7//dy1/qn87Mrj8vs/d04Wzp3V6LJgRghuAAAU3m33PJb3/fUPUqsl/99rz89rVi1vdEkwowQ3AAAKa9vwaG762g/zpe9tzMoTFudjr1uVE4+yAQntR3ADAKCQ7n5kML/++XXZ8NT2vPNlp+WGl5+eWTYgoU0JbgAAFMrYeC1/dsdP8t9u/3F6uufk8299fi4+9ahGlwUNJbgBAFAYjw/tyI1fuDvffejJXHXusfnAa87NoiNsQAKCGwAAhfB39z6e3/zyDzIyNp4/uua8XHvB8pRKpUaXBYUguAEA0FDbd47m92+7P391V1/OW74oH33dqpxy9PxGlwWFIrgBANAw9z46lBs+vy4PP7Etb3/JT+XGy5+X2V02IIGnq9u/FY888khe+tKX5uyzz86KFSvy0Y9+tF63AgCgyYyP1/LJOx7Ka/70zmwbHs3nfuXivPdVZwpt8AzqNuLW1dWVD3/4w1m9enW2bNmSCy64IJdffnnOPvvset0SAIAmsKlSzbu/dHf+z4NP5BVnL81//fnzsmT+7EaXBYVWt+B27LHH5thjj02SdHd356yzzsqjjz4quAEAtLF/vK+c//Lle7J952g+8Jpzc91FJ9iABA7AjKxxW79+fdatW5eLL754Jm4HAEDBVEfG8oFv3J//+d0NOfvYhfnYdStzWk93o8uCplH34LZ169b8/M//fD7ykY9k4cKF+7x+yy235JZbbkmSbN68ud7lAAAww+5/vJJf//y6/Li8NW950Sl5zyvPyJyuzkaXBU2lVKvVavW6+MjISF796lfniiuuyI033vicx69ZsyZr166tVzkAAMygWq2WT39nfT74v3+UhXNn5b/9wvm59HnHNLosKLRnykR1G3Gr1Wr5lV/5lZx11lkHFNoAAGgdT2wdznu+dHe++cDmvOzMnvzRNefl6AVzGl0WNK26Bbc777wzn/nMZ3Luuedm5cqVSZIPfOADufLKK+t1SwAACuBbD2zKf/7S3alUR/P//uyKvOH5J9mABA5T3YLbi170otRxFiYAAAVTHRnLH/3dA/nUnQ/njKXd+dxbnp8zltmABKbDjOwqCQBAa3uwvCXv/Kt1+VH/lrzxBSfnva86M3Nn2YAEpovgBgDAIavVavnsv/XlD267L/PndOXP/+OavPyspY0uC1qO4AYAwCF5atvO/Jdb78k/3l/Oi08/Oh/+hfPT0z230WVBSxLcAAA4aN9+8Inc+MXvZ3D7SH77qrPy5heeko4OG5BAvQhuAAAcsJ2j4/nw7Q/kz+54KD91zPz8xZsuzIrjFjW6LGh5ghsAAAfkoc1bc8Pn1+XeRyv5xYtPzO9cdXbmzbYBCcwEwQ0AgP0a3L4z6x4ZzLq+wazrG8hdDz+VebM782dvuCBXrFjW6PKgrQhuAABkdGw8Py5vTW/fwERQe2QgD23eliTpKCVnLluY1154Qn71stOybJENSGCmCW4AAG3oia3DUyNpvX0DuWfjULbvHEuSHL1gdlaesCTXXLA8q05YkvOWL8r8Of5shEbybyAAQIvbOTqe+x+vZF3fwNTUx76ntidJujpKWXHcwvzCmhOy6sTFWX3ikixfMi+lkh0ioUgEt2cxMjaeR57anlKplFKSUikppZRdn2OlUvb7WilJSknH1Gt7H5NSpo571mvvcb2OUmnqfgAAz6Z/qDo1kraubzA/eHQow6PjSZKlC+dk9YlL8obnn5RVJy7OOccvytxZNhiBohPcnsXjg9W87MP/0ugy9uuZgl/2CHodu4JlaXfw2zNMdkwGyL2e65i4zj7nZu/wuNe5U6/v+dre5+xzbvY4tzRx7kTNu4/vKCUdHaXd308e31Ha+/3teexe5+5xfGep9Kyv7/d6e9bSsbu+vY9NOjt2193Zsfcxe9a/12t7Xe85rjFZ+z7X2OM1oKBqteTf/i15/PHk2GOTiy/O1H+hg2lSHRnLDx8bmpz2OJjevoE8PlRNkszu6si5xy+aDGlLsvqkxTl20bwGVwwcCsHtWRy1YHY+8tqVqaWWWm3i/39rSWq1WmpJUsvu17Lr9d2PU6tlvLb7+D3Pz36O3/Nxdt1n8rXx2t7X3d/99no8efz4Hs+N1/b4Onn98fGJ18d3vb9aber18b2us/vr+OSbH586fuL5ZN/jxsb3vN7kubt+Lpm4/+73tvta4+O7v999vX3vOT6+9/32d3w72BXq9gqqewS+vcLf1PelySCYPb7f47mO0h7X2Ts07u+8ieOzx/e7A+re5+4OtJ3PWFf2vldHKV0du4/v3OO6nXvcb9exu77vfNqxHXs8v/f5SVdHx8T73M919nzPRr05YN/4RvK2tyWDg0lHRzI+nixenPzZnyVXXtno6mhStVotGwd27LGByGDue2woI2MT/4e3fMm8rDn5yKw+cXFWnbgkZx3bnTldRtOgFQhuz2L+nK783KrjG10Gh6FWe+5gt1cQfPrxk8FwbOq13YH06deZeK42GVYnrju2x3XGn+O1vY7bo7Zd5+z5Wm3y8X6vMRlodx03NvXaRFAem/q+lrF9ats7cE89Pz6x29jYHqF6n2Mm7z1x/N61Pdd5zWRXSN4rNO4nQHZ1Pj1AToTPXSG0ozR5TEdHOkuZ+DoZIPc8p7NUSmfnHuc80/V2hdtd1+vsSOfTjn+m605cqyNdk4+7Ojoyq3Pi+VmdHVPvZ3/HCLLP4BvfSK65JtmxY+/nt26deP7WW4U3Dsj2naO5Z+PQHpuIDOaJrcNJknmzOnPe8kX5lRedmlUnLs6qExenp9tuj9CqBDda2tRUzPjjssh2B739hMup4JeMjo9PvT62R+Ace1r4HBvP1PNj+1yn9rTzd99/dHz3sVPXnjo/e50/Pj5x/L733nX/8Ynr1WoZHZv8Or673tHxWoZHxjM6Pra71j1q2/VeR8fH977e5LnjtdrUf2FvpI5S0tXZMRnmSvt+v0fQe/r3u4Jh157BsGN3oJ319GMmrznx3OTznR2ZvUeonD353NTrHaXM6urIrMnXZ+35Wmcpszo6Mqtr8rjJax+2Wi25/vp9Q9suO3ZMjMT19Zk2yV5qtVrWP7k9vRsGsu6RiRG1H/Vvydjkf+E65ej5ufT0o7PqpCVZdcLinLmsO12dHQ2uGpgpghvQcB0dJeH6EO0ZRMfG9w6HU0FwbDII7ic8Tn0/VsvI+PjUsaOTz018Hd/7617PPe3x+Ph+zqtNvTY2XsvI2Hh2ju57/K7X9jp+V13jMxNUdwXR2VOhczIYPj3wdXZMhr7do4+7guRpP/lB3vbkQJ5t3GPkyYH846e+mi3nXZBZXbsC5cR9p8Jl19Med3ZkdtfTHnd2WOfahEbHxjM8Op4dI2P50eNbdm8i8shgBrePJEkWzOnKyhMW51cv+6msOnFxVp6wJEfOn93gyoFGEtwAmtiu0NsOG8Ltmqo8MhkIR0bHMzIZ/EbGxjMytjv4jYyPZ2QyHO56bXRsPDunguF4dk4+t/v1yWP3e83xjEzec/c1xzM8Mp6tY6MT549PHN/9wPqMPkfGrI7V8pW//b/5+wdnHfbPZc8RyDlTwe7Zw96szomRxlmTo5RT53TtftzVuXv67d7Tcfedsrtrqu+uY58+RXjXdNznOqZzj2Onc01prVbLzsmwtHOP/009Hhub+n6vY8bGMzwylp1je5wz+Xvf67nRsanjn36N4f1cZ39TxE/vWZArzl42OeVxSU7rWTA9I8BAyxDcAGgKpVJpcj3gZEqd09h6ntG/zk2+cnMy/MyHLJjdkQ/+pyvyu6svnAigk6FyV1AcGX3a48k/+EfGatk5OjbxdSp07nr+aY8nr7Pn463Do5PXr+1xz8n77AoeY+NTm2Q12v6C3Z5BsGOPdaFdHaWMjdeeFqh2B7DpUColsyfD8Jyujszp6szsyZHR2V27n58/p2uP4zqnnp/6usc1Tj1mQc4/YXEWzTv8EA+0NsENAKbTxRcnixZNbETyDEqLF+fIl764sGvcxsYnguDe03DH95qG+/RpuROPJ9Zk7jp2z3WZ+z4e3+/r+7vermOf7f4jY+Pp7ChNBak5szoyu7Nzr0C1Kzzt7/Xdz+0ZsvZ+3YY8QCMJbgAwnUql5JZb9r+rZJLMmzfREqDAAaCzo5R5s9tg/i1AE7EVEQBMtyuvnNjyf/nyZMGCZOHCia/Ll2sFAMAhMeIGAPVw5ZUTW/7fdVfy2GPJccclF11U6JE2AIpLcAOAeimVJta8AcBhMlUSAACg4AQ3AACAghPcAAAACk5wAwAAKDjBDQAAoOAENwAAgIIT3AAAAApOcAMAACg4wQ0AAKDgBDcAAICCE9wAAAAKTnADAAAoOMENAACg4AQ3AACAghPcAAAACk5wAwAAKDjBDQAAoOAENwAAgIIT3AAAAApOcAMAACg4wQ0AAKDgBDcAAICCE9wAAAAKTnADAAAoOMENAACg4AQ3AACAghPcAAAACk5wAwAAKDjBDQAAoOAENwAAgIIT3AAAAApOcAMAACi4Uq1WqzW6iF2OPvronHzyyc953NDQUBYtWnRI9ziUczdv3pxjjjnmkO7H/h3O77BIivQ+ZrqWet1vOq87Hdc61Gv4rCmGIv07ejiK9j5msp563qtInzf+tml+Rfv39FAV6X20698269evzxNPPLHvC7Um9Na3vnVGz73gggsO+X7s3+H8DoukSO9jpmup1/2m87rTca1DvYbPmmIo0r+jh6No72Mm66nnvYr0eeNvm+ZXtH9PD1WR3oe/bfbWlFMlf+ZnfqYh5zJ9WuX3UKT3MdO11Ot+03nd6bjWoV6jSP9stLNW+T0U7X3MZD31vFeRPm/8bdP8WuX3UKT34W+bvRVqqmRRrVmzJmvXrm10GUCL81kDzBSfN9B8mnLEbaZdf/31jS4BaAM+a4CZ4vMGmo8RNwAAgIIz4gYAAFBwghsAAEDBCW4AAAAFJ7gdhvvvvz9vf/vbc8011+TjH/94o8sBWthXvvKVvPWtb81rX/va3H777Y0uB2hRDz30UH7lV34l11xzTaNLAZ6mbYPbm9/85vT09OScc87Z6/m/+7u/yxlnnJHTTjstH/rQh571GmeddVY+8YlP5Itf/GLuvPPOepYLNLHp+Lz5uZ/7uXzyk5/MJz7xiXzhC1+oZ7lAk5qOz5pTTz01f/7nf17PMoFD1La7St5xxx1ZsGBBfvmXfzn33ntvkmRsbCzPe97z8g//8A9Zvnx5LrzwwvzVX/1VxsbG8r73vW+v8z/1qU+lp6cnX/va1/Lxj388b3jDG/KLv/iLjXgrQMFN1+dNkrz73e/O61//+qxevXrG3wdQbNP5WXPNNdfk1ltvnfH3ADyzrkYX0CiXXnpp1q9fv9dzd911V0477bSceuqpSZLXve51+epXv5r3ve99ue222/Z7nauvvjpXX311rrrqKsEN2K/p+Lyp1Wp573vfm1e96lVCG7Bf0/W3DVBMbTtVcn8effTRnHDCCVOPly9fnkcfffQZj//Wt76VG264IW9729ty5ZVXzkSJQIs42M+bP/mTP8k//uM/5tZbb80nPvGJmSgRaAEH+1nz5JNP5u1vf3vWrVuXD37wgzNRInCA2nbEbTpcdtllueyyyxpdBtAGbrjhhtxwww2NLgNocUcddZT/OAQFZcRtD8cff3weeeSRqccbN27M8ccf38CKgFbl8waYCT5roHUIbnu48MIL8+CDD+bhhx/Ozp078/nPfz5XX311o8sCWpDPG2Am+KyB1tG2we26667LJZdckgceeCDLly/Pn//5n6erqyv//b//91xxxRU566yz8gu/8AtZsWJFo0sFmpzPG2Am+KyB1ta27QAAAACaRduOuAEAADQLwQ0AAKDgBDcAAICCE9wAAAAKTnADAAAoOMENAACg4AQ3AApjwYIFM3q/F7zgBTN6v8HBwfzpn/7pjN4TgNYguAHQskZHR5/19e985zszek/BDYBDJbgBUGg/+clP8spXvjIXXHBBXvziF+dHP/pRkuTrX/96Lr744qxatSo//dM/nXK5nCS56aab8oY3vCEvfOEL84Y3vCE33XRT3vzmN+eyyy7Lqaeemo997GNT1941wvetb30rl112Wa655pqceeaZef3rX59arZYk+cY3vpEzzzwzF1xwQW644Ya8+tWv3qfGT3/607n66qvzspe9LC9/+cuzdevWvPzlL8/q1atz7rnn5qtf/WqS5L3vfW9+8pOfZOXKlXnPe96TJLn55ptz4YUX5rzzzsv73//++v0gAWhqXY0uAACezfXXX59PfOITOf300/Nv//Zv+dVf/dX88z//c170ohflX//1X1MqlfI//sf/yB/90R/lwx/+cJLkvvvuy7e//e3MmzcvN910U370ox/lm9/8ZrZs2ZIzzjgj73jHOzJr1qy97rNu3br88Ic/zHHHHZcXvvCFufPOO7NmzZq87W1vyx133JFTTjkl11133TPW2dvbm3vuuSdHHnlkRkdH8zd/8zdZuHBhnnjiiTz/+c/P1VdfnQ996EO599578/3vfz9Jcvvtt+fBBx/MXXfdlVqtlquvvjp33HFHLr300rr9PAFoToIbAIW1devWfOc738m111479dzw8HCSZOPGjXnta1+bxx9/PDt37swpp5wydczVV1+defPmTT2+6qqrMmfOnMyZMyc9PT0pl8tZvnz5Xve66KKLpp5buXJl1q9fnwULFuTUU0+duvZ1112XW265Zb+1Xn755TnyyCOTJLVaLb/1W7+VO+64Ix0dHXn00UenRgT3dPvtt+f222/PqlWrpt7vgw8+KLgBsA/BDYDCGh8fz+LFi6dGqPb0zne+MzfeeGOuvvrqfOtb38pNN9009dr8+fP3OnbOnDlT33d2du53HdqBHPNs9rzn5z73uWzevDnf+973MmvWrJx88smpVqv7nFOr1fK+970vb3vb2w7qXgC0H2vcACishQsX5pRTTsmXvvSlJBNB5+67706SDA0N5fjjj0+S/OVf/mVd7n/GGWfkoYceyvr165MkX/jCFw7ovKGhofT09GTWrFn55je/mQ0bNiRJuru7s2XLlqnjrrjiinzqU5/K1q1bkySPPvpoNm3aNL1vAoCWYMQNgMLYvn37XlMYb7zxxnzuc5/LO97xjvzBH/xBRkZG8rrXvS7nn39+brrpplx77bVZsmRJXvayl+Xhhx+e9nrmzZuXP/3TP80rX/nKzJ8/PxdeeOEBnff6178+P/MzP5Nzzz03a9asyZlnnpkkOeqoo/LCF74w55xzTl71qlfl5ptvzv33359LLrkkycRmKZ/97GfT09Mz7e8FgOZWqu3aNgsA2MfWrVuzYMGC1Gq1/Nqv/VpOP/30/MZv/EajywKgzZgqCQDP4pOf/GRWrlyZFStWZGhoyHo0ABrCiBsAAEDBGXEDAAAoOMENAACg4AQ3AACAghPcAAAACk5wAwAAKDjBDQAAoOD+/329yDnlwwZHAAAAAElFTkSuQmCC\n", "text/plain": [ - "
" + "
" ] }, "metadata": {}, @@ -388,56 +364,31 @@ } ], "source": [ - "lower_lr, upper_lr = 1e-5, 1e-2\n", + "%matplotlib inline\n", + "lower_lr, upper_lr = 1e-3, 1e-0\n", "optimizer = torch.optim.Adam(model.parameters(), lower_lr)\n", "lr_finder = LearningRateFinder(model, optimizer, loss_function, device=device)\n", "lr_finder.range_test(train_loader, val_loader, end_lr=upper_lr, num_iter=20)\n", - "ax=plt.subplots(1, 1, facecolor='white')[1]\n", + "steepest_lr, _ = lr_finder.get_steepest_gradient()\n", + "ax=plt.subplots(1, 1, figsize=(15,15), facecolor='white')[1]\n", "_ = lr_finder.plot(ax=ax)" ] }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "lr w/ steepest gradient: 5.455595e-04\n", - "lower: 1e-04, upper: 1e-03\n" - ] - } - ], - "source": [ - "steepest_lr = lr_finder.get_steepest_gradient()[0]\n", - "lower_lr = 10 ** floor(log10(steepest_lr))\n", - "upper_lr = 10 ** ceil(log10(steepest_lr))\n", - "print(f\"lr w/ steepest gradient: {steepest_lr:e}\")\n", - "print(f\"lower: {lower_lr:1.0e}, upper: {upper_lr:1.0e}\")" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Model training\n", + "## Live plotting\n", "\n", - "Execute a typical PyTorch training that run epoch loop and step loop, and do validation after every epoch.\n", - "Will save the model weights to file if got best validation accuracy." + "This function is just a wrapper around `range`/`trange` such that the plots are updated on every iteration." ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ - "%matplotlib notebook\n", - "# def get_colour(q):\n", - "# return plt.rcParams['axes.prop_cycle'].by_key()['color'][q]\n", - "\n", "def plot_range(data, wrapped_generator):\n", " plt.ion()\n", " for q in data.values():\n", @@ -447,8 +398,9 @@ " ax.legend()\n", " fig = ax.get_figure()\n", " fig.show()\n", - " \n", + "\n", " for i in wrapped_generator:\n", + " yield i\n", " for q in data.values():\n", " for d in q.values():\n", " if isinstance(d, dict):\n", @@ -457,42 +409,51 @@ " ax.legend()\n", " ax.relim()\n", " ax.autoscale_view()\n", - " fig.canvas.draw()\n", - " yield i" + " fig.canvas.draw()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training\n", + "\n", + "The training looks slightly different from a vanilla loop, but this is only because it loops across each of the different learning rate methods (standard, steepest and cyclical), such that they can be updated simultaneously" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ - "def train(max_epochs, axes, data):\n", - " for z, d in enumerate(data.keys()):\n", - " data[d][\"model\"] = densenet121(\n", - " spatial_dims=2, in_channels=1,\n", - " out_channels=num_classes).to(device)\n", + "def get_model_optimizer_scheduler(d):\n", + " d[\"model\"] = get_new_net()\n", "\n", - " if \"lr_lims\" in data[d]:\n", - " data[d][\"optimizer\"] = torch.optim.Adam(\n", - " data[d][\"model\"].parameters(), data[d][\"lr_lims\"][0])\n", - " # In the paper referenced at the top of this notebook, a step\n", - " # size of 8 times the number of iterations per epoch is suggested.\n", - " step_size = 8 * len(train_loader)\n", - " data[d][\"scheduler\"] = torch.optim.lr_scheduler.CyclicLR(\n", - " data[d][\"optimizer\"], base_lr=data[d][\"lr_lims\"][0], \n", - " max_lr=data[d][\"lr_lims\"][1], step_size_up=step_size,\n", - " cycle_momentum=False,\n", - " )\n", - " else:\n", - " data[d][\"optimizer\"] = torch.optim.Adam(\n", - " data[d][\"model\"].parameters(), data[d][\"lr_lim\"])\n", + " if \"lr_lims\" in d:\n", + " d[\"optimizer\"] = torch.optim.Adam(\n", + " d[\"model\"].parameters(), d[\"lr_lims\"][0])\n", + " d[\"scheduler\"] = torch.optim.lr_scheduler.CyclicLR(\n", + " d[\"optimizer\"], base_lr=d[\"lr_lims\"][0], \n", + " max_lr=d[\"lr_lims\"][1], step_size_up=d[\"step\"],\n", + " cycle_momentum=False,\n", + " )\n", + " elif \"lr_lim\" in d:\n", + " d[\"optimizer\"] = torch.optim.Adam(\n", + " d[\"model\"].parameters(), d[\"lr_lim\"])\n", + " else:\n", + " d[\"optimizer\"] = torch.optim.Adam(\n", + " d[\"model\"].parameters())\n", + " \n", + "\n", + "def train(max_epochs, axes, data):\n", + " for d in data.keys():\n", + " get_model_optimizer_scheduler(data[d])\n", "\n", " for q, i in enumerate([\"train\", \"auc\", \"acc\"]):\n", " data[d][i] = {\"x\":[], \"y\":[]}\n", " data[d][i][\"line\"], = axes[q].plot(\n", " data[d][i][\"x\"], data[d][i][\"y\"], label=d)\n", - "# get_colour(z), label=d)\n", "\n", " val_interval = 1\n", " \n", @@ -545,9 +506,9 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "metadata": { - "scrolled": true + "scrolled": false }, "outputs": [ { @@ -1514,7 +1475,7 @@ { "data": { "text/html": [ - "" + "" ], "text/plain": [ "" @@ -1527,27 +1488,43 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|██████████| 100/100 [04:36<00:00, 2.77s/it]\n" + "100%|██████████| 100/100 [03:11<00:00, 1.91s/it]\n" ] } ], "source": [ - "fig, axes=plt.subplots(3, 1, figsize=(10, 10), facecolor='white')\n", + "%matplotlib notebook\n", + "fig, axes=plt.subplots(3, 1, figsize=(10,10), facecolor='white')\n", "for ax in axes:\n", " ax.set_xlabel('Epoch')\n", "axes[0].set_ylabel('Train loss')\n", "axes[1].set_ylabel('AUC')\n", "axes[2].set_ylabel('ACC')\n", "\n", + "# In the paper referenced at the top of this notebook, a step\n", + "# size of 8 times the number of iterations per epoch is suggested.\n", + "step_size = 8 * len(train_loader)\n", + "\n", "max_epochs = 100\n", "data = {}\n", - "data[\"Default LR\"] = {\"lr_lim\": 1e-5}\n", + "data[\"Default LR\"] = {}\n", "data[\"Steepest LR\"] = {\"lr_lim\": steepest_lr}\n", - "data[\"Cyclical LR\"] = {\"lr_lims\": (0.8*steepest_lr, 1.2*steepest_lr)}\n", + "data[\"Cyclical LR\"] = {\"lr_lims\": (0.8*steepest_lr, 1.2*steepest_lr), \"step\": step_size}\n", "\n", "train(max_epochs, axes, data)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "Unsurprisingly, both `Steepest LR` and `Cyclical LR` show quicker convergence of the loss function than `Default LR`.\n", + "\n", + "There's not much of a difference in this example between `Steepest LR` and `Cyclical LR`. A bigger difference may be apparent in a more complex optimisation problem, but feel free to play with the step size, and the lower and upper cyclical limits." + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -1559,7 +1536,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ From 4f318e0108663500c0304ea68e7833afcad8c4ef Mon Sep 17 00:00:00 2001 From: Richard Brown <33289025+rijobro@users.noreply.github.com> Date: Wed, 20 Jan 2021 12:46:17 +0000 Subject: [PATCH 3/4] delete unnecessary file --- modules/._learning_rate.ipynb | Bin 4096 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 modules/._learning_rate.ipynb diff --git a/modules/._learning_rate.ipynb b/modules/._learning_rate.ipynb deleted file mode 100644 index d489ed98b676fe49e4b919e63f73d6dc10171041..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4096 zcmZQz6=P>$Vqox1Ojhs@R)|o50+1L3ClDJkFz{^v(m+1nBL)UWIUt(=a103v0xIE$ z=wR3bWJjQB0htaG7hqtJO3u&KODrhJN!80qEG{W6PEAQkEJ;-k2!`r81Ef8G#v#<@ zCMM?q1)Wqm5Bh0E*Q%TmS$7 From 6dcbea41b5225e21c5da354de67d8beb9ab1a3e6 Mon Sep 17 00:00:00 2001 From: Richard Brown <33289025+rijobro@users.noreply.github.com> Date: Wed, 20 Jan 2021 12:56:34 +0000 Subject: [PATCH 4/4] PEP8 --- modules/learning_rate.ipynb | 112 ++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 51 deletions(-) diff --git a/modules/learning_rate.ipynb b/modules/learning_rate.ipynb index 8082b3e4d1..cf013467c9 100644 --- a/modules/learning_rate.ipynb +++ b/modules/learning_rate.ipynb @@ -90,14 +90,10 @@ "import os\n", "import shutil\n", "import tempfile\n", + "\n", "import matplotlib.pyplot as plt\n", - "from math import ceil, floor, log10\n", - "import torch\n", "import numpy as np\n", - "from sklearn.metrics import classification_report\n", - "from torch.utils.data import DataLoader\n", - "from tqdm import trange\n", - "\n", + "import torch\n", "from monai.apps import MedNISTDataset\n", "from monai.config import print_config\n", "from monai.metrics import compute_roc_auc\n", @@ -109,13 +105,12 @@ " CenterSpatialCropd,\n", " Compose,\n", " LoadImaged,\n", - " RandFlipd,\n", - " RandRotated,\n", - " RandZoomd,\n", " ScaleIntensityd,\n", " ToTensord,\n", ")\n", "from monai.utils import set_determinism\n", + "from torch.utils.data import DataLoader\n", + "from tqdm import trange\n", "\n", "print_config()" ] @@ -235,7 +230,7 @@ } ], "source": [ - "# Set fraction of images used for testing to be very high, then don't use it. In this way, we can reduce the number \n", + "# Set fraction of images used for testing to be very high, then don't use it. In this way, we can reduce the number\n", "# of images in both train and val. Makes it faster and makes the training a little harder.\n", "def get_data(section):\n", " ds = MedNISTDataset(\n", @@ -244,18 +239,19 @@ " section=section,\n", " download=True,\n", " num_workers=10,\n", - " val_frac=.0005,\n", + " val_frac=0.0005,\n", " test_frac=0.995,\n", " )\n", " loader = DataLoader(ds, batch_size=30, shuffle=True, num_workers=10)\n", " return ds, loader\n", "\n", + "\n", "train_ds, train_loader = get_data(\"training\")\n", "val_ds, val_loader = get_data(\"validation\")\n", "\n", "print(len(train_ds))\n", "print(len(val_ds))\n", - "print(train_ds[0]['image'].shape)\n", + "print(train_ds[0][\"image\"].shape)\n", "num_classes = train_ds.get_num_classes()" ] }, @@ -284,14 +280,14 @@ ], "source": [ "%matplotlib inline\n", - "fig, axes = plt.subplots(3, 3, figsize=(15, 15), facecolor='white')\n", + "fig, axes = plt.subplots(3, 3, figsize=(15, 15), facecolor=\"white\")\n", "for i, k in enumerate(np.random.randint(len(train_ds), size=9)):\n", " data = train_ds[k]\n", " im, title = data[\"image\"], data[\"class_name\"]\n", - " ax = axes[i//3, i%3]\n", + " ax = axes[i // 3, i % 3]\n", " im_show = ax.imshow(im[0])\n", " ax.set_title(title, fontsize=25)\n", - " ax.axis('off')" + " ax.axis(\"off\")" ] }, { @@ -309,15 +305,19 @@ "source": [ "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", "loss_function = torch.nn.CrossEntropyLoss()\n", + "\n", + "\n", "def get_new_net():\n", " return DenseNet(\n", - " spatial_dims=2, \n", - " in_channels=1, \n", - " out_channels=num_classes, \n", - " init_features=2, \n", - " growth_rate=2, \n", - " block_config=(2,)\n", + " spatial_dims=2,\n", + " in_channels=1,\n", + " out_channels=num_classes,\n", + " init_features=2,\n", + " growth_rate=2,\n", + " block_config=(2,),\n", " ).to(device)\n", + "\n", + "\n", "model = get_new_net()" ] }, @@ -327,7 +327,7 @@ "source": [ "# Estimate optimal learning rate\n", "\n", - "Use MONAI's `LearningRateFinder` to get an initial estimate of a learning rate. Assume that it's in the range 1e-5, 1e0. If that weren't the case (which we'd notice in the plot), we could just try again over a larger/different window. \n", + "Use MONAI's `LearningRateFinder` to get an initial estimate of a learning rate. Assume that it's in the range 1e-5, 1e0. If that weren't the case (which we'd notice in the plot), we could just try again over a larger/different window.\n", "\n", "We can plot the results and extract the learning rate with the steepest gradient." ] @@ -370,7 +370,7 @@ "lr_finder = LearningRateFinder(model, optimizer, loss_function, device=device)\n", "lr_finder.range_test(train_loader, val_loader, end_lr=upper_lr, num_iter=20)\n", "steepest_lr, _ = lr_finder.get_steepest_gradient()\n", - "ax=plt.subplots(1, 1, figsize=(15,15), facecolor='white')[1]\n", + "ax = plt.subplots(1, 1, figsize=(15, 15), facecolor=\"white\")[1]\n", "_ = lr_finder.plot(ax=ax)" ] }, @@ -432,39 +432,41 @@ "\n", " if \"lr_lims\" in d:\n", " d[\"optimizer\"] = torch.optim.Adam(\n", - " d[\"model\"].parameters(), d[\"lr_lims\"][0])\n", + " d[\"model\"].parameters(), d[\"lr_lims\"][0]\n", + " )\n", " d[\"scheduler\"] = torch.optim.lr_scheduler.CyclicLR(\n", - " d[\"optimizer\"], base_lr=d[\"lr_lims\"][0], \n", - " max_lr=d[\"lr_lims\"][1], step_size_up=d[\"step\"],\n", + " d[\"optimizer\"],\n", + " base_lr=d[\"lr_lims\"][0],\n", + " max_lr=d[\"lr_lims\"][1],\n", + " step_size_up=d[\"step\"],\n", " cycle_momentum=False,\n", " )\n", " elif \"lr_lim\" in d:\n", - " d[\"optimizer\"] = torch.optim.Adam(\n", - " d[\"model\"].parameters(), d[\"lr_lim\"])\n", + " d[\"optimizer\"] = torch.optim.Adam(d[\"model\"].parameters(), d[\"lr_lim\"])\n", " else:\n", - " d[\"optimizer\"] = torch.optim.Adam(\n", - " d[\"model\"].parameters())\n", - " \n", + " d[\"optimizer\"] = torch.optim.Adam(d[\"model\"].parameters())\n", + "\n", "\n", "def train(max_epochs, axes, data):\n", " for d in data.keys():\n", " get_model_optimizer_scheduler(data[d])\n", "\n", " for q, i in enumerate([\"train\", \"auc\", \"acc\"]):\n", - " data[d][i] = {\"x\":[], \"y\":[]}\n", - " data[d][i][\"line\"], = axes[q].plot(\n", - " data[d][i][\"x\"], data[d][i][\"y\"], label=d)\n", + " data[d][i] = {\"x\": [], \"y\": []}\n", + " (data[d][i][\"line\"],) = axes[q].plot(\n", + " data[d][i][\"x\"], data[d][i][\"y\"], label=d\n", + " )\n", "\n", " val_interval = 1\n", - " \n", + "\n", " for epoch in plot_range(data, trange(max_epochs)):\n", - " \n", + "\n", " for d in data.keys():\n", " data[d][\"epoch_loss\"] = 0\n", " for batch_data in train_loader:\n", " inputs = batch_data[\"image\"].to(device)\n", " labels = batch_data[\"label\"].to(device)\n", - " \n", + "\n", " for d in data.keys():\n", " data[d][\"optimizer\"].zero_grad()\n", " outputs = data[d][\"model\"](inputs)\n", @@ -476,31 +478,36 @@ " data[d][\"epoch_loss\"] += loss.item()\n", " for d in data.keys():\n", " data[d][\"epoch_loss\"] /= len(train_loader)\n", - " data[d][\"train\"][\"x\"].append(epoch+1)\n", + " data[d][\"train\"][\"x\"].append(epoch + 1)\n", " data[d][\"train\"][\"y\"].append(data[d][\"epoch_loss\"])\n", "\n", " if (epoch + 1) % val_interval == 0:\n", " with eval_mode(*[data[d][\"model\"] for d in data.keys()]):\n", " for d in data:\n", - " data[d][\"y_pred\"] = torch.tensor([], dtype=torch.float32, device=device)\n", + " data[d][\"y_pred\"] = torch.tensor(\n", + " [], dtype=torch.float32, device=device\n", + " )\n", " y = torch.tensor([], dtype=torch.long, device=device)\n", " for val_data in val_loader:\n", " val_images = val_data[\"image\"].to(device)\n", " val_labels = val_data[\"label\"].to(device)\n", " for d in data:\n", " data[d][\"y_pred\"] = torch.cat(\n", - " [data[d][\"y_pred\"], data[d][\"model\"](val_images)], dim=0)\n", + " [data[d][\"y_pred\"], data[d][\"model\"](val_images)],\n", + " dim=0,\n", + " )\n", " y = torch.cat([y, val_labels], dim=0)\n", - " \n", + "\n", " for d in data:\n", " auc_metric = compute_roc_auc(\n", - " data[d][\"y_pred\"], y, to_onehot_y=True, softmax=True)\n", - " data[d][\"auc\"][\"x\"].append(epoch+1)\n", + " data[d][\"y_pred\"], y, to_onehot_y=True, softmax=True\n", + " )\n", + " data[d][\"auc\"][\"x\"].append(epoch + 1)\n", " data[d][\"auc\"][\"y\"].append(auc_metric)\n", - " \n", + "\n", " acc_value = torch.eq(data[d][\"y_pred\"].argmax(dim=1), y)\n", " acc_metric = acc_value.sum().item() / len(acc_value)\n", - " data[d][\"acc\"][\"x\"].append(epoch+1)\n", + " data[d][\"acc\"][\"x\"].append(epoch + 1)\n", " data[d][\"acc\"][\"y\"].append(acc_metric)" ] }, @@ -1494,12 +1501,12 @@ ], "source": [ "%matplotlib notebook\n", - "fig, axes=plt.subplots(3, 1, figsize=(10,10), facecolor='white')\n", + "fig, axes = plt.subplots(3, 1, figsize=(10, 10), facecolor=\"white\")\n", "for ax in axes:\n", - " ax.set_xlabel('Epoch')\n", - "axes[0].set_ylabel('Train loss')\n", - "axes[1].set_ylabel('AUC')\n", - "axes[2].set_ylabel('ACC')\n", + " ax.set_xlabel(\"Epoch\")\n", + "axes[0].set_ylabel(\"Train loss\")\n", + "axes[1].set_ylabel(\"AUC\")\n", + "axes[2].set_ylabel(\"ACC\")\n", "\n", "# In the paper referenced at the top of this notebook, a step\n", "# size of 8 times the number of iterations per epoch is suggested.\n", @@ -1509,7 +1516,10 @@ "data = {}\n", "data[\"Default LR\"] = {}\n", "data[\"Steepest LR\"] = {\"lr_lim\": steepest_lr}\n", - "data[\"Cyclical LR\"] = {\"lr_lims\": (0.8*steepest_lr, 1.2*steepest_lr), \"step\": step_size}\n", + "data[\"Cyclical LR\"] = {\n", + " \"lr_lims\": (0.8 * steepest_lr, 1.2 * steepest_lr),\n", + " \"step\": step_size,\n", + "}\n", "\n", "train(max_epochs, axes, data)" ]