diff --git a/modules/learning_rate.ipynb b/modules/learning_rate.ipynb new file mode 100644 index 0000000000..cf013467c9 --- /dev/null +++ b/modules/learning_rate.ipynb @@ -0,0 +1,1579 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Optimal learning rates\n", + "\n", + "In this tutorial, we'll use the MedNIST dataset to explore MONAI's `LearningRateFinder` and use it to get an initial estimate of a learning rate.\n", + "\n", + "We then employ one of Pytorch's cyclical learning rate schedulers to vary the learning rate over the course of the optimisation. This has been shown to give improved results: https://arxiv.org/abs/1506.01186. We'll compare this to the optimiser's (ADAM) default learning rate and the learning rate suggested by `LearningRateFinder`.\n", + "\n", + "This 2D classification is fairly easy, so to make it a little harder (and faster), we'll use a small network, only a subset of the images (~250 and ~25 for training and validation, respectively), we'll crop the images (from 64x64 to 20x20) and we won't use any random transformations. In a more difficult scenario, we probably wouldn't want to do any of these things.\n", + "\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Project-MONAI/tutorials/blob/master/modules/learning_rate.ipynb)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup environment" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "!python -c \"import monai\" || pip install -q monai[pillow, tqdm]\n", + "!python -c \"import matplotlib\" || pip install -q matplotlib" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup imports" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "MONAI version: 0.4.0+66.g028a965.dirty\n", + "Numpy version: 1.19.5\n", + "Pytorch version: 1.7.1\n", + "MONAI flags: HAS_EXT = False, USE_COMPILED = False\n", + "MONAI rev id: 028a965b208e2ef4265d51ec9c7ec8bd43b83359\n", + "\n", + "Optional dependencies:\n", + "Pytorch Ignite version: 0.4.2\n", + "Nibabel version: 3.2.1\n", + "scikit-image version: 0.18.1\n", + "Pillow version: 8.1.0\n", + "Tensorboard version: 2.4.0\n", + "gdown version: 3.12.2\n", + "TorchVision version: 0.8.2\n", + "ITK version: 5.1.2\n", + "tqdm version: 4.51.0\n", + "lmdb version: 1.0.0\n", + "psutil version: 5.8.0\n", + "\n", + "For details about installing the optional dependencies, please visit:\n", + " https://docs.monai.io/en/latest/installation.html#installing-the-recommended-dependencies\n", + "\n" + ] + } + ], + "source": [ + "# Copyright 2020 MONAI Consortium\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License.\n", + "\n", + "import os\n", + "import shutil\n", + "import tempfile\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import torch\n", + "from monai.apps import MedNISTDataset\n", + "from monai.config import print_config\n", + "from monai.metrics import compute_roc_auc\n", + "from monai.networks.nets import DenseNet\n", + "from monai.networks.utils import eval_mode\n", + "from monai.optimizers import LearningRateFinder\n", + "from monai.transforms import (\n", + " AddChanneld,\n", + " CenterSpatialCropd,\n", + " Compose,\n", + " LoadImaged,\n", + " ScaleIntensityd,\n", + " ToTensord,\n", + ")\n", + "from monai.utils import set_determinism\n", + "from torch.utils.data import DataLoader\n", + "from tqdm import trange\n", + "\n", + "print_config()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup data directory\n", + "\n", + "You can specify a directory with the `MONAI_DATA_DIRECTORY` environment variable.\n", + "This allows you to save results and reuse downloads.\n", + "If not specified a temporary directory will be used." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/home/rbrown/data/MONAI\n" + ] + } + ], + "source": [ + "directory = os.environ.get(\"MONAI_DATA_DIRECTORY\")\n", + "root_dir = tempfile.mkdtemp() if directory is None else directory\n", + "print(root_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set deterministic training for reproducibility" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "set_determinism(seed=0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define MONAI transforms and get dataset and data loader" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "transforms = Compose(\n", + " [\n", + " LoadImaged(keys=\"image\"),\n", + " AddChanneld(keys=\"image\"),\n", + " ScaleIntensityd(keys=\"image\"),\n", + " CenterSpatialCropd(keys=\"image\", roi_size=(20, 20)),\n", + " ToTensord(keys=\"image\"),\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Verified 'MedNIST.tar.gz', md5: 0bc7306e7427e00ad1c5526a6677552d.\n", + "file /home/rbrown/data/MONAI/MedNIST.tar.gz exists, skip downloading.\n", + "extracted file /home/rbrown/data/MONAI/MedNIST exists, skip extracting.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading dataset: 100%|██████████| 249/249 [00:00<00:00, 913.43it/s]\n", + "Loading dataset: 100%|██████████| 25/25 [00:00<00:00, 1242.09it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Verified 'MedNIST.tar.gz', md5: 0bc7306e7427e00ad1c5526a6677552d.\n", + "file /home/rbrown/data/MONAI/MedNIST.tar.gz exists, skip downloading.\n", + "extracted file /home/rbrown/data/MONAI/MedNIST exists, skip extracting.\n", + "249\n", + "25\n", + "torch.Size([1, 20, 20])\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# Set fraction of images used for testing to be very high, then don't use it. In this way, we can reduce the number\n", + "# of images in both train and val. Makes it faster and makes the training a little harder.\n", + "def get_data(section):\n", + " ds = MedNISTDataset(\n", + " root_dir=root_dir,\n", + " transform=transforms,\n", + " section=section,\n", + " download=True,\n", + " num_workers=10,\n", + " val_frac=0.0005,\n", + " test_frac=0.995,\n", + " )\n", + " loader = DataLoader(ds, batch_size=30, shuffle=True, num_workers=10)\n", + " return ds, loader\n", + "\n", + "\n", + "train_ds, train_loader = get_data(\"training\")\n", + "val_ds, val_loader = get_data(\"validation\")\n", + "\n", + "print(len(train_ds))\n", + "print(len(val_ds))\n", + "print(train_ds[0][\"image\"].shape)\n", + "num_classes = train_ds.get_num_classes()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Randomly pick images from the dataset to visualize and check" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA00AAANWCAYAAAAiEmd0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAACK2ElEQVR4nOzdd5xdVb3//88+fXrPhPSEBEjoTYKBoCAgRVEQAQMMCCIqKqj3Eq6RgCBYkPIDFQUpQapU4YIFDChFMPQWAiEJkJ7p5Zw5bf3+yHfOzWRmVvIOGRLG1/PxyOORZL/3Ouvssvb+nH3O3oFzzhkAAAAAoF+hLd0BAAAAANiaUTQBAAAAgAdFEwAAAAB4UDQBAAAAgAdFEwAAAAB4UDQBAAAAgAdFEz4yp5xyigVBYKeccsqW7goAAACw0SiatgIXXHCBBUFgQRBsMLt48eJC9qabbhr8zgHYotYdH9b/U1xcbJMmTbKGhgZ7+umnt3RXP1JXXnmlXXDBBfbSSy95c+PGjSssr6qqKkulUt78ihUrLBqNFub51Kc+1Sfz+OOP97s+IpGI1dTU2Cc/+Un78Y9/bGvWrNmovvFBEjA4crmc3XXXXXbyySfbdtttZ5WVlRaLxWzYsGG233772XnnnWevvfZar3mOOOIIC4LAxo0bZ+3t7d72TzjhBAuCwIYPH95nf+9vjAiFQlZeXm677LKLfetb37I33nhjs79nDB6KJgD4mKivry/8qaurs3Q6be+8847NmTPHpk2bZhdccMGW7uJH5sorr7QLL7xwg0XTulpaWuy+++7zZm6++WbLZrMb3WZVVVVhnZSVlVlTU5M988wzNnv2bJsyZYq98MILG90WgM3nX//6l02ZMsWOO+44u+WWW+ztt9+2rq4uKysrs8bGRnvqqafspz/9qe288852zDHHWDqdNjOz6667zqqqqmzJkiX2ve99b8D277rrLrvjjjvMzOy3v/2t1dbW9psrKSkpjBE1NTXW0dFhr776qv3617+23XbbzW644YbN/+YxKCiaAOBjYsWKFYU/q1atsu7ubnvyySdtzz33NDOzCy+88D/uitPGGjdunJmZ3Xjjjd5czxX8nvyG3HvvvYV10tzcbC0tLXb55ZdbLBaz1atX27HHHmuZTOZD9ByA6sEHH7RPfepTtmDBAqupqbFLL73UFixYYOl02hobGy2dTtu///1vmzlzppWXl9u9995rXV1dZmY2YsQIu/rqq83M7Prrr7dHHnmkT/srVqywb37zm2ZmdvLJJ9tRRx01YF9+8IMfFMaI1atXWzKZtPvvv99Gjx5tmUzGvv71r9tbb701CEsBmxtFEwB8TIXDYZs2bZrdf//9hf974IEHtlyHtmLHHHOMlZSU2GOPPWbvvfdev5mnn37a5s+fb+PHj7fp06dv0utUVFTYOeecY7NmzTIzs3fffdfmzp27yf0GoHn77bftxBNPtO7ubpsyZYq99NJLNnPmTJs0aVIhEw6Hba+99rJLL73UFi1a1KfomTFjhn3xi180M7PTTz/dWlpaek3/2te+Zo2NjTZq1Ci76qqrpP7F43E76qij7NZbbzUzs2w2azfffPMmvFN81CiahpAVK1bY1VdfbUcddZRNnjzZKioqrKioyCZOnGinn366vf766wPOu/5NGu6++2771Kc+ZdXV1VZcXGy77babXXXVVZbP5719uPXWW23atGlWVlZmFRUVts8++9jvfvc7c85tzrcKYB2jRo2ympoaMzPr6OjoNa3nN1E9v82555577JBDDrFhw4ZZKBTq85W+1atX26xZs2z33Xe3iooKSyQSNmHCBDvttNO8Y8i//vUvO/fcc23//fe3sWPHWiKRsMrKSps6dar97Gc/69OvdSWTSbvsssts3333taqqKotGo1ZXV2dTpkyxhoYGu+eee/q8nyVLlpiZ2amnntrndwP9KS0ttWOPPdby+fyAJyg9X5PpGQ8/jM9+9rOFv/uWG4DNa9asWdbW1maJRMLuu+8+GzVqlDdfXV1t999/v1VUVPT6/2uvvdZqa2tt2bJldtZZZxX+/4YbbrCHHnrIzNZeiaqsrNykfu63335WUlJiZowRHxeRLd0BbD4zZ84snAxEIhErLy+3rq4uW7hwoS1cuND+8Ic/2K233mrHHHOMt52zzjrLfvWrXxV+sJhMJu3ll1+2s88+21544YV+Tzicc3baaacVvvoSBIFVVlbavHnz7LnnnrO5c+daPB7f/G8agC1dutQaGxvNzGz77bcfMPf973/fLr/88sL+GQr1/tzs0UcftWOPPbbwqWo0GrVYLGaLFi2yRYsW2R/+8Ae77rrr7OSTT+7T9r777lv4e3FxsRUXF1tzc7M9++yz9uyzz9qcOXNs7ty5NmzYsF7ztbe32/77728vv/yyma0dOyoqKqylpcXWrFljb775pj3xxBOFcau0tNTq6+tt9erVls/nrby83IqKijZqOZ166ql200032U033WSzZs3qVRh1dXXZXXfdZaFQyBoaGmz27Nkb1eZA1v2gKJfLfai2AGyclStX2t13321ma68Wbbfddhs97/oflAwbNsx+85vf2LHHHls4d9pjjz3snHPOMTOzM844ww499NDN0m/GiI8HrjQNIRMnTrRf/OIX9uqrr1oymbTGxkbr7u621157zWbMmGHd3d3W0NBgy5YtG7CNP/3pT3bdddfZ5Zdfbs3Nzdbc3Gxr1qyx008/3czM5syZY3//+9/7zHf11VcXCqazzjrLVq1aZU1NTdbU1GQXXHCB3XnnnXxtCNjMcrmcPfPMM4WvkQwbNqzfgsbM7Pnnn7fLL7/czj33XFu5cqU1NTVZZ2ennXrqqWZm9uqrr9rnP/95a2lpsa997Wv2xhtvWDKZtI6ODluyZIl985vftHQ6baeddprNmzevT/uf+9zn7M4777Tly5dbZ2enNTU1WVdXl9177722/fbb2xtvvGFnnnlmn/muuuoqe/nll626utruueceSyaT1tzcbN3d3bZ06VKbM2eOHXLIIYV8z+8DRo8eXZh/3d96rVixYsDlNX36dJs4caK9++679sQTT/Sa9sc//tHa29vtwAMPtLFjx25gyW/Yn//858LfJ0yY8KHbA7Bhc+fOLXwjpmdc/DC+9KUv2fHHH29mZmeeeaadeOKJ1tbWZuPHj7df/vKXH6rtf/7zn9bZ2WlmjBEfGw5b3OzZs52ZOTNz9fX13j+1tbWF7I033ii9zhFHHOHMzF100UV9pjU0NGyw3T333NOZmTv99NN7/X8ymXTV1dXOzNxJJ53U77wzZ84stN/Q0CD1G/hPNtD4UFdX58LhsDMzV15e7mbMmOEWL17snf973/vegK9z4IEHOjNz55133oCZ73znO87M3FFHHSW9hw8++MDF43EXBIFbsmRJr2mHHXaYMzN3ySWXSG2OHTt2o8bBntzs2bOdc85dfPHFzszcySef3Cs3ffp0Z2bu1ltvdc7935h4wAEH9Glz7ty5hWU6d+7cXtNaWlrcFVdc4WKxWGGdJZNJb98YE4HNY9asWYV9c+nSpZulzcbGRjd8+PBCu0EQuMcff3yD8/Xke8aeHqlUyt1///1u9OjRhczzzz+/WfqKwcWVpq3MypUrvX829NwPnyOOOMLMzJ588skBM6NHj7aGhoZ+p33+8583M7NXXnml1///9a9/taamJjMzO//88/udd+bMmZZIJOQ+A/g/644Fq1evLnylo6ury1pbW23lypUDzhsKhezcc8/td9rixYvt73//u0UiEfvBD34wYBs9V7EeffRR6eskI0eOtF133dWcc33u7tfze4Dly5dvdHsfRkNDg4VCIbv77rsLz2BZuHCh/fOf/7TKyko7+uijpfaOPvpoGz58uA0fPtyqqqqssrLSzjnnHEun01ZWVmZ33nknYx/wEen5mrLZ2t8qbQ7V1dX2la98pfDv448/3g444ICNnv+yyy4rjBF1dXVWVFRkX/jCF+z9998vTN9jjz02S18xuCiatjLOOe+fRYsWeed/+eWX7Zvf/KbtsssuVl5ebqFQqPDj6J7bY37wwQcDzr/33nsP+APoESNGmJkVCqQePV/VGT16tE2cOLHfeSsqKgq3RQawadYfD5LJpL344ovW0NBgDz30kE2fPr3XnfTWNXHixD6/J+rx1FNPmZlZPp+3KVOmFA7w6//publBZ2dnr5OTnnlvu+02+/znP29jxoyxoqKiXjdneO6558ys7/hz5JFHmpnZNddcYyeccILdf//9H+rDoQ0ZNWqUHXzwwdbV1WV33nmnma29Dblzzo4//ni5wGlubi4UsuveYWu33Xazt956Szq5ArD1mT9/vv36178u/PuRRx7x/sxhfZ2dnb0++Hb/7/eOVVVV9tRTT9n3v//9zd5nDA6KpiHkmmuusT322MN+85vf2KuvvmodHR1WUVFReKhaeXm5mVnhO7T9KSsrG3BaJLL2viHrP3Nk1apVZrb202SfDd3BBoAmkUjYbrvtZtdff7198YtftO7ubjvllFOsra2tT3aggsnMCicA+Xx+o6909zzTpOfvn/nMZ2zGjBn24IMP2vvvv2/5fN6qq6sL4080GjWzvuPPV77yFfvud79rQRDYHXfcYV/84hetrq7OJk2aZN/61rfs+eef/1DLqD9f/epXzWxtsZTP523OnDm9/l8xd+7cQhHb2NhoDz30UOE2x9/4xjf4gTfwEeq5i6hZ3w94N0Uul7OGhgZLpVK200472ejRowu/+9xYs2fPLowRnZ2d9txzz9nnPvc5a25utlNOOUUqwLBlUTQNEW+++aadffbZls/n7dhjj7XnnnvOUqmUNTc3F34cffnll5uZcftvYAjqOYi3trbaww8/3Gd6OBwecN6eE/v6+voNXu3u+bPuw19/8pOf2Ny5c62oqMiuuOIKW7JkiaVSKWtsbCyMP/vss4+Z9T/+XHnllfbWW2/ZJZdcYocddphVVlbaO++8Y7/+9a9tr732srPPPvtDLJm+jjrqKKuqqrKnn37afvWrX9n7779vO+64o+29994fqt3q6mo74ogjbO7cuVZfX28PPPCAXXTRRZup1wA2ZMcddyz8/cUXX/zQ7f30pz+15557zqLRqM2ZM8euu+46MzN7+OGHCw/CVhQXF9vee+9t999/vx100EH29ttv24wZMzgv+5igaBoi7r77bsvlcjZ58mS74447bO+997ZYLNYr47ur1IfR8wn20qVLvbkNTQew6da949uGvsa7vuHDh5uZ2Zo1a7xXogdyxx13mNna3zSeffbZNmbMmD5f893Q+DNx4kQ777zz7OGHH7bGxkZ75pln7Atf+IKZrb1D3p/+9Ce5XwOJx+OF3yj0/Iar5y6Cm8OwYcPs0ksvNbO1J12LFy/ebG0DGNinP/3pwqMU7rvvvg/V1iuvvGI//vGPzcwKz6479NBDC3cTPuecczb5vCYUCtlvfvMbi0Qi9vjjjxfGUGzdKJqGiJ4fFO666659nr3S49FHHx2U195rr70KfVi4cGG/mba2tkH5mg2Atdb9rVDPAxM31rRp08xs7RWnRx55RH7tnvFn991373f64sWL7Z133tno9kKhkE2dOtXuvvtuGzNmjJmZ/e1vf+uTMdv0K+c9X8VLp9MWiUTspJNO2qR2BnLyySfbtttua93d3QPeIAfA5lVfX194ptttt91mCxYs2Oh51x1LMpmMNTQ0WDqdtj322MP+53/+pzDtl7/8pY0ZM8ZaWlrsjDPO2OS+Tpo0yWbMmGFma4uybDa7yW3ho0HRNET0PMn61Vdf7fck4pFHHrHHH398UF774IMPtqqqKjOzAb+K8vOf/9ySyeSgvD6AtScIPXo+yNhYkyZNsk996lNmZvbDH/7QWltbvfn1fyvQM/70PKB2fTNnzhywre7u7gGnhcPhwhXz9T8M6vmN5ro3X1DsscceduGFF9r3v/99u+KKK7y/+doU4XC4cLfC2267zebPn79Z2wfQv4svvthKS0stmUza0UcfvcGrQc3NzXbMMcf0Gvcuuugie+mllywej9ucOXMKv+k2Wzv2XH/99Wa29mt6Pc+o3BQzZ860UChk77777odqBx8NiqYhoueuVq+//rp961vfKpzUdHZ22m9/+1v70pe+1OsHkptTUVGR/ehHPzIzs5tvvtnOPvvswp212tra7KKLLrJLLrmkcGthAJvPihUrbNasWXbzzTebmdnUqVNt3333ldu5+uqrrbS01BYsWGBTp061Bx54wFKpVGH60qVL7ZZbbrGDDjqoz63Le8afiy++2O69997CJ6aLFi2yr3zlK3bXXXcVPlhZ3z777GPf+c537PHHH+/11cBly5bZt7/97cIVqsMPP7zXfDvttJOZrf1qcnNzs/x+zdZ+nfCyyy6zs846a5Pm35CGhgYbOXKk5XI5u+CCCwblNQD0tt1229ktt9xisVjMXn/9ddttt93sZz/7Wa+r3blczl588UU7//zzbcKECXbvvfcWpj3//POFr9deeOGFvX4n1ePggw8uXGX6MF/T22GHHQqPObj44ostnU5vUjv4iAzyc6CwEdZ9+OSGLFq0aMCH0B5//PGFaWbmKisrCw+/3HPPPd3VV1/tzMyNHTu2T7s9D3L0PWTxxhtvHHD+XC7nTjrppMJrh0IhV1VVVXj9448/fqNeA0BvvodfV1RU9Nrnd9555z4PdOyZv7+HtK7vySef7PUQx3A47GpqalxRUVGv11n/AdeLFy929fX1hemRSKRX3y655BJ3wAEH9Pugx54HvNr/e2hkZWWlKykp6fV655xzTp++PvHEEy4IgkI/t9lmGzd27Ng+49P6D7fdWJv6cNv1XXHFFYX39sorr/TbN8ZEYPN78skn3cSJE3uNJbFYzFVXV7tQKNRr3DnhhBNcOp12qVTK7bjjjs7M3NSpU102mx2w/ba2tsI+fNhhh/WZ3tP+hsaeF154oZC95pprPuzbxiDiStMQcuutt9qVV15pu+yyi8XjccvlcrbzzjvbpZdeak899ZSVlpYO2muHQiGbM2eOzZkzx6ZOnWpFRUWWzWZtjz32sGuvvbbXV4cAbJr1bwHe1dVlw4cPt0MPPdSuu+46mzdvXuF5apti2rRptmDBArvsssts+vTpVllZaS0tLRYOh23y5Ml24oknFsaZdY0dO9bmzZtnp512WuH1E4mEHXnkkfaXv/zFzjvvvAFf84477rALL7zQDjroIBs/fryl02nLZDI2duxYO+644+yxxx4r3PlzXdOnT7f//d//tc985jNWWVlpK1eutCVLltiSJUs2+f0PhjPOOMNqa2vNOWezZ8/e0t0B/mNMmzbN5s+fb7fffrvNmDHDJk6caIlEwtrb2626utr2228/++EPf2hvvvmm3XbbbRaNRu3888+3119/3YqKiuymm27y3nW0rKzMfv/731sQBPbII4/YDTfcsEn93H333QtX0i+55JJeV/ixdQmc4z6HAAAAADAQrjQBAAAAgAdFEwAAAAB4UDQBAAAAgAdFEwAAAAB4RHwTD4nPkBoLot7m+srntfz246X42ydXSPmSbf0PdFxf27IyKT/6r1Lcih96Qcq7vHZPjyAUDGr74dISKd94VN9nIfis/szAD8Xsj0tq22fdvwa+a06/+b8skvLZ5SukvIW0/vwte4fW/sfMwaFjt3QXAGyiv+X/uKW7MGgOG6E996tzjzFSfvk07ViWi2vH7lyxeG4WEc8NSrJSvq66TcqPLNXO5YYn2qV8ZbRLypeFtbvhVYS19qNBTsq35xNSviOn5VP5qJTPOO3cRtWd1/aX/2/32wecxpUmAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCIbM7GgojWXK6jU8qHO1NSvmhllZRvqyiR8hZ1Urx1bFTKl9UPk/LZpcukfBBLSHmX0pZ/UFYq5ZPDAq39kLb8nRa3vLa6zMJhcQZNENKWDwDgo5ddsVLKt42ZIOWdeKhR8xbRDpaR0oyUr61ql/Ljy5ukfHWsS8pHQjkpPyqm9acsrJ07lYWSUj4WaP1vy2vnfk2Bdi6XctrJU7d8sqVZnq7YbG1xpQkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCK+iaGSIqkxl+qW8kE4LOWtsUWKhzLbSPnSmi4pn057F18fHWO195sdWSPlbekyLR+NavlUSoq7ilKt+Ton5fNZseYPa+2HMlrzlsuJM4gCPuNYV2TkCCmfW7lKyjtxfarjmctmpfygC2n9DyXiUj6fTEp5c9r+KhPfbxDVxnvLa/13mbTW/mALAikeKi4epI58/OQP2F3Kpyu0Ze0CcdtSDx3RvBQPhbV8d0bblxpTJVI+HtbG1mGRdilfFtbOhcpC2tgXCwb5XGKQdee1c8uM08biqLh8aqMdUt6HszAAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8Ij4Jianbic1Fvvzv6V8qKxMylsokOKlS/NSviWvtb/PmMVS/qmlU6R8uiou5ROJhJTPt7dLeVWmpkTLV2Wl/M4Tlkr515aMkPL5sHf36MOl01I+iIjtZzNSfqhrv0HbP3LX7ynly+57XspboH0Gpa7/IK69X3NOi2e0/S+f6pbyan9C4njm1PfbrfXfdeek/MeeuDzznZ2D1JGPn2yROLaLH1+HtEON5WNa3mW2rs/TwyHtXC4SaPlooO3bZaGklI8NcvuqRKCdS6j5snBKyqfyUSmvyrjwZmtr69ozAAAAAGArQ9EEAAAAAB4UTQAAAADgQdEEAAAAAB4UTQAAAADgQdEEAAAAAB4UTQAAAADgQdEEAAAAAB4UTQAAAADgQdEEAAAAAB4UTQAAAADgEfFNbNwxKjU2+s3RUj675H0pH0okpHysLSflkx1xKb93+RIp/8+K7aR81zBt+RfX1kh5t3K1lLdQIMUzpd7N60MbU9ws5V/tGiPlQ1kpbhZoy8flnda+E/ND3L1T/iDlTzjzeCkfmlsh5XNrGqW8ymXVDfLjLZ9KbekufDjieBAqLdXaz2nHt3xXl5QPV2rbvwV8BtsjXR6W8qGM1n4op21bQW7rOnbEo9pYFgtp23pJpFvKl4UHd6xJBNoKTgTa8kkE2vJJOa39qNif8pC2PFNOO9dVteW12sGHUQ4AAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCK+iV3b5KXGVn5mlJQf9kCXlM+taZTykY6Mll9aJOU/2KVKypfVdEr5bHGllHdlxVp+qbZ8whXlUj6XEGvykLa9xUNa/0NdWn+iSSflLaf13/I5LY9e3siUSPmHdrhXyh+69zelfNGTaSlvGW37zafF7T0WHdz2S7TxRpVvbx/U9i0IBrd90aC/X1GupVXKR0aOGKSefPykS7VtKxAPHbmEdmzKa0OBBQnt2BSPa2NHIpKV8sURbWyNh7T21XOJj7tE8PE+94gG2vrdnLjSBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeEd/Eqh2apMaSq2ulfMe0baV80QONUj62rEXKlywtlvIvNI3W2o+npXxXcSDlc2UJKW/OaflwWGteLcnF7izsqJPy4W5teUY7s1LeZbW8BVp/LOAzjnWNDndI+XhQKuW7hnmHx77td2j9kddnPqflLTqo7efb27X2RaFibTzOd3VpLyCOf0FE2x5cTl1fGrU/4Trt+JxduVrKd+42UsoPZbmYlu+u0bbFbGleyrsibVssKe2W8pVFKSlfEdPyJRHt3CkaaO83EWSkfExsPxpo5wYJuX1te8jIJ2dbl4zTxr7N6eO95AAAAABgkFE0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeER8E/epXyI19vCIaimfaAxL+dKJ46W8C2vth7udlF/RViblh5V1SPmVE3JSvvSDYilf0TxBymfrtPfbPkpb/pHSbim/Olki5V0gxS3UnZfy+c4u7QWctr2Zaf0Z6ipD2mc+72W1/a+7Wtxg1PXptP1b5eTtSxNEY2Lee7jpI59MSvlBJx5PLDfI6zeblfLZ5SsGqSdrvXcYn8H2yMW1sSNdl9FeIKzt25EibVspimn9KYpo+UhI2zfiIa3/iZDWn2ig9acu3C7lE2L7ZeLyiUpps/ZBPpdIOW2s19vX3nFbPrHZXptRDgAAAAA8KJoAAAAAwIOiCQAAAAA8KJoAAAAAwIOiCQAAAAA8KJoAAAAAwIOiCQAAAAA8KJoAAAAAwIOiCQAAAAA8KJoAAAAAwIOiCQAAAAA8Ir6Jc9+bKDW2026LpfzClROkfNtuw6R8uNtJ+UxpIOVDgdZ+TaJTyreNb5byTY21Uj6bqJfy6XJt+bTtmJHyu45cLuXT+bCUX1aVlfK5Iq39aD4n5VVBWOvPUPdEShsPDkisGqSebJpwZYWUzydTUj5UWiLlc93dUt5l0oOat0Abb4J4XMq7tNh/cfmowuXlUt5ltfEs39Ul5YNoTMoXbdMh5YeybLE4Q1g7l7CQlg+F81r7H3PRQDsWJ0LauUpCbD8aaMs/KqV1CfHcNSNunoMt7bbcuRBXmgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAI+KdOq9CaqzxgKSUz+zSIeVbM6VSPtbmpHwuIcWttlh7vyWRtJQvT6Sk/JoxWvtmMS2uLU6LlGakfGdW609jZ7GUt7D2BrrLw1K+pKpKyueam6V8EPHvrv9pOvNxKR8PtOWXmqqNT8kvfELKr9xL276cFrdQOpDyI57Sxo+id9ZIedfcKuVzrW1SPohp44fr7pbyqtAuO0j5JUdWS/mu8dr4GqS1z0hrxmrj0+7VS6X8UJYt1Y41oTZtbMqXZ7V8Tlv3ubw2dqTz4uA0yOIhbd+IBtryTImDcTTIS3mt92ZRMZ9y2vpV3+9giwU5KZ9y6hIaGFeaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMAj4p2Y1Bprai+R8jtss0rKv/0JJ+XTC0qlfJDT2n9vWY2Uz+S1GjXvAilvWS3fXZWX8qGMFLcRNa1SviKmbXBLmyukfJAMS/l0mbj8B5nLZrd0F7YqYdO237s6Rkn543d4XsrfccKeUj70pjY+5cLa+BROSXFb/Dlt/3CRYVI+yNZL+UiHNl6G0tr+Gm2X4la8Ulv+nSO1/qQrtfaL341K+eSInJSPRbTxZlFbtZQfysJd2rrPxbV1b93avuESW9exTFUS6ZbyiUA8WRGlnPfUuY+6IC3ly4LBvp6hHTsTYa3/7XltrFmd05anKiOuLx+uNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAR8Q3MdrupMbaWuPaqw/T4qVF3VJ+TXWRlI+0h6V8bV2blJ9at1jKd+S05dmeTEj5rg9KpXyuPC/ld6peLuXrYu1SPhRo2+e/V20r5S0Q4yXFWr5de78um5XyQ91lvzheyle9nZLyoSdelPLj429p7VdWSPncylVSXhUZO1rKv3+Mlu/YRRu/c7molJ/06/elfPb9D6R8ZMI4KW/7bSPFt3kyKeVDT74k5SPbDJfyuZG1Uj5TLh7/D9HiHye5hHZskkW09sORnJQvjaelfEVMG1urYtq2XhbW2o8G2vvNOO+pcB+VIW0sKwu06xOlIXFfUuW1/pvTzv1U0WBwz202Z/tcaQIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAj4hvYqY8kBqLrolK+TeKtpHy44evkfL5UVLcWt+qlvLdGe/i66Mm2inlq6JdUr66RMt3hUqlvEXzUjwS5KR8Jh+W8mWRbikfSmmfEeQj2vafq6uU8u6DpVI+VFYm5Yc8p8XbxiSkfPUuO0j5UJu2/7l4TMpHirT+d06ul/LpvLZAY21avuT1uJQPp6S4dU0ZLuU7PjNayjfupr1fV5KR8sm6Yik/sn2ylHcrGqV863ba8SHRpI33Q1kopx070sXasgsSWj4e17bF0ph2bK2Ka2NfiXjsjornEqpokBXz2rlQNNDObdR8xn28972M086ltySuNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAR8Q3sWsbJzUW6tZePJcKS/nty1dJ+fLqpJS/fcU+Ur6jPSHll3VXSvlQkJfynemolHchbf1aVquxWzJFUr40rG1AzWmtfVW6Usu3TyqT8mWvene/PvIdHVJ+qMvHtHzForSUd2++K+WzGa39yDbDpXzn7qOlfPMkbTyoeV3b/6rma+Nr67ba/tpdGUj5JYdr+1N0eKeUL3umVMqPvGGBlM+1tUn5oLZGyudbWqV89VxtvHdV5VJ+KMuHxWNrUU6Kx4syUr4kro1NxRFxLBPPVeKhrJjX3m9CzA+2jNOWT8Zp20O3095vSu2PlDZLOe3cPuW0Y5Uq47Rjgw9XmgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAI+KbmK3NaK11D24N1pQulvKV0S4pX1yt5btaiqR8KMhL+XgoK+XzeXH5B1rcQk6Kjy9ulPIHlb0u5d9oO1TK5yu05ZksFpen8+5OfZTtur2UD15+S8oPddvcv0jKZ5evkPJBsTbehIoSUt4lk1I+1pyW8iMea5fy+dfmS3lV7coJ2gzNrVI8O2mUlF/5iVIpH8pJcQtKtO3H2tqkuOvolPJBRBuf1P3F1PwQlqnSjvVBWDu2RqPasSwa0vqjyjrtWNmZjUv57khUyqsSgXaumwjE9RWEpfzWJuW0k8WUeC6UdtrySTlte2jPacdmH640AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIBHxDfxwCnzpcYef2eS9uqr4lL8raY6KT+6uFnK15R2SfnkB2VSPh7KSvn9y96S8n8PictfFETyUn7Hog+k/K6xpJRvShVL+VA8J+XDxRkpr/XebMkRFVK+aO+9xVcY2rLLV0j5cG2NlM+3tkv5oLhIyufWNEr5yOuLtPbb2qR8qFjbn/Jd2niZX6yNB8GUbaV8+LV3pfzINdrxpGWPYVI+N0Lb3kzcnvNpbXxShRIJKZ/v7h6kngx9YfHYOtiy+bCU7855TyX7iATa+804rT+qWKCdGwy2aDC47zejLn/T8olAO9dVl//qbLmUT7molPfhShMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeEQ2Z2Nj6puk/JLW4VK+oysh5UfFmqX89PqclL/trWFSfnmqQsrvXLNKypfF01K+KeSkfBDW8uWhlJRvzWvLv6mzWMqHxPcbi2e19qvzUj4Vj2n5MYGUH+qCqLb8cmsapXwooY03bkSdlM/tNFbKh1d3Snl7vU2K51PdUj5UViblLZOR4kGnNn5oe7eZW7pCypeXF0n5UFtSyudCYSkfLi/V2m9plfIW1cZXS2nraygLMoM7Vufz2ufdGTE/2OIh7dgaDbRzg2igta9KOW39Zpx2bpBx2vvtdtrYmhL7o77frU13PrrZ2tq69iQAAAAA2MpQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhEfBPfbq2TGhtT1izll5TUSvns8mIpv3y7Cil/UPnrUv62yr2l/OpkqZSvC3tXTx/bV66U8u8lqqV8OJKT8mWhpJRPuUDKR8Naf6IlYl58vy6m9T+f0z6zyIvLZ6hzmbSUD6IxKZ/v7pby9sp8KR7bfqKUD1rbpXwokZDy+VRKygcRbXzKtWv9j6S09WuD3J/wwg+kvEtnpLzltfHG4nEpLm//4vJRt7ehLB/Pa/nusJQPhbX2szGtfVU8nJXykZC2rcdD4r40yDJOO3annHju4bT3m3Ha9qAuTfX9tue1saBNzCe24PbAlSYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8Ij4Ji5dVSk1dtDwt6T80pEVUn5J+3Ap/+iy7aX816qflvLjR6yR8ul8WMqHxJp2z7IlUv7J0glSPpfT+rN7LCvl38lqyycInJQfVdEq5VtSRVI+lfXuTn04F0j5r+38lJTHelxeiodKS6V8vqNDyucWLJTy4YpyKR+qqdbysaiUz1VryyddN1HKR95aJeVdMinlLdD2PwuL43eFNn6ocqsbpXyoKCHlXSYt5YNEXMoPZUFGO1a6opyUD4W0Y59KPVdRxUPaucFgSwQZKR8NtGPJYIsG2vYWFY+Fg01d/qp4aPO1z5UmAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCI+Ca65pjU2MutI6X8l0a+IOWvavy0lG9uLZHyrfmolP/KyOek/AOrdpPyf01WS/ld4u9L+bqyTim/T91iKV8c0rafvOuW8hOqGqX88s5yKR8L56S8c4GU//z2r0j5sbE1Uh69BRHvcNeHSyalfLiyUspbSNteglJtPGvbY4SUb9ohLOWj2vBhta+mtBmy4v6XzWrti4JEQpshph1PrE3bHiwvLp9ubXxVuVx+UNv/OHFF2rpJlKalfGVpl5Qvj2nrvjKmjX0VUS0fDbTlM9hSTttXE4ET89r1iXig9afbZaT8YFOXp6o9VyTlO3Li2O3BlSYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8Aicc25LdwIAAAAAtlZcaQIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omfGROOeUUC4LATjnllC3dFQAAgK0a501bF4qmrcAFF1xgQRBYEAQbzC5evLiQvemmmwa/cwC2qHXHh/X/FBcX26RJk6yhocGefvrpLd3Vj9SVV15pF1xwgb300kve3Lhx4wrLq6qqylKplDe/YsUKi0ajhXk+9alP9ck8/vjj/a6PSCRiNTU19slPftJ+/OMf25o1azaqb5wQARrOm7AlUDQBwMdEfX194U9dXZ2l02l75513bM6cOTZt2jS74IILtnQXPzJXXnmlXXjhhRssmtbV0tJi9913nzdz8803Wzab3eg2q6qqCuukrKzMmpqa7JlnnrHZs2fblClT7IUXXtjotgAAWy+KJgD4mFixYkXhz6pVq6y7u9uefPJJ23PPPc3M7MILL/yPu+K0scaNG2dmZjfeeKM31/NJdE9+Q+69997COmlubraWlha7/PLLLRaL2erVq+3YY4+1TCbzIXoOANgaUDQBwMdUOBy2adOm2f3331/4vwceeGDLdWgrdswxx1hJSYk99thj9t577/Wbefrpp23+/Pk2fvx4mz59+ia9TkVFhZ1zzjk2a9YsMzN79913be7cuZvcbwDA1oGiaQhZsWKFXX311XbUUUfZ5MmTraKiwoqKimzixIl2+umn2+uvvz7gvOv/2PDuu++2T33qU1ZdXW3FxcW222672VVXXWX5fN7bh1tvvdWmTZtmZWVlVlFRYfvss4/97ne/M+fc5nyrANYxatQoq6mpMTOzjo6OXtN6vvvf89uce+65xw455BAbNmyYhUKhPl/pW716tc2aNct23313q6iosEQiYRMmTLDTTjvNO4b861//snPPPdf2339/Gzt2rCUSCausrLSpU6faz372sz79WlcymbTLLrvM9t13X6uqqrJoNGp1dXU2ZcoUa2hosHvuuafP+1myZImZmZ166ql9flvUn9LSUjv22GMtn8/bzTff3G/mhhtuMLP/Gw8/jM9+9rOFv/uWG4Ath/MmKCJbugPYfGbOnFk4GYhEIlZeXm5dXV22cOFCW7hwof3hD3+wW2+91Y455hhvO2eddZb96le/slAoZOXl5ZZMJu3ll1+2s88+21544YV+Tzicc3baaacVvvoSBIFVVlbavHnz7LnnnrO5c+daPB7f/G8agC1dutQaGxvNzGz77bcfMPf973/fLr/88sL+GQr1/tzs0UcftWOPPdZaWlrMzCwajVosFrNFixbZokWL7A9/+INdd911dvLJJ/dpe9999y38vbi42IqLi625udmeffZZe/bZZ23OnDk2d+5cGzZsWK/52tvbbf/997eXX37ZzNaOHRUVFdbS0mJr1qyxN99805544onCuFVaWmr19fW2evVqy+fzVl5ebkVFRRu1nE499VS76aab7KabbrJZs2b1Koy6urrsrrvuslAoZA0NDTZ79uyNanMg657w5HK5D9UWgMHBeRMUXGkaQiZOnGi/+MUv7NVXX7VkMmmNjY3W3d1tr732ms2YMcO6u7utoaHBli1bNmAbf/rTn+y6666zyy+/3Jqbm625udnWrFljp59+upmZzZkzx/7+97/3me/qq68u7PhnnXWWrVq1ypqamqypqckuuOACu/POO/naELCZ5XI5e+aZZ+yLX/yimZkNGzas34LGzOz555+3yy+/3M4991xbuXKlNTU1WWdnp5166qlmZvbqq6/a5z//eWtpabGvfe1r9sYbb1gymbSOjg5bsmSJffOb37R0Om2nnXaazZs3r0/7n/vc5+zOO++05cuXW2dnpzU1NVlXV5fde++9tv3229sbb7xhZ555Zp/5rrrqKnv55Zeturra7rnnHksmk9bc3Gzd3d22dOlSmzNnjh1yyCGF/A9+8ANbsWKFjR49ujD/ur/1WrFixYDLa/r06TZx4kR799137Yknnug17Y9//KO1t7fbgQceaGPHjt3Akt+wP//5z4W/T5gw4UO3B2Dz47wJEoctbvbs2c7MnJm5+vp675/a2tpC9sYbb5Re54gjjnBm5i666KI+0xoaGjbY7p577unMzJ1++um9/j+ZTLrq6mpnZu6kk07qd96ZM2cW2m9oaJD6DfwnG2h8qKurc+Fw2JmZKy8vdzNmzHCLFy/2zv+9731vwNc58MADnZm58847b8DMd77zHWdm7qijjpLewwcffODi8bgLgsAtWbKk17TDDjvMmZm75JJLpDbHjh27UeNgT2727NnOOecuvvhiZ2bu5JNP7pWbPn26MzN36623Ouf+b0w84IAD+rQ5d+7cwjKdO3dur2ktLS3uiiuucLFYrLDOksmkt2+MiYCG8yZsCVxp2sqsXLnS+2dDz/3wOeKII8zM7MknnxwwM3r0aGtoaOh32uc//3kzM3vllVd6/f9f//pXa2pqMjOz888/v995Z86caYlEQu4zgP+z7liwevXqwte+urq6rLW11VauXDngvKFQyM4999x+py1evNj+/ve/WyQSsR/84AcDttFzFevRRx+VvnI2cuRI23XXXc051+fufpWVlWZmtnz58o1u78NoaGiwUChkd999t7W3t5uZ2cKFC+2f//ynVVZW2tFHHy21d/TRR9vw4cNt+PDhVlVVZZWVlXbOOedYOp22srIyu/POOxn7gEHEeRM+KhRNWxnnnPfPokWLvPO//PLL9s1vftN22WUXKy8vt1AoVPhx9De/+U0zM/vggw8GnH/vvfce8AfQI0aMMDMr7Og9er6qM3r0aJs4cWK/81ZUVBRuiwxg06w/HiSTSXvxxRetoaHBHnroIZs+fXqvO+mta+LEiX1+T9TjqaeeMjOzfD5vU6ZMKRQB6//publBZ2dn4TdUPfL5vN122232+c9/3saMGWNFRUW9bs7w3HPPmVnf8efII480M7NrrrnGTjjhBLv//vs/1EnOhowaNcoOPvhg6+rqsjvvvNPM1t6G3Dlnxx9/vHyS0tzcXDg56/ktmJnZbrvtZm+99ZYdcMABm7P7ANbDeRM+KtwIYgi55ppr7Lvf/W7hTi09P6ju+SFhMpm0trY26+zsHLCNsrKyAadFIms3l/WfObJq1SozW/tpss+oUaM2/CYAbLREImG77babXX/99dbU1GT33XefnXLKKfbee+9ZeXl5r+xABZOZFb6vn8/nvVer1tXV1dXr70ceeWSvW2vHYjGrrq62aDRqZmtPGjKZTJ/x5ytf+Yo999xzdvXVV9sdd9xhd9xxh5mtLfIOOeQQ++pXv7rZTxy++tWv2l/+8he78cYb7atf/arNmTOn8P+quXPnFu5M2PNg2//+7/+2l156yb7xjW/YPffcY+FweHN2H8BmwnkTFFxpGiLefPNNO/vssy2fz9uxxx5rzz33nKVSKWtubi78OPryyy83M+M2lsAQ9LWvfc3MzFpbW+3hhx/uM9134t7zVbv6+voNfmrb82fdh7/+5Cc/sblz51pRUZFdccUVtmTJEkulUtbY2FgYf/bZZx8z63/8ufLKK+2tt96ySy65xA477DCrrKy0d955x37961/bXnvtZWefffaHWDJ9HXXUUVZVVWVPP/20/epXv7L333/fdtxxR9t7770/VLvV1dV2xBFH2Ny5c62+vt4eeOABu+iiizZTrwFsTpw3QUXRNETcfffdlsvlbPLkyXbHHXfY3nvvbbFYrFfGd1epD6PnE+ylS5d6cxuaDmDTrXvHtw19HWV9w4cPNzOzNWvWeD9RHUjP1aHzzz/fzj77bBszZkyfr6tsaPyZOHGinXfeefbwww9bY2OjPfPMM/aFL3zBzNbeIe9Pf/qT3K+BxONx+8pXvmJmVvgNV89dBDeHYcOG2aWXXmpmZj/96U9t8eLFm61tAJsH501QUTQNEe+//76Zme266659nr3S49FHHx2U195rr70KfVi4cGG/mba2Nnv++ecH5fUB9P7OfUlJiTTvtGnTzGztFadHHnlEfu2e8Wf33Xfvd/rixYvtnXfe2ej2QqGQTZ061e6++24bM2aMmZn97W9/65Mx2/RPgHu+ipdOpy0SidhJJ520Se0M5OSTT7Ztt93Wuru7B/yhN4Ath/MmqCiahoiKigozW/uslf5OIh555BF7/PHHB+W1Dz74YKuqqjIzG/CrKD//+c8tmUwOyusDMLvtttsKf+85IG+sSZMmFX6X88Mf/tBaW1u9+fV/1Nwz/vQ8oHZ9M2fOHLCt7u7uAaeFw+HCJ7/rn9T0/GZr3ZsvKPbYYw+78MIL7fvf/75dccUV3t98bYpwOFy4W+Ftt91m8+fP36ztA/hwOG+CiqJpiOi5q9Xrr79u3/rWtwonNZ2dnfbb3/7WvvSlL1lNTc2gvHZRUZH96Ec/MjOzm2++2c4+++zCnbXa2trsoosusksuuaRwa2EAm8+KFSts1qxZhSfOT5061fbdd1+5nauvvtpKS0ttwYIFNnXqVHvggQcslUoVpi9dutRuueUWO+igg/rcurxn/Ln44ovt3nvvtWw2a2Zrvyb4la98xe66667CCcL69tlnH/vOd75jjz/+eK+vBi5btsy+/e1vF65QHX744b3m22mnncxs7Vdsmpub5fdrtvbrhJdddpmdddZZmzT/hjQ0NNjIkSMtl8vZBRdcMCivAWDTcN4EFUXTEHHQQQfZ8ccfb2Zmv/nNb6ympsaqqqqsoqLCzjzzTJs8efKgHrS/+93vFr7ectVVV9mwYcOsurraqqur7fzzz7fjjjvOjjrqqEF7feA/wfq3AK+srLRtttnGfvKTn5iZ2c4772z33HPPgLe/9dlpp53sz3/+sw0fPtzmz59vX/jCF6y0tNRqa2utuLjYRo0aZSeffHK/T7a/+OKLrb6+3trb2+2YY46xoqIiq6ystAkTJtjtt99uP/nJT2yXXXbp93VbWlrs6quvtk9/+tNWVlZmVVVVVlpaaiNHjrRrrrnGzMzOOeccO/TQQ3vNd8YZZ1gQBPb0009bXV2djRgxwsaNG9frBhVbWiwWK/xm6q677rJXX311C/cIQA/Om6CiaBpCbr31Vrvyyittl112sXg8brlcznbeeWe79NJL7amnnrLS0tJBe+1QKGRz5syxOXPm2NSpU62oqMiy2aztsccedu211/b66hCATbP+Qxu7urps+PDhduihh9p1111n8+bNKzwXZFNMmzbNFixYYJdddplNnz7dKisrraWlxcLhsE2ePNlOPPHEwjizrrFjx9q8efPstNNOK7x+IpGwI4880v7yl7/YeeedN+Br3nHHHXbhhRfaQQcdZOPHj7d0Om2ZTMbGjh1rxx13nD322GOFO1ita/r06fa///u/9pnPfMYqKytt5cqVtmTJEluyZMkmv//BcMYZZ1htba0552z27NlbujsA1sF5ExSB4z6KAAAAADAgrjQBAAAAgAdFEwAAAAB4UDQBAAAAgAdFEwAAAAB4RHwTP7vdf2utxaJafuUaKf7e13aQ8tk92qX80ZP6fzDjQEbGtWeD1EXapPzjrZOl/L+Wj5Xy6ax39fcx4jItv3q3Einfsldayh+y0+tS/hNli6R8xoWl/MpMhZS/653dpXw8mpXyLx15sZT/uJl2zGVSvvi+ZwepJ9gqhLT91fI5Kf7e7E9K+dzkDim/x+gPpHzeabeVH1/SKOWXp8qlvOoP+/x+UNvfkibe1f/DSgcSCmn348qktWNx1RMJKV/3XIuUd6+/reWz2rFMFa7UjsXJqdtJ+fcO1caa8gktUr66pEvKL3prGyk/7gFt7Is/85aUd6mBH1jen0CsHfLrPM9vY4R20mqHv7wy8P7LlSYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8Ih4pza2SI0FFWVS3kX8L7++UFqKWyattZ8Ta8iw5cW8k/I5F0j5ZHdMykejWSnfXROX8uG09n7FxWN5p62v99I1Ur4q0inlFye19tXtszulrd+hbumBWn7SfYPTDwyOUHGxlM93dWntl5RI+Z0PfkvK//ut8VL+5WUjpfwpO/xLyj/VtK2UT4QzUj4SaMfDoSzToY3V1fVt2gsUdUvxTEmRlE8N1/aNxLta+66jQ8qbE08OaquleC6unWvlK7V9o0s8dgeB9n5jjdq5UKQ9JeWDsNZ+PqstH5fRTu7DVVVSvn1ShZT34UoTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhEfBNzzc1SY+HaKu3Vk1o8lNXy4XBeyrdkiqR8VzQu5ZektXw4cFI+1RGT8t1dxVLebR+W8jltcVq4SFvBu5R+IOU/SGvbZyLISPnlXeVSPiOur3CbtvyHuoP3eUXKv1dSIuXzyZSUN6eNN+a0/fvjLohq23u+q2uQerJWqKxUyr+1plrKjxjZJOXrijql/Kp0mZSvT7RL+e68Nt60pcUBfwgre0Pb1t2wQMrn8trn3Z2jtbGp5g0pbjZ+pBQPL2+U8kFYe7+5Cu3cJl2qtR+Op6V8ptt7qt1He07sj9O2n1BKO9fKtbZJ+cGWnzBCyqfLNt/1Ia40AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIBHxDcxv//uUmOhFxZo+boaLZ92Uj7blJDyq0eWSvnJJculfEdO60/OBVI+FM1L+eIR7VI+WRnX2i9JSfnPjHpHaz/ULeXz4vLsymvvd3RJi5RfVa9tb/FRWSk/1O1e+p6Uf/arR0n5+quflvJbnVBYioerKqS8S4n7X2enlB9s2RUrpXz0wQlS/kfn3SDlX+gaJ+VXZcqkfHde2x6Kwhkp3xWKSfmhrHiVdixe2agdC2RV2rGjZduolK/Ml0j5WFjbFoNcTsqna4qkfOu2g3v9IJbQ9qVcVls+qnSNdi6aqK6S8uqxIVSpHXtSFVr/28dq534+XGkCAAAAAA+KJgAAAADwoGgCAAAAAA+KJgAAAADwoGgCAAAAAA+KJgAAAADwoGgCAAAAAA+KJgAAAADwoGgCAAAAAA+KJgAAAADwoGgCAAAAAI+Ib+KiL8SlxrZbUS/lswuXSHkLRkpxF81L+bpEh5aPtEv5vNNq1O2LV0r5d7eplfJdmaiUrynpkvKV8aSU3698gZRX7VC0XMpnXFjKj0i0SPmlZRVSPhHOSPmhbrfEe1J+4pe17Sv54Bgpn12s9UcVrtS2F5fqlvK5xiYprwri2vHEdWv9DyUSUl4VTmv5spA2/r3Srh3fQuak/L6V70r5l9pHS/m8C6T8UBbu1taNJbVjTVCck/LxUm1fatvWe2rYR7QzJuXzcfHzeu1UzpJ1Wv+767XludPIFVK+Lq6dW85vGSblV5RrY1/bWG19xdcMl/KhZu3cODOyWso37qgdS5IjslLehytNAAAAAOBB0QQAAAAAHhRNAAAAAOBB0QQAAAAAHhRNAAAAAOBB0QQAAAAAHhRNAAAAAOBB0QQAAAAAHhRNAAAAAOBB0QQAAAAAHhRNAAAAAOAR8U0MjeySGmvea5iUL3/7XSmfKQukfLg0I+XTee/i6GNlpkLKt+cSUr422i7lK2JJKZ/MRKV8cTQt5RMRbfnHgpyULwtp73eH2Eop35LX1ldZOCXlV5eWSfmWdJGUH+pypo0HZ434u5T/RsOZUn78XXEpn3vzbS3f0irlI8Prpby1h6V4vrNTygcRbXy1vNPiKW3/C5eXS/k1h2rtv5+pkfKTS1dI+YVdtVL+qZZtpXzeafuXevwcyrJF2ufRgctL+W1Hr5LymZy2by8bpa379uZSKe9C2rYiborWXS3OoA019kGrdu7XEteO3ctXVUr5aJf2frPFWr5rtLZ+gxElUr5tjLY9tG2nnStGKrRzVx+uNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAR8Q3Mbu6SGqsbZxWg6W/vq+W37NDyo+vbZbyw+LtUn5UrFHKZ5x3cfdRF26T8otLaqV8Oq/1pz6hLZ8RiRYpr0q5qJRvyhVL+bQLS/lEkJHyI+ItUr4+pm0PQ93qXLmULwslpfwDX/2FlP/29OOkfOMd2vhXO+cFKZ9dsVLKy4JAiuc7OwepI2uFKyuk/PIZO0r5K/a5Scq/mRop5YvD3VK+KKyNN++1V0v54SWMNx+VcId27tTUpZ2bRcN5KR+L5aR8qtJJ+WRGGztMa966q7QZIjUpKb9DzSopn8pp51pLQ5VSPq+dqlhO23wsWa29QC4htl+vbQ9FI7RaYHRVi5T34UoTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhEfBNdaVZqrGus0/IT81J+REWHlFflXSDlM867+PrJh6X8imyFlC8Op6X8qOIWKV8Xa5fy20S19nNOq+GLg5yUb8snpHzKRbV8XstXhJNSPhRo+8tQ15bT1qeaX5qpkvL3bX+vlC++MCbl/+ebu0j5O+d+UsrXvqSNf/FWbf8Td1drnaCNrx37aPvTrD3vlvLDwtr49/e0Nn6/16ltb1Ux7f1uW75GyqtSWW38G8ry2qHeIl3avtf0QaWUHz1htZSvKNa2rc6qIimvHlvDSW35ZEu1Y+V+4xZJ+e1KVkn5VekyKf9aZBspr25veW1otXS5tvy7taHMuodpB4eEeK6+urNEyvtwpQkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCK+idO2Xyg1tqitWsq3dhVJ+bwLpHw0lJPyocBJ+YwLa+1bXmzfu3r6KA6lpfzYRKOULw2npHx1pEPK10XapPzwcKeUD5u2fhdnK6R8KhSV8uXi8kRvrbkSKd+Vj0n5ifGVUv7RZKWUHxdtkvKz6uZJ+f869l9S/v2jtc/QouJ4NjaijWdPprT1GwvU8V7r/x8aPynli8LaeFyX0MbLbF47/ixPlkv5ylhSymcdn8H26K7WzlXyMe3YFCrLSPnp9e9I+e68tq8+I6XNlmZrpbwLtG3dlWWl/HYlq6T8NtEWKV8baZfy/yzaVsq3xoqlfEYbCiwf07bnVL22/KtHtUj5WEQb69uTCSnvwygHAAAAAB4UTQAAAADgQdEEAAAAAB4UTQAAAADgQdEEAAAAAB4UTQAAAADgQdEEAAAAAB4UTQAAAADgQdEEAAAAAB4UTQAAAADgQdEEAAAAAB4R38RJJaukxtoyCSlfEU9JeVVtokPK510g5VP5qJTfIb5cyrfki6X8mmyZlI8GOSlfFkpK+ZJQt5QfGdbWl0r9hCDltPWbCDJSPue0Ho2LNkn5oa49p403Hbm4lE/FtPUfDvJS/u30MCn/RndYyg+PtEr5EZF2Kb84Uynl72ubIOX3KF4s5W9d8wkpXxLRxqfyiDb+zW+vl/J5cTyIhbNSvjymHW8bUyVSPm/a8XMo6650Uj4X09qPxbV1rx7ro2EtP6JUG2uWWq2UzxdpY2t1rTaWlYa1fWNRd52U3ybWIuUzOW2sD7TNzbLF2vLMJbR9u6iuS8rvP+JdKb+4o0bKF0e1czMfrjQBAAAAgAdFEwAAAAB4UDQBAAAAgAdFEwAAAAB4UDQBAAAAgAdFEwAAAAB4UDQBAAAAgAdFEwAAAAB4UDQBAAAAgAdFEwAAAAB4UDQBAAAAgEfEN7E+2io1tk2Rlm/NFEn5YfF2Kd+d9769PorDaSlfHemQ8m35hJSvDHVJ+W2izVI+47Tlkxdr7EnRRimvKgsFUr4iFJPyq/Pa9jbYRoRzW7oLW5UHLjtQyqdqtO3lztF5KT9yx5VS/pBt3pTyOaftfztXLJPy/06NkfLRICvlW3PaeP9Y2xQpHw9p/SkNd0v54pB2fBhToo3HyVxUyqvU421JVFs+b64cLuWHsnStNlZH2sJSvrJUOzdoy2rnHpGQNvZNLFkt5f8dGS/lLa0tn22rtHOPv67SxpqRxdq5bt5px57OVm19WUJbXxZ2UtwFWn6fUUukvDoWV8W17T+Z3XxjK1eaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMAj4pvYmiuWGisNd0v5dN778n3knVbjVUaTUj4a5KT88EirlE8EGSlfGUpL+eKQtvxfTY2W8qqUC0v5qOWlfHtey5tpy7MrXyblQ4HWn858XMq35p2UHyalP37axgdSXtydrGiVNt4sc/VS/g//2kbKZ0u17WtOMF3KV2/XJOV3q1sq5f+1bKyUTyVjUj7bEZXyFtH2p1iJtgFFItrxRJXNauNrJqUdb2ue0ManzBRteQ5lseqUlM9ki6R8KNCWdZvYfnWsU8o3ZkqkvPpxvSvW9qXxJY1S/tWWEVL+mJp5Uv6n7x4m5a1b27eDhDjWtGtjwbZTlkn5fcrflfK3vf8JKT+uXFu/qzvF7dODK00AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4BHxTawId0mNVRRp+TXRMinfmi2S8vXRNinfnktI+USQkfIxy0v5nAsGtf2ycFLKq6Jif2KBlo9qi8dyzkn5kNgfdXsYHmmV8tUhPuNYV2pMWspXPxeV8t2V2gaWWKWtn0STtj0mP9su5YPnKqR815paKf+P4hopH2/Slqcboe1/tW9o7Ve82y3lwykpbqGunJQPUtr4EaS1vOvolPI2TFu/df/Mau1/T4t/nJQWaxtLs2nnNnnx3KA6pq37yUXLpPy87HgpL54aWKRY27Z2L14i5bvz3lPhPtRj/XvLq6W8xbWxQzy1sVCVduz8ZO27Uj4R0pbPfsMWSnnVP1dvt9na4iwMAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwivomjY41SYzmn1WB1kXYpvyRdK+WjQU7Kh4O8lG/PJ6T8iLD2fttdVMqHzQ1uXlw+4UBrvzqkbT+hIJDyXXlte0gEGSkfM2355Ezrfzzw7q7/cb68xzwp37RTiZQ/pkZr/4YV+0n5V/+6vZSvKUlK+dVl5VI+1qJtj0Ud4vbboo0HNYctl/KrmkZK+eo3slI+/PYHUt51dkn5oE47vqXH1Un5VO02Ur5oZUrKR99dIeWHstHlrVK+KVop5XN57Vi5e/ESKV8W1saayqi2ratG1rZI+cqw1p+9SxdJ+X8nx0t5VbRIO/dw4vZQVdEp5aeXzpfyr6ZGS/njq56T8qe80iDlg66wlPfhShMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeER8E3NOq6macqVSfnS0Ucp3RmJSfk22XMoXh9JSviVXIuX3iLVL+XBe60+lWAIvzATaDOL2kHNa+8WhqJQPiTV/qyWlfMpp/clYXsongqzYfk7KD3VPrpwg5XevWSrlr/ngQCn/5eH/lvIL966R8ksX10r5Uc9r22PZghYpn6kt1vIl3sNNH9HzK6X8uMXvSnmXFfenmiopHlRXau13dEnx6MsLpXyoXTv+qFyFdrwdyvaqWiLlXykeKeWzOe3YNy66Rsq/3q31pzYqntsUa8e+A+rflvItOW1sqgl3SPl3UvVS3sRTrdLibinfmYxL+c+PflXK14U7pfynit+S8j9f9lkp37y0Qspb1Gl5D640AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIAHRRMAAAAAeFA0AQAAAIBHxDexJVcsNabmR0SapfzwSKuUb82VSPlokJXynfmYlA8FgZSvDDkpXxHS+lMZ6pLyGQuLea0mj4jthwOt/Zy2OGU58f0Wi9tbIohK+aFu5WvDpPwT2yakfC6nrc8fz/uSlJ94Z6eUr/33c1I+iHiH9z5cPC7lY42lUj5apo3Hrkgbz8yJO3gmreVXrZHi+WRKygfi8SFIaOsrVKqtL7U/LtUt5YeyYdE2KR8vykj5dFbbt0NBXsqrdogvl/LDq7Xls2fxIim/Olsu5XeIrZDy3Xlt+Ycj2vLP5rVjz7bDtLHpzKrnpXxrXhtbW5w2dj/11rZSPijRzp0C8VzahytNAAAAAOBB0QQAAAAAHhRNAAAAAOBB0QQAAAAAHhRNAAAAAOBB0QQAAAAAHhRNAAAAAOBB0QQAAAAAHhRNAAAAAOBB0QQAAAAAHhRNAAAAAOAR8U1sypVKjVWHO6R8l4tL+QnRNVK+Jd8i5bvyWn/yYs25MpeX8nWhQMrnnJPyNeFOKd+eT0j5jNu6avKwtjitPOiW8qFAW/5hMa+u36GubJG2fSW7KqR8xdtS3Gr/8YGUzy1bKeXDVVVS3iLe4b2vTFqKZ1eI/c/WSHl7XxufsumMlA/CYSnvslr7pu6v0ZgUF4czc8mk1n5ZmdY+41NBdz4q5XcdsVTKL+vQxjL13GZ4tEXKj400S/ndarT3OzLSIuXDgXauNSKSlfJdeW1fHVbdJuUTYn9mjn1Yyi/LaWPfLjHt3O8riz4t5SPxnJTPZbRjfziite+zdZ3VAgAAAMBWhqIJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAI+Kb2JwtkRqbEFsl5dvzRVK+LMhK+eGRVin/fqZGyqta8jEpv20kLOXz5qR8cdAt5dultFmn095vt0tL+eJAa19VHNK2t6i4/BOBFLfi0OC+34+b2teSUj73trY/xZq1/SO75H0pH0okpLwqt3q1lI+MGinlQ+mMlHfJlJRXhYq05RnEtP3JJbXtLZ/S3q/afwtr27PLauNZrrlZyofKyqT8UPZBukrKf6rqLSn/WmyUlG/La9tWLMhJedXupUukfH1YPTdokvKqbRPa2PpSSFtfv5l0u5RXzz2eTY2W8mMjy6T8/MZhUj4W144lWXHsy2a0vA9XmgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAI+KbuKirRmpsfHy1lN+/6F0p/2p6mJTfObZKypcH3VL++dRYKZ9yUSnfnO+U8mWhmJSfENX680J3lZTfMbZMyocDrYbPubyUHxkulvKvZ9JSvt1p/R8dyUh5rCfnpHjigzYpH7R3Sfl8IqHlUykpb2ltewnXVGvtZ7NSPCjS3q8T+59PassniGrbgzkt73LaeBNEtfHYpbXxxonLR6X2P9/ePkg9+fiJBjkpXxfRlt0upe9L+cqwNpap3s9WSPlPiud+2p6nW5ELD2r7f9vxHim/Ujy2Pdo1QcpPiS+V8g90jJbyRTFtrO/OeEuRPsJhbYtw+UDK+3ClCQAAAAA8KJoAAAAAwIOiCQAAAAA8KJoAAAAAwIOiCQAAAAA8KJoAAAAAwIOiCQAAAAA8KJoAAAAAwIOiCQAAAAA8KJoAAAAAwIOiCQAAAAA8IpuzsTeTI6T8Hon3pPzwSKuUz0tps7JQRsqPjjaKr6DJOCfl38rkpPzkaFTKj4w0S/kKcXl25bUaPmNpKV8aaO+3JMhK+bT4GcSwcImUzzht/cal9MdQEEjxfJG2/kMru6S8y6kjjiYIae/XpbX9zzLa9m5if4KYtvytU4vL8oO7vsxp7TunLc/BFoTFz1TdZj2d+FgbFWuS8u35hJR/o0s71yoPJaX8Pon3pfzTmbFS/q1urf/Hlb8h5Wsj2rY4Lx2T8ieI/VmQ0fall7q15RkNtHODskA7NrzQOU7KJ9PaWF9enJLyrZ1FUj4IaefSPlxpAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAACPwDnntnQnAAAAAGBrxZUmAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAKAj4lTTjnFgiCwU0455WP9GgAw1C1evNiCILAgCGzx4sVbujvYDCiaPkZyuZzddddddvLJJ9t2221nlZWVFovFbNiwYbbffvvZeeedZ6+99lqveY444ggLgsDGjRtn7e3t3vZPOOEEC4LAhg8fbmvWrOk1rWfHX/dPKBSy8vJy22WXXexb3/qWvfHGG5v9PQNDWXNzsxUVFRX2qbfffntLdwmbQVtbm1199dX2uc99zsaOHWslJSVWVFRko0aNssMPP9wuv/xyW7FiRSE/bty4fsfYjflzwQUXbLk3CmyiCy64oLANb8i6xcdNN900+J3bAl555RU799xz7ROf+ITV19dbLBaziooKmzJlijU0NNgDDzxgmUzGzMwef/zxTR4vKOA+nMiW7gA2zr/+9S9raGiwBQsWFP4vGo1aWVmZNTY22lNPPWVPPfWU/fSnP7Wjjz7abr/9dovFYnbdddfZTjvtZEuWLLHvfe97dt111/Xb/l133WV33HGHmZn99re/tdra2n5zJSUlVlpaamZri7jGxkZ79dVX7dVXX7XrrrvOrr32WvvqV7+6md89MDTdeuutlkqlCv++4YYb7NJLL92CPcKH9fvf/97+67/+y5qbmwv/V1RUZPF43JYuXWpLly61Rx55xH74wx/arFmz7Ic//KHV1dX12g56JJNJa2trMzOz2tpaC4fDfTI94zGAj5/29nY788wz7fbbbzfnnJmt/ZC6oqLCksmkvfnmm/bmm2/anDlzbNttt7Vbb73VYrGY1dfX99teU1OTZTIZi0ajVl1d3W+mv3EEG8lhq/enP/3JxeNxZ2aupqbGXXrppW7BggWF6dls1v373/92M2fOdOXl5c7MXHNzc2H6H/7wB2dmzszcww8/3Kf95cuXu5qaGmdm7uSTT+63Dz3zz549u9f/p1Ipd//997vRo0c7M3ORSMTNnz9/s7xvYKjbbbfdnJm5b3/7287M3DbbbOOy2eyA+YaGBmdmrqGhYdD69FG8xlD1P//zP4WxcvLkye7mm292K1euLEzv6upyf/nLX9yJJ57owuGw23XXXb3t3XjjjYX2Fi1aNLidBz5Cs2fPLmzbG7Jo0aJC9sYbbxz8zm0m6/a7v/23qanJ7bjjjs7MXBAE7vjjj3dPPPGE6+7uLmSWLl3qrr/+erfLLrs4M3NXXHGF9zUPOOAAZ2bugAMO2LxvBs455/h63lbu7bffthNPPNG6u7ttypQp9tJLL9nMmTNt0qRJhUw4HLa99trLLr30Ulu0aJEdddRRvdqYMWOGffGLXzQzs9NPP91aWlp6Tf/a175mjY2NNmrUKLvqqquk/sXjcTvqqKPs1ltvNTOzbDZrN9988ya8U+A/ywsvvGAvvfSSVVZW2s9//nMbP368LV++3B5++OEt3TVsgjvvvNMuueQSMzP78pe/bC+++KKdfPLJNmzYsEKmqKjIDjnkELvlllvspZdesh133HFLdRfAFjZjxgx7/fXXLRKJ2J133mm33367TZ8+3WKxWCEzYsQIO+200+yll16yX//615ZIJLZgj0HRtJWbNWuWtbW1WSKRsPvuu89GjRrlzVdXV9v9999vFRUVvf7/2muvtdraWlu2bJmdddZZhf+/4YYb7KGHHjIzs+uvv94qKys3qZ/77beflZSUmJnZ66+/vkltAP9Jfv/735uZ2XHHHWeJRMJOPvlkM1u7T24M55xde+219olPfMLKy8utvLzc9ttvP7vttts2OO+tt95q06ZNs7KyMquoqLB99tnHfve73xW+HrIh9957rx155JGF797X19fbkUceaffdd9+A86x/g4mbbrrJ9t13X6uoqLCqqir7zGc+Y//4xz8K+Ww2a1dffbXtueeeVl5ebhUVFXb44YfbCy+84O1bPp+3W2+91Q4//PBC/+rq6uyQQw7p9RWY9fX8ruimm26ydDptv/jFL2zXXXe1kpISq6iosAMPPND+/Oc/9ztvOp22//qv/zIzsylTptjNN99s8Xjc28+ddtrJbrnlFm8GwMZ57bXX7IwzzrBJkyZZcXGxlZaW2i677GI//OEP+/xGu0cmk7E//elPdsYZZ9hee+1l22yzTeF34oceeqh3vOixdOlS+/rXv26jR4+2eDxuo0aNslNPPdXeeecd73yPPPKIPfLII2Zmdv7559uxxx7rzQdBYN/4xjfsjDPO8OYwyLbshS74rFixwoVCIWdm7rTTTvvQ7f3xj38sXCq+99573eLFiwtf5zvjjDO889oAX8/rkc/nXUlJiTMzd8QRR3zovgJDWTKZdJWVlc7M3FNPPeWcc27hwoUuCAIXiUTcihUr+p1v3a/OHXfccc7MXCgUclVVVS4IgsJ+euqpp7p8Pt9n/nw+70499dRCLggCV1VVVRhnjj/+eO/X87q7uwuvu+5r98xvZu6EE05w6XTa2/eev0ciEVdWVlaYNxKJuAcffNClUil3yCGHODNzsVisMLaYmSsuLnbz5s3rd/k0Nja66dOnF7Jm5ioqKnr9+/Of/3yvr7/0GDt2rDMzd/XVV7t99tnHmZmLRqOutLS01/L6/e9/32feu+66q5C55ZZb+u3bpuDreRiqNufX8372s5/1GoOKi4tdLBYr/HubbbZxL7zwQp/55s6d22tsKC8v7zUemZk79thjXS6X67dfzz//vKuqqipki4qKCuNFeXm5u/POOwfcfw8//PDC+NTR0bFRy2xj8PW8wUXRtBW7/fbbCzvcQw89tFnaPP74452ZuWHDhrn99tvPmZkbP368a29v9863oaLpiSeeKGS+/e1vb5a+AkNVz+8MJ06c2Ov/999/f2dm7uc//3m/8/UUGxUVFS4IAnfRRRe51tZW55xzq1atcmeddVZhP7zqqqv6zH/VVVcVpp911llu9erVzjnnWlpa3AUXXOCCICgUc/0VTd///vcLxcOPfvSjwm8nm5qaev2e59xzzx2w75WVla6oqMj99re/dV1dXc455+bPn+/23HNPZ2Zu3Lhx7qyzznLV1dXurrvucul02uXzeTdv3jy37bbbOjNz06ZN69N+NpstnDDstttu7sEHH3SdnZ3OOec6OjrczTff7IYNG+bMzJ199tl95u8pmqqqqtzIkSPd/fffXyj+5s+f76ZOnerMzJWWlrqWlpZe8379618vFJE962NzoGjCULW5iqbrr7++sF/+5Cc/ccuXL3fOrR0P5s2b5w488EBnZm7UqFF9znOeffZZ9/Wvf9397W9/67XfNjY2uquuuqrwoXJ/Y2lbW5sbM2aMMzM3ZswY99e//rXwQdXTTz/tdtxxx8JYuv7+m8lkCsXVl770pY1dZBuFomlwUTRtxWbNmlXY4ZYuXbpZ2mxsbHTDhw/v9cnp448/vsH5Biqa1r8RhJm5559/frP0FRiqPv3pTzszcz/+8Y97/f91113nzMztsMMO/c7XU3iYmfvRj37Ub+bEE090Zuaqq6tdMpks/H8ymXTV1dXOzNxJJ53U77wzZ84stL9+0fTBBx+4SCTizMydd955/c7/ve99r3CFZtmyZQP2/Q9/+EOfed95551en/D+85//7JN57LHHCtPff//9XtPmzJlTWHbrFzU95s2b54IgcLFYrNcNGpz7v6IpHo+7N998s8+8q1atcolEot/+93wANWnSpH5fd1NRNGGoWrdoqq+v9/6pra3tt2hqa2srFCZ//vOf+32dTCZT+EBmQzdRWF/Pt3O23XbbPtN+9rOfFa6Ev/HGG32mL1++vNdVqHX333XHup/85CdSnzaEomlw8ZumrVhjY2Ph7wPdOlJVXV1tX/nKVwr/Pv744+2AAw7Y6Pkvu+wyGz58uA0fPtzq6uqsqKjIvvCFL9j7779fmL7HHntslr4CQ9G7775beM7GSSed1Gval7/8ZSsqKrL58+fb008/PWAbRUVF9oMf/KDfaeeff76Zrb317N/+9rfC///1r3+1pqamXpn1zZw5c8AfGt9zzz2WzWYtkUjYzJkz+83MmjXL4vG4ZTIZu/vuu/vNjBkzptcY1GPbbbe1iRMnmpnZ/vvvb/vtt1+fzAEHHFD4rdArr7zSa1rPb8S+8Y1v9PlNZ48999zTdtxxR0un0zZ37tx+M1/60pdshx126PP/dXV1tu+++/b72j1j9eYap4H/JCtXrvT+Geg3Sffcc4+1tLTY7rvvboceemi/mUgkYieccIKZmf3lL3+R+nXEEUeYmdnChQt7PVfNzAqPaDn22GNt8uTJfeYdPny4nXnmmf22OxjndvhoUDT9h5k/f779+te/Lvz7kUcesWXLlm30/J2dnb0GMvf/fiRZVVVlTz31lH3/+9/f7H0GhpIbb7zRnHO2//7727hx43pNKy8vty984Qtm9n9FQH/22msvKy8v73fapEmTCjeMmTdvXuH/e/4+evToQnGyvoqKCttzzz37ndYz/9577z3ga1dVVdlee+3V57XX7/tAD7TsefbI3nvv3e/0cDhceIbcus9ByuVy9q9//cvM1j40s+eDnf7+vPXWW2ZmtmTJkn5fY5999un3/83W3snKzArFJ4APz6391tOAfxYtWtTvfE899ZSZmb355pveff7HP/6xmfW/z7e3t9svfvELO+CAA2zYsGEWi8UKD4EtLi4u5D744IPC39PptL366qtmZnbggQcO+L580/DxRNG0FaupqSn8fXMcpHO5nDU0NFgqlbKddtrJRo8ebS0tLfa1r31to9uYPXt2YSDr7Oy05557zj73uc9Zc3OznXLKKVIBBvynyefzhSfa99wtb30NDQ1mtvaB0x0dHf1mRo4c6X2dnumrVq0q/F/P3zc070B36FTnX/e111VWVjbgvJFIZKMzmUym8H9NTU3W3d1tZmuLKd+n1j3zdXV1bXL/1n1ts/8bqymmgI9Oz/lGKpXy7vM9D4hef59fsGCBTZkyxf77v//b/vGPf9jq1astGo1aXV2d1dfX93qAbGdnZ+HvTU1Nls1mzcw/Hg40lm7uczt8dCiatmLrPsPjxRdf/NDt/fSnP7XnnnvOotGozZkzx6677jozM3v44YcLJ3KK4uJi23vvve3++++3gw46yN5++22bMWPGRt+2GPhP85e//KXwieXpp59e+ERz3T+f/exnzcyso6PD7rrrri3Z3Y+NXC5X+PsjjzyywU+unXN2wQUXbLbX7xmrFy5caO3t7ZutXQAD69nvjzvuuI3a5xcvXtxr/lNPPdU++OADGzdunP3xj3+0xsZG6+zstFWrVtmKFSts6dKlhezmPK8ZO3aslZaWmtnmObfDR4eiaSv26U9/2kKhtavI9/yTjfHKK68ULlHPmjWr8B3g008/3czMzjnnnF4DhCIUCtlvfvMbi0Qi9vjjjxe+6wugN99X7pT8hvbVnunrPli15+8bO+/6euZf92sq/emZvu5rD7aamprCVaCBvnY3mA466CAzW3sl8YEHHvjIXx/4TzR8+HAz27R9/v333y/8bvT222+3L33pS31+X7T+75h6VFdXWzgcNjP/eDrQtEgkYtOnTzczs7/97W+9rmJh60bRtBWrr6+3Y445xszMbrvtNluwYMFGz7vupyKZTMYaGhosnU7bHnvsYf/zP/9TmPbLX/7SxowZYy0tLR/qoWmTJk2yGTNmmNnaoqzn0jWAtVavXm1/+tOfzMzs7rvvtvb29gH/PPfcc2Zm9vTTTxd+g7OuefPmDfjVvXfeeadQuPT8vmjdv7///vu2cOHCfudta2uz559/vt9p6/5WqbW1td9MS0tLr98+fVSi0ah94hOfMDOzBx988CN73R5HHXVU4as4l156qaVSqY2aL5/PD2a3gCFt2rRpZmb2/PPP2/Lly6V5e25eZWa2++6795t59NFH+/3/WCxmu+yyi5nZgDeUMTP7+9//PuC0b33rW2Zm1traapdffvkG+9uDMWPLomjayl188cVWWlpqyWTSjj766A1+Stzc3GzHHHNMr5Oaiy66yF566SWLx+M2Z86cwieyZmt/eH799deb2dqv6d14442b3NeZM2daKBSyd99990O1AwxFt9xyi2UyGauoqLDPfe5zVlpaOuCfvffeu3AHt/6uNiWTSbvsssv6fZ2LL77YzNZ+GnrwwQcX/v/ggw+2qqoqM1s7JvTn5z//uSWTyX6nHXPMMRaJRCyVStnPfvazfjOXXHKJdXd3WzQaLXzg81Hp+dDn4Ycftocfftib3dy/I4jFYvbzn//czMzeeOONwodUPm+88caAv2sDsGHHHnusVVZWWiaTse9973ver9Dl83lraWkp/HvdO2y+/PLLffLt7e2FsbQ/xx13nJmZ/fGPf+z3g61Vq1bZtddeO+D8hx9+uB1yyCFmZvbjH/94wLuNrut3v/td4WcV2DIomrZy2223nd1yyy0Wi8Xs9ddft912281+9rOf2TvvvFPI5HI5e/HFF+3888+3CRMm2L333luY9vzzz9ull15qZmYXXnhhr99J9Tj44IMLJxwf5mt6O+ywgx199NFmtvbEbUMnDcB/kp7i56ijjrJYLLbB/LHHHmtmZnPmzOlz5baiosIuuugiu/TSSwu/oVmzZo1997vftZtvvtnMzH70ox/1un14UVGR/ehHPzIzs5tvvtnOPvvswq1v29ra7KKLLrJLLrnEKisr++3PyJEj7bvf/a6Zrf195OzZswsnIS0tLfajH/3IfvGLX5iZ2fe+9z3bZpttNvgeN6cTTzzRPvOZz5hzzr74xS/axRdf3OvGNJ2dnTZ37lz71re+ZRMmTNjsr3/CCSfYf//3f5vZ2pt47L777nbLLbfY6tWrC5lUKmWPPfaYnXrqqbbrrrvaa6+9ttn7AfynqKystCuvvNLM1t4C/IgjjrBnn322cDUmn8/bm2++ab/85S9txx13tIceeqgw7+TJk23MmDFmZvbVr3611xX2Z555xj71qU/1ukPn+r7xjW/YqFGjrLu72z772c/aY489Vijann32WfvMZz6zwatCt912m02ePNmy2ax9+ctfthkzZtg///nPXjeaWb58ud18882255572te//vUBP9TCR2RwHwOFzeXJJ590EydO7PXwx1gs5qqrq10oFOr1sNoTTjjBpdNpl0ql3I477ujMzE2dOtVls9kB229rays83PGwww7rM72n/fUfbru+F154oZC95pprPuzbBoaEZ555prBfPPjggxs1zyuvvFKY5/7773fO/d8DYhsaGtxxxx3nzMyFw2FXVVXlgiAo5E8++WSXy+X6tJnL5dxJJ51UyIVCIVdVVeXC4bAzM3f88cf3eo31dXd3uy9/+ct95l93DOoZf9bna7dHz4MZfeNMzzi17kMue7S2trojjzyy1zhZXl7uKisrey2fSCQitau8h9/+9reFB272/CkuLu73/37+858P2I5zPNwWQ9e6D7fdkEWLFvX7cNsev/nNb1wsFitk4vG4q6mpcdFotNc+t/5DqR988MHCA7t79sni4mJnZq6kpMQ9+uijhWlz587t87r//ve/e+3XxcXFrrS01JmZKysrc3feeecG99/W1lb35S9/udf4FASBq6qqKjxMu+fP5MmT3bx587zLiofbDi6uNH1MTJs2zebPn2+33367zZgxwyZOnGiJRMLa29uturra9ttvP/vhD39ob775pt12220WjUbt/PPPt9dff92KiorspptuKvxwsT9lZWX2+9//3oIgsEceecRuuOGGTern7rvvbocffriZrf2qzsZ+tx8YynquMlVUVBS+krEhO++8c+Ghif19Re/222+3X//617b77rtbNpu1kpIS23fffW3OnDl28803F24is65QKGRz5syxOXPm2NSpU62oqMiy2aztsccedu2119ptt93m7VMsFrM777zT7r77bjvssMOspqbG2tvbraamxg477DC79957C+PPllBeXm4PPvigPfzww3bcccfZmDFjrLu727q6umzkyJF2yCGH2KWXXtrv12k2lzPOOMMWL15sV155pR1++OE2evRoc85ZMpm0kSNH2mGHHWZXXXWVLVmyxP7rv/5r0PoB/Kc488wz7a233rIf/OAHtuuuu1o8HreWlhYrLS21vfbay7797W/b3/72t8JDbnsceeSR9o9//MOOOOIIq6ystGw2a7W1tXbqqafa888/X7jBy0D22msve+WVV+z000+3kSNHWjabtYqKCmtoaLAXXnih8DtLn/LycrvzzjvtxRdftB/84Ae21157WW1trbW3t1s0GrXJkydbQ0ODPfTQQ/bqq68O+Bw9fDQC57g/NAAAAAAMhCtNAAAAAOBB0QQAAAAAHhRNAAAAAOBB0QQAAAAAHhHfxENiJ/gm9+HWe5bIhoTrh0l5q67YcGYd2apiKZ+sT2w4tI4PjshJ+d23WyLl961+V8oXh7TnIpWFtPv9l4jt14Q7pPyC9HAp/9c1U6T86yu058bkF5RK+YoFUtyqX2+X8rmEd3ft47F//FDKf9x8doeZ2gzqk9Sj2vJOjSyX8h0jN/yspnWp9zotWaGNx0VLte3RRQe+G2d/UsO08TiS1MbXTJm2vrrLtQWaqtHyyWHaPZby4g0HS5cEUr7mDe1OpvFFqzccWle3dnx4ZNk1WvsfI5/d5ltbugsANtGfl/9qwGlcaQIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAj4hvYhCLSY2Fqqq0V6+ukOJBc5uUD72zRMqX7DJJygfdpVK+MxOX8okgK+U7cgkpPym2QsovztRJ+egg9781XSTlY1GtP51xJ+WLGvNSPh8LS3kLB1p+iHPF2v4UdCS19uNRKd82VhsvU3Xa+ix9X9u+Yi1pKZ8r0/a/TKn38NFHkNf2p+bttPWbHKYtz2yx1p98VMuHRnVJ+ZqKTinf2jlMyseWtUr5zIhqKR9p0d4vAHzccKUJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwomgAAAADAg6IJAAAAADwivokuk5UaCypKpXzLLjVSvmSF1n7s3aiUbxurtR8f1iXlJ5WvlvIrM+VSfrvEcinflNPe77io1n9VU7ZEymfzWs2fyYalfDgZaPlUTsrnEt7dr4/Yqk4pP9St3rtCykc7tP3JiR8pNe6prf+iZdr6767QOpSqj0v54qVJKd8+pkjKW16Ldw3X9r/U2LSUD7doy79q+yYp39xWLOVXz6+V8pEdO6R8+45a+9F27fgf7tTGV2y6IKHt25YTd76stu6dmJcF2tgXhMXBO9DGGpfVxvpB709eXL85rf/mnJaPaGNrIOYHfXvz4EoTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhEfBND1ZVSY91jq6V8ujyQ8h2jiqR8dPvRUr59vBS3HYatlvLbF6+Q8qEgL+VTLiblayJNUj4W5KR8cahbyk8uWiblV1eUSvlU1ru597GyskTKt0zSlr8La9t/eUlYyg912SJt+YVTWvtt22rtx9doeSd+ZJXXNi8LdTsp3z6uWMq3ieNlpEtbPvmY1v8goo2X+dq0lG9+Szu+RTq195st0d5v7N/a+Ff25NtSfs3hE6W8Op5h0+WbWqR8EIjrJiwea8LiYBaIeaft2y6jnatYXtv31PfrnLb81fWl5p26/E1b/rL8ILe/GXGlCQAAAAA8KJoAAAAAwIOiCQAAAAA8KJoAAAAAwIOiCQAAAAA8KJoAAAAAwIOiCQAAAAA8KJoAAAAAwIOiCQAAAAA8KJoAAAAAwIOiCQAAAAA8Ir6JQSwmNZYpC2v5kkDK5729/fAyVTkpX59ol/LxUEbKJ4K0lhfbV9WFOwe1/WiQlfITitZI+c6KuJRfEauW8qG09hlErDMv5aMd2vIZ6tTxwIkfEVW9qa2firfapPya3SukfD4qxS2U0fqfLtUWaKxNG78TjU7Kd47U2i96KyHlU8O05aNSt7eyhdoMdS9q43H7fhOkfKZMW/7ZNu34j00XKi/TZgi0dSnnQ2I+r40Flhf31Zx2LudyYvt5rX35/QZiXqWuL9P27UDdfj5GuNIEAAAAAB4UTQAAAADgQdEEAAAAAB4UTQAAAADgQdEEAAAAAB4UTQAAAADgQdEEAAAAAB4UTQAAAADgQdEEAAAAAB4UTQAAAADgQdEEAAAAAB4R30SXTkuNBTntxXMJLR9t1/Ilq7QOtaS0GnJ1qlTKrykqk/IVkS4pHxNXQNiclA+J+S7n3bz6yIj5nNPWV1c2KuWDjPiZQqDFY23i+kpmtRcY4tp31Manjs6wlC97V8tHu7TxIFWnbTCxFm3/y5SJ+1NcilvJUq0/6TLt/ZYt0tpP1mvtR9u0vLp8Qlmt/dLl2niQroxJ+eX7icunVYpbyXItjw8hn5fiTsxbTjyZc9q+ankxHxIPriFt7JbbFxenOXF9Oa0/QTC4+UGnbj9bEFeaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMAj4p1aWS41linWarB4k5PyQU6KWzau9cdFtP60Z+JSvtv5F/f6wpaX8qrqcIeU7xL7n7NAyteF26T8EquV8iu7yqS8S2gbXNPuUtxCWXF7SA3u9vBxU17Tqc1Qo8Vrp2jtv7+6SspnWqW4jfqLtj8VrUhJ+UhXVMrHm7ql/LL9tf2vYlFWyjcerOVjC4qkfMX72vEhpy1O69gmLOXbx2vHt6Ll2vaTKdfebz6itY9N5zKZLd2F3kLatmvithIE4ral5kVOfb+qvLbvOfHyh7w8B5lz2vvdkrjSBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeEd/EzknVUmOdI7QaLNrmpHykW8t3VwZSPlydkvIjS1qkfGlYaz8a5KT86GijlC8JslJeFQ3yUj5mWn5UTHu/21askfKqru6YlG9KVUr5WId3d/2P09GRkPKRqLY/dcWjUr6mskPKr0yHpXymWNu+SpMZKR/KavtfPqb1P9Gojd+NO2rLPy8eH2pe17aHVLV2fOsco/VHFR7TKeUji0qlfOVCbXvARyeIx8UZtHMhCw/y5+k5bdtyGW0ss+601n5WOxcKlZZo7efFfSmnjU0mxl1YG7sDdfsZwrjSBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeEd/EdJlWU+Xi4ot7X72vIKnlk8MCKV9b1S7lxxc3SvnqcKeUT4QyUr4k0PIhc1I+EeSlfElIW/45l5PyleEuKb9PxSIpHw9p/Xlu+RgpP3zHVVJ+ZaZeyg91xSXdUj4a1tbnylUVUl5VUp6S8iunh6V8tKtMyidrtfG+ca+slK+ZJ8Wta4Q23tQ+GZXype91SPnWbbXlmS3Wtrd4k7Z+s4tLpLwTj7dlC1qkfNsOldoLYJO5bm3sM6cd6y0v5p22r1ogfl4f1vJBPKblE+LJa0wba4K0dm7mctrYYTlx+YtcWBubZOr2Jp5bbk5caQIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAD4omAAAAAPCgaAIAAAAAj4hvYjSZlxorW6K9eNl7KSmfLo9K+WxxIOW7M97F0UdHNi7lh5e0SPmSIC3lo4G2vipDWr7dactTFRP735XXln9rtljKh8T+VBUnpXxrMiHlSyc3S/mhLpmMSfnuwEn5SCwn5W2Jtn11hbX+RMu18aDkA2377dimTMqfMvUpKf+n5w+Q8sXLtM/0Spdpy6dluxIpH2/U1lfJUvH4UyXFLdaq5csXa+NT54QKKZ+s4TPYj0oQ1459Lp3RXiCf1fJRbSy2Sm2syQ4rl/LJem35dJdr227tkyukfJDTxmLLiOvLie2r53J5beyz0OCeK25JjHIAAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4EHRBAAAAAAeFE0AAAAA4BHxTQxyWmNFqzMfpi8b1LxdVMpnqrX+FMW0/LBYu5RXpV1YyleHtBUWDgIpH3VOyue0uExdPktS1VK+PZOQ8mPKmqR8W7xIyi9YXSflh7pQoG1g4Uheyuey2mdK+TFJKe/WxKV80Kb1Z+GXpLiFu7Xl+eefTpfy9a9o+0fnhAopn1jeIeUjXdr+l49qy79lYkxr33s07qv65WYp76LaeLlmf235lL4/yAM+Nl13txR3Oe1cIj9plJRvnp2S8nfudI2UHx8tlfJrcp1S/tEu7f2e9+QxUn7SDVkpH3r2NSkfrquV8i6jnRvnVjZK+fDE8VLe2rX1tTlxpQkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCiaAAAAAMCDogkAAAAAPCLeiV05qbGOkTEpny4PpHyq1kl5C2v5sli3lJ+cWCrlh4fbpHxjrkTKtztteVZqcYuK+USg1eThQFtfNZEOLR/tlPLxUHZQ8+r7HVnVKuWHulxO277iiYyU715ZLOVjTVp/8pO6pPyee74n5Wvj2v7x38PmSvm/d42T8r+4/stSfvT/rpbytqZFikdCdVLehbUBsOJdKW5BXsuHWrXxrHWvEVI+rx3OLSfmselct3auYsNqpHiuvkLK73r1y1L+F8NflPJmpWJeUxvWzrWOL2uW8oce8v9J+Zk7HyTln//t3lK+5oZ/SfnwxPFSPlKqLU9LatuzWAlsVlxpAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAAAPiiYAAAAA8KBoAgAAAACPiG9itCUlNZYbF5Pyiaa8lO8YHWjtV3RL+UhI68+yTJWUrwl3SPnKcJeULw6clDfTlqcqGmg1eVRsvyRIS/naqLb8S8Pa9hMNZbV8kJPyIxMtUn7IEzf3kLh/uCJt/Qx/Rhs/PigpkvLTd10g5dvzCSk/KlIq5Q8sXizlL9d2V8vUav2JxLURJFek5cPJjJSPNWvHz1C7lldlirXxOBfT9pdO8fiMTeeS2rYShMNSPv1rbd3PGva0lDfTxr4PstqxOxFo22JtuETKq6rCxVL+t6OekfLHf1Vbni2v7CTl3fzFWj6jjZVBIi7lLS7mNyOuNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAB0UTAAAAAHhQNAEAAACAR8Q3sXnHcqmxUE578XRpIOXdqJSU33Pk+1J+ZFGLlK8Md0n5RJCV8tWhtJSvDHlXZx8pp62wsJTWZVx+UNsvDnVvVe2Ho26QevKfIRzRtpdYRNveg05tiy9euEbKR/epl/KnViyW8i9rw4d9+d3PSvl5L0yU8pPve0/KpyZpy6e7tkjKRzsyUj7U2inlXXFCzMelvEW18b50qbZBtG2r9Sc1WtzgsMlClRVSvnu74VL+8Z1ukPILMtpYXBpo+ZJA+3w/Z4N7bO122tixLKudG4yPlkr5O8b/XWv/1DOk/A4/EE/uRUFZmZR36S031nClCQAAAAA8KJoAAAAAwIOiCQAAAAA8KJoAAAAAwIOiCQAAAAA8KJoAAAAAwIOiCQAAAAA8KJoAAMD/3569/NhZ13EcP8+59Jy5dG6daYsUSG1tqZCoiUSjCRtiXMjGS0Li3oSwwo1bY+J/4MaYmLgyLA0GE+Ml7hCNEjQKBYFSWqcU2sJczpzLnOdx4QKSJj/yKUwK5PXanvf85pnn/p0DQIGhCQAAoMDQBAAAUGBoAgAAKOiWPmzvN9Fih7bqqL9xpvjrb1JPsxnvwtZa1B/rb0X9SmcY9aMm+3vfmHXC9cdRf0+3F/XD1jTqU9NWdr7NWlXYZ+dPr5pFfWq+nR2vi+MjB7QlH0+LC6Oon+xn11N3Jztfqr3w+vvxX6L+6888FvVVnV1Pc6/eiPqzg3eifnzqaNQf+vPzUT986P6or/vZ8e1cz+7f1TA7P5vDc1FfD7Lt6b+xE/XtST/r+wd7v+RdzSg7ty4/OIj6i/vZuXKmtxj1qdXO/IGun+pX2bvTyV7Wj5vsXavbyp5t93329aiv+9m9oHVkJcqb7d1s/dvIN00AAAAFhiYAAIACQxMAAECBoQkAAKDA0AQAAFBgaAIAACgwNAEAABQYmgAAAAoMTQAAAAWGJgAAgAJDEwAAQEG39OHy+e1osfH6XNQvXq6jfvtMJ+rnetOoX+0No/5wey/qB9V+1B/rTKJ+ocpm4H5VPPwfWKeqor7dzA5oS/6vV2XrZ2dPbhb+z2K5m51vn3SddhP1b19civqzT9yI+und61HfProc9YO3RlE/Wh9E/WxtIeq7m9n+6bWz8706eVfUX3ooW79Zze6vc+ePRf3Gs9kdZOGFq1FfL2fHq+lk9+PZILu+WmHOrau3dqJ+1s8Ozt3dxWz9JnuXuzrL3rXuCLdnWGfXdq/K3i1T6fqzJjte7epgL75mL3v3aO9ls0A9zNav5rP1P0y+aQIAACgwNAEAABQYmgAAAAoMTQAAAAWGJgAAgAJDEwAAQIGhCQAAoMDQBAAAUGBoAgAAKDA0AQAAFBiaAAAACrqlD8frc9Fi7f3mA23M++kuTaL+7NLVqF/vbkf9dp3tn1G1H/Vr7ezvHbXqqF9sZcerU1VR3w5n8kHVifqFKts/C+1x1I/qXtTHwstltbt7MNvxMXXt2mLUr/81Ox/b17aivp4/lK2/PcrWf+1S1C+srkR9s5Ttz1Y725/VMLv+Xvze0ah//Gu/ifrP9K9E/e/uvy/qf730pag/fXEQ9dV4GvVNP7uf9Xay+/14kt2/uXWdI6tRv/7cwb6bXdgfRv1B/7d+vp3diw/arMnezQ56+/91/kTUn1vI7t3NcC/q22sr2fqjbHs+TL5pAgAAKDA0AQAAFBiaAAAACgxNAAAABYYmAACAAkMTAABAgaEJAACgwNAEAABQYGgCAAAoMDQBAAAUGJoAAAAKuqUP+9dG0WL7hw9F/Vufr6J+dXk36q9P5qN+tpDNkMc7W1E/aoq7+yZ1VOemzexA1+9W2fFthzN8r8q2v1ftR/0s3P5peHw3uttR/8pkI+o/6TpX+lG//vTVqK83VqK+u3kj6pv5QdTXXzyX9XUT9VUT9rOsbw8nUX/PFy5H/QNzr0T9pNWJ+kfWnon63997Nur3l7LzoXfl7ahvzbInyspL2f11eDy7/3HrmvRY/vb5qD/9y0ej/j/f/WnUf9S8U+9F/XJ7Luo3Z8OoP9FdjPond7N33VNPhO9+3fDa3stmh2Y0zta/jXzTBAAAUGBoAgAAKDA0AQAAFBiaAAAACgxNAAAABYYmAACAAkMTAABAgaEJAACgwNAEAABQYGgCAAAoMDQBAAAUdEsfbn51KVps6979qF+983rUr80No/6OwTtRf7i9F/WTcOYcVNn+ebs+FPWz9jTql6O61apbddg3UT9usv3zPqfvTQZVtn/qKju+oyY7Xtdmi1FfN/7H8V4nn8yu1+bSZvYLTt8d5eOTG1Hff+1a1Hd2e1E/WZuL+u4f/xb1o288EPVzl2dR/9pzn4r6yac7UX+8sxv10/D6276UPT9PvHoh6md3rkd952r2PFz6+3+z9UfHo771eJbzrupQdi9oppOoP/Pz7N70ucuPRf2pb74U9T87+auoX+8sRP20yd5tZmF/ops963/45n1R/9RPHoz6jaefjfqml51v7SOr2frbO1Hf6mbvfh8mb2EAAAAFhiYAAIACQxMAAECBoQkAAKDA0AQAAFBgaAIAACgwNAEAABQYmgAAAAoMTQAAAAWGJgAAgAJDEwAAQEG39OE933olWuzc0pWof3lnPeqP9HejfrE7jvpO1UR9ar69H/W9VrY9K+1sBq5bddT3qs6B9qntehD1o6YX9Yc7e1G/1tmJ+gvTjaifNf7H8V7dN7ejfvzlc1E/+PelqM/OrlarPjyX/cDLr0d5f3g06jcf+0q2/sNXo/7RU3+I+h/94+Go/8EL34n63T9l+6cuPi1vdtc/s/t9s7QY9dUsez7Uq9n6bz6wGvV7x6qo59Y1k2n2A53wWbyZXdt3/uKtqB89tRb13z77/ai/fi67WHdPzKJ+6aVsfy68kb1rLb24FfVHXz8f9a1BP8qrufBZNc3ufa3q43Pv8BYGAABQYGgCAAAoMDQBAAAUGJoAAAAKDE0AAAAFhiYAAIACQxMAAECBoQkAAKDA0AQAAFBgaAIAACgwNAEAABRUTdM0t3sjAAAAPqp80wQAAFBgaAIAACgwNAEAABQYmgAAAAoMTQAAAAWGJgAAgIL/AfTB8u317ZIPAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%matplotlib inline\n", + "fig, axes = plt.subplots(3, 3, figsize=(15, 15), facecolor=\"white\")\n", + "for i, k in enumerate(np.random.randint(len(train_ds), size=9)):\n", + " data = train_ds[k]\n", + " im, title = data[\"image\"], data[\"class_name\"]\n", + " ax = axes[i // 3, i % 3]\n", + " im_show = ax.imshow(im[0])\n", + " ax.set_title(title, fontsize=25)\n", + " ax.axis(\"off\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define loss function and network" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", + "loss_function = torch.nn.CrossEntropyLoss()\n", + "\n", + "\n", + "def get_new_net():\n", + " return DenseNet(\n", + " spatial_dims=2,\n", + " in_channels=1,\n", + " out_channels=num_classes,\n", + " init_features=2,\n", + " growth_rate=2,\n", + " block_config=(2,),\n", + " ).to(device)\n", + "\n", + "\n", + "model = get_new_net()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Estimate optimal learning rate\n", + "\n", + "Use MONAI's `LearningRateFinder` to get an initial estimate of a learning rate. Assume that it's in the range 1e-5, 1e0. If that weren't the case (which we'd notice in the plot), we could just try again over a larger/different window.\n", + "\n", + "We can plot the results and extract the learning rate with the steepest gradient." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Computing optimal learning rate: 90%|█████████ | 18/20 [00:14<00:01, 1.26it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Stopping early, the loss has diverged\n", + "Resetting model and optimizer\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA24AAANgCAYAAABUWzsXAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABTAUlEQVR4nO39eZTkdX0v/j+ru2djpmdh6RlgWAOyDMvMMIC4IGoQBUPiDUSJMVeNoiZXYvB6o/kmkXyTqDfE+1Vzf9HgjTFXvXHBxIV4E7JouKIJd+wRRBCJMNMM0DUDdHfNVj291O+P7umZYQaYpavrU1WPxzme7qr6LK/qhrKfvJdXqVar1QIAAEBhdTS6AAAAAJ6d4AYAAFBwghsAAEDBCW4AAAAFJ7gBAAAUnOAGAABQcF2NLmBPRx99dE4++eRGlwEAANAQ69evzxNPPLHP84UKbieffHLWrl3b6DIAAAAaYs2aNft93lRJAACAghPcAAAACk5wAwAAKLhCrXEDAIB2NTIyko0bN6ZarTa6FGbA3Llzs3z58syaNeuAjhfcAACgADZu3Jju7u6cfPLJKZVKjS6HOqrVannyySezcePGnHLKKQd0jqmSAABQANVqNUcddZTQ1gZKpVKOOuqogxpdFdwAAKAghLb2cbC/a8ENAACaUa2W/Ou/Jn/zNxNfa7W63OYjH/lItm/fXpdrH6jBwcH86Z/+6Yzd7+STT55qgv2CF7zgkK/z6U9/Oo899ti01CS4AQBAs/nGN5ITT0wuvzx54xsnvp544sTz06xVgtvo6Oghnfed73znkO8puAEAQLv6xjeSa65JNm5Mtm5NKpWJrxs3Tjx/iOFt27Ztueqqq3L++efnnHPOyRe+8IV87GMfy2OPPZaXvvSleelLX5okuf3223PJJZdk9erVufbaa7N169Ykyfe+97285CUvyQUXXJArrrgijz/+eJLksssuy6//+q9n5cqVOeecc3LXXXdN3e/Nb35zLrrooqxatSpf/epXkyQ//OEPc9FFF2XlypU577zz8uCDD+a9731vfvKTn2TlypV5z3ves0/tv//7v58zzjgjL3rRi3Ldddflj//4j6fu/a53vStr1qzJRz/60Xz961/PxRdfnFWrVuWnf/qnUy6XkyRPPvlkXvGKV2TFihV5y1vektoeo5cLFiyY+v7mm2/OhRdemPPOOy/vf//7kyTr16/PWWedlbe+9a1ZsWJFXvGKV2THjh259dZbs3bt2rz+9a/PypUrs2PHjkP6vUypFcgFF1zQ6BIAAKAh7rvvvuc+aHy8Vjv++FptYmLk/v+3fPnEcQfp1ltvrb3lLW+Zejw4OFir1Wq1k046qbZ58+ZarVarbd68ufbiF7+4tnXr1lqtVqt96EMfqv3e7/1ebefOnbVLLrmktmnTplqtVqt9/vOfr73pTW+q1Wq12kte8pKp6/7Lv/xLbcWKFbVarVZ73/veV/vMZz5Tq9VqtYGBgdrpp59e27p1a+0//af/VPvsZz9bq9VqteHh4dr27dtrDz/88NR5T3fXXXfVzj///NqOHTtqlUqldtppp9VuvvnmqXu/4x3vmDr2qaeeqo1P/mw++clP1m688cZarVarvfOd76z93u/9Xq1Wq9Vuu+22WpKp9zx//vxarVar/f3f/33trW99a218fLw2NjZWu+qqq2r/8i//Unv44YdrnZ2dtXXr1tVqtVrt2muvnXpfL3nJS2r/9//+32f8me/vd/5MmUg7AAAAaBb/9m/J0NCzHzM4mNx1V3LxxQd16XPPPTfvfve785u/+Zt59atfnRe/+MX7HPOv//qvue+++/LCF74wSbJz585ccskleeCBB3Lvvffm8ssvT5KMjY3l2GOPnTrvuuuuS5JceumlqVQqGRwczO23356vfe1rU6Nj1Wo1fX19ueSSS/KHf/iH2bhxY/7Df/gPOf3005+17jvvvDM/+7M/m7lz52bu3Ln5mZ/5mb1ef+1rXzv1/caNG/Pa1742jz/+eHbu3Dm1Ff8dd9yRv/7rv06SXHXVVVmyZMk+97n99ttz++23Z9WqVUmSrVu35sEHH8yJJ56YU045JStXrkySXHDBBVm/fv2z1nwoBDcAAGgWjz+edDzHaqeOjuQQ1lU973nPS29vb77xjW/kt3/7t/Pyl788v/u7v7vXMbVaLZdffnn+6q/+aq/nf/CDH2TFihX57ne/u99rP30HxVKplFqtli9/+cs544wz9nrtrLPOysUXX5y//du/zZVXXpk/+7M/y6mnnnrQ72eX+fPnT33/zne+MzfeeGOuvvrqfOtb38pNN910wNep1Wp53/vel7e97W17Pb9+/frMmTNn6nFnZ+fhT4vcD2vcAACgWRx7bDI+/uzHjI8nxx130Jd+7LHHcsQRR+SXfumX8p73vCe9vb1Jku7u7mzZsiVJ8vznPz933nln/v3f/z3JxDq1H//4xznjjDOyefPmqeA2MjKSH/7wh1PX/sIXvpAk+fa3v51FixZl0aJFueKKK/Inf/InU+vJ1q1blyR56KGHcuqpp+aGG27Iz/7sz+aee+7Zq4ane+ELX5ivf/3rqVar2bp1a2677bZnfI9DQ0M5/vjjkyR/+Zd/OfX8pZdemv/1v/5XkuR//+//nYGBgX3OveKKK/KpT31qak3fo48+mk2bNj3rz/TZ6j5YRtwAAKBZXHxxsmjRxGYkz2Tx4uSiiw760j/4wQ/ynve8Jx0dHZk1a1Y+/vGPJ0muv/76vPKVr8xxxx2Xb37zm/n0pz+d6667LsPDw0mSP/iDP8jznve83HrrrbnhhhsyNDSU0dHRvOtd78qKFSuSJHPnzs2qVasyMjKST33qU0mS3/md38m73vWunHfeeRkfH88pp5yS2267LV/84hfzmc98JrNmzcqyZcvyW7/1WznyyCPzwhe+MOecc05e9apX5eabb56q+8ILL8zVV1+d8847L0uXLs25556bRYsW7fc93nTTTbn22muzZMmSvOxlL8vDDz+cJHn/+9+f6667LitWrMgLXvCCnHjiifuc+4pXvCL3339/LrnkkiQTm5Z89rOfTWdn5zP+TN/4xjfm7W9/e+bNm5fvfve7mTdv3sH+WqaUarU6NXw4BGvWrMnatWsbXQYAAMy4+++/P2edddZzH7hrV8n9TcebNy+59dbkyiunv8BDdNlll+WP//iPs2bNmrrdY+vWrVmwYEG2b9+eSy+9NLfccktWr15dt/tNl/39zp8pE5kqCQAAzeTKKyfC2fLlyYIFycKFE1+XLy9caJsp119/fVauXJnVq1fn53/+55sitB0sUyUBAKDZXHll0tc3sXvkY49NrGm76KLkaZuAFMG3vvWtut9j1/q0Via4AQBAMyqVDnrLf5qXqZIAAFAQBdp+gjo72N+14AYAAAUwd+7cPPnkk8JbG6jVannyySczd+7cAz7HVEkAACiA5cuXZ+PGjdm8eXOjS2EGzJ07N8uXLz/g4wU3AAAogFmzZuWUU05pdBkUlKmSAAAABSe4AQAAFJzgBgAAUHCCGwAAQMEJbgAAAAUnuAEAABSc4AYAAFBwghsAAEDBCW4AAAAFJ7gBAAAUnOAGAAC0jZ2j49lUqWZsvNboUg6K4AYAALSN+x6v5KIP/FO+9cCmRpdyUAQ3AACgbfQPVZMkSxfObXAlB0dwAwAA2ka5IrgBAAAUWrlSTVdHKUfNn93oUg6K4AYAALSN/ko1Pd1z0tFRanQpB0VwAwAA2samynCWLmquaZKJ4AYAALSR/ko1S7sFNwAAgMIqD1WzzIgbAABAMW0bHs2W4dH0LJzT6FIOmuAGAAC0hV2tAJY1WSuARHADAADaRLkynERwAwAAKKxdI249ghsAAEAx9e+aKmlzEgAAgGIqV6qZP7szC+Z0NbqUgya4AQAAbaFcqTZl8+1EcAMAANpEuTLclBuTJIIbAADQJvqHqlkquAEAABTT+Hgtm7YIbgAAAIU1sH1nRsZqWbpwTqNLOSSCGwAA0PKmWgEYcQMAACimTZXhJLGrJAAAQFHtGnGzxg0AAKCg+ocmgltPtzVuAAAAhbRpSzVHL5idWZ3NGYGas2oAAICD0Mw93BLBDQAAaAPlynDT7iiZCG4AAEAbKFeq6RHcAAAAiml4dCxPbttpxA0AAKCoNm+Z7OG2sDl3lEwENwAAoMWVd/Vwa9Lm24ngBgAAtLhyZWLEzVRJAACAgtrVfFs7AAAAgIIqV6qZ3dmRJUfManQph0xwAwAAWtpEK4A5KZVKjS7lkAluAABAS+uvVJt6fVsiuAEAAC1uU2W4qXeUTAQ3AACghdVqtfRXqlnaLbgBAAAU0pbh0WzfOZZli5q3+XYiuAEAAC1sU6X5WwEkghsAANDC+ocmmm8LbgAAAAVVnhxxs6skAABAQfWbKgkAAFBs5Uo1C+d2Zd7szkaXclgENwAAoGWVK9Usa/IebongBgAAtLD+ynDTT5NMBDcAAKCFbapUBTcAAICiGhuvZdOW4Sxd2NzNtxPBDQAAaFFPbh3O2Hit6VsBJIIbAADQosqV1mi+nQhuAABAi2qVHm6J4AYAALSo8mRw0w4AAACgoMqVajpKyVHzZze6lMMmuAEAAC2pf6iaY7rnpKuz+WNP878DAACA/ShvGW6JHSUTwQ0AAGhR5aFqegQ3AACA4uqvVI24AQAAFFV1ZCxDO0aydOGcRpcyLQQ3AACg5ZRbqIdbIrgBAAAtqFwZTtIaPdwSwQ0AAGhB/UbcAAAAiq08JLgBAAAUWrlSzdxZHVk4t6vRpUwLwQ0AAGg5u1oBlEqlRpcyLQQ3AACg5WyqDLfMNMlEcAMAAFpQf6UquAEAABRVrVabmCrZIq0AEsENAABoMUM7RrJzdDw93XMaXcq0EdwAAICWsquHmxE3AACAgipXhpMky6xxOzCDg4O55pprcuaZZ+ass87Kd7/73XreDgAAoOWabydJXbvR/fqv/3pe+cpX5tZbb83OnTuzffv2et4OAABgaqpkz8LWWeNWt+A2NDSUO+64I5/+9KeTJLNnz87s2bPrdTsAAIAkSblSzZIjZmVOV2ejS5k2dZsq+fDDD+eYY47Jm970pqxatSpvectbsm3btnrdDgAAIMlEcGulaZJJHYPb6Ohoent78453vCPr1q3L/Pnz86EPfWif42655ZasWbMma9asyebNm+tVDgAA0CbKleGW2lEyqWNwW758eZYvX56LL744SXLNNdekt7d3n+Ouv/76rF27NmvXrs0xxxxTr3IAAIA20V+pZmm34HZAli1blhNOOCEPPPBAkuSf/umfcvbZZ9frdgAAABkZG88TW4eztMVG3Oq6q+Sf/Mmf5PWvf3127tyZU089NX/xF39Rz9sBAABt7omtw6nVkqUttKNkUufgtnLlyqxdu7aetwAAAJjSP9nDrZWabyd1bsANAAAwk8qV4SSt1Xw7EdwAAIAWUp5svi24AQAAFFR/pZqujlKOmj+70aVMK8ENAABoGeVKNT3dc9LRUWp0KdNKcAMAAFpGuVJtuVYAieAGAAC0kHJluOV2lEwENwAAoIWUh6ottzFJIrgBAAAtYtvwaLYMjwpuAAAARbW7FcCcBlcy/QQ3AACgJfRPBjdr3AAAAApqU2U4SewqCQAAUFT9U1MlBTcAAIBC6h+qZsGcriyY09XoUqad4AYAALSETVuq6WnBjUkSwQ0AAGgR/UPVltyYJBHcAACAFlGuDAtuAAAARTU+XpucKim4AQAAFNJT23dmZKyWZda4AQAAFFO5hVsBJIIbAADQAqaCWws2304ENwAAoAWUK8NJYnMSAACAouofqqZUSo7ptsYNAACgkMqVao6aPyezOlsz4rTmuwIAANpKuVLN0hbdUTIR3AAAgBbQ38LNtxPBDQAAaAGbKtWW3VEyEdwAAIAmNzw6lie37czSbsENAACgkDbtagWwyBo3AACAQtq0ZaL5do81bgAAAMXUP9TazbcTwQ0AAGhy5crEiJvgBgAAUFDlSjWzuzqy+IhZjS6lbgQ3AACgqfVPNt8ulUqNLqVuBDcAAKCplSvVlm4FkAhuAABAkytXhlu6+XYiuAEAAE2sVqulXKm29MYkieAGAAA0sS3Do9m+cyxLF7Zu8+1EcAMAAJpYeWiiFcBSI24AAADFVK5MNN8W3AAAAAqqvw2abyeCGwAA0MTKFVMlAQAACq1cqWbh3K7Mm93Z6FLqSnADAACaVv9QNctavIdbIrgBAABNrLxluOWnSSaCGwAA0MTKQ1XBDQAAoKjGxmvZvHW45XeUTAQ3AACgST25dThj47UsXTin0aXUneAGAAA0pf42aQWQCG4AAECTKleGk8SukgAAAEVlxA0AAKDgykPVdJSSoxdY4wYAAFBI5Uo1x3TPSWdHqdGl1J3gBgAANKX+SrUtWgEkghsAANCkNlWG22J9WyK4AQAATaq/UhXcAAAAiqo6MpahHSNt0QogEdwAAIAmVJ5sBdDT3fo7SiaCGwAA0IT6hyaCmxE3AACAgipvGU4Su0oCAAAUVXlyxK1HcAMAACim/ko182Z1ZuHcrkaXMiMENwAAoOmUK9UsXTgnpVKp0aXMCMENAABoOuU26uGWCG4AAEATKleG22ZHyURwAwAAmkytVku/ETcAAIDiGtw+kp2j44IbAABAUZW3TLQCWLpwToMrmTmCGwAA0FT6J3u4tUvz7URwAwAAmsymynCSmCoJAABQVP2ViRG3HlMlAQAAiqm/Us2R82dnTldno0uZMYIbAADQVDZVqunpbp/RtkRwAwAAmkx/pdpWzbcTwQ0AAGgy5cpwW+0omQhuAABAExkZG88TW4fTI7gBAAAU0+Ytw6nV2quHWyK4AQAATaQ82QpgaRu1AkgENwAAoInsDm5G3AAAAAqpXBlOErtKAgAAFFV/pZpZnaUcecTsRpcyowQ3AACgaZSHqunpnpuOjlKjS5lRghsAANA0yluq6WmzjUkSwQ0AAGgi/UPVtmsFkAhuAABAE9lUGW67HSUTwQ0AAGgS24ZHs2V4VHADAAAoqv7JHm7LFlnjBgAAUEhTzbe7jbgBAAAU0lRwa7Pm24ngBgAANIlyZThJrHEDAAAoqv6hahbM6cqCOV2NLmXGCW4AAEBTKFeqWdqGzbcTwQ0AAGgSE8Gt/aZJJoIbAADQJMqV4SwT3AAAAIppfLyWTVuqbbmjZCK4AQAATeCp7TszMlbL0m5r3AAAAAqpf2iih9syI24AAADFtGnLRHDrscYNAACgmPqHJppv25wEAACgoMqVakql5Bhr3AAAAIqpXKnmqPlzMquzPSNMe75rAACgqfRXqlm2qD1H2xLBDQAAaALlynCWdrfn+rZEcAMAAJpAudK+zbcTwQ0AACi44dGxPLVtZ9vuKJkIbgAAQMFtqky0Ali60Bo3AACAQipXJppvLzXiBgAAUEzlqRE3wQ0AAKCQ+idH3KxxAwAAKKhNlWpmd3Vk8RGzGl1KwwhuAABAofVXqlm6cE5KpVKjS2kYwQ0AACi0/qFqW0+TTAQ3AACg4DZtGU6P4AYAAFBMtVrNiFsENwAAoMC2DI9mx8iY4NboAgAAAJ5JeWiiFUDPwjkNrqSxBDcAAKCw9HCbILgBAACFVa4MJ0mWCm4AAADFVN414rZIcAMAACikcqWaRfNmZe6szkaX0lCCGwAAUFj9Q9UsbfONSRLBDQAAKLBypdr269sSwQ0AACiwcmVYcIvgBgAAFNTYeC2btw63fSuARHADAAAK6omtwxkbr2Vpm+8omQhuAABAQe1qBbC02+YkghsAAFBI/UN6uO0iuAEAAIVU3jKcJDYnieAGAAAUVHmoms6OUo5eYKqk4AYAABRSf6WaYxbMSWdHqdGlNJzgBgAAFNJE822jbYngBgAAFNREcLO+LRHcAACAgipXhu0oOUlwAwAACqc6MpahHSNG3CYJbgAAQOHs6uEmuE0Q3AAAgMIpV3YFN5uTJIIbAABQQP2TwW2ZEbckghsAAFBAmyrDSZKlNidJIrgBAAAF1F+pZt6sznTP6Wp0KYUguAEAAIXTX6lm2aK5KZVKjS6lEAQ3AACgcDZVqunptjHJLoIbAABQOLtG3JgguAEAAIVSq9VSrgzbUXIPghsAAFAog9tHsnN0PD2C2xTBDQAAKBQ93PYluAEAAIVSngxuSxfanGQXwQ0AACiU3cHNiNsughsAAFAo5cpwkqTHiNsUwQ0AACiU/ko1R86fnTldnY0upTAENwAAoFDKQ1XTJJ9GcAMAAAqlvKVqY5KnEdwAAIBC6R/SfPvpBDcAAKAwRsbG8+S2YVMln0ZwAwAACmPzluHUaloBPF1XPS9+8sknp7u7O52dnenq6sratWvreTsAAKDJ9U/2cFu2yBq3PdU1uCXJN7/5zRx99NH1vg0AANACNk0Gt55uI257MlUSAAAojP6hXSNugtue6hrcSqVSXvGKV+SCCy7ILbfcst9jbrnllqxZsyZr1qzJ5s2b61kOAABQcOUtw5nVWcqRR8xudCmFUtepkt/+9rdz/PHHZ9OmTbn88stz5pln5tJLL93rmOuvvz7XX399kmTNmjX1LAcAACi48lA1Pd1z09FRanQphVLXEbfjjz8+SdLT05PXvOY1ueuuu+p5OwAAoMn1VzTf3p+6Bbdt27Zly5YtU9/ffvvtOeecc+p1OwAAoAWUK1WtAPajblMly+VyXvOa1yRJRkdH84u/+It55StfWa/bAQAALaBcGc6LTz+m0WUUTt2C26mnnpq77767XpcHAABazNbh0WwdHrWj5H5oBwAAABRCebKHmzVu+xLcAACAQigP7QpuRtyeTnADAAAKobxFcHsmghsAAFAI/UPDSZJlgts+BDcAAKAQypVquud0Zf6cuu2h2LQENwAAoBDKlWp6bEyyX4IbAABQCP2VqlYAz0BwAwAACmFTZThLuwW3/RHcAACAhhsfr6VcqWapEbf9EtwAAICGe2r7zoyO1+wo+QwENwAAoOH6p5pv25xkfwQ3AACg4coVzbefjeAGAAA0XLky0XxbcNs/wQ0AAGi4/ko1pVJyTLepkvsjuAEAAA23qVLN0QvmZFaniLI/fioAAEDD9VeqNiZ5FoIbAADQcP1DVa0AnoXgBgAANNymLcPpEdyekeAGAAA01PDoWJ7attOI27MQ3AAAgIbaNNkKQHB7ZoIbAADQULuab/fYnOQZCW4AAEBD9U8Gt2WLjLg9E8ENAABoqPLkVMml3YLbMxHcAACAhipXqpnd1ZHFR8xqdCmFJbgBAAANVa5M9HArlUqNLqWwBDcAAKCh+oeqWWpjkmcluAEAAA1VrlSzVCuAZyW4AQAADVOr1VKuDAtuz0FwAwAAGqZSHc2OkTHNt5+D4AYAADTMpskebkv1cHtWghsAANAwu5pvL+22OcmzEdwAAICG6R+aCG7LjLg9K8ENAABomE1bhpPE5iTPQXADAAAapn+omkXzZmXurM5Gl1JoghsAANAw/ZWqHSUPgOAGAAA0zKZKNT0LbUzyXAQ3AACgYYy4HRjBDQAAaIix8Vo2bxm2MckBENwAAICGeGLrcMZrmm8fCMENAABoiKkebkbcnpPgBgAANES5MhHcltqc5DkJbgAAQEPsCm5G3J6b4AYAADREuTKczo5SjlpgxO25CG4AAEBD9FeqOWbBnHR2lBpdSuEJbgAAQEOUK1U7Sh4gwQ0AAGiIcqWapd2mSR4IwQ0AAGiI/qFqlhlxOyCCGwAAMON27BxLpTqapXaUPCCCGwAAMON293AT3A6E4AYAAMy4fj3cDorgBgAAzLjdI242JzkQghsAADDjpoKbzUkOiOAGAADMuHJlOEfM7kz3nK5Gl9IUBDcAAGDG9VeqWbpwbkqlUqNLaQqCGwAAMOPKQ1Xr2w6C4AYAAMy48paqVgAHQXADAABmVK1WS7kyrBXAQRDcAACAGTW4fSQ7R8eNuB0EwQ0AAJhR/VM93AS3AyW4AQAAM2pXcFu2yOYkB0pwAwAAZtSmyeDW023E7UAJbgAAwIzqHxpOYqrkwRDcAACAGVXeUs1R82dndpc4cqD8pAAAgBlVHqqmx2jbQRHcAACAGdVfqWbZQhuTHAzBDQAAmFHlyrD1bQdJcAMAAGbMyNh4ntwmuB0swQ0AAJgxm7cMp1ZLli0S3A6G4AYAAMyYXc23l1rjdlAENwAAYMaUh3YFNyNuB0NwAwAAZky5IrgdCsENAACYMf2V4czqLOXII2Y3upSmIrgBAAAzZlOlmp7uuenoKDW6lKYiuAEAADOmv1K1MckhENwAAIAZ01+pagVwCAQ3AABgxmyqDKenW3A7WIIbAAAwI7YOj2br8KgRt0MguAEAADNiVyuAZVoBHDTBDQAAmBG7mm/32JzkoAluAADAjOg34nbIBDcAAGBGlCvDSZKlgttBE9wAAIAZUa5U0z2nK/PndDW6lKYjuAEAADOiXKlmqR0lD4ngBgAAzIj+SjVLbUxySAQ3AABgRpSHqta3HSLBDQAAqLvx8Vo2bRkW3A6R4AYAANTdk9t2ZnS8phXAIRLcAACAuitP9nAz4nZoBDcAAKDudgc3m5McCsENAACou/7J4LZMO4BDIrgBAAB1V64Mp1RKjl5gxO1QCG4AAEDdlYeqOXrBnMzqFEEOhZ8aAABQd+UtVTtKHgbBDQAAqLv+oaqNSQ6D4AYAANRduVLVCuAwCG4AAEBdDY+OZWD7iOB2GAQ3AACgrjZVhpPEGrfDILgBAAB1NdV8Ww+3Qya4AQAAdbWr+bbNSQ6d4AYAANRV/9BEcDNV8tAJbgAAQF1t2jKc2V0dWTRvVqNLaVqCGwAAUFf9QxPNt0ulUqNLaVqCGwAAUFflStU0ycMkuAEAAHVVrlTTY2OSwyK4AQAAdVOr1dJvxO2wCW4AAEDdVKqjqY6MZ6ngdlgENwAAoG40354eghsAAFA3erhND8ENAACom6kRN5uTHBbBDQAAqJvdwc2I2+EQ3AAAgLopV4azaN6szJ3V2ehSmprgBgAA1I1WANNDcAMAAOqmXKnaUXIaCG4AAEDdlCvVLO22McnhEtwAAIC6GB0bz+Ytw1lmxO2wCW4AAEBdPLltZ8ZrSY81bodNcAMAAOpC8+3pI7gBAAB10V8R3KaL4AYAANTFpqnm2zYnOVyCGwAAUBf9lWo6O0o5aoHgdrgENwAAoC7KleEcs2BOOjtKjS6l6QluAABAXWi+PX0ENwAAoC76h6pZZn3btDig4LZt27aMj48nSX784x/na1/7WkZGRupaGAAA0NzKlWqW2lFyWhxQcLv00ktTrVbz6KOP5hWveEU+85nP5I1vfGOdSwMAAJrVjp1jqVRHBbdpckDBrVar5Ygjjshf//Vf51d/9VfzpS99KT/84Q/rXRsAANCkylOtAAS36XDAwe273/1uPve5z+Wqq65KkoyNjdW1MAAAoHlpvj29Dii4feQjH8kHP/jBvOY1r8mKFSvy0EMP5aUvfWm9awMAAJrUrhG3ZYtsTjIdug7koJe85CV5yUtekiQZHx/P0UcfnY997GN1LQwAAGheu4JbjxG3aXFAI26/+Iu/mEqlkm3btuWcc87J2WefnZtvvrnetQEAAE2qf2g4R8zuTPecAxor4jkcUHC77777snDhwnzlK1/Jq171qjz88MP5zGc+U+/aAACAJlXeMtEKoFQqNbqUlnBAwW1kZCQjIyP5yle+kquvvjqzZs3yCwAAAJ5ReaiapZpvT5sDCm5ve9vbcvLJJ2fbtm259NJLs2HDhixcuLDetQEAAE2qv1K1o+Q0OqDgdsMNN+TRRx/NN77xjZRKpZx00kn55je/We/aAACAJlSr1bKpMqyH2zQ6oOA2NDSUG2+8MWvWrMmaNWvy7ne/O9u2bat3bQAAQBMa2D6SnWPjgts0OqDg9uY3vznd3d354he/mC9+8YtZuHBh3vSmN9W7NgAAoAnt7uEmuE2XA9qb8yc/+Um+/OUvTz1+//vfn5UrV9arJgAAoIn1TwY3m5NMnwMacZs3b16+/e1vTz2+8847M2/evLoVBQAANK/y0K7gZsRtuhzQiNsnPvGJ/PIv/3KGhoaSJEuWLMlf/uVf1rUwAACgOZUrw0mSnm7BbbocUHA7//zzc/fdd6dSqSRJFi5cmI985CM577zz6locAADQfPor1Rw1f3Zmdx3QBD8OwEH9JBcuXDjVv+2//bf/VpeCAACA5rapUjVNcpodcgSu1WrTWQcAANAi+itVG5NMs0MObqVSaTrrAAAAWkS5UtUKYJo96xq37u7u/Qa0Wq2WHTt21K0oAACgOY2MjeeJrTttTDLNnjW4bdmyZabqAAAAWsCmLRM7Shpxm162eQEAAKZNebL59jKbk0wrwQ0AAJg2u5pv99icZFoJbgAAwLTpN+JWF4IbAAAwbcqV4czqLGXJEbMbXUpLEdwAAIBpU65U09M9Nx0d2odNJ8ENAACYNnq41YfgBgAATJv+SjVLbUwy7eoe3MbGxrJq1aq8+tWvrvetAACABisPVbPUxiTTru7B7aMf/WjOOuuset8GAABosK3Do9m2c0xwq4O6BreNGzfmb//2b/OWt7ylnrcBAAAKoH9IK4B6qWtwe9e73pU/+qM/SkeHpXQAANDqNk32cDPiNv3qlqhuu+229PT05IILLnjW42655ZasWbMma9asyebNm+tVDgAAUGf9U8HN5iTTrW7B7c4778zXvva1nHzyyXnd616Xf/7nf84v/dIv7XPc9ddfn7Vr12bt2rU55phj6lUOAABQZ/1G3OqmbsHtgx/8YDZu3Jj169fn85//fF72spfls5/9bL1uBwAANNimynC653Rl/pyuRpfSciw+AwAApkX/UDVLNd+uixmJwpdddlkuu+yymbgVAADQIOUtVTtK1okRNwAAYFqUh6rpsTFJXQhuAADAYRsfr2XTlmEjbnUiuAEAAIftyW07Mzpes6NknQhuAADAYStrBVBXghsAAHDYdgW3ZXaVrAvBDQAAOGy7m2/bnKQeBDcAAOCwlYeqKZWSYxYIbvUguAEAAIetXBnO0QvmpKtTxKgHP1UAAOCw9Vc0364nwQ0AADhs5UrVjpJ1JLgBAACHbSK4Wd9WL4IbAABwWKojYxnYPmKqZB0JbgAAwGHZvGU4iebb9SS4AQAAh2Wqh5vm23UjuAEAAIelPBncTJWsH8ENAAA4LP1DkyNuNiepG8ENAAA4LOVKNXO6OrJo3qxGl9KyBDcAAOCwlCvDWbpwbkqlUqNLaVmCGwAAcFj6K1Xr2+pMcAMAAA5LuVK1o2SdCW4AAMAhq9VqE8Gt28Yk9SS4AQAAh6yyYzTVkfEsM+JWV4IbAABwyMpbJloB9FjjVleCGwAAcMh29XCzOUl9CW4AAMAh668IbjNBcAMAAA7ZpsquqZI2J6knwQ0AADhk/ZVqFh8xK3NndTa6lJYmuAEAAIesXBnO0m7TJOtNcAMAAA6Z5tszQ3ADAAAOWf9QNcusb6s7wQ0AADgk24ZH88TW4Ry7aF6jS2l5ghsAAHBI7t44mPFacv4JixpdSssT3AAAgEOyrm8wSbLqhCWNLaQNCG4AAMAh6d0wkFOPmZ8l82c3upSWJ7gBAAAHrVarZd0jg1l9otG2mSC4AQAAB239k9vz1LadgtsMEdwAAICD1rthIEmy+qTFjS2kTQhuAADAQevtG8iCOV05vae70aW0BcENAAA4aL19g1l5wuJ0dpQaXUpbENwAAICDsnV4NA/0V7L6xMWNLqVtCG4AAMBBueeRicbbq06yMclMEdwAAICD8r1dG5NovD1jBDcAAOCg9PYN5LSeBVl0xKxGl9I2BDcAAOCA7W68vbjRpbQVwQ0AADhgDz2xLYPbRzTenmGCGwAAcMB2N94W3GaS4AYAAByw3r7BdM/tymnHLGh0KW1FcAMAAA7Yur6BrDxhcTo03p5RghsAAHBAtlRH8kB5i/VtDSC4AQAAB+TuR4ZSq1nf1giCGwAAcEB6+yY2Jll5wuLGFtKGBDcAAOCA9PYN5PSeBVk0T+PtmSa4AQAAz2l8vJbeDQPWtzWI4AYAADynh57Ymkp1NBdY39YQghsAAPCcejcMJklWn7S4oXW0K8ENAAB4Tr19A1k4tyunHq3xdiMIbgAAwHPq7RvIqhOXaLzdIIIbAADwrCrVkTy4aauNSRpIcAMAAJ7V9/sGJxtvL250KW1LcAMAAJ5Vb99ASiWNtxtJcAMAAJ5Vb99gntfTne65Gm83iuAGAAA8o/HxWtb1DZgm2WCCGwAA8Ix+snlrtlRHs8rGJA0luAEAAM/oexsGkiQXnCS4NZLgBgAAPKPevoEsPmJWTj16fqNLaWuCGwAA8Ix6+waz6oTFKZU03m4kwQ0AANivoe0j+XeNtwtBcAMAAPZr3SMT69tWW9/WcIIbAACwX719g+koJedrvN1wghsAALBf6/oG8ryl3Vkwp6vRpbQ9wQ0AANjH+Hgt3+8bNE2yIAQ3AABgHw9u2potw6M2JikIwQ0AANhHb9/kxiQnLm5sISQR3AAAgP343oaBLDliVk7ReLsQBDcAAGAfvX0DWX3iEo23C0JwAwAA9jK4fWce2rzNxiQFIrgBAAB7Wdc3mCRZZX1bYQhuAADAXnr7BiYaby9f3OhSmCS4AQAAe+ntG8iZyxZmvsbbhSG4AQAAU8amGm8vbnQp7EFwAwAApvy4vCXbdo5pvF0wghsAADBld+Ntwa1IBDcAAGBK74bBHDl/dk466ohGl8IeBDcAAGDKur6BrD5xscbbBSO4AQAASZKntu3MQ09ovF1EghsAAJBkYrQtsb6tiAQ3AAAgycTGJJ0dpZy3fFGjS+FpBDcAACDJxMYkZx3bnSNma7xdNIIbAACQ0bHx3L1x0DTJghLcAACAPFDeku0abxeW4AYAAKS3bzCJjUmKSnADAACybsNAjl4wOyccOa/RpbAfghsAAJDevoGsOnGJxtsFJbgBAECbe3LrcNY/ud00yQIT3AAAoM2tm1rftrihdfDMBDcAAGhz3+sbSFdHKectX9zoUngGghsAALS53g0DOfu4hZk3u7PRpfAMBDcAAGhjo2PjuWfjkPVtBSe4AQBAG/tR/5bsGBnLKuvbCk1wAwCANtbbN5BE4+2iE9wAAKCN9W4YyDHdc7J8icbbRSa4AQBAG+vtG8zqExdrvF1wghsAALSpJ7YOp+8pjbebgeAGAABtqnfD5Pq2kwS3ohPcAACgTfX2Daaro5Rzj1/U6FJ4DoIbAAC0qd6+gaw4flHmztJ4u+gENwAAaEMjY+O5Z+PExiQUn+AGAABt6P7HK6mOjNuYpEkIbgAA0IZsTNJcBDcAAGhDvX2DWbpwTo5bNLfRpXAABDcAAGhDvX0DWX3iEo23m4TgBgAAbWbTlmo2Duywvq2JCG4AANBmejcMJklWn7S4oXVw4AQ3AABoM+v6BjKrs5QVx2m83SwENwAAaDO9fQNZcZzG281EcAMAgDayc3Q892wcsr6tyQhuAADQRu5/vJLh0fFcoH9bUxHcAACgjXxvqvH24sYWwkER3AAAoI309g3k2EVzc+yieY0uhYMguAEAQBtZ1zdofVsTEtwAAKBNlCvVPDq4I6tOXNzoUjhIghsAALSJ3qn1bUbcmo3gBgAAbaK3byCzOzuy4riFjS6FgyS4AQBAm+jtG8w5xy/MnC6Nt5uN4AYAAG1g5+h4fvCoxtvNSnADAIA28MPHhrJzdNz6tiYluAEAQBvo7RtMklwguDUlwQ0AANpAb99Ajl88L0sXzm10KRwCwQ0AANpA74YB/duamOAGAAAt7vGhHXl8qGpjkiYmuAEAQIvr3TCYROPtZia4AQBAi+vtG8icro6cfazG281KcAMAgBbX2zeQc49flNld/vxvVn5zAADQwoZHx/LDRyumSTY5wQ0AAFrYvY9WsnNsPKvtKNnUBDcAAGhh6/oGksSOkk1OcAMAgBa2q/F2j8bbTU1wAwCAFta7YTAXWN/W9AQ3AABoUY8N7kh/pWp9WwsQ3AAAoEV9b8Pk+jYjbk1PcAMAgBbV2zeQubM6cpbG201PcAMAgBbV2zeY845fnFmd/uxvdn6DAADQgqojY7nvsaGsOmlxo0thGghuAADQgu59dCgjYzX921qE4AYAAC2oV+PtliK4AQBAC+rdMJgTjpyXY7rnNLoUpoHgBgAALaZWq6W3b8BoWwsR3AAAoMU8Orgjm7YM5wL921qG4AYAAC2mt28wifVtrURwAwCAFtO7YSDzZnXmzGXdjS6FaSK4AQBAi+ntG8h5yxelS+PtluE3CQAALWSi8XYlq61vaymCGwAAtJB7Ng5ldFzj7VZTt+BWrVZz0UUX5fzzz8+KFSvy/ve/v163AgAAJu1qvL3qxMWNLYRp1VWvC8+ZMyf//M//nAULFmRkZCQvetGL8qpXvSrPf/7z63VLAABoe70bBnLSUUfk6AUab7eSuo24lUqlLFiwIEkyMjKSkZGRlEqlet0OAADa3kTj7UHTJFtQXde4jY2NZeXKlenp6cnll1+eiy++eJ9jbrnllqxZsyZr1qzJ5s2b61kOAAC0tI0DO/LE1uGsNk2y5dQ1uHV2dub73/9+Nm7cmLvuuiv33nvvPsdcf/31Wbt2bdauXZtjjjmmnuUAAEBL272+zYhbq5mRXSUXL16cl770pfm7v/u7mbgdAAC0pd4NAzlitsbbrahuwW3z5s0ZHBxMkuzYsSP/8A//kDPPPLNetwMAgLbX2zeY85cv1ni7BdVtV8nHH388//E//seMjY1lfHw8v/ALv5BXv/rV9bodAAC0te07R3Pf45W8/SWnNroU6qBuwe28887LunXr6nV5AABgD/dsHMqYxtstyxgqAAC0ABuTtDbBDQAAWkDvhsGccvT8HDl/dqNLoQ4ENwAAaHK1Wi3r+gaySv+2liW4AQBAk+t7anue3LbT+rYWJrgBAECT27W+TXBrXYIbAAA0ud4Ng5k/uzNnaLzdsgQ3AABocr19A1l54uJ0dpQaXQp1IrgBAEAT275zND/q32KaZIsT3AAAoInd/YjG2+1AcAMAgCa2u/H24sYWQl0JbgAA0MR6Nwzk1GPmZ/ERGm+3MsENAACaVK1Wy7pHBk2TbAOCGwAANKn1T27PUxpvtwXBDQAAmlTvhsnG2yctbmwh1J3gBgAATaq3byAL5nTl9B6Nt1ud4AYAAE2qt28wK0/QeLsdCG4AANCEtg6P5oH+SlZrA9AWBDcAAGhC9zwymPFasvokG5O0A8ENAACa0FTj7RMEt3YguAEAQBPq7RvMaT0LsuiIWY0uhRkguAEAQJOp1Wrp7Ruwvq2NCG4AANBkHnpiWwa3j2i83UYENwAAaDK7G28Lbu1CcAMAgCbT2zeY7rldOe2YBY0uhRkiuAEAQJNZ1zeQlScsTofG221DcAMAgCaypTqSB8pbrG9rM4IbAAA0kbsfGUpN4+22I7gBAEAT6e0bSKmUrDxhcaNLYQYJbgAA0ER6+wZyes+CLJqn8XY7EdwAAKBJjI/Xsq5v0Pq2NiS4AQBAk3joiW0Z2qHxdjsS3AAAoEnsbry9uLGFMOMENwAAaBK9fQNZOLcrpx6t8Xa7EdwAAKBJ9PYNZNWJSzTebkOCGwAANIFKdSQPbtpqfVubEtwAAKAJfL9vcLLx9uJGl0IDCG4AANAENN5ub4IbAAA0gd6+wTyvpzvdczXebkeCGwAAFNxE4+2BrD7J+rZ2JbgBAEDB/WTz1mypjmb1iYsbXQoNIrgBAEDB9fbtarxtxK1dCW4AAFBwvRsGs/iIWTn16PmNLoUGEdwAAKDgvtc3kFUnLE6ppPF2uxLcAACgwIa2j+TfNd5ue4IbAAAU2LpHrG9DcAMAgELr7RtMRyk5X+Pttia4AQBAga3rG8jzlnZnwZyuRpdCAwluAABQUOPjtXy/b9A0SQQ3AAAoqgc3bc2W4dFcYGOStie4AQBAQWm8zS6CGwAAFFTvhoEcOX92Tj7qiEaXQoMJbgAAUFC9Gm8zSXADAIACGty+Mz/ZvM00SZIIbgAAUEjr+gaTJKtOXNzQOigGwQ0AAAqot29govH28sWNLoUCENwAAKCAevsGcuayhZmv8TYR3AAAoHDGphpvL250KRSE4AYAAAXz4/KWbNs5ltUabzNJcAMAgIKZarwtuDFJcAMAgILp3TCYo+bPzkkabzNJcAMAgIJZ1zeQVScu0XibKYIbAAAUyMC2nXnoiW02JmEvghsAABTIukesb2NfghsAABRI74bBdHaUct7yRY0uhQIR3AAAoEC+t2EgZx3bnSNma7zNboIbAAAUxOjYeO7eOGiaJPsQ3AAAoCAeKG/Jdo232Q/BDQAACqK3bzCJjUnYl+AGAAAF8b31T+XoBbNzwpHzGl0KBSO4AQBAAdz76FC+fs/jufzsZRpvsw/BDQAAGmx4dCzv/uLdOWr+7Lz3lWc2uhwKyB6jAADQYB/7pwfzQHlLPvXGNVl0xKxGl0MBGXEDAIAGuvuRwXz8Wz/JtRcsz8vOXNrocigowQ0AABqkOjKWd3/p7ixdODe//eqzG10OBWaqJAAANMj/948/zr9v2pq/fPNFWTTPFEmemRE3AABogO9tGMgn73go1110Ql7yvGMaXQ4FJ7gBAMAMq46M5T1fujvHLpqX37ryrEaXQxMwVRIAAGbYH//9A3noiW357K9cnO65pkjy3Iy4AQDADFq7/qn8+Z0P55eef2JedPrRjS6HJiG4AQDADNmxcyz/+Ut35/jF8/K+V5kiyYEzVRIAAGbIH/39j7L+ye35q7c+P/Pn+FOcA2fEDQAAZsC/PvRk/uLO9fmPl5yUS37qqEaXQ5MR3AAAoM62DY/mv9x6T0488oj85qvObHQ5NCHjswAAUGf/9e9+lEcGtucL11+SI2b7E5yDZ8QNAADq6Dv//kT+53c35E0vOCUXnXJko8uhSQluAABQJ1uHR/OeW+/JKUfPz3uuOKPR5dDEjNMCAECdfOAb9+exoR259e2XZN7szkaXQxMz4gYAAHVwx48353/9W1/e+uJTc8FJpkhyeAQ3AACYZpXqSN775XvyU8fMz42XP6/R5dACTJUEAIBp9oe33Z/+SjVffscLMneWKZIcPiNuAAAwjb75wKZ8Ye0jedtLfiqrTlzS6HJoEYIbAABMk6EdI3nfl3+Q5y1dkHf99OmNLocWYqokAABMk9+/7b5s3jqcW375gszpMkWS6WPEDQAApsE/3V/Ord/bmHe85Kdy3vLFjS6HFiO4AQDAYRrcvjPv++sf5Mxl3Xnny09rdDm0IFMlAQDgMP3e1+/LU9t25lNvvNAUSerCiBsAAByGv/9hf/5m3aP5tZeelnOOX9TocmhRghsAAByip7btzP/zNz/I2ccuzK+91BRJ6sdUSQAAOETv/9oPM7RjJP/zzRdndpcxEerHP10AAHAIvvGDx/P1ux/LDS87PWcft7DR5dDiBDcAADhIT2wdzm9/5d6ce/yivP2yn2p0ObQBwQ0AAA5CrVbL73zl3mytjuaPrz0/szr9SU39+acMAAAOwm33PJ7/fW9/3nX56TljWXejy6FNCG4AAHCANm2p5ne+em/OP2Fxrn/xqY0uhzYiuAEAwAGo1Wr57b+5N9t3juXD156XLlMkmUH+aQMAgAPwtbsfy+33lfPuy5+X03pMkWRmCW4AAPAcNlWq+d2v/jCrTlyct5giSQMIbgAA8CxqtVp+629+kOrIWP742vPT2VFqdEm0IcENAACexV/3Ppp/vH9T3nPFGfmpYxY0uhzalOAGAADPoH+ompu+/sNcePKSvOmFpzS6HNqY4AYAAPtRq9Xy3r++JyNj47n5GlMkaSzBDQAA9uNLazfmWw9szntfeWZOPnp+o8uhzQluAADwNI8O7sjv33ZfLj7lyPzyJSc3uhwQ3AAAYE+1Wi3v/fI9GavVcvM156fDFEkKQHADAIA9/NVdj+T/PPhE3nflWTnxqCMaXQ4kEdwAAGDKI09tzx/+7X154WlH5fUXndjocmCK4AYAAEnGx2v5zS/fkyT5rz9/nimSFIrgBgAAST73bxvynZ88mf/nqrOzfIkpkhSL4AYAQNvre3J7PvCNH+XFpx+d6y46odHlwD4ENwAA2tr4eC3vufXudHWU8l9//ryUSqZIUjyCGwAAbe1/fnd9/u3hp/I7rz47xy2e1+hyYL8ENwAA2tb6J7blQ3/3o1x2xjG5ds3yRpcDz0hwAwCgLY2N1/Kfv3R3ZnV25EP/wRRJiq2r0QUAAEAj/MWdD2fthoF8+Nrzs2zR3EaXA8/KiBsAAG3nJ5u35ua/fyA/fVZP/sPq4xtdDjwnwQ0AgLaya4rk3Fmd+cBrzjVFkqZgqiQAAG3lf/yfh7KubzAffd3K9Cw0RZLmYMQNAIC28WB5Sz78Dz/OFSuW5urzj2t0OXDABDcAANrC6Nh4/vOX7s782Z35g58zRZLmYqokAABt4c/ueCh3bxzKn1y3Ksd0z2l0OXBQjLgBANDyftRfyUf+8ce58txlefV5xza6HDhoghsAAC1tZHKK5MK5s/L7P3uOKZI0JVMlAQBoaZ/41k9y76OVfPz1q3PUAlMkaU5G3AAAaFn3PVbJx/75wfzM+cflVeeaIknzEtwAAGhJO0cnpkgumjc7/+/VKxpdDhwWwQ0AgJYztGMkv/75dbnv8Uo+8JpzsmT+7EaXBIfFGjcAAFrKXQ8/ld/4wvdTrlTzvledmVesWNbokuCwCW4AALSE0bHxfOyfHsx//+a/54Qjj8it73hBVp6wuNFlwbQQ3AAAaHqPPLU9v/75dentG8zPr16e3/vZFVkwx5+6tA7/NAMA0NS+su7R/PZX7k2plHzsulW5+vzjGl0STDvBDQCAprSlOpLf/eoP8zfrHs2ak5bkI69bmeVLjmh0WVAXghsAAE3nexsG8q4vrMtjg9X8xk8/L7/20p9KV6cN02ldghsAAE1jbLyW/983/z0f/acHc+yiufni256fC046stFlQd0JbgAANIVHB3fkNz7//dy1/qn87Mrj8vs/d04Wzp3V6LJgRghuAAAU3m33PJb3/fUPUqsl/99rz89rVi1vdEkwowQ3AAAKa9vwaG762g/zpe9tzMoTFudjr1uVE4+yAQntR3ADAKCQ7n5kML/++XXZ8NT2vPNlp+WGl5+eWTYgoU0JbgAAFMrYeC1/dsdP8t9u/3F6uufk8299fi4+9ahGlwUNJbgBAFAYjw/tyI1fuDvffejJXHXusfnAa87NoiNsQAKCGwAAhfB39z6e3/zyDzIyNp4/uua8XHvB8pRKpUaXBYUguAEA0FDbd47m92+7P391V1/OW74oH33dqpxy9PxGlwWFIrgBANAw9z46lBs+vy4PP7Etb3/JT+XGy5+X2V02IIGnq9u/FY888khe+tKX5uyzz86KFSvy0Y9+tF63AgCgyYyP1/LJOx7Ka/70zmwbHs3nfuXivPdVZwpt8AzqNuLW1dWVD3/4w1m9enW2bNmSCy64IJdffnnOPvvset0SAIAmsKlSzbu/dHf+z4NP5BVnL81//fnzsmT+7EaXBYVWt+B27LHH5thjj02SdHd356yzzsqjjz4quAEAtLF/vK+c//Lle7J952g+8Jpzc91FJ9iABA7AjKxxW79+fdatW5eLL754Jm4HAEDBVEfG8oFv3J//+d0NOfvYhfnYdStzWk93o8uCplH34LZ169b8/M//fD7ykY9k4cKF+7x+yy235JZbbkmSbN68ud7lAAAww+5/vJJf//y6/Li8NW950Sl5zyvPyJyuzkaXBU2lVKvVavW6+MjISF796lfniiuuyI033vicx69ZsyZr166tVzkAAMygWq2WT39nfT74v3+UhXNn5b/9wvm59HnHNLosKLRnykR1G3Gr1Wr5lV/5lZx11lkHFNoAAGgdT2wdznu+dHe++cDmvOzMnvzRNefl6AVzGl0WNK26Bbc777wzn/nMZ3Luuedm5cqVSZIPfOADufLKK+t1SwAACuBbD2zKf/7S3alUR/P//uyKvOH5J9mABA5T3YLbi170otRxFiYAAAVTHRnLH/3dA/nUnQ/njKXd+dxbnp8zltmABKbDjOwqCQBAa3uwvCXv/Kt1+VH/lrzxBSfnva86M3Nn2YAEpovgBgDAIavVavnsv/XlD267L/PndOXP/+OavPyspY0uC1qO4AYAwCF5atvO/Jdb78k/3l/Oi08/Oh/+hfPT0z230WVBSxLcAAA4aN9+8Inc+MXvZ3D7SH77qrPy5heeko4OG5BAvQhuAAAcsJ2j4/nw7Q/kz+54KD91zPz8xZsuzIrjFjW6LGh5ghsAAAfkoc1bc8Pn1+XeRyv5xYtPzO9cdXbmzbYBCcwEwQ0AgP0a3L4z6x4ZzLq+wazrG8hdDz+VebM782dvuCBXrFjW6PKgrQhuAABkdGw8Py5vTW/fwERQe2QgD23eliTpKCVnLluY1154Qn71stOybJENSGCmCW4AAG3oia3DUyNpvX0DuWfjULbvHEuSHL1gdlaesCTXXLA8q05YkvOWL8r8Of5shEbybyAAQIvbOTqe+x+vZF3fwNTUx76ntidJujpKWXHcwvzCmhOy6sTFWX3ikixfMi+lkh0ioUgEt2cxMjaeR57anlKplFKSUikppZRdn2OlUvb7WilJSknH1Gt7H5NSpo571mvvcb2OUmnqfgAAz6Z/qDo1kraubzA/eHQow6PjSZKlC+dk9YlL8obnn5RVJy7OOccvytxZNhiBohPcnsXjg9W87MP/0ugy9uuZgl/2CHodu4JlaXfw2zNMdkwGyL2e65i4zj7nZu/wuNe5U6/v+dre5+xzbvY4tzRx7kTNu4/vKCUdHaXd308e31Ha+/3teexe5+5xfGep9Kyv7/d6e9bSsbu+vY9NOjt2193Zsfcxe9a/12t7Xe85rjFZ+z7X2OM1oKBqteTf/i15/PHk2GOTiy/O1H+hg2lSHRnLDx8bmpz2OJjevoE8PlRNkszu6si5xy+aDGlLsvqkxTl20bwGVwwcCsHtWRy1YHY+8tqVqaWWWm3i/39rSWq1WmpJUsvu17Lr9d2PU6tlvLb7+D3Pz36O3/Nxdt1n8rXx2t7X3d/99no8efz4Hs+N1/b4Onn98fGJ18d3vb9aber18b2us/vr+OSbH586fuL5ZN/jxsb3vN7kubt+Lpm4/+73tvta4+O7v999vX3vOT6+9/32d3w72BXq9gqqewS+vcLf1PelySCYPb7f47mO0h7X2Ts07u+8ieOzx/e7A+re5+4OtJ3PWFf2vldHKV0du4/v3OO6nXvcb9exu77vfNqxHXs8v/f5SVdHx8T73M919nzPRr05YN/4RvK2tyWDg0lHRzI+nixenPzZnyVXXtno6mhStVotGwd27LGByGDue2woI2MT/4e3fMm8rDn5yKw+cXFWnbgkZx3bnTldRtOgFQhuz2L+nK783KrjG10Gh6FWe+5gt1cQfPrxk8FwbOq13YH06deZeK42GVYnrju2x3XGn+O1vY7bo7Zd5+z5Wm3y8X6vMRlodx03NvXaRFAem/q+lrF9ats7cE89Pz6x29jYHqF6n2Mm7z1x/N61Pdd5zWRXSN4rNO4nQHZ1Pj1AToTPXSG0ozR5TEdHOkuZ+DoZIPc8p7NUSmfnHuc80/V2hdtd1+vsSOfTjn+m605cqyNdk4+7Ojoyq3Pi+VmdHVPvZ3/HCLLP4BvfSK65JtmxY+/nt26deP7WW4U3Dsj2naO5Z+PQHpuIDOaJrcNJknmzOnPe8kX5lRedmlUnLs6qExenp9tuj9CqBDda2tRUzPjjssh2B739hMup4JeMjo9PvT62R+Ace1r4HBvP1PNj+1yn9rTzd99/dHz3sVPXnjo/e50/Pj5x/L733nX/8Ynr1WoZHZv8Or673tHxWoZHxjM6Pra71j1q2/VeR8fH977e5LnjtdrUf2FvpI5S0tXZMRnmSvt+v0fQe/r3u4Jh157BsGN3oJ319GMmrznx3OTznR2ZvUeonD353NTrHaXM6urIrMnXZ+35Wmcpszo6Mqtr8rjJax+2Wi25/vp9Q9suO3ZMjMT19Zk2yV5qtVrWP7k9vRsGsu6RiRG1H/Vvydjkf+E65ej5ufT0o7PqpCVZdcLinLmsO12dHQ2uGpgpghvQcB0dJeH6EO0ZRMfG9w6HU0FwbDII7ic8Tn0/VsvI+PjUsaOTz018Hd/7617PPe3x+Ph+zqtNvTY2XsvI2Hh2ju57/K7X9jp+V13jMxNUdwXR2VOhczIYPj3wdXZMhr7do4+7guRpP/lB3vbkQJ5t3GPkyYH846e+mi3nXZBZXbsC5cR9p8Jl19Med3ZkdtfTHnd2WOfahEbHxjM8Op4dI2P50eNbdm8i8shgBrePJEkWzOnKyhMW51cv+6msOnFxVp6wJEfOn93gyoFGEtwAmtiu0NsOG8Ltmqo8MhkIR0bHMzIZ/EbGxjMytjv4jYyPZ2QyHO56bXRsPDunguF4dk4+t/v1yWP3e83xjEzec/c1xzM8Mp6tY6MT549PHN/9wPqMPkfGrI7V8pW//b/5+wdnHfbPZc8RyDlTwe7Zw96szomRxlmTo5RT53TtftzVuXv67d7Tcfedsrtrqu+uY58+RXjXdNznOqZzj2Onc01prVbLzsmwtHOP/009Hhub+n6vY8bGMzwylp1je5wz+Xvf67nRsanjn36N4f1cZ39TxE/vWZArzl42OeVxSU7rWTA9I8BAyxDcAGgKpVJpcj3gZEqd09h6ntG/zk2+cnMy/MyHLJjdkQ/+pyvyu6svnAigk6FyV1AcGX3a48k/+EfGatk5OjbxdSp07nr+aY8nr7Pn463Do5PXr+1xz8n77AoeY+NTm2Q12v6C3Z5BsGOPdaFdHaWMjdeeFqh2B7DpUColsyfD8Jyujszp6szsyZHR2V27n58/p2uP4zqnnp/6usc1Tj1mQc4/YXEWzTv8EA+0NsENAKbTxRcnixZNbETyDEqLF+fIl764sGvcxsYnguDe03DH95qG+/RpuROPJ9Zk7jp2z3WZ+z4e3+/r+7vermOf7f4jY+Pp7ChNBak5szoyu7Nzr0C1Kzzt7/Xdz+0ZsvZ+3YY8QCMJbgAwnUql5JZb9r+rZJLMmzfREqDAAaCzo5R5s9tg/i1AE7EVEQBMtyuvnNjyf/nyZMGCZOHCia/Ll2sFAMAhMeIGAPVw5ZUTW/7fdVfy2GPJccclF11U6JE2AIpLcAOAeimVJta8AcBhMlUSAACg4AQ3AACAghPcAAAACk5wAwAAKDjBDQAAoOAENwAAgIIT3AAAAApOcAMAACg4wQ0AAKDgBDcAAICCE9wAAAAKTnADAAAoOMENAACg4AQ3AACAghPcAAAACk5wAwAAKDjBDQAAoOAENwAAgIIT3AAAAApOcAMAACg4wQ0AAKDgBDcAAICCE9wAAAAKTnADAAAoOMENAACg4AQ3AACAghPcAAAACk5wAwAAKDjBDQAAoOAENwAAgIIT3AAAAApOcAMAACi4Uq1WqzW6iF2OPvronHzyyc953NDQUBYtWnRI9ziUczdv3pxjjjnmkO7H/h3O77BIivQ+ZrqWet1vOq87Hdc61Gv4rCmGIv07ejiK9j5msp563qtInzf+tml+Rfv39FAV6X20698269evzxNPPLHvC7Um9Na3vnVGz73gggsO+X7s3+H8DoukSO9jpmup1/2m87rTca1DvYbPmmIo0r+jh6No72Mm66nnvYr0eeNvm+ZXtH9PD1WR3oe/bfbWlFMlf+ZnfqYh5zJ9WuX3UKT3MdO11Ot+03nd6bjWoV6jSP9stLNW+T0U7X3MZD31vFeRPm/8bdP8WuX3UKT34W+bvRVqqmRRrVmzJmvXrm10GUCL81kDzBSfN9B8mnLEbaZdf/31jS4BaAM+a4CZ4vMGmo8RNwAAgIIz4gYAAFBwghsAAEDBCW4AAAAFJ7gdhvvvvz9vf/vbc8011+TjH/94o8sBWthXvvKVvPWtb81rX/va3H777Y0uB2hRDz30UH7lV34l11xzTaNLAZ6mbYPbm9/85vT09OScc87Z6/m/+7u/yxlnnJHTTjstH/rQh571GmeddVY+8YlP5Itf/GLuvPPOepYLNLHp+Lz5uZ/7uXzyk5/MJz7xiXzhC1+oZ7lAk5qOz5pTTz01f/7nf17PMoFD1La7St5xxx1ZsGBBfvmXfzn33ntvkmRsbCzPe97z8g//8A9Zvnx5LrzwwvzVX/1VxsbG8r73vW+v8z/1qU+lp6cnX/va1/Lxj388b3jDG/KLv/iLjXgrQMFN1+dNkrz73e/O61//+qxevXrG3wdQbNP5WXPNNdfk1ltvnfH3ADyzrkYX0CiXXnpp1q9fv9dzd911V0477bSceuqpSZLXve51+epXv5r3ve99ue222/Z7nauvvjpXX311rrrqKsEN2K/p+Lyp1Wp573vfm1e96lVCG7Bf0/W3DVBMbTtVcn8effTRnHDCCVOPly9fnkcfffQZj//Wt76VG264IW9729ty5ZVXzkSJQIs42M+bP/mTP8k//uM/5tZbb80nPvGJmSgRaAEH+1nz5JNP5u1vf3vWrVuXD37wgzNRInCA2nbEbTpcdtllueyyyxpdBtAGbrjhhtxwww2NLgNocUcddZT/OAQFZcRtD8cff3weeeSRqccbN27M8ccf38CKgFbl8waYCT5roHUIbnu48MIL8+CDD+bhhx/Ozp078/nPfz5XX311o8sCWpDPG2Am+KyB1tG2we26667LJZdckgceeCDLly/Pn//5n6erqyv//b//91xxxRU566yz8gu/8AtZsWJFo0sFmpzPG2Am+KyB1ta27QAAAACaRduOuAEAADQLwQ0AAKDgBDcAAICCE9wAAAAKTnADAAAoOMENAACg4AQ3AApjwYIFM3q/F7zgBTN6v8HBwfzpn/7pjN4TgNYguAHQskZHR5/19e985zszek/BDYBDJbgBUGg/+clP8spXvjIXXHBBXvziF+dHP/pRkuTrX/96Lr744qxatSo//dM/nXK5nCS56aab8oY3vCEvfOEL84Y3vCE33XRT3vzmN+eyyy7Lqaeemo997GNT1941wvetb30rl112Wa655pqceeaZef3rX59arZYk+cY3vpEzzzwzF1xwQW644Ya8+tWv3qfGT3/607n66qvzspe9LC9/+cuzdevWvPzlL8/q1atz7rnn5qtf/WqS5L3vfW9+8pOfZOXKlXnPe96TJLn55ptz4YUX5rzzzsv73//++v0gAWhqXY0uAACezfXXX59PfOITOf300/Nv//Zv+dVf/dX88z//c170ohflX//1X1MqlfI//sf/yB/90R/lwx/+cJLkvvvuy7e//e3MmzcvN910U370ox/lm9/8ZrZs2ZIzzjgj73jHOzJr1qy97rNu3br88Ic/zHHHHZcXvvCFufPOO7NmzZq87W1vyx133JFTTjkl11133TPW2dvbm3vuuSdHHnlkRkdH8zd/8zdZuHBhnnjiiTz/+c/P1VdfnQ996EO599578/3vfz9Jcvvtt+fBBx/MXXfdlVqtlquvvjp33HFHLr300rr9PAFoToIbAIW1devWfOc738m111479dzw8HCSZOPGjXnta1+bxx9/PDt37swpp5wydczVV1+defPmTT2+6qqrMmfOnMyZMyc9PT0pl8tZvnz5Xve66KKLpp5buXJl1q9fnwULFuTUU0+duvZ1112XW265Zb+1Xn755TnyyCOTJLVaLb/1W7+VO+64Ix0dHXn00UenRgT3dPvtt+f222/PqlWrpt7vgw8+KLgBsA/BDYDCGh8fz+LFi6dGqPb0zne+MzfeeGOuvvrqfOtb38pNN9009dr8+fP3OnbOnDlT33d2du53HdqBHPNs9rzn5z73uWzevDnf+973MmvWrJx88smpVqv7nFOr1fK+970vb3vb2w7qXgC0H2vcACishQsX5pRTTsmXvvSlJBNB5+67706SDA0N5fjjj0+S/OVf/mVd7n/GGWfkoYceyvr165MkX/jCFw7ovKGhofT09GTWrFn55je/mQ0bNiRJuru7s2XLlqnjrrjiinzqU5/K1q1bkySPPvpoNm3aNL1vAoCWYMQNgMLYvn37XlMYb7zxxnzuc5/LO97xjvzBH/xBRkZG8rrXvS7nn39+brrpplx77bVZsmRJXvayl+Xhhx+e9nrmzZuXP/3TP80rX/nKzJ8/PxdeeOEBnff6178+P/MzP5Nzzz03a9asyZlnnpkkOeqoo/LCF74w55xzTl71qlfl5ptvzv33359LLrkkycRmKZ/97GfT09Mz7e8FgOZWqu3aNgsA2MfWrVuzYMGC1Gq1/Nqv/VpOP/30/MZv/EajywKgzZgqCQDP4pOf/GRWrlyZFStWZGhoyHo0ABrCiBsAAEDBGXEDAAAoOMENAACg4AQ3AACAghPcAAAACk5wAwAAKDjBDQAAoOD+/329yDnlwwZHAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "%matplotlib inline\n", + "lower_lr, upper_lr = 1e-3, 1e-0\n", + "optimizer = torch.optim.Adam(model.parameters(), lower_lr)\n", + "lr_finder = LearningRateFinder(model, optimizer, loss_function, device=device)\n", + "lr_finder.range_test(train_loader, val_loader, end_lr=upper_lr, num_iter=20)\n", + "steepest_lr, _ = lr_finder.get_steepest_gradient()\n", + "ax = plt.subplots(1, 1, figsize=(15, 15), facecolor=\"white\")[1]\n", + "_ = lr_finder.plot(ax=ax)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Live plotting\n", + "\n", + "This function is just a wrapper around `range`/`trange` such that the plots are updated on every iteration." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "def plot_range(data, wrapped_generator):\n", + " plt.ion()\n", + " for q in data.values():\n", + " for d in q.values():\n", + " if isinstance(d, dict):\n", + " ax = d[\"line\"].axes\n", + " ax.legend()\n", + " fig = ax.get_figure()\n", + " fig.show()\n", + "\n", + " for i in wrapped_generator:\n", + " yield i\n", + " for q in data.values():\n", + " for d in q.values():\n", + " if isinstance(d, dict):\n", + " d[\"line\"].set_data(d[\"x\"], d[\"y\"])\n", + " ax = d[\"line\"].axes\n", + " ax.legend()\n", + " ax.relim()\n", + " ax.autoscale_view()\n", + " fig.canvas.draw()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training\n", + "\n", + "The training looks slightly different from a vanilla loop, but this is only because it loops across each of the different learning rate methods (standard, steepest and cyclical), such that they can be updated simultaneously" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "def get_model_optimizer_scheduler(d):\n", + " d[\"model\"] = get_new_net()\n", + "\n", + " if \"lr_lims\" in d:\n", + " d[\"optimizer\"] = torch.optim.Adam(\n", + " d[\"model\"].parameters(), d[\"lr_lims\"][0]\n", + " )\n", + " d[\"scheduler\"] = torch.optim.lr_scheduler.CyclicLR(\n", + " d[\"optimizer\"],\n", + " base_lr=d[\"lr_lims\"][0],\n", + " max_lr=d[\"lr_lims\"][1],\n", + " step_size_up=d[\"step\"],\n", + " cycle_momentum=False,\n", + " )\n", + " elif \"lr_lim\" in d:\n", + " d[\"optimizer\"] = torch.optim.Adam(d[\"model\"].parameters(), d[\"lr_lim\"])\n", + " else:\n", + " d[\"optimizer\"] = torch.optim.Adam(d[\"model\"].parameters())\n", + "\n", + "\n", + "def train(max_epochs, axes, data):\n", + " for d in data.keys():\n", + " get_model_optimizer_scheduler(data[d])\n", + "\n", + " for q, i in enumerate([\"train\", \"auc\", \"acc\"]):\n", + " data[d][i] = {\"x\": [], \"y\": []}\n", + " (data[d][i][\"line\"],) = axes[q].plot(\n", + " data[d][i][\"x\"], data[d][i][\"y\"], label=d\n", + " )\n", + "\n", + " val_interval = 1\n", + "\n", + " for epoch in plot_range(data, trange(max_epochs)):\n", + "\n", + " for d in data.keys():\n", + " data[d][\"epoch_loss\"] = 0\n", + " for batch_data in train_loader:\n", + " inputs = batch_data[\"image\"].to(device)\n", + " labels = batch_data[\"label\"].to(device)\n", + "\n", + " for d in data.keys():\n", + " data[d][\"optimizer\"].zero_grad()\n", + " outputs = data[d][\"model\"](inputs)\n", + " loss = loss_function(outputs, labels)\n", + " loss.backward()\n", + " data[d][\"optimizer\"].step()\n", + " if \"scheduler\" in data[d]:\n", + " data[d][\"scheduler\"].step()\n", + " data[d][\"epoch_loss\"] += loss.item()\n", + " for d in data.keys():\n", + " data[d][\"epoch_loss\"] /= len(train_loader)\n", + " data[d][\"train\"][\"x\"].append(epoch + 1)\n", + " data[d][\"train\"][\"y\"].append(data[d][\"epoch_loss\"])\n", + "\n", + " if (epoch + 1) % val_interval == 0:\n", + " with eval_mode(*[data[d][\"model\"] for d in data.keys()]):\n", + " for d in data:\n", + " data[d][\"y_pred\"] = torch.tensor(\n", + " [], dtype=torch.float32, device=device\n", + " )\n", + " y = torch.tensor([], dtype=torch.long, device=device)\n", + " for val_data in val_loader:\n", + " val_images = val_data[\"image\"].to(device)\n", + " val_labels = val_data[\"label\"].to(device)\n", + " for d in data:\n", + " data[d][\"y_pred\"] = torch.cat(\n", + " [data[d][\"y_pred\"], data[d][\"model\"](val_images)],\n", + " dim=0,\n", + " )\n", + " y = torch.cat([y, val_labels], dim=0)\n", + "\n", + " for d in data:\n", + " auc_metric = compute_roc_auc(\n", + " data[d][\"y_pred\"], y, to_onehot_y=True, softmax=True\n", + " )\n", + " data[d][\"auc\"][\"x\"].append(epoch + 1)\n", + " data[d][\"auc\"][\"y\"].append(auc_metric)\n", + "\n", + " acc_value = torch.eq(data[d][\"y_pred\"].argmax(dim=1), y)\n", + " acc_metric = acc_value.sum().item() / len(acc_value)\n", + " data[d][\"acc\"][\"x\"].append(epoch + 1)\n", + " data[d][\"acc\"][\"y\"].append(acc_metric)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "application/javascript": [ + "/* Put everything inside the global mpl namespace */\n", + "/* global mpl */\n", + "window.mpl = {};\n", + "\n", + "mpl.get_websocket_type = function () {\n", + " if (typeof WebSocket !== 'undefined') {\n", + " return WebSocket;\n", + " } else if (typeof MozWebSocket !== 'undefined') {\n", + " return MozWebSocket;\n", + " } else {\n", + " alert(\n", + " 'Your browser does not have WebSocket support. ' +\n", + " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", + " 'Firefox 4 and 5 are also supported but you ' +\n", + " 'have to enable WebSockets in about:config.'\n", + " );\n", + " }\n", + "};\n", + "\n", + "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n", + " this.id = figure_id;\n", + "\n", + " this.ws = websocket;\n", + "\n", + " this.supports_binary = this.ws.binaryType !== undefined;\n", + "\n", + " if (!this.supports_binary) {\n", + " var warnings = document.getElementById('mpl-warnings');\n", + " if (warnings) {\n", + " warnings.style.display = 'block';\n", + " warnings.textContent =\n", + " 'This browser does not support binary websocket messages. ' +\n", + " 'Performance may be slow.';\n", + " }\n", + " }\n", + "\n", + " this.imageObj = new Image();\n", + "\n", + " this.context = undefined;\n", + " this.message = undefined;\n", + " this.canvas = undefined;\n", + " this.rubberband_canvas = undefined;\n", + " this.rubberband_context = undefined;\n", + " this.format_dropdown = undefined;\n", + "\n", + " this.image_mode = 'full';\n", + "\n", + " this.root = document.createElement('div');\n", + " this.root.setAttribute('style', 'display: inline-block');\n", + " this._root_extra_style(this.root);\n", + "\n", + " parent_element.appendChild(this.root);\n", + "\n", + " this._init_header(this);\n", + " this._init_canvas(this);\n", + " this._init_toolbar(this);\n", + "\n", + " var fig = this;\n", + "\n", + " this.waiting = false;\n", + "\n", + " this.ws.onopen = function () {\n", + " fig.send_message('supports_binary', { value: fig.supports_binary });\n", + " fig.send_message('send_image_mode', {});\n", + " if (fig.ratio !== 1) {\n", + " fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n", + " }\n", + " fig.send_message('refresh', {});\n", + " };\n", + "\n", + " this.imageObj.onload = function () {\n", + " if (fig.image_mode === 'full') {\n", + " // Full images could contain transparency (where diff images\n", + " // almost always do), so we need to clear the canvas so that\n", + " // there is no ghosting.\n", + " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", + " }\n", + " fig.context.drawImage(fig.imageObj, 0, 0);\n", + " };\n", + "\n", + " this.imageObj.onunload = function () {\n", + " fig.ws.close();\n", + " };\n", + "\n", + " this.ws.onmessage = this._make_on_message_function(this);\n", + "\n", + " this.ondownload = ondownload;\n", + "};\n", + "\n", + "mpl.figure.prototype._init_header = function () {\n", + " var titlebar = document.createElement('div');\n", + " titlebar.classList =\n", + " 'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n", + " var titletext = document.createElement('div');\n", + " titletext.classList = 'ui-dialog-title';\n", + " titletext.setAttribute(\n", + " 'style',\n", + " 'width: 100%; text-align: center; padding: 3px;'\n", + " );\n", + " titlebar.appendChild(titletext);\n", + " this.root.appendChild(titlebar);\n", + " this.header = titletext;\n", + "};\n", + "\n", + "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n", + "\n", + "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n", + "\n", + "mpl.figure.prototype._init_canvas = function () {\n", + " var fig = this;\n", + "\n", + " var canvas_div = (this.canvas_div = document.createElement('div'));\n", + " canvas_div.setAttribute(\n", + " 'style',\n", + " 'border: 1px solid #ddd;' +\n", + " 'box-sizing: content-box;' +\n", + " 'clear: both;' +\n", + " 'min-height: 1px;' +\n", + " 'min-width: 1px;' +\n", + " 'outline: 0;' +\n", + " 'overflow: hidden;' +\n", + " 'position: relative;' +\n", + " 'resize: both;'\n", + " );\n", + "\n", + " function on_keyboard_event_closure(name) {\n", + " return function (event) {\n", + " return fig.key_event(event, name);\n", + " };\n", + " }\n", + "\n", + " canvas_div.addEventListener(\n", + " 'keydown',\n", + " on_keyboard_event_closure('key_press')\n", + " );\n", + " canvas_div.addEventListener(\n", + " 'keyup',\n", + " on_keyboard_event_closure('key_release')\n", + " );\n", + "\n", + " this._canvas_extra_style(canvas_div);\n", + " this.root.appendChild(canvas_div);\n", + "\n", + " var canvas = (this.canvas = document.createElement('canvas'));\n", + " canvas.classList.add('mpl-canvas');\n", + " canvas.setAttribute('style', 'box-sizing: content-box;');\n", + "\n", + " this.context = canvas.getContext('2d');\n", + "\n", + " var backingStore =\n", + " this.context.backingStorePixelRatio ||\n", + " this.context.webkitBackingStorePixelRatio ||\n", + " this.context.mozBackingStorePixelRatio ||\n", + " this.context.msBackingStorePixelRatio ||\n", + " this.context.oBackingStorePixelRatio ||\n", + " this.context.backingStorePixelRatio ||\n", + " 1;\n", + "\n", + " this.ratio = (window.devicePixelRatio || 1) / backingStore;\n", + " if (this.ratio !== 1) {\n", + " fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n", + " }\n", + "\n", + " var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n", + " 'canvas'\n", + " ));\n", + " rubberband_canvas.setAttribute(\n", + " 'style',\n", + " 'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n", + " );\n", + "\n", + " // Apply a ponyfill if ResizeObserver is not implemented by browser.\n", + " if (this.ResizeObserver === undefined) {\n", + " if (window.ResizeObserver !== undefined) {\n", + " this.ResizeObserver = window.ResizeObserver;\n", + " } else {\n", + " var obs = _JSXTOOLS_RESIZE_OBSERVER({});\n", + " this.ResizeObserver = obs.ResizeObserver;\n", + " }\n", + " }\n", + "\n", + " this.resizeObserverInstance = new this.ResizeObserver(function (entries) {\n", + " var nentries = entries.length;\n", + " for (var i = 0; i < nentries; i++) {\n", + " var entry = entries[i];\n", + " var width, height;\n", + " if (entry.contentBoxSize) {\n", + " if (entry.contentBoxSize instanceof Array) {\n", + " // Chrome 84 implements new version of spec.\n", + " width = entry.contentBoxSize[0].inlineSize;\n", + " height = entry.contentBoxSize[0].blockSize;\n", + " } else {\n", + " // Firefox implements old version of spec.\n", + " width = entry.contentBoxSize.inlineSize;\n", + " height = entry.contentBoxSize.blockSize;\n", + " }\n", + " } else {\n", + " // Chrome <84 implements even older version of spec.\n", + " width = entry.contentRect.width;\n", + " height = entry.contentRect.height;\n", + " }\n", + "\n", + " // Keep the size of the canvas and rubber band canvas in sync with\n", + " // the canvas container.\n", + " if (entry.devicePixelContentBoxSize) {\n", + " // Chrome 84 implements new version of spec.\n", + " canvas.setAttribute(\n", + " 'width',\n", + " entry.devicePixelContentBoxSize[0].inlineSize\n", + " );\n", + " canvas.setAttribute(\n", + " 'height',\n", + " entry.devicePixelContentBoxSize[0].blockSize\n", + " );\n", + " } else {\n", + " canvas.setAttribute('width', width * fig.ratio);\n", + " canvas.setAttribute('height', height * fig.ratio);\n", + " }\n", + " canvas.setAttribute(\n", + " 'style',\n", + " 'width: ' + width + 'px; height: ' + height + 'px;'\n", + " );\n", + "\n", + " rubberband_canvas.setAttribute('width', width);\n", + " rubberband_canvas.setAttribute('height', height);\n", + "\n", + " // And update the size in Python. We ignore the initial 0/0 size\n", + " // that occurs as the element is placed into the DOM, which should\n", + " // otherwise not happen due to the minimum size styling.\n", + " if (width != 0 && height != 0) {\n", + " fig.request_resize(width, height);\n", + " }\n", + " }\n", + " });\n", + " this.resizeObserverInstance.observe(canvas_div);\n", + "\n", + " function on_mouse_event_closure(name) {\n", + " return function (event) {\n", + " return fig.mouse_event(event, name);\n", + " };\n", + " }\n", + "\n", + " rubberband_canvas.addEventListener(\n", + " 'mousedown',\n", + " on_mouse_event_closure('button_press')\n", + " );\n", + " rubberband_canvas.addEventListener(\n", + " 'mouseup',\n", + " on_mouse_event_closure('button_release')\n", + " );\n", + " // Throttle sequential mouse events to 1 every 20ms.\n", + " rubberband_canvas.addEventListener(\n", + " 'mousemove',\n", + " on_mouse_event_closure('motion_notify')\n", + " );\n", + "\n", + " rubberband_canvas.addEventListener(\n", + " 'mouseenter',\n", + " on_mouse_event_closure('figure_enter')\n", + " );\n", + " rubberband_canvas.addEventListener(\n", + " 'mouseleave',\n", + " on_mouse_event_closure('figure_leave')\n", + " );\n", + "\n", + " canvas_div.addEventListener('wheel', function (event) {\n", + " if (event.deltaY < 0) {\n", + " event.step = 1;\n", + " } else {\n", + " event.step = -1;\n", + " }\n", + " on_mouse_event_closure('scroll')(event);\n", + " });\n", + "\n", + " canvas_div.appendChild(canvas);\n", + " canvas_div.appendChild(rubberband_canvas);\n", + "\n", + " this.rubberband_context = rubberband_canvas.getContext('2d');\n", + " this.rubberband_context.strokeStyle = '#000000';\n", + "\n", + " this._resize_canvas = function (width, height, forward) {\n", + " if (forward) {\n", + " canvas_div.style.width = width + 'px';\n", + " canvas_div.style.height = height + 'px';\n", + " }\n", + " };\n", + "\n", + " // Disable right mouse context menu.\n", + " this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n", + " event.preventDefault();\n", + " return false;\n", + " });\n", + "\n", + " function set_focus() {\n", + " canvas.focus();\n", + " canvas_div.focus();\n", + " }\n", + "\n", + " window.setTimeout(set_focus, 100);\n", + "};\n", + "\n", + "mpl.figure.prototype._init_toolbar = function () {\n", + " var fig = this;\n", + "\n", + " var toolbar = document.createElement('div');\n", + " toolbar.classList = 'mpl-toolbar';\n", + " this.root.appendChild(toolbar);\n", + "\n", + " function on_click_closure(name) {\n", + " return function (_event) {\n", + " return fig.toolbar_button_onclick(name);\n", + " };\n", + " }\n", + "\n", + " function on_mouseover_closure(tooltip) {\n", + " return function (event) {\n", + " if (!event.currentTarget.disabled) {\n", + " return fig.toolbar_button_onmouseover(tooltip);\n", + " }\n", + " };\n", + " }\n", + "\n", + " fig.buttons = {};\n", + " var buttonGroup = document.createElement('div');\n", + " buttonGroup.classList = 'mpl-button-group';\n", + " for (var toolbar_ind in mpl.toolbar_items) {\n", + " var name = mpl.toolbar_items[toolbar_ind][0];\n", + " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", + " var image = mpl.toolbar_items[toolbar_ind][2];\n", + " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", + "\n", + " if (!name) {\n", + " /* Instead of a spacer, we start a new button group. */\n", + " if (buttonGroup.hasChildNodes()) {\n", + " toolbar.appendChild(buttonGroup);\n", + " }\n", + " buttonGroup = document.createElement('div');\n", + " buttonGroup.classList = 'mpl-button-group';\n", + " continue;\n", + " }\n", + "\n", + " var button = (fig.buttons[name] = document.createElement('button'));\n", + " button.classList = 'mpl-widget';\n", + " button.setAttribute('role', 'button');\n", + " button.setAttribute('aria-disabled', 'false');\n", + " button.addEventListener('click', on_click_closure(method_name));\n", + " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", + "\n", + " var icon_img = document.createElement('img');\n", + " icon_img.src = '_images/' + image + '.png';\n", + " icon_img.srcset = '_images/' + image + '_large.png 2x';\n", + " icon_img.alt = tooltip;\n", + " button.appendChild(icon_img);\n", + "\n", + " buttonGroup.appendChild(button);\n", + " }\n", + "\n", + " if (buttonGroup.hasChildNodes()) {\n", + " toolbar.appendChild(buttonGroup);\n", + " }\n", + "\n", + " var fmt_picker = document.createElement('select');\n", + " fmt_picker.classList = 'mpl-widget';\n", + " toolbar.appendChild(fmt_picker);\n", + " this.format_dropdown = fmt_picker;\n", + "\n", + " for (var ind in mpl.extensions) {\n", + " var fmt = mpl.extensions[ind];\n", + " var option = document.createElement('option');\n", + " option.selected = fmt === mpl.default_extension;\n", + " option.innerHTML = fmt;\n", + " fmt_picker.appendChild(option);\n", + " }\n", + "\n", + " var status_bar = document.createElement('span');\n", + " status_bar.classList = 'mpl-message';\n", + " toolbar.appendChild(status_bar);\n", + " this.message = status_bar;\n", + "};\n", + "\n", + "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n", + " // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n", + " // which will in turn request a refresh of the image.\n", + " this.send_message('resize', { width: x_pixels, height: y_pixels });\n", + "};\n", + "\n", + "mpl.figure.prototype.send_message = function (type, properties) {\n", + " properties['type'] = type;\n", + " properties['figure_id'] = this.id;\n", + " this.ws.send(JSON.stringify(properties));\n", + "};\n", + "\n", + "mpl.figure.prototype.send_draw_message = function () {\n", + " if (!this.waiting) {\n", + " this.waiting = true;\n", + " this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n", + " }\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", + " var format_dropdown = fig.format_dropdown;\n", + " var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n", + " fig.ondownload(fig, format);\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_resize = function (fig, msg) {\n", + " var size = msg['size'];\n", + " if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n", + " fig._resize_canvas(size[0], size[1], msg['forward']);\n", + " fig.send_message('refresh', {});\n", + " }\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n", + " var x0 = msg['x0'] / fig.ratio;\n", + " var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n", + " var x1 = msg['x1'] / fig.ratio;\n", + " var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n", + " x0 = Math.floor(x0) + 0.5;\n", + " y0 = Math.floor(y0) + 0.5;\n", + " x1 = Math.floor(x1) + 0.5;\n", + " y1 = Math.floor(y1) + 0.5;\n", + " var min_x = Math.min(x0, x1);\n", + " var min_y = Math.min(y0, y1);\n", + " var width = Math.abs(x1 - x0);\n", + " var height = Math.abs(y1 - y0);\n", + "\n", + " fig.rubberband_context.clearRect(\n", + " 0,\n", + " 0,\n", + " fig.canvas.width / fig.ratio,\n", + " fig.canvas.height / fig.ratio\n", + " );\n", + "\n", + " fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n", + " // Updates the figure title.\n", + " fig.header.textContent = msg['label'];\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n", + " var cursor = msg['cursor'];\n", + " switch (cursor) {\n", + " case 0:\n", + " cursor = 'pointer';\n", + " break;\n", + " case 1:\n", + " cursor = 'default';\n", + " break;\n", + " case 2:\n", + " cursor = 'crosshair';\n", + " break;\n", + " case 3:\n", + " cursor = 'move';\n", + " break;\n", + " }\n", + " fig.rubberband_canvas.style.cursor = cursor;\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_message = function (fig, msg) {\n", + " fig.message.textContent = msg['message'];\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n", + " // Request the server to send over a new figure.\n", + " fig.send_draw_message();\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n", + " fig.image_mode = msg['mode'];\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n", + " for (var key in msg) {\n", + " if (!(key in fig.buttons)) {\n", + " continue;\n", + " }\n", + " fig.buttons[key].disabled = !msg[key];\n", + " fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n", + " }\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n", + " if (msg['mode'] === 'PAN') {\n", + " fig.buttons['Pan'].classList.add('active');\n", + " fig.buttons['Zoom'].classList.remove('active');\n", + " } else if (msg['mode'] === 'ZOOM') {\n", + " fig.buttons['Pan'].classList.remove('active');\n", + " fig.buttons['Zoom'].classList.add('active');\n", + " } else {\n", + " fig.buttons['Pan'].classList.remove('active');\n", + " fig.buttons['Zoom'].classList.remove('active');\n", + " }\n", + "};\n", + "\n", + "mpl.figure.prototype.updated_canvas_event = function () {\n", + " // Called whenever the canvas gets updated.\n", + " this.send_message('ack', {});\n", + "};\n", + "\n", + "// A function to construct a web socket function for onmessage handling.\n", + "// Called in the figure constructor.\n", + "mpl.figure.prototype._make_on_message_function = function (fig) {\n", + " return function socket_on_message(evt) {\n", + " if (evt.data instanceof Blob) {\n", + " /* FIXME: We get \"Resource interpreted as Image but\n", + " * transferred with MIME type text/plain:\" errors on\n", + " * Chrome. But how to set the MIME type? It doesn't seem\n", + " * to be part of the websocket stream */\n", + " evt.data.type = 'image/png';\n", + "\n", + " /* Free the memory for the previous frames */\n", + " if (fig.imageObj.src) {\n", + " (window.URL || window.webkitURL).revokeObjectURL(\n", + " fig.imageObj.src\n", + " );\n", + " }\n", + "\n", + " fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n", + " evt.data\n", + " );\n", + " fig.updated_canvas_event();\n", + " fig.waiting = false;\n", + " return;\n", + " } else if (\n", + " typeof evt.data === 'string' &&\n", + " evt.data.slice(0, 21) === 'data:image/png;base64'\n", + " ) {\n", + " fig.imageObj.src = evt.data;\n", + " fig.updated_canvas_event();\n", + " fig.waiting = false;\n", + " return;\n", + " }\n", + "\n", + " var msg = JSON.parse(evt.data);\n", + " var msg_type = msg['type'];\n", + "\n", + " // Call the \"handle_{type}\" callback, which takes\n", + " // the figure and JSON message as its only arguments.\n", + " try {\n", + " var callback = fig['handle_' + msg_type];\n", + " } catch (e) {\n", + " console.log(\n", + " \"No handler for the '\" + msg_type + \"' message type: \",\n", + " msg\n", + " );\n", + " return;\n", + " }\n", + "\n", + " if (callback) {\n", + " try {\n", + " // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n", + " callback(fig, msg);\n", + " } catch (e) {\n", + " console.log(\n", + " \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n", + " e,\n", + " e.stack,\n", + " msg\n", + " );\n", + " }\n", + " }\n", + " };\n", + "};\n", + "\n", + "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", + "mpl.findpos = function (e) {\n", + " //this section is from http://www.quirksmode.org/js/events_properties.html\n", + " var targ;\n", + " if (!e) {\n", + " e = window.event;\n", + " }\n", + " if (e.target) {\n", + " targ = e.target;\n", + " } else if (e.srcElement) {\n", + " targ = e.srcElement;\n", + " }\n", + " if (targ.nodeType === 3) {\n", + " // defeat Safari bug\n", + " targ = targ.parentNode;\n", + " }\n", + "\n", + " // pageX,Y are the mouse positions relative to the document\n", + " var boundingRect = targ.getBoundingClientRect();\n", + " var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n", + " var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n", + "\n", + " return { x: x, y: y };\n", + "};\n", + "\n", + "/*\n", + " * return a copy of an object with only non-object keys\n", + " * we need this to avoid circular references\n", + " * http://stackoverflow.com/a/24161582/3208463\n", + " */\n", + "function simpleKeys(original) {\n", + " return Object.keys(original).reduce(function (obj, key) {\n", + " if (typeof original[key] !== 'object') {\n", + " obj[key] = original[key];\n", + " }\n", + " return obj;\n", + " }, {});\n", + "}\n", + "\n", + "mpl.figure.prototype.mouse_event = function (event, name) {\n", + " var canvas_pos = mpl.findpos(event);\n", + "\n", + " if (name === 'button_press') {\n", + " this.canvas.focus();\n", + " this.canvas_div.focus();\n", + " }\n", + "\n", + " var x = canvas_pos.x * this.ratio;\n", + " var y = canvas_pos.y * this.ratio;\n", + "\n", + " this.send_message(name, {\n", + " x: x,\n", + " y: y,\n", + " button: event.button,\n", + " step: event.step,\n", + " guiEvent: simpleKeys(event),\n", + " });\n", + "\n", + " /* This prevents the web browser from automatically changing to\n", + " * the text insertion cursor when the button is pressed. We want\n", + " * to control all of the cursor setting manually through the\n", + " * 'cursor' event from matplotlib */\n", + " event.preventDefault();\n", + " return false;\n", + "};\n", + "\n", + "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n", + " // Handle any extra behaviour associated with a key event\n", + "};\n", + "\n", + "mpl.figure.prototype.key_event = function (event, name) {\n", + " // Prevent repeat events\n", + " if (name === 'key_press') {\n", + " if (event.which === this._key) {\n", + " return;\n", + " } else {\n", + " this._key = event.which;\n", + " }\n", + " }\n", + " if (name === 'key_release') {\n", + " this._key = null;\n", + " }\n", + "\n", + " var value = '';\n", + " if (event.ctrlKey && event.which !== 17) {\n", + " value += 'ctrl+';\n", + " }\n", + " if (event.altKey && event.which !== 18) {\n", + " value += 'alt+';\n", + " }\n", + " if (event.shiftKey && event.which !== 16) {\n", + " value += 'shift+';\n", + " }\n", + "\n", + " value += 'k';\n", + " value += event.which.toString();\n", + "\n", + " this._key_event_extra(event, name);\n", + "\n", + " this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n", + " return false;\n", + "};\n", + "\n", + "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n", + " if (name === 'download') {\n", + " this.handle_save(this, null);\n", + " } else {\n", + " this.send_message('toolbar_button', { name: name });\n", + " }\n", + "};\n", + "\n", + "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n", + " this.message.textContent = tooltip;\n", + "};\n", + "\n", + "///////////////// REMAINING CONTENT GENERATED BY embed_js.py /////////////////\n", + "// prettier-ignore\n", + "var _JSXTOOLS_RESIZE_OBSERVER=function(A){var t,i=new WeakMap,n=new WeakMap,a=new WeakMap,r=new WeakMap,o=new Set;function s(e){if(!(this instanceof s))throw new TypeError(\"Constructor requires 'new' operator\");i.set(this,e)}function h(){throw new TypeError(\"Function is not a constructor\")}function c(e,t,i,n){e=0 in arguments?Number(arguments[0]):0,t=1 in arguments?Number(arguments[1]):0,i=2 in arguments?Number(arguments[2]):0,n=3 in arguments?Number(arguments[3]):0,this.right=(this.x=this.left=e)+(this.width=i),this.bottom=(this.y=this.top=t)+(this.height=n),Object.freeze(this)}function d(){t=requestAnimationFrame(d);var s=new WeakMap,p=new Set;o.forEach((function(t){r.get(t).forEach((function(i){var r=t instanceof window.SVGElement,o=a.get(t),d=r?0:parseFloat(o.paddingTop),f=r?0:parseFloat(o.paddingRight),l=r?0:parseFloat(o.paddingBottom),u=r?0:parseFloat(o.paddingLeft),g=r?0:parseFloat(o.borderTopWidth),m=r?0:parseFloat(o.borderRightWidth),w=r?0:parseFloat(o.borderBottomWidth),b=u+f,F=d+l,v=(r?0:parseFloat(o.borderLeftWidth))+m,W=g+w,y=r?0:t.offsetHeight-W-t.clientHeight,E=r?0:t.offsetWidth-v-t.clientWidth,R=b+v,z=F+W,M=r?t.width:parseFloat(o.width)-R-E,O=r?t.height:parseFloat(o.height)-z-y;if(n.has(t)){var k=n.get(t);if(k[0]===M&&k[1]===O)return}n.set(t,[M,O]);var S=Object.create(h.prototype);S.target=t,S.contentRect=new c(u,d,M,O),s.has(i)||(s.set(i,[]),p.add(i)),s.get(i).push(S)}))})),p.forEach((function(e){i.get(e).call(e,s.get(e),e)}))}return s.prototype.observe=function(i){if(i instanceof window.Element){r.has(i)||(r.set(i,new Set),o.add(i),a.set(i,window.getComputedStyle(i)));var n=r.get(i);n.has(this)||n.add(this),cancelAnimationFrame(t),t=requestAnimationFrame(d)}},s.prototype.unobserve=function(i){if(i instanceof window.Element&&r.has(i)){var n=r.get(i);n.has(this)&&(n.delete(this),n.size||(r.delete(i),o.delete(i))),n.size||r.delete(i),o.size||cancelAnimationFrame(t)}},A.DOMRectReadOnly=c,A.ResizeObserver=s,A.ResizeObserverEntry=h,A}; // eslint-disable-line\n", + "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", + "\n", + "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", + "\n", + "mpl.default_extension = \"png\";/* global mpl */\n", + "\n", + "var comm_websocket_adapter = function (comm) {\n", + " // Create a \"websocket\"-like object which calls the given IPython comm\n", + " // object with the appropriate methods. Currently this is a non binary\n", + " // socket, so there is still some room for performance tuning.\n", + " var ws = {};\n", + "\n", + " ws.close = function () {\n", + " comm.close();\n", + " };\n", + " ws.send = function (m) {\n", + " //console.log('sending', m);\n", + " comm.send(m);\n", + " };\n", + " // Register the callback with on_msg.\n", + " comm.on_msg(function (msg) {\n", + " //console.log('receiving', msg['content']['data'], msg);\n", + " // Pass the mpl event to the overridden (by mpl) onmessage function.\n", + " ws.onmessage(msg['content']['data']);\n", + " });\n", + " return ws;\n", + "};\n", + "\n", + "mpl.mpl_figure_comm = function (comm, msg) {\n", + " // This is the function which gets called when the mpl process\n", + " // starts-up an IPython Comm through the \"matplotlib\" channel.\n", + "\n", + " var id = msg.content.data.id;\n", + " // Get hold of the div created by the display call when the Comm\n", + " // socket was opened in Python.\n", + " var element = document.getElementById(id);\n", + " var ws_proxy = comm_websocket_adapter(comm);\n", + "\n", + " function ondownload(figure, _format) {\n", + " window.open(figure.canvas.toDataURL());\n", + " }\n", + "\n", + " var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n", + "\n", + " // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n", + " // web socket which is closed, not our websocket->open comm proxy.\n", + " ws_proxy.onopen();\n", + "\n", + " fig.parent_element = element;\n", + " fig.cell_info = mpl.find_output_cell(\"
\");\n", + " if (!fig.cell_info) {\n", + " console.error('Failed to find cell for figure', id, fig);\n", + " return;\n", + " }\n", + " fig.cell_info[0].output_area.element.on(\n", + " 'cleared',\n", + " { fig: fig },\n", + " fig._remove_fig_handler\n", + " );\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_close = function (fig, msg) {\n", + " var width = fig.canvas.width / fig.ratio;\n", + " fig.cell_info[0].output_area.element.off(\n", + " 'cleared',\n", + " fig._remove_fig_handler\n", + " );\n", + " fig.resizeObserverInstance.unobserve(fig.canvas_div);\n", + "\n", + " // Update the output cell to use the data from the current canvas.\n", + " fig.push_to_output();\n", + " var dataURL = fig.canvas.toDataURL();\n", + " // Re-enable the keyboard manager in IPython - without this line, in FF,\n", + " // the notebook keyboard shortcuts fail.\n", + " IPython.keyboard_manager.enable();\n", + " fig.parent_element.innerHTML =\n", + " '';\n", + " fig.close_ws(fig, msg);\n", + "};\n", + "\n", + "mpl.figure.prototype.close_ws = function (fig, msg) {\n", + " fig.send_message('closing', msg);\n", + " // fig.ws.close()\n", + "};\n", + "\n", + "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n", + " // Turn the data on the canvas into data in the output cell.\n", + " var width = this.canvas.width / this.ratio;\n", + " var dataURL = this.canvas.toDataURL();\n", + " this.cell_info[1]['text/html'] =\n", + " '';\n", + "};\n", + "\n", + "mpl.figure.prototype.updated_canvas_event = function () {\n", + " // Tell IPython that the notebook contents must change.\n", + " IPython.notebook.set_dirty(true);\n", + " this.send_message('ack', {});\n", + " var fig = this;\n", + " // Wait a second, then push the new image to the DOM so\n", + " // that it is saved nicely (might be nice to debounce this).\n", + " setTimeout(function () {\n", + " fig.push_to_output();\n", + " }, 1000);\n", + "};\n", + "\n", + "mpl.figure.prototype._init_toolbar = function () {\n", + " var fig = this;\n", + "\n", + " var toolbar = document.createElement('div');\n", + " toolbar.classList = 'btn-toolbar';\n", + " this.root.appendChild(toolbar);\n", + "\n", + " function on_click_closure(name) {\n", + " return function (_event) {\n", + " return fig.toolbar_button_onclick(name);\n", + " };\n", + " }\n", + "\n", + " function on_mouseover_closure(tooltip) {\n", + " return function (event) {\n", + " if (!event.currentTarget.disabled) {\n", + " return fig.toolbar_button_onmouseover(tooltip);\n", + " }\n", + " };\n", + " }\n", + "\n", + " fig.buttons = {};\n", + " var buttonGroup = document.createElement('div');\n", + " buttonGroup.classList = 'btn-group';\n", + " var button;\n", + " for (var toolbar_ind in mpl.toolbar_items) {\n", + " var name = mpl.toolbar_items[toolbar_ind][0];\n", + " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", + " var image = mpl.toolbar_items[toolbar_ind][2];\n", + " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", + "\n", + " if (!name) {\n", + " /* Instead of a spacer, we start a new button group. */\n", + " if (buttonGroup.hasChildNodes()) {\n", + " toolbar.appendChild(buttonGroup);\n", + " }\n", + " buttonGroup = document.createElement('div');\n", + " buttonGroup.classList = 'btn-group';\n", + " continue;\n", + " }\n", + "\n", + " button = fig.buttons[name] = document.createElement('button');\n", + " button.classList = 'btn btn-default';\n", + " button.href = '#';\n", + " button.title = name;\n", + " button.innerHTML = '';\n", + " button.addEventListener('click', on_click_closure(method_name));\n", + " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", + " buttonGroup.appendChild(button);\n", + " }\n", + "\n", + " if (buttonGroup.hasChildNodes()) {\n", + " toolbar.appendChild(buttonGroup);\n", + " }\n", + "\n", + " // Add the status bar.\n", + " var status_bar = document.createElement('span');\n", + " status_bar.classList = 'mpl-message pull-right';\n", + " toolbar.appendChild(status_bar);\n", + " this.message = status_bar;\n", + "\n", + " // Add the close button to the window.\n", + " var buttongrp = document.createElement('div');\n", + " buttongrp.classList = 'btn-group inline pull-right';\n", + " button = document.createElement('button');\n", + " button.classList = 'btn btn-mini btn-primary';\n", + " button.href = '#';\n", + " button.title = 'Stop Interaction';\n", + " button.innerHTML = '';\n", + " button.addEventListener('click', function (_evt) {\n", + " fig.handle_close(fig, {});\n", + " });\n", + " button.addEventListener(\n", + " 'mouseover',\n", + " on_mouseover_closure('Stop Interaction')\n", + " );\n", + " buttongrp.appendChild(button);\n", + " var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n", + " titlebar.insertBefore(buttongrp, titlebar.firstChild);\n", + "};\n", + "\n", + "mpl.figure.prototype._remove_fig_handler = function (event) {\n", + " var fig = event.data.fig;\n", + " if (event.target !== this) {\n", + " // Ignore bubbled events from children.\n", + " return;\n", + " }\n", + " fig.close_ws(fig, {});\n", + "};\n", + "\n", + "mpl.figure.prototype._root_extra_style = function (el) {\n", + " el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n", + "};\n", + "\n", + "mpl.figure.prototype._canvas_extra_style = function (el) {\n", + " // this is important to make the div 'focusable\n", + " el.setAttribute('tabindex', 0);\n", + " // reach out to IPython and tell the keyboard manager to turn it's self\n", + " // off when our div gets focus\n", + "\n", + " // location in version 3\n", + " if (IPython.notebook.keyboard_manager) {\n", + " IPython.notebook.keyboard_manager.register_events(el);\n", + " } else {\n", + " // location in version 2\n", + " IPython.keyboard_manager.register_events(el);\n", + " }\n", + "};\n", + "\n", + "mpl.figure.prototype._key_event_extra = function (event, _name) {\n", + " var manager = IPython.notebook.keyboard_manager;\n", + " if (!manager) {\n", + " manager = IPython.keyboard_manager;\n", + " }\n", + "\n", + " // Check for shift+enter\n", + " if (event.shiftKey && event.which === 13) {\n", + " this.canvas_div.blur();\n", + " // select the cell after this one\n", + " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", + " IPython.notebook.select(index + 1);\n", + " }\n", + "};\n", + "\n", + "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", + " fig.ondownload(fig, null);\n", + "};\n", + "\n", + "mpl.find_output_cell = function (html_output) {\n", + " // Return the cell and output element which can be found *uniquely* in the notebook.\n", + " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", + " // IPython event is triggered only after the cells have been serialised, which for\n", + " // our purposes (turning an active figure into a static one), is too late.\n", + " var cells = IPython.notebook.get_cells();\n", + " var ncells = cells.length;\n", + " for (var i = 0; i < ncells; i++) {\n", + " var cell = cells[i];\n", + " if (cell.cell_type === 'code') {\n", + " for (var j = 0; j < cell.output_area.outputs.length; j++) {\n", + " var data = cell.output_area.outputs[j];\n", + " if (data.data) {\n", + " // IPython >= 3 moved mimebundle to data attribute of output\n", + " data = data.data;\n", + " }\n", + " if (data['text/html'] === html_output) {\n", + " return [cell, data, j];\n", + " }\n", + " }\n", + " }\n", + " }\n", + "};\n", + "\n", + "// Register the function which deals with the matplotlib target/channel.\n", + "// The kernel may be null if the page has been refreshed.\n", + "if (IPython.notebook.kernel !== null) {\n", + " IPython.notebook.kernel.comm_manager.register_target(\n", + " 'matplotlib',\n", + " mpl.mpl_figure_comm\n", + " );\n", + "}\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 100/100 [03:11<00:00, 1.91s/it]\n" + ] + } + ], + "source": [ + "%matplotlib notebook\n", + "fig, axes = plt.subplots(3, 1, figsize=(10, 10), facecolor=\"white\")\n", + "for ax in axes:\n", + " ax.set_xlabel(\"Epoch\")\n", + "axes[0].set_ylabel(\"Train loss\")\n", + "axes[1].set_ylabel(\"AUC\")\n", + "axes[2].set_ylabel(\"ACC\")\n", + "\n", + "# In the paper referenced at the top of this notebook, a step\n", + "# size of 8 times the number of iterations per epoch is suggested.\n", + "step_size = 8 * len(train_loader)\n", + "\n", + "max_epochs = 100\n", + "data = {}\n", + "data[\"Default LR\"] = {}\n", + "data[\"Steepest LR\"] = {\"lr_lim\": steepest_lr}\n", + "data[\"Cyclical LR\"] = {\n", + " \"lr_lims\": (0.8 * steepest_lr, 1.2 * steepest_lr),\n", + " \"step\": step_size,\n", + "}\n", + "\n", + "train(max_epochs, axes, data)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "Unsurprisingly, both `Steepest LR` and `Cyclical LR` show quicker convergence of the loss function than `Default LR`.\n", + "\n", + "There's not much of a difference in this example between `Steepest LR` and `Cyclical LR`. A bigger difference may be apparent in a more complex optimisation problem, but feel free to play with the step size, and the lower and upper cyclical limits." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cleanup data directory\n", + "\n", + "Remove directory if a temporary was used." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "if directory is None:\n", + " shutil.rmtree(root_dir)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}