diff --git a/tutorials/generative/anomaly_detection/anomaly_detection_with_transformers.ipynb b/tutorials/generative/anomaly_detection/anomaly_detection_with_transformers.ipynb new file mode 100644 index 00000000..4aacdd09 --- /dev/null +++ b/tutorials/generative/anomaly_detection/anomaly_detection_with_transformers.ipynb @@ -0,0 +1,1105 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f6090d00", + "metadata": {}, + "source": [ + "# Anomaly Detection with Transformers\n", + "\n", + "This tutorial illustrates how to use MONAI to perform image-wise anomaly detection with transformers based on the method proposed in Pinaya et al.[1].\n", + "\n", + "Here, we will work with the [MedNIST dataset](https://docs.monai.io/en/stable/apps.html#monai.apps.MedNISTDataset) available on MONAI, and similar to \"Experiment 2 – image-wise anomaly detection on 2D synthetic data\" from [1], we will train a general-purpose VQ-VAE (using all MEDNIST classes), and then a generative models (i.e., Transformer) on `HeadCT` images.\n", + "\n", + "Finally, we will compute the log-likelihood of images from the same class (in-distribution class) and images from other classes (out-of-distribution).\n", + "\n", + "[1] - [Pinaya et al. \"Unsupervised brain imaging 3D anomaly detection and segmentation with transformers\"](https://doi.org/10.1016/j.media.2022.102475)" + ] + }, + { + "cell_type": "markdown", + "id": "8b27924f", + "metadata": {}, + "source": [ + "### Setup environment" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "01787b4b", + "metadata": {}, + "outputs": [], + "source": [ + "!python -c \"import seaborn\" || pip install -q seaborn\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "id": "56afab18", + "metadata": {}, + "source": [ + "### Setup imports" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b6b0c79f", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/media/walter/Storage/Projects/GenerativeModels/venv/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-03-11 22:41:26,292 - A matching Triton is not available, some optimizations will not be enabled.\n", + "Error caught was: No module named 'triton'\n", + "MONAI version: 1.2.dev2304\n", + "Numpy version: 1.23.5\n", + "Pytorch version: 1.13.1+cu117\n", + "MONAI flags: HAS_EXT = False, USE_COMPILED = False, USE_META_DICT = False\n", + "MONAI rev id: 9a57be5aab9f2c2a134768c0c146399150e247a0\n", + "MONAI __file__: /media/walter/Storage/Projects/GenerativeModels/venv/lib/python3.10/site-packages/monai/__init__.py\n", + "\n", + "Optional dependencies:\n", + "Pytorch Ignite version: 0.4.10\n", + "ITK version: 5.3.0\n", + "Nibabel version: 4.0.2\n", + "scikit-image version: 0.19.3\n", + "Pillow version: 9.3.0\n", + "Tensorboard version: 2.11.0\n", + "gdown version: 4.6.0\n", + "TorchVision version: 0.14.1+cu117\n", + "tqdm version: 4.64.1\n", + "lmdb version: 1.4.0\n", + "psutil version: 5.9.4\n", + "pandas version: 1.5.3\n", + "einops version: 0.6.0\n", + "transformers version: 4.21.3\n", + "mlflow version: 2.1.1\n", + "pynrrd version: 1.0.0\n", + "\n", + "For details about installing the optional dependencies, please visit:\n", + " https://docs.monai.io/en/latest/installation.html#installing-the-recommended-dependencies\n", + "\n" + ] + } + ], + "source": [ + "# Copyright 2020 MONAI Consortium\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "# http://www.apache.org/licenses/LICENSE-2.0\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License.\n", + "import os\n", + "import tempfile\n", + "import time\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import seaborn as sns\n", + "import torch\n", + "from monai import transforms\n", + "from monai.apps import MedNISTDataset\n", + "from monai.config import print_config\n", + "from monai.data import DataLoader, Dataset\n", + "from monai.utils import first, set_determinism\n", + "from torch.nn import CrossEntropyLoss, L1Loss\n", + "from tqdm import tqdm\n", + "\n", + "from generative.inferers import VQVAETransformerInferer\n", + "from generative.networks.nets import VQVAE, DecoderOnlyTransformer\n", + "from generative.utils.enums import OrderingType\n", + "from generative.utils.ordering import Ordering\n", + "\n", + "print_config()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "de0ed372", + "metadata": {}, + "outputs": [], + "source": [ + "# for reproducibility purposes set a seed\n", + "set_determinism(42)" + ] + }, + { + "cell_type": "markdown", + "id": "ad40db27", + "metadata": {}, + "source": [ + "### Setup a data directory and download dataset\n", + "\n", + "Specify a `MONAI_DATA_DIRECTORY` variable, where the data will be downloaded. If not\n", + "specified a temporary directory will be used." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "42fa255d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/tmp/tmp83zh5r1m\n" + ] + } + ], + "source": [ + "directory = os.environ.get(\"MONAI_DATA_DIRECTORY\")\n", + "root_dir = tempfile.mkdtemp() if directory is None else directory\n", + "print(root_dir)" + ] + }, + { + "cell_type": "markdown", + "id": "10054720", + "metadata": {}, + "source": [ + "### Download training data" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "7db7ac32", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "MedNIST.tar.gz: 59.0MB [00:04, 13.2MB/s] " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-03-11 22:41:31,031 - INFO - Downloaded: /tmp/tmp83zh5r1m/MedNIST.tar.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-03-11 22:41:31,104 - INFO - Verified 'MedNIST.tar.gz', md5: 0bc7306e7427e00ad1c5526a6677552d.\n", + "2023-03-11 22:41:31,105 - INFO - Writing into directory: /tmp/tmp83zh5r1m.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading dataset: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 47164/47164 [00:25<00:00, 1816.60it/s]\n" + ] + } + ], + "source": [ + "train_transforms = transforms.Compose(\n", + " [\n", + " transforms.LoadImaged(keys=[\"image\"]),\n", + " transforms.EnsureChannelFirstd(keys=[\"image\"]),\n", + " transforms.ScaleIntensityRanged(keys=[\"image\"], a_min=0.0, a_max=255.0, b_min=0.0, b_max=1.0, clip=True),\n", + " transforms.RandAffined(\n", + " keys=[\"image\"],\n", + " rotate_range=[(-np.pi / 36, np.pi / 36), (-np.pi / 36, np.pi / 36)],\n", + " translate_range=[(-1, 1), (-1, 1)],\n", + " scale_range=[(-0.01, 0.01), (-0.01, 0.01)],\n", + " spatial_size=[64, 64],\n", + " padding_mode=\"zeros\",\n", + " prob=0.5,\n", + " ),\n", + " ]\n", + ")\n", + "train_data = MedNISTDataset(root_dir=root_dir, section=\"training\", download=True, seed=0, transform=train_transforms)\n", + "train_loader = DataLoader(train_data, batch_size=256, shuffle=True, num_workers=4, persistent_workers=True)" + ] + }, + { + "cell_type": "markdown", + "id": "ec356258", + "metadata": {}, + "source": [ + "### Visualise some examples from the dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "33d7c3dc", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAClCAYAAADBAf6NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAACnR0lEQVR4nO29525kWbIdvNJ7R88y3VU9tmdGI80VdIH7KHpIvYAAQX+uPkAjSMLV+Pbd1UVPpnfMZOb3g1iR60Ttk0yyWEVWVwZAsIp58ph99o5YscLsxHw+n2Mta1nLWtaylrV8tJJ86BtYy1rWspa1rGUtDytrMLCWtaxlLWtZy0cuazCwlrWsZS1rWctHLmswsJa1rGUta1nLRy5rMLCWtaxlLWtZy0cuazCwlrWsZS1rWctHLmswsJa1rGUta1nLRy5rMLCWtaxlLWtZy0cu6VUPTCQS7/I+HkwSiYQ9WzKZtH8nEgnM53OwJ1PcMYlEAslkMnLMKv/W6+q5/TGh3xTtF6X3wd+z2QxXV1eR5+D980c/8+fjOfW6s9nsjXFJJpP2t/l8jslkgtlshsvLS0wmE1xdXWE6nb5xjfcle3t7kf/rvfJnNpu98b1l7zBuPfA8Okb+nP77ce837n3zHSQSCaRSqTe+k0wmkclkkM1mMZvNMBqNMJ/PI+8+mUxG5oefJ6F1od8NzQsem0qlcHl5icvLS6RSKWSzWQCwa6TTabtvPQfvO5VKRa7BY1KpFBKJhM0lHqM/fjxSqRRyuVxkLnN9pNNppNNpG6PZbGbnnk6nNj6TyQQA8K//+q9vvM93LT9VvftYhTpR/w8s11vL9PJjkVXuaWUw8NjkPhZJyKjHnTfOiMcp9rh/h37f5VlCL5cGQn/fh/B8cdcFoovoQ1Zg+gw3PffbvrebwJ1ef1Xx9xsCOcu+e9OcUQCl1/PHLPuufs+DAgIBGn4aeQBvrE/9LJ1OGzjStZVKpewzD9z5fQIlnmc2m9l5bjN+a/npSmht+DlFiXME7iIhR23ZPb2N3DsYCHk5qxrI93Wsv18qGSoNel7q0XrAwBcexwzE/T/OCwRWMzj+GP2/Nx70eD3giJtgoWP0+6FJHgJH3vObTCY2jg8hfD9xACr0Nw+obgKJcbLKYo0z+jcBAs7RZDKJbDaLZDKJq6sr82jJCPAdqvEEEPGElQVYNib8ngfC/K3MiF9P0+kU8/kc5XIZpVIJ+Xwe5XI5CMrptXtGAoDNL31PnG/8Ti6XM2aEx2QyGQDAYDDAeDxGKpVCKpXC1dUVxuOxPT/HfTabodvt4ujo6FF6e2u5X9H5TIDIeeUZUc7dVCqFfD5v81GPC7Gu/m+hzynT6RSXl5fGsuraCtmRt52j9woG4rznuN93/ew+jvGKRJWF0t3e0Htl4RVs3Jj4+/H3AiwmXJx4wx/6LHSMej6h74fG5qbz+rFV48lrpVIpM0wPKcuM6bK/3QTO3pYJ8SyEXmNVVoBzVQ0vcD2XvFeroI6iQOGm+wz93xtqfqbrgmuF8zudTqNQKKBcLqPRaLzxnAwXcD3qGkwkEsjn88hkMri6usLV1VUQ0BeLRZRKJUynU4zHYztnIpHAxcVFhIG4uroypU9QxbEj2HroObyW9yfL7AdwPZ8zmQxKpRKy2SzK5XIkVMq5HgoZKljQzwG8MccYnqJTqjZL7+m+gOrKYKBYLL7xt2We8Cr/vu/jVv2OCo0+sHgZOgnS6fgh0ti8l1U9/xBKXPYdjXt6b08VusZRKToRGROld6mf6zPQePAYVfb6maLi0PUek9w01nGfqZH2nvQyw+3PGwJSFA+s4s4zn88tHp5Op1GpVJBKpdDv93F5eWmGUr+rf9NzhMZGFY8HsV4p8b6XPbfODyq38XiMfr8PYLGGlO4nSOePsh/8XD0yBaODwcDAPEHpdDpFIpHA7u4ustksSqUSyuUyLi8v0Ww2MR6PcXZ2hsFg8Mb6WocLPg7xa9WDXOa07O/v4ze/+Q2KxSJ2dnYMnBIE+Fwcrj1dBwrceQy/S1bq+PgYo9EIJycnGI/Hb4AM/f22sjIYKJVKAOK98MckqyhmCpWEGgjSkIw36vf4Qqmg1PON85RD119GF8UhPlXKvD9Sql55qnelylhpp+l0agl+OoFDLEjoGXmPngbmZzzvQ4OB0HsA3jR8/Jv+9sfzfPr+b/Lk/TVCnrTea9z79/MlnU6jVCohk8mgVqtFgCuTOPW7fD8Uzh1gMa/1eE0K1GcOgQXee9yzeyA6nU4xGo3sfAxXUNl6EJJMJlEqlSIefT6fR6FQAICIYk0mkxiPx5jP58jn86hWqwBgCY1PnjzB/v4+Njc3sbe3h36/jx9++AGdTgf/9//+XwwGgwjgJjhZy09bQmveO3zZbBa5XA7Pnz/HP//zP6PRaODly5fI5XKYTqdm9Ols8Tf1bAgUUE9yzU4mE0wmE5ycnCCTyaDT6aDb7UaSWePu923k1tUEIW/gMQKCOImj7W9Sfj4WpF6Jxse9EvEZ0XEVAyqaBOXPRUWoGde8vr8Pf/+cnJysoZ8QmlU0GsqPCHmRaigfen54ui70Pvl56HdI+LzKDHgg589/m3PTGw2Ns85VAr5CoYB0Om1G8PLy8g3mhsrGAwJ/f3HjsmwcQs/IewyxC/Tuqdy0QkDvLZVKGUBQIBACU4nEda4Az0tFPBqNkE6nUSwWkcvlUK/XsbGxgXK5jEwmg3w+j0ajgXQ6jXq9boxFt9u1KoN1mODjFA/gy+UyqtUqtra2sLOzg3q9jq2tLWSz2QgjFwcGqH91XVLvet08Ho9RLBYxmUxszqtOXyarHONlZTAQiik+tJJ/G9HBBaKekzfcVKQ6uLlczrwS4HosSHHSU0+n08hkMpEEL2UcfOY0j9Xv84ffV+rfe5CqhOfzOUajEcbjcQRRekWtQGA0GmE6nRrV3O120ev1Ih6lem08D8+lGeAEA4+BXvXUuAcDQBgwqOjiUgXBf+v68Mf6a/h/+3Mq3a3n1nunB1woFFAsFtFoNJDNZpHJZCxBbjAYYDqd2u/xeGzzmMyCF71vn7BEsBliLDxwoQENgVV+TtoeWIABPi/HIZ/Po1gsRtaAjjOvwXVXrVZRKBTQ6/XQ7/cxmUxwfn6OUqmETz75BI1GA5999hmeP39u58hms6hWqxgOh2i1WigWi/jLX/6Cw8NDDIdDS8Rcy09fvE3TuU9W6fnz5/jtb3+L3/72t6jVanj27BkymQxGo5F59VqaqkBcQYE/Rp2x2WyGTCaDk5MTA7nD4TAS3l12/3exzR99nwFVuqFn9H9Tml4NNLAIL9CToSFXMMD/+38rKODf+X2ey7MMvH/+5k8oByDuGdPptE0wJlMB12CHMplM3lCIPhFGvb/bINj3IX6MQqyAfr7M6Mc9jzeGej5/D8vuMe6+/f3puamsFDjOZjObc5eXlwAWZXOazOfDP6F7WeUZPAsUAkMhZsPPEQVJDHVtbm4im82+AQY0zAFcr8lyuYx8Pm9KnJ695ggUCgUbI4Iq5hVUKhX0+/3IWOn9ruXjEs4jAu1arYatrS3U63WUSiUUi0VkMplIhYt+l/MKQGTtATAW4fLyMgICliXzvkv5YPsMvK0QqVGhKG2jjIHS88DC4G9sbGBjYyNSYqJGXI18KEbsPbDZbGYUb5zyDKFHRZbeyOn9UJHmcjlTsmQm8vk8AKBSqQC4btKTSCSsadBgMMDFxQXG47H97nQ6EQo1nU7buUNJiw8pHhx5r9f/XsVgqyH2v0PUfujfoXPHGVTevz4H2RyCuPl8jmKxiHw+b2VJl5eX9v9cLofxeIzRaBRhgRSAKojzIQ9m7vtxoIfuw0Q6/vP53OZ3oVBAPp83Jat5F3wuzqFPPvkE//k//2dUq1ULAfixu7q6wnA4tBJB5sLw+ZrNJgqFAn7xi1+gVqvZPPfjn06n8eLFC2xvb+P169f45ptvTE+swcDHISHnIJ1OY29vD7VaDX/4wx/wu9/9Dp988oklDpIN0DCBZyOpJ3u9Hnq9HlqtFk5PT3F5eYlWq2XOWCKRQKlUQqFQwMnJCXq9njECvjFY3P3f1Qn7KMGAKjdlBkLeolf2pItIXxaLRdTr9UhzkzgvHlgoRp9cx8lCY8//++QSJgBycmhcyhu6fD5vZVj5fD7YmEUpXAIE/uaE7vV6mM/nGA6HGA6HABax5pCXGhrvh5QQqPLJdf5YSlw8XH+HjGfcPfjPQ+fnuWgk43Ie+O71/WsIimCPx0wmEyQS1x38UqmU/V3B6k0lhqFn0Hfvs/D9/FbwCizCdQpseXwicV0i+OLFCzQajUg9t8pkMkGv18NkMkGz2Yz0V2Aor1AooFaroVqtWsmil2QyiUqlglwuh1KpFOmXsQYDH6dorkC9Xsf29jaePHmCzc1NFAqFSDhU9bV3ALhWx+MxhsMhut0uzs/PMRqNcHFxgel0aszeeDxGtVq1cK3m+txWl97m+I8SDFD4okk1+mQ7X07H2A0zSqmcWMccUtpeAWrsSBU6PwuVpOhvBQtxXi5lOBwaAMhmsxEgoCGJbDZrtd8sVyMoILBpNBpoNBrY3d3FbDZDq9VCv99Hp9PB+fk5JpMJut2uNW+hcMGQCnsI8TSdBwMqfsGFgGHoGA8I4s6tskr4IeSh69xiCOfs7MzmJFsR08MoFouYTqfI5/O4vLxELpezd6VNdzT+HgKy/l49UNZ5ynnnQRiACHCmN0UDrkB0Op3i4OAA//W//lfUajVsb28jl8vZMbVaDZubmxHGazAYoN1uW2MjAoJCoYBSqWRGHogCH94n5z77FDDf4iGbZq3l/Yhn5ZhsWqvV8B//43/EkydP8Pvf/x6/+MUvLDxAds4nW3NtDgYDXF5e4uzsDOPxGIeHh2g2m2g2m8YM9Pv9SFiP4Yd2u43j42P0ej1j8lYBBXfVsx8lGFAlp7F6ojz1zHVglXLP5XIWV6fi8kkipCr1Mypdn8VPUTDiPdlVkaGnrJX69QqdhkObZ9CgkPkgKmaiVSqVQrfbxXA4xOHhoZW/NJvNiNEvFArI5XKPIhM7NJZxx+lvL3Gx9VUBQehacZ95ys+zMJyj4/EYrVYLmUwGlUoF+Xwe2WzWgBiNNL0OPj/DY5eXl6ZsvHfvk0VvGg+eW5kjT7MrACfDNRqNACz6DADX6+rs7Ax//OMfUS6X8eLFC5RKJQPkT548QbFYRDabtT4oo9EI3W43Ei7J5/MGlBie0HtRcEg2gMcyp2YNBn764gFwJpOxyoFf/epXePnyJX7+85/j2bNndjzZWjXUXJej0QjtdhvD4RCvX79Gv9/Hjz/+iPPzc7TbbZyfn5tt4PXS6TS63a4lApPpImhW3bUsz+cu8lGCASDaAx14c1MfT3XqC55Opzg7O3ujlltflqJEpfI1g1Rj/Sohw6+Ua8hbVVnFq+NxvD4RKr21ZDKJ4XBoMa7JZIJMJoNerxephGAWe7vdBnDd6rXVaplyJzB6SPEgaFVkHUfn3+TFhxiAuO/FvSd/zdC1WClAJXJ1dWVNiEajEXq9XsQLp7dMpqjf7yOZTJqyUYUDvNmdMDTv9P7jgAF/+3mpsX3OF94/AOsXcHJygna7jdFoZEaaSYL7+/t2DqVUR6MROp2OeWYEPiGwpvel4RktYVzLT1980nMul8PW1hZ2d3exvb2Nra0tA5N0+rSdt+YF9Pt99Ho9HB8fGwjo9/s4PT1Fu902UM6KIGABhDXHhj835V7dNU9A5aMFA5r96Q24etM0wvys0+lgPp+j2+3i8PDQPgNuLuuIo6AVfITkJoXkr6fNgjyr4FHlfL7owEiqlscwhpXP53F6eopsNmsx1adPn2JzcxNPnjxBvV5Hu91GtVpFp9PBX//6V0O9RM0PKcvGAQgj7JuYg2XX8ue9CXjEHXfTtSaTCTqdTqR8lTv0cV5mMhmUy2Vks1lsbm4in8+jUqmg1+vZd5mgxHJE7XTGUJLezzIAQPHgGnizK+Z0OrVr93o9JBKLVsM8nslW/D9BTT6fx9XVFX72s59Z2ICgYjabWRiEz6VZ3SHxa0wrgEKAfS0/TdG1WCwW8cknn+Dp06d48eIFPv30U9RqNSSTSQvPqe5nIuHp6amFA7777jv0ej38+OOPGAwG6Pf7GI1GyGazNteZgA7AgCzBRK/Xe4N5CN3rfchHCwZuMpLei1HkpShQFcUybykEAvwxy+51VeqZx8d5t6HnC3nNCnKURmapWqfTAbDo3T4ej60l7s7ODnK5HFqtFjqdzoN7V3EADIh2i9T/q4QM3irvI44JCAGym74bN480PgkA4/HYgC4bDynIJVNA4EB6nd39fHMUPz7LQJQfx9CYaYgDgAFyAg7tReCPBWCKmMpXzwMsqFaOgSbarsIGqWguzhoMfDxCAMzEa02+JiOnibsUAoRut4uLiwvLDej3+xgOhwZOeY24MJxPCo+T+wIBlI8WDHiKX8MC3rOmkmOGNoV/j8tO5jE8v37ntvGeOIMRMi5Ky/NvfCZ//6FwBEXzHxg6YJjg9PQU6XQatVrNkgs///xzZLNZvHz5EpPJBH/84x/xpz/9CcB1RcJDij5rHMJWuQnQ3dc93Qbk8V4U0FBpdLtdU1j0hmnUOT8535kPQmAwHo8tRplIJNDpdDAcDq1yJW6+8jM1lhrz57H+eD02m80agzGfzy1MoKVUXJNsPqQhAeYENBoNAAswxDyYTqeDVqtlvRb8OtRxnM/nNlac8z50spafrlAvML+kWq1ic3MT9XrdwCrZAK4xynw+R7vdRqvVwpdffok///nP6Ha7ODg4sPkKLOY8c7XIWJGhZV4Z5z8l5Lzdt3y0YABYPqh+4L3HoyAhpNDjzr3M8NwFFMQZFAUKnhHwzxX3jDyOlDGVsraO9cloqVQK1WoVs9nMMnHVaDyErHpd7wUvO08cQ+DBlZ47dC+h66mxX4U18hUpjJGzjSm9GK1/Zq+JYrFoSaNUhJlMxsBf6NlvCqus+q4VeBMMxLFIPobv8260kZfGXieTCcbjsSWyhhghD1jIihEIPNS8Xcv7FepJrgtWobBSBVhUzChApB5kmI0huF6vh8Fg8Aag1qR1nlNZPk0gf5/y0YIBekWe5tcXpHFTfZHAQmlopjRfsH+5cZSQKjEVT+WHvFr+KJuhzZJCRoe/vbKOMzheqc7nczMS/GGSVrvdRr/ft7rwWq2G3/3ud/jDH/6A//N//g/+y3/5L5Yk9tASF/NeVeJCDMuOC92DZ55C31UDGQKimpU/nU7tfbCNLo0845PMtGdWfqPRsCZEqVQKjUbD9jpgv/Ver2dgMAR+9f5ViWlVghpcnbtseKVrUdtf8xkZHqBHxnwD5hqQ6WBC1jfffIPvv/8ew+EQzWYTW1tb+Pzzz23r2VwuF1G69NDIiJyenuLo6AjtdvuNTodreZzi9SX/BoTXoteB3JL4xYsXePnyJfb29vDpp5+iXq8bK6VhAi0bH4/HODo6wuHhoSW8TqdTC8ORBVDRcBcAm/dkpDRUoM+26lj4Z7xJPtoZ7puiePEUeqidr6c/tXe7ggG9lgcbIeXKv+kEUJpS7y1E8a/iTfrrxY0Rn8m3zKRyZC0tDVCxWES1WkU6ncbz58/x8uVLNJvNB9/17SbDHPq3fi80T1ZlEeKu5yn/Ze8t7r41wW0+v96PQhsIzedzDAYDXF1dodvtYjqdWs8JliIyNjqZTFAqlSJGOZFIoNvt2lz19xpSuh5wLptfZAT4XQ3f6XxmeEGVMTOtmQSpeQftdts6CPb7fcznc0veIgABom2UqZiZ6MVwyUPnvKxlNYnTpZQQG6rHsUR1a2sLL168wNbWFjY2NmyNcA76SrHLy0uMx2N0u13LEWBYgMmt7OGiTeTIOvEedcMiX47rn+W2umIV+WjBQFzyYMhrDnn0IRreGw5tTXzbTV5Cx/KccUzCfWft87mYWa1GRxErGQRmhn/55Zc4OjrCxcUFvv32W/zlL3958IqC28oyqp+fh9B3HNCKOyaOZfD/D30nNE/4bi4vLw0EcB8NZuBr/TyTBukVsbb64uICAKxdMD1xbX7lc2hCY+TBqrILIcXK+cZzkZVSdo3XPDo6wn//7//dFDgbLaXTaXzzzTc4Ozsz4JDNZvH1119jMBjgV7/6VWT/AgUCX331lWWDa+z2octj13KzcI6oblImF3iz0oqSyWTw9OlT7Ozs4Pe//z3+03/6TyiXy9jb27NN6chOaZhA5y93xtzZ2THmmSCAOrLdbkfApjJh6mz5H3+/y8TrkVXlowUD6m2rUtDPgTfBgI8hxtFOaijVgIeSkbwHFQpBcBKz3MkrVCpoPdbf312F4Q8aGQ1/MOuWNe/z+XXZZTKZxLfffot6vW5K+UMWj8pvWpyh4ylx3w29szimgv/2SUb8XBtgAYiAAU3S49ysVqsAENkydTweI5/PWwUJ+2yocYwDA57N4rEeQHBeaOyfSpzMBo/RrYxTqRRev36N77//PlI2+eTJE5RKJRwcHODk5CRyT19++SVarRa2t7exs7MTARgcs3/84x/49ttv8fr168gOdLehW9fyMKLGn/rX75Sp80rzZ7LZLD755BN89tln+Kd/+if8y7/8i4XLgGgPGW0PTGPOJNhCoYD9/X1sbW2hUChgY2Mjovtfv36Ni4sLa0WsDa20isdXwYRCH8tCATexciH5IMHAfS3MkKLVHw8E/He998ZzeJofiO7trkjRgxIVVVShfAD10ENo8G2BgNJYyhKoKDrWa3LR9Ho926DmoSRkiOM88tsIAVLoOvwciN+uOO4+lzEN/vzKQvlzEbCxbwD3lSgUCshkMubN0KPWPBrmgTCGzu+xSRGvy2vcBH48gCJrxi6efg7zPCEmjN+nImWfBdZo83xsEENmpN1uYz6f48cff7SM8UKhgKurKwwGA2utzeMKhYKFF9by+EUdqFXXNIFAPp+3vQfK5bIBT/XYtXOs6kJWoGxubkZyS3K5nIFs5gBwDc7nc9s0TEsVtUmbMh3+OUOhDpW76LMPDgzcJxDwA+pR2LIOZF5JccLo93gNrQOnV8YJOBqNLONUPRBPqXogoAY4ZBDugxHgxOT9kgmgt6b7JvD6er/MIxgOhw/uWd2GZltFfFgodP445B4CmVpj7L8furY3mmSgtPeFehn0kPL5vDE85XIZs9nMNtsqFArWzjidTltXStLsZAharRZ6vV6Ewuf1Q+PDMVDASAq1VqtFFKvOKXpbTPbr9Xpm/JnwpwwVaVg2U9rY2LA9NmazGV69emVj/N1339mmM71eD99//z263S6++OILNJtNzGYzbG1tWa+MdWnh4xdlUv2a9GtQdSbznD799FP8+te/xv7+PrLZLABY/J6Amsl9nJfz+dy2u/7FL36B2WxmIJM5MVdXV7i4uMBoNEKhUECz2USj0bD9B9ibYDgcYjAYWI8MzmvgTV2yDOzc1Rn84MDAu5AQylqFiok7V8iD9kaezSwAROJQIe8oFIq4iWq+bwmNj0/A9IyID3U8lCx7dzreNx1HWTbGOm9CC1iPCbFKq0pozsbRhRx/Gm5WD3D7VW2EQnaAddYAjCpl/kAmk7GEKgLBm5SvHsO660wmY7XaBCsKZnymv3plCngIgpT2V4qYn5Ohuri4iJTI9vt9HB0dWde38Xhs3+dzPPQcXsvNsorH7PVmIpEww667vPoKGH6X3/GOj1accHMw5opNp1MDDgQKPIab3PluuLxOHDMdei6Vj4IZuC8JKfe4jHf1+uMUsE4Q1jpr8hG9M9avNhoNVCoVDIdD202O+x34cIVOEH8fel2Nl92H8Dl4XZ2YNBpq7PWzRCKxUhetxyKr0Io+MSn0HnTfBl+Gyh81wHESYp5CyolJd2ok/XG8PnM5lK0ile7jlo1GA6VSCRcXF0aVn5ycGEOQy+UwHA7R6XRsjmunQ/8swCI/oNlsWpa+93z43Joro42AOP7MzNbxZ9Y2Gw4xBOEZtD/96U8RD470LM+t98/M8DUYeLyi4NoDbg9WOU/5eTqdtqZppVLJwIBu4T6fL8p7Of9D65zX1mMILDgXq9WqHU+2uFAoWG8Q6lVN8PVlvXFO4Ns6hx8tGADisy5D3lpIGXjF7JOf/DHAQrmxnItKSb/vwUAcMPD3et/iPXy9xxD48F7xu7y320gI3d/2e/x/nPeh3igXNdG+erv8rYl/cffkGYS454jzyn0og8aYscp8Pm9JguqNz2Yz2/VwOp1auWEul7OQAZ9BwaKCEx0Tf58sUw2NIcMN/DxU4885qBUICor9s/AYgv1Op2N9FXTH0lQqhVKpZOCDwO0xsFtrWV1Ceic03+iR5/N525JYw12hda4VYnFOmA/x8jrMXSkUCgZECT606ozX4Jp9X3r0owYDFG/cFFWqJ55IXG+kojkB7FQGRNGn9yKJFrmZCmNFjMtS6ei9eEOi96uTjaJ0713jRiq6EDxrouUw6q15MPOYgMEyCjvuWBW+E++JqBGjd0FQACzijnFlQ1qloWxKCDCEDH7ofkNKiucbj8fWrIcG//Xr1yiXy9jf3wew8G4KhQKePXtmeS2dTgf5fB79fh+lUgmVSsX2qmCNviowLatVw8x71OchUNDnUtYiNO4KHAgQlG3QcWJog8lcnn1JJBK20yM/89dey+MVvu9QiM4fR6Ncr9fx8uVLbG9vY3d3F9Vq1cJnumZ1XXrHTBOkFcTOZjNzCDj3yDiws2GxWMTx8TFqtZptDc9za6KuMmGhZ7qrw6Py0YGB0CTxYMCXAapCYS02jwntKMVGQxRFkFT80+nUqCDutBZS8B4M0OCG4ky+3Os+FZgfNz82qvRvQ2O9L7mPa4c8XQqfP5fLWa07PWjd/EffsXYy4/9DbFDcsyxTejqf+bmWRgHXFGmv18N8PsfZ2RmGwyGq1aolP3Gub21tGYDVtTKdTlEul00ZsgkQu1Sqx00FqfetjIdXsP7+9Vm53rT6gXOfY+lLeBOJhLWV1fFTMKFjSG9NQe5aHq94/eLfq18zTGCtVqvY29vDzs4O6vW6rV1t/OMrpoBo5QKTCylc+wyj8m8E1wDMhiSTSezu7lq3T4YMJpMJksmk9Qrx4bNV5TZ674MDA/e1KL1iAhZ0rM9IBRY10qR2PAWp3wslgahC4fd8pynGlbSuX3MO+H9VmipUuvfpiasnzOehYg8xEP7e7ouleBtZxgjc9p7UePGcGh4g5chqEfUc/OYjBIb0HMgyKTWtQFD/rzF1/UzvS+emv3dei62GO50OptMpjo6OMBwOrUxKe/TXajV7LrJZZLf4nFRgrNEHFiySn7P6N88u+XenovS+Z2v0+8pKEazpGCigJojQ895lfqzlYSQOOHvhfMjn89jY2MD29jaePn2K3d1dyxug0VUdTeZOM/15Xc034d/UPvC6NOicdwQNlUoFAKz5F9eUbh7mK9RWGQ9ed1X54MCAytss1NDA+gnFl6nJIOVy2WqTSSGydE69G15DXz4NPCeZ1pOye5WiSiop1qdOp1Mrr9JkLZ77vlv+qtL0lDOvRcXMZ1Kjp5vGPLRSvUlR3HR/N80VxgNLpRJqtRry+TxKpVIEyHmqUT/jLoPal9wDQIIEAga9J+8xqBLSuDiN4dXVlc1b3V99OByiUCjgs88+Q6VSsRp8ANje3kYicd2euN/vYzAYoNvtYjKZoFwumzLr9/totVpotVpWORCiMcma8D7JRhCExIXINK+A4lk9KnTNOfCMn5YdUpkzxMOx5/0+9PxdS7zcBuRzPZTLZTx9+hRPnz7Fr3/9a+zt7WFvbw+1Wi2yBgmEuT6poymqy2mwtTuhJgrz2px3bFK0tbVlieVsn93v95HP59FutwHAQhe3nYc/aWaA8jaLM+67odCB/qiEUB9rU/3xVGCh8yiAoHfiwxIacwVg2c9xGen3Scd7GtqfO+TxecX80PK2QMCfxwMBvn9lgTSRkIaJffg1rKMJe2roufugskMEA5qcqImdnmYPvTMPaKiwSEuy1K/f79smPR44sythInFdpsgQB3MmaEzH43GE5dCYp94nEG0Tq+sq7ncoPOKfOW7t6vvyDJYCgNvOj7U8HlnGFMznc0sc1ES+kDPF+cjmQJwjdNqAsK2Ic4C4tnu9ngHmi4sL29+FYWOCUQ2/ekB/3/LBggHK2y5UT6vynPxh4ocmFNEQ84feB1EjPRd+7mOUiUQicg7SyLopytXVFUqlUqRedT6fG1IdDoembNkilrkH+lz3ociURvXiGyUBsAZFHjA9NrnNPYXiyip8R4yzs5+5igID7/HyvIw/jsdjDIdD8575t+l0ahvuMPmUSmNZolNIUXE+zedzq69nox/uWVCr1fDZZ59ZE6LZbIZarYZSqYTz83MDKYyPlkolq0Ag03B+fo7JZGIbJQFvbu6lCbkaI/WG3yvauLBD3Pf5HpQh4d90PDQUsZYPT+LWK/+WzWbRaDSwubmJer1um6upY8P5NJlMcHJyguFwaMxwpVLBxsbGGxS+Vg5o0zquy16vh9FohNevX+Prr7/GxcUF/vrXv0b2/SBI6ff7kTBXaC7eJyj44MHAfSMkTwnG0cNqAGn82bmNySFEgar4VREBC6XIfwMwykl7wJNZ4Lm1yQqpeU8r3/f4hOhSNTaqrD2N+9ByH57eKgxDKMGHC1kT0kLMAnANpNQge0aA84nC5ES9tjIOvO+4d6cKj+ejx9/r9axZin6XbYRzuZyxHbxvPidzJ4AFk0Bwo3OFzx8y7jo+y6j60Pr0540TfyzHYw0EPhy5C3WeTC5KvDWcqcdwLjBMywochkBVzy4TZcbG4zH6/T663a51y2T3QR5LHeHPH1q/9ykfLBhYJY63bLDiBpkKgef3JX70wtnUhJ3a0uk0KpWKgQEqbqV6VVGNRiOMx+NI1imN+Wg0wnA4tHgRFSxjXWQhWP9Nqpf5BKSZyD7w3KHnX+ZlrSJqyCia/Oav8VMQT9VRQZCloQesxpFgkcpHvYd0Om30OoXMABXIbDazxLx2u22xej1mPp8bezCZTCJ5CApivXc9mUzQarUslsnYZafTQTabRa/XQz6ft7wCBTXaXvvq6gr9fh/j8diSDS8vL7GxsWH7vQ+HQ2M2CEJ4Lj8XFSgD0cRaZeRCAEDfkS+51Vwd/Q0sgJs21VpV6a/lcYgyRHHvjWuOlTPAYp8VgunxeIzz83P0ej18/fXXaDabxoptb28DQITZIuhVxomMHZm8w8NDnJ+f4/Xr13j16pWtX89kaV5YKDl4VbmN3v1gwQDwdsjIe/f6OwQUNIkrkUiYEU8mk6YQaaj5Q4VMhaSgguemp6elUVTimpFNxZdIJCw5TfdlZzyLio/fu+3EWbaAeL5QzEonXcg7/qmJ92rVi2CtPBe4dhbTZFJNmGPTEyqE8XiMXC4XmQ+cU5qESg+FHnpo3vnQAe9blQub8NAAUxnyh5U0WlarrBi/SzCiiXi5XM7ADe9XGTMFlCEmTv/ugasa+BDb4Fk5/S6fwb9X3o9mla/lcYvXSXEME0WZAZ1TnhEgQ3ZxcWEdYtnKe2Njw3Q9Q4R6Hq0ooGPI5FoCeq5TnXMAIt99X/PvgwUDb4vUfQKTKiQqTz/BqCQZoyVNSkTIl04auFqtmsfHieJj+vTqp9Mput0uLi8v0e12bRL2ej2bRFTazHY9Pz+PeJ6c2OVyGQDsWE5qVcKhull9ztuM8yro8zEzAyGWKQR44o5VsEeDp70kqCxo0DlvcrlcJETEeaWeqQJUet/ZbNYoeOB6vjabTVxeXtp+6fTip9OpVb8QVHiFxzlEsKneyHg8xunpKUqlkoHe0Whk4Yl6vR4JeaVSKducqt/vGxPC7oXj8RiVSgWDwcCUrIZCgDcrfUIhNU0AjAOwceBn2fvUa2o+zFoetyiwDeka/o0AoFQqmX7W6hTVx9yzgkl+ZM8A4OTkxEA42bInT56gUqlEKljI0p6fn2M4HOL4+BjHx8dotVoRHUFnUBliJhXyHKqz38W8/ODAwF2o7LjzUPkC0eQ+Drx6DfpvzSbN5XIol8tmCJgZDgC1Wg1Pnz5FPp9HrVaLNI3I5XKRbFRSUuPxGCcnJ+h2u7bvdbfbxfHxMUajke0ix53V9D62trasvI3eGBU3G2kwY3XVSbVKnHYVeWgwcBPA8cDQMyDLGBMaDtKBZI5IZROkTafXO+7RsygWiwYkCRQIBGhYlT7k54VCwVioXC6HyWSCw8NDDAYDu28afSaWUrkxAZVKjw2CGN4iGOCzcl6ORiNsb2+jUCgYtel3HeS12dyH96AbHE2nU9RqNQyHQ/O02MEwxF7o2CuLoCBpGUsV+n/cfPDhBe8wrOVxin//ywABQTLBALcr1mocrg2CgYuLC1xcXFjYLJG4zqlptVrGCPudCvP5fKRU+OTkBP1+H8fHxzg5ObGEXWBRns2QMtk4ggGChmUdCJfJqvP3QcGAf2mhGLOPG/rjgEW3KVLk6imzrapuHqGZ/qqAvOfEc7NblXrhnCwAUC6Xra95IpGwLm6NRgO1Ws2+o7HRcrlszSYAGKPA+DDjy9lsFhsbG9jY2LDtY8fjMZrNpjWMISPR6/XM6Cv9lUpdb5DEsSRT4PMZKD5z9UP3jm57/zcdr96oeq3z+dyMKw0uFRAN2GQyiSTbkQVKJBIGEJi0x2spEOUc4twol8t2LdL5nU7HEp60W990OrVcFjIMBCP0bvhb8xcIKvr9fgScUDHyfubzOcrlsv0duO7QyeejguQzUemORiMbGzJYSvsrYOdY+/BCKMSnusOHGTwbEAcg1kDgwxMFp/o3AtVKpYJSqWShVgJgZYbpSNEzp/N0eXlpyX4My3F/j93dXWPPqIdZPXB0dIRer4eTkxOcn5+j3+8bAFYdTrvEUJ828FKQfhtZ9TvvFAzEUc+estOXMJ/PI/uV+9I0VQR86Uz0oTHXxL6NjQ3kcjk0Gg1rGpTNZu0lswsbyz3YYIWeDBXm/v4+UqmUeVmk8zc3N7GxsYFMJoN6vY5cLoft7W2Uy2VsbGxgc3PTJgUnTyqVws7ODvb29uzZNExQq9XQ6XSMSmWi1nQ6NYT5l7/8Bd99950xCOwLD8DGr1aroV6vW7et+fy6hIxJaNzKVndKpHJnqIN/e5u4VUjZfsiiXiPHhXOSyXEaHqDnwb/n83nkcjkMBgP0+32k02k0m02bQ6x7LpfLkRwDpTqTyet92Dc2NpBKpfDkyRNcXV3h9PQU3W4Xh4eH+P777y3HRelIvnP2Ruc9aZWDp+knk4mdm4CF84xJh0xwBa53JiwWi7bb4Ww2M8aAY1Sv17G3t4fBYIBXr16h3+/j9evX6Ha7ts4ViFO0yYvPw1AwT9Dhx2+Z4fegYS2PX7zjQsCq65N/q9Vq1mBoa2vLGFtgEcIdj8fodrvodDrGzBK8D4dDXF5eRtZupVJBsVhErVYDsKjm6ff7OD09Ra/Xw1dffYVOp2NhB2V5CYDJ2mm+AW2iryJaVW7znZXBwNt6hzQq/sWF/gaEG/csE3oqwLWnThqo0Wggn8+jXq9bBimV3+XlpWX09/t9nJ+f28tRoEHFTNqWSE1zC5hgxQ0o6ClpPTgA6xugz8jvUyEz7KCxU7arrVQqSKVS2NjYsJgsaSS/YYY2k9ENYJjvACxa4vpEFfV6P3Rm4C4SemYNH8R9x4MmggbtU8G/E+1zPhBA8DsEE/p9ZcNo6HSeFAoFzOdz1Go1bG5uGlugXSw5H7iPAnNaGMbSTGhlJaiw2EuBHrw3tnwWMgH8zeMY2qIxB4BSqWS/lV4Nxfo1pBd6X/zObSlVHxpaswMfroQAHxkw9m+hnvZzjDkstA801sCbjoCud9oObjbU6/VwdnaGfr+PZrOJbrdr3TtZMab9YQgG9Do3VUXcp7xXZkCNvz4gFYL2eCbNokYZWOy8xvNTqWYyGTx58sR2WdvY2EC9XsfW1pYpJo3Z0/viy2NCx9nZmXn+xWIRW1tbaDQa+NnPfoZ0Oo3z83OL47RaLVPU+XweT548iYCG09NT/OMf/zAwkMlk8POf/xz1et1oIvUgqRBp4NmhipvEzGYzVKtVVCoVlMtl/OY3v8Hx8TG+++479Pt9HB4e2gQm3XR8fGzXYMiBDTdIUzOWy53pSAvHMTt3kQ9ZoWrIyP/dGx56BZxDariUZte9DBjGSafTODs7sw1UGo2G5YCoASXzpeEIAgu2Q97a2sIvfvELDIdDnJycWC7KcDg0VozXj3tmhhfIRFBZMSmQSa7sKZBMJiO7Fm5sbKBUKlmJ4cXFhYGJbDZrY8bmTKPRCNVqFYPBACcnJ7YWuXsbcyY4Dr4UDIjuTaDP4sMtofdLBuw2TshaHo/EGUzOmWw2i52dHbx8+RJPnjxBo9Ew8EwPfD6fYzAYoNPpoNls4uTkxPSiCpMQc7kcNjY2kM/nbd6+evXKQgwnJycYjUa29shCh/rC+BD1+wIBlHcGBlaJuwILxKaDHVqMod/qEZGuIT2/s7ODarWKzc1NU2gAIkqzXC6bh8xyKMZMmZxVKBTs2HQ6bR3flEYHEGEGqJwGg4ElSbFUTHtdUyHT06JCnc+vk674PR0bItlMJmO0VL/ft8Su4XCIRCJhIRNSqmQy6vW6fZ9laMxqZ5zYU60/dQ9pWX5EyCvl30MsF/8dileSalS6n3OJlDjBLQCbAwQO8/k8MpdV+M5IqxeLRSQSCdv1jH0tFBiSEaAi0ufV6gY+g3orVGJMTtXvTadTU74ADMywb4YabH6vVCrZeDCE0u127Zy8tube6HPrOvTvRUHbMnYn9PdlIGItj1d0TmjoM5/PW85ALpeLtHrnPCMrwHwXsms6N5iHxjAyE4bZUIgMwOnpqeV4Md+AOl0rCFTfqu74SYABT73w32yAop367GakxMMnTJBe5UvQvtKbm5soFot48eIFKpUKtre3Ua/XrYzEe7qJRAJnZ2d4/fo1er0eTk9PMRgMjB3Y29vD/v6+gQrGeKmQSKtrHTknytXVFQ4PD9FqtXB4eIiDgwMUi0Xs7e1ZOdhoNMLf//53/Nu//RvK5bLFrn7729/ajnepVAqVSgWJRMK8N7IFOjZkCEajEXZ3dzEcDnFwcIBOp2OliQQn9O44ETn+tVrNlDoTEbVRDcfsbefDhyZ+/obCJkrb++P8uXz8kgAtmUxaK+rhcIhsNmsKI5PJoN/vmyJLp9NoNBqmSNjwqlqtWtKs5tswt4B/Vw+IwOPq6spAg2Yua5dLba/KHATGTtWjZoUEgQIZAipW5r5oIyCelyBCEw5brVYEzPLeNJziDT7H2Oc93BTm8bLOHfgwxK83rZTiDwEw81RoH5gHpmu72+3i/PzcEvyAxZrlfKDt0dDe0dERJpMJms0mms0mRqOR7Qbq2YBQSPEmR+RdyzsNE3j0T6OsyT9UJEC0d7t2zFPPhYCB5VX1eh3Pnz9HuVy2ndYajYZlWVN5cIc2SrfbxdnZGdrtNo6OjiyRcD6fY2Njw3pP7+zsmMKigQxNED7rbDbDxcUFjo6OcHx8jKOjI2xsbODTTz+1LW0nkwl+/PFHHB4eYnNzE/1+Hzs7O/jlL38ZQbE6Luw212w2LTdgPp+jVCqhXq/j8vIS5XI5Ut7IhjaaT0AwQ0TM8ITuiKfZsvcxIR9aoYbiziG5CZGHsnnjauJD51VgxbH13fHYpAdY5BAw70M7GLIdMLBgpbQCAECkmoGMAQA7B1ujct8A7pimgECfMQQGxuOxAW5ei4lWZMPohZXLZfvMe21kBQBYsyUCkfPzcwCL1suheRliYvh3Hf9QLDkkeo41M/BhiepoBYfMGavX68ZWATAHie+bdL7OU58/w62HlfVrtVqRknCCZWUd9J4ocSzV+5Y7JRDeVrnr8VREuk2vR/i+oQjBQSKRsJKQ7e1t7OzsoFar4fnz55YkyNIOr9h9ZUIul7PWkpubm3YMM/2r1SqKxSIqlYol4jEZMJVK4eLiIlISMp/PrUXw5uZmpCHF1tYWXrx4YR5fp9PB4eEhvvvuOzPibAGrVLLmOgCLTWyGwyFarRZ6vZ7RUoo+GW7ghhq8d01SYbyXBgdY7H+gTATj3m8rDw0IVG6ay3GLVr1/VR5xDIK/hlLp/nu+oobzn/kjNN66NwUNOecys/q5xjSkpP0GdC1wYyEgui+BPifnt0/a4/2Ox2NcXFxgOBxiY2PDsv8ZgqCyrdfrqFQq5nURrPMe5/O55VCQ6QCAXq+HwWBgfTjm87ndI0X3/Fg13+Uxzcm1vBshoGRlGENomrNCppTzgUacoWPqcTpltEdkDlidc3p6avpYNx7iffhQ1W1Yqvch77zPgI//aw8Aeq1UgFRQmkXNgWeNNJMCP/vsM7x8+RK1Wg3Pnj2LUKCqtFURayMh5gsUCgXU63WjkHK5HHZ3d62ZCpXjxcWFKaVsNovDw0MAiCjLSqViNBRLDJ89e4ZGo4Ff//rXmM1m+NOf/oTz83N89913+Nvf/mYlVvl83hL/6IERNNGoT6dTlMtljEYjfPXVVxbaODg4MCosmUxaSSWNOVkFTmCWUXY6nch1qtWq5RuocfLv8T7mwmOVZehd2Z9QbPmm7+kxPkeGP8wTGY/H6PV6yGQytucFaUnuW5HP53F6eopcLoednR0LmTF3hk1QSLO3223LJWGWf6VSsXAE5x7DVGTDWOpH0Z7pZN14H9Vq1dYVEwpPT0+RTCaxt7cXyd3hrpsM/VEYsioWiygWi0ilUpar0+v1TNlqqJFAlgBDlbAHMXF5H/oelBVYhwset8Tl7ug7ZBltqVRCpVKJ9LygfSErTUdpMplYUjfnIRkvAmBtFsdybW0jz/vwfTDeplT7Xcl7bTqkC40vIVQ6QY+YQEABBLedbDQaEbqHMUdOBq01VWDBY3ldKkTtQcDEEk4mKkaGNHxrSAIbzSFgPIpgg2GLUqmEwWBgPQBqtZoltGiJVoim1ISvcrmMWq0WQahEtwQ+9LgARDxC9Qz5Hmg0tJxSWyu/LW21KlX/rmTZ9VeN1YWQ/F2f6abv6TskSAQQoeo5n6fTqSkiAFYxwAZInO+cC9pPQPdJYF6CelJkhhQAEGByffFeGW5IJpOWdMtz8plmsxmKxSI2Nzctn0XF7x8wnU6tnLZer0fWrgdo/K1AIDSuPlyj/w7NkzUQ+HDEAwHObU3U9rlqnPsKDsig8jgNE3ANcFMuMnS+U6Deh8pD6sFlcqswwV3DA/o9rbf2m+lQydXrdYuDszsZPeR/9+/+nSX47e/v28vWMiOWXtEoK61Lz4Ux9VB2NhOpOEnUcPb7fbTbbSsRAa7bDjORS2tZK5UKarUaisUidnd3MZ1O0Wq1UCqVMBwOkcvl8OzZM/zyl7/E1taWZVSHPBGODXfaIu16fn6OcrlsZSxEpZeXl1bypS1o2ciIAIDvgZQYPVCOKUMX7FPwNvIhK1X16m+7mFcx/N6TpVLSRDwCzMFgYO+WFQK66yCTT6vVKmq1Gl68eBHpKTAejyNtiAkUCF5pzCuVink+GobiJis0nJwn0+kUX3/9NRKJBD777DP8/Oc/tw1d5vO5eVR7e3v47LPPcHh4iG+//dZYEOa6FAoFY7V432zhzMZEDEvwe8oGAPElhp6i9YBbj/mQ5+vHJOpgqjOo4VZ2fGXXQdX5BAfUhSwtVOdRnUiC3oODA2MECJY17O31uAeay3TJQzhPt2IGbrNAQkBA/6Y5Ah4saHtTxks1hs9/M+lIPV2t4eZmMLx3GmqCAK35BqIlTESFfPksBeSPJlcxrEGlxPOwXJAxKmZ2X15eotFooNfrWbKiZwZCY55ILNoZkzm5vLy0zFgmf6kn6SecTlb+nwmdfGZtCENj8bbykMxA6Nohpa9jFYfsdd7q+fU7/u+ryDI2QvNIuCao8LjbIBk00uUaiqAHzo6GvDcqTKXR+d61l0GxWLQ5omEMBdm8f43p8940h4fhLKVtGYrjNXz5IIXHlkolA6eM4WoXTSDqyfn15EXv/7F6bWu5uxAQ6E6z3i4RgNNpUv2u9oGsgf5osq0Xr1/078vul7/f53x86zDBqgowFDv1lDiNqvZGn06nqFar+Pf//t9b5QCTi/r9fsRzYsiA4QNS/qpI6U0cHBzg9PQUlUoFu7u7b1Q3aFy41+vh1atXaLVa+Oabb3BwcGDeGRNSUqkUjo+Pbc+A8XhsCVMEINlsFs+ePcPW1hY2Nzfx+eefWymkKl+Oj5+USudyx61qtYqdnR2Mx2O8ePECo9EIh4eH6HQ6aLVaaDabFkcltauhAyr22Wxm7TU5/swjYAvOu07MEDB837IMECwTPSZUTeCvseq9eAMV8hKUotTz666AbGZVrVZtr4JcLhdpp9psNpHP5/Hs2TNUKhUDnxpnZ5KfhsMIaHd3dyP3eXJygmazieFwGOmvDiySUC8uLvCXv/zFcnoIAIBFiWw+n8dvfvMbDAYD/PDDD9aohQ4AASqffXt7G5eXlygUCvbs7XbbvDlNkqSiVvDr47WeSo57P2uG4HFLiLXjv/le6UjS0VFHgMcwV4D6MJvNolarYTab2XbDurcA82907vC6vrzRg+Y4ZyMOCLwPUPBOwwSUEO3Nv/O3L9NThZRKpbC9vY3NzU3U63WjR5khT6pUu7MVCgUDAxqaIL1/fn6O77//3vYs8LEkfWntdhsnJydmXNvtdsQTI7U/GAwiOwXO53OLUzFjn6wGk73Y6pj37ycM6XyOEz0nJlwx54FhATbJAGATWNmTEBNCz9OHbajACbbeZkI+BmZg1Xu4zX0um9erLGYFGqFzesCs4QP+Znx/Pp8bBUoPnglPDF3xGM2p4f0p1ckfbVCkVS2cM35rbM5htlsFYPt6+LybcrlspbUMc3DuqpLm98iwJRIJa/DFZ2VFDdkxBfUhEMjfnl6Oe7drQPC4JcSA6non20yw6tlozn96+gQSXFfcII4hBIZeaeT9WtJ5pfe0TNQe3sRmvQu5U5hgVaUaAgHe29WyQb4ANmrQEg9mKAOwemju+EYvg5sSaT4AsPCA+/0+/r//7//D69ev0e/3MRgMImEFKiKtz+f2qt9++631D+CmMtVq1fIb2GyF8dpkMont7W1jMTgONPqcaLpznE4gTjIqP/3hu9BxZg1tNpvF3t4eSqWShRHYzjiRSFiSJEtfut3uG14VcwrIDrCHN99dHFXu780vjsfGDFBWWaSJxGJrXn1OBVQhLzR07pvYBc+kqHes3yc9PpvN0G63rdKA85eAkQCCm6ZwLRSLRTQaDcznc+unztCaMmWaV8I46tbWlvX0GI/HODs7s8RCBZ2TyQQHBwdWpcM1wuc6Ozuz/BfOWTICnIMMGbLiAViEAdLp682XqtWqdfbk/foQgn9HOuYamlCWUsHzWh6nhICcZwjoJNLh8+uYpbpkujS0O5lM0Ol00Ol0LF+GoWTVhSGdGBeW0t+PRd5JNUEIYXNgOID0FKiogEVcn+VvVAjsZ84ugKQF2eTh+PgY5XIZv/3tb83A6kungbu4uMD/+l//C3/9618t27larVryFJPlfCfEbreLH374Aefn57ZzG3scEBBQgZLC1zIvfUYqaM1u1bwGKisqRU2EClHLqsgY253NZqhUKhgOh+h2u2i32wYGCFq04xyVIxUpwy/lctlyHMhsqLHzJXbKqvjFoVTa+5abgIAqhWUePIGmMlc0kLrJSKhCJnRdntdfx9OOcTkbnFdkCBKJBHq9npWXNhqNCOA+Pz+3/ILRaGRlhcC1t59IJGwXN4IBTSIlMGd1DJ+T2dRMTmVpKo334eGh7bDI3Bh2fmu321YlBMDYBDbPUjDAe+c4s/5b9Yp2WARgoECZD60EonCdKRigHgglI67lcUkc6OYP54pWfXGdcQ1TX7LsVsPGvV7PNhviGlM2AFise7V3Xrdo6C+kzx9S7j1M4JVciPLUf3uKBViUbnCAxuMxDg4OMBqNDK21221DatzW9/vvv0e/37eNijSb/vz8HOfn5xgOh5hMJqjVanj69Ck2NjbMg6AyIADp9Xpot9sGOmhA9ZmoLDWTW8MdcchQFY9XTFoZoWOkiikUW+JxNFiNRgP7+/vIZrPodDqRNsabm5sGQqjkmZTFf6uRyefzkeRIz1aE3q0HDQ852UNGV5VFHOvCDHvWvRMUAYttrvW3r8G/iY0IhQniGJWQwvDrktf3tdMKMNiEinOF649AnMcrw8DrMjzG+2Xp4ObmprFO2noVWDCAvV4PACy/QceRISqOM/+mc4v345MZ9T363BaGMPTd6DvXd+Az0Pmu10DgwxDvcOgao372x6iu1d4XykyzrwfbtPPcmnAeEq9PQrYvJHFMw7uWW4cJvNwmZBD67bN+dUHT+HW7Xfy///f/rKwuk8lgOBxa7+cff/zRPPONjQ387ne/wyeffILt7W0UCgUMBgN8++23OD09td7++/v7+Od//mejVjlZSItyB8ODgwMcHBzg7OzMDKoKPWml4L0nEaLTfaIgsKj/1j7spEpJUbPtbMh4cdIzNpxMJlGv13FwcGD5A5xg9XrdmmpcXV2h1+vh5OTEQiqTycTYGOC6SRPHxbMSpFHjqLq4ufO+xaN1IDr2PIYyn89tLwClGdXQ+fMQCJCqDo1BaLGrJ6tNuJQpCIVoQseQLtceAYnEoucEe7EzEZEhJobGuKlKIpGwRlpqpAkK6f2zNevV1ZUBc7Zk5Zjys1arhc8++8zOqRQtKxe4yZJvHsRkYOaxMOEQWMR9NamL1/VhgpAu4vhRsTOHiN9fy+MWZUhDdD3bd/tqG7JJymwxibXX69kutdzHxoMB4M29EOIMPXWy2gD+3d+3nvN9yXtLIAydT88bZzCTyetmOOyeRjBAZUda8urqCs1mE/P53JIE2T6YdGS327WaUyYYhrwujTlqnJz3poqDSkab+vgXGJocob/5yakNLHhdemVkIELGhucmNcZMWm4So5Rpt9u1nQy5JTPvRSsQ6CV5CnvZew39/bHJTXOaXiKNnm6049ks/ubYqbHz4he9L43V+ch71Hnov893zvdKxUe2SoGbvkOfMEhPiesmnU5jMBi80YtDS2p1rTKvR/eBJ3DiebX8VePxqkg1jKjzX+eisi+qTLVEkVnhXFc+tMWx80m1TDbTRl5refyi+lzBv4IFiupu9mahzeDPYDAwcODBPc8Ruv6ye1NmQO2qv+eQbXyX8tbMwG2/pxSqtrpVutzHdmazGb744ovIZ2rUqFS++OIL5PN5bG9vWzvhTCaDg4MD/PWvf0W328X+/j5evnxpDYvUKFOxdDodmwRUPvRG9Hr0ROid5PP5yH7tlJCy0ZgTlTDjzmy36scwkUgYZV2pVGwXRH5Oo0P0y17129vb+M1vfmOtZXn/qVTK9megR8dnZwtj4Lq+m+EUMiu8pk5Y9Y712eMM4vsUjjGNQRxTpR651twXCgVrDKWxZ+3bDyCYO0CamqLMkYIHHSdljRRkKmOmx/sSVHaoVKOtu6yRCdPWwzxvKnXddvXw8BDZbBYXFxfIZrPGkDBnQOP5BI1bW1sAYG252+02Xr16ZQr36uoKp6enGI1G1mab969rwYcLOG4KOFjWy7Uzm80sh4DPpgBaWzH78STA4N84DlwHazDwuEWBc4gZ0KovdeKo0waDAZrNJlqtlpWtckdbLXlVI85zhxhRD9q9g+v1DrBghpW5WJYvdN9ya2bgbW/CD5QOJhUq2+ECsHIjlhES9asHTvaA/aRJ9fR6PcsC5f7S+/v7theAggF9AYoQ1ZvwlDj/psYhLk68jBlQxa810nw2b8TYJdAbNh1X9aK4kQ0pfvZ+Z6LZxsYGRqORlSYyeUvL1+jl+ixc/271t1+YD6VQ9fo3IXkFOPyc711BHsfHJwRxPvDfaqh4PgINXfB+y271/Ak4NF+DBk4V23w+jxh+bhjEe9G22QoAvMdEip6x//l8Hkng9fS9jikNMADzrBUYA4uNYMha8Dn9e+H3NZTjFSnZEB0fAJbsyfdGgKBNZfRdk/lQMJBMJiO08Fo+DPFAgL89K8B1Q6aKbLOyVwyZUsf78u9VxbMTcQyCghW99/cx/24FBmgM4hSqKghVjrpQPfVJg5VMJq3enpn6VFTcI1oVJmkdnoNgYDgc4uuvv7btUwuFgr1Uxja5r0G9Xo/EQFmjfHZ2hvPzc7TbbcsgpYHc2NiwfQzoMXOLVqVK2fZYAQt/q6JXA8I+Be122/awp0Lks/qmGfT+9Rr8jEqfLWaZ6Mj2r8PhEPl8Hp988gnS6TS+/PJLpFKLTWHozXnglU6nI0pSvVn9WTbp37fofQHxSoLiaWltxMRxvbq6svg2Q04M74S8Xs234A/nHpM4NZudxovboDL5le+E7whYzKVCoWAtfFnWyrWoyYI08Ozqx2MUmGj4i/0K0uk0Tk9PrXtgo9Gw5iz0vIBrlox5FtzngtskK7hVbwhYGHyCUF03ZDb0mf3z666l8/l1T4VqtRp5D94oeOZO72PNDHwYokZU2T3NSeO7Vps0Ho9xfHyMbrdrDePocHLd+6RxXi8kqvf0h+srlO+l9kDnmQ9phebufcrKYEAzwpcpeo9o9O9xipgKlwmC29vb2NraihgebvFLb4UGnsL8gclkgpOTk0gvffYEYFkdFVmhULBkK80g5cS4uLjA8fFxpJd7qVRCsVi0vAMCAd1TgP3VtQ0rJfQyOTnJSDBEQYNALw2AKeRisWgJVXHvQxcDz0NWgKEAGgSWmo3H44jX6jfe8OWO3rjp3+IYhIeSOFYq7j49rUfQypJQjpFm3pNNYmdIVVKMk6t3SqPL5FR6y+w/wQTV6XRqzU6YD0NPBYAZTs5r7i2gFSi8f+Y80FgDb5aM0jtWBoRho263a7t8AogAD44lWQg2O+IWr6PRKBIG49zn97QXvIJeDwY4dnw/NN7cdIzOAgG7AmU1EPo3jgMQBSVrMPC4hXMnRKl7vaiMHcE4q9LYVVDtks+xAeLLgkP2zZes+vNxvflzqH6Ku959y8pgQBF6aHC8QeAi529P9fFYGrZ8Po+9vT3bUEIXZyJxXR/P5kSkrYnaWNdMQ8WmOTTKhUIBT548QbVaxdOnT7G9vW07Bmq3QMaJms0mzs/PDVzwevSyi8Wi7TbImDv3IaASpxdGRerpTb5sTkjugkXvT+umqZB5PL035jQUi8U3ELBOIPW62OAIgMXACDr29vaQTqfxww8/RMaf1wJgdCrjr7wez69zIpSw8xCySmggTgiI2KsCWPTv165mnnbnd308WhvqqGKiF8qcGAJOzSXRXA52llTPmHOBmdO6AyGASI3+fD63pkPAosc/RRNYtSMbx+7q6nrrYrZmnc/nlk/AihfvpbFVsoIdfq5hDgC2nvg524xnMhlj0JRJ4frns+dyOQtR8L69FwbEt3zWNftY2K21hIXvKJSB73WvOjkE1gQDBJz8PqtuQiEs/Xec17/sXr2+BqJlwbRz+kzvWn+uDAb8Q3gA4EMBStMoGKBXRc9IKffnz5/btrwaByQYoLLgbmaMcbMemeCg0+ng4uICtVrNFPaLFy+wubmJly9fYnNz00BCp9Oxnc/Yz//09NR2ACS1y3prel0bGxvY2tpCsVi0HQsJBthumIrfj6MKk1joMQ0Gg0isVieienmj0cieWXMHEolEpPTL5zjozohkO1hG9ezZMxSLRfzlL3+JJKgxxkslzm6NSiUr1Qq86Xl7APm+xVNsccf4/5OxSSaTlnzJNZDL5YyCptHXBawlfRwfgisyNFQ+NF6scqlUKtjc3LR74Ty5urqKMAMaKgMQKZ/SXIf5fG7MAu+/Wq1ib28PAKwjJd8X9zYYj8fodDoAEPFqGJZj7L/b7SKfz+NnP/uZsRtkxjge1WrVmDN2IvRggOCZ+Rl8jkwmg62tLWQyGVxcXBhQpyfH8adjoXNOvS1gAZz4b1XinrVcA4HHLz48SvHMngICgoHhcIiLiwtrysYE4fl8bqFh7+j683O+c+3xM85rnd+8D64PnacaGmPSrDbk8zrMP+/byq1yBvS3Fx8j8cBBX4RmPtOboHc7nU6NslHlQENDGpYDx/7rCiBosBqNBp4+fYrd3V3bNCiRSBgTwLas3gAXCgWrw+ff6IVQ2fBFEmywk6DvfujpdD+mVPLqiXE8/bgq++G97pvem3o8ZAgIsOhxailiqVSKhE8YnqC3yXtQulav4+fGY6FZV1XufF98P5ybakwBmFFmFz6fqc5rkgHQ8wNvMmj8jIZex47rSJPdNFyh3ru+b/aKoHIjq8a4PhWanodzm9UxjPN7AOg9LQIHziHOV2ARiuD88546QTfBkwJIjjnvlaE53hv1BO+FjI06J5qf4MMGITCwlg9TlHnl/OOc1zmruw5Op9NIpRrXPM/HOUn9DizYT+pC2h2uc8/EaViPjpw6Ud6J8GECnoff12e9aTxWkVvnDGi5gyo0/1tjefw+AKvXpOFsNBr45S9/aaV7nU4HX375JV69emXXYixUFZwOuFYaJBIJo+4///xz/PM//zPq9TpevnyJXC5nDEC/37eKAYIBnufJkyfY29szQ6AUMVsPU5mxb0Eul7NdsbSHgXpSOg6U8XhsjADpXypMX0FAxa4ARL1xfSe8lsZVlcoiPXxycoJXr14BuI79JpNJ7O3tYTQa4fT0FGdnZ+Yp8hmVqtVs77j7AB6HcvWe3k2LiF58p9Mxulm76jE2zaRLJnNq5j2wyLNQkKChJ84jri+GjHSuJBKL5Dhu/8tnuLq6shbS/D+flZUkZJMYHmBiHdtx03ATGGryIBXm2dlZJMFVPSZ+59tvv0UymcTGxobtzfHkyRMkEtd9LLihFjcbYzyec4jJjxwzMlP8fyqVsuZeHL/xeIxWq2XHcltvvgPv3XkgHWK1/Dx4LGB2LW9K6N3wnTKJnOBUS2F7vZ7ZgclkYu3jGS5WL58MIHeLVXDMdapN67RXDLAIASgrQAZLr8N1FHKu9VnVwfaf6RjcRm7NDIQu6i+uxscDBdIfHAR6AkxqG41G9jI0XqjnVdFOb1zojDeybI57BfAajBXRm6MR5g9DArPZzBLv+EJo+EOthxUE+RfoS5o4BtxlUNv/6tgRTeq54xiCEPPg/877UqpWS+EIbhqNBnq9nrEcSn9rnJbn9/PDU1qPRZn6BRIKH+h4alyeRkmP1/fAEEIikTAKkqEwHUNtUOUNlR8zb7T0Pfkwkt6zejW6flQZ6TzTvh7+Grx3Vvvo/YcYAuCahWAIgV6WeuV+HXvmSsMM+my6xnkMn5cOAY/VMfNj6NdMXIK0Z3nW8viE7yuOxldbw/fOuaJMrNoi3URO9R0dTa20ohPp70nLV3kNZSnIytEhIHiIm6/veg6uDAZoMLRRkDf4ntpWpaMdzFgdQKTGtqiHh4e2nSkpUiqifr8PILzhjSrTVCqFer2OZ8+e4fnz5/jkk0+MhZjNZmi1WoYI6d0yce/Vq1cYjUZWLkjvu1QqYW9vD9lsFjs7OygUChYvzWaztmcBEaOOAXBNJXNXNz4bDYgmqNEbpNfDH4oqSoId3QWO70ZLEkM5C2RaMpkMer0e8vm8KfdUKoXf/OY3ePnyJf76179aXgbpX90GVA0a/8+5op6vGr2HFF1UNwFaYMHiMFGThl03w+K6YBLrcDhEJpPB6ekpvv/+e5s//K1bWquxZ7UImYV8Pm+dIT3YVK9ZqXsFxQAMhHiDCEQrWLS9MsEy/0ZgnUql8Pz5c6RSKQPP3W4XFxcXVoGjIRJWRzDvgM3AuEY0MZbPTSaD40NDT5DM/B0yY8Bi7XtQpVSv6gx977o+/Dk4zhoqXMvjFAVyQNQOcb1yW3vNhWKHTIaoy+WysU6syqH+4hogGODc7/f7ePXqlVWzaQib7BqwcBq4OyfbfAOL/jKcf5znyuQqKPfA477k1n0GVNmEqAoVfRguXi5wIiPNru71etZKmLWevI6CEP/bA4R8Po9KpWI/NF6cADTkVGr8f6vVsuY7VKRUTroTIWvLOQmpdNTg6XjQ0DP7mm1eVVFr1raGXzTuqUZBu8dRsavCCsWodcxIe/s4VzJ53UVuPp/j9PQUR0dHSCavk+f4eQiQ8RoA3jD+IQ/2IWRVdK3HMVTAsdMOg6T3NRZNNklzKehRb2xsWOIqe0PwvHz3LBdVg+fZCO/ZcnzVmOl34uhC9V4YniJlqowb+1SUy+XI5l/ZbNYqTZiPwLGbTqfWH4HKdXt7G5lMxsZQ56w2Q/LMF89LnRAqbQ39W+e0vlP9zXepIRa+Vz7HYwCya1kunkHyIDqUM8B3zdCB5nvxWCbxMjxaqVSsJb7vNspGRcoAc+5qkiEd4Wq1at/TRnB+rVPn+rDhfcut2hHrjYVoYr/4uHA5iEwcmk6n1ksgl8uhXC5bdjp7B/hzeyOrJURUtqz5/93vfodPP/0Uu7u7BgBOT08xGAzw5ZdfotlsRpgBbkDBuCM9OSo+emuMT1JZ1Ot1DIdDdDodU37AQrFzcrRaLfz444/o9/s4Ojoy75HZ5OVyOUjVUhlRSdF4q7HQRi8cL/4maAgZD16vXq/j008/tSTBRCJhm9bMZjMUCgUcHh7apNfkSL6nZZOYzMZDKtRQuISyyqLimDGeByzq+rmdKUGB3zM9mUxa0uFoNLLqGe6AqB0NAaDdbltFyunpaZB5U+aIgEGVDo0pcwz8Fr5M9uN5FDRo0x7NN5nNZsYakHmigpxOp9jf37eafI2Z8v6n06mV6xJgq8JUqt/TqlrupWWOrCZQZoLrlOE/BdNU1L7XAxDVNwDsfa6bDn144sN8nvXR+H06nUa1WsV0OkWpVLIkQjICzHeh88T5Nh6PcXZ2hna7jYuLCysJ9+E/DfMyT4AMBFkyDZHx/pX5ol1U5m0Vue2cvRUY4I3Gxd24uNWY0SDRYDOD/enTp3j+/Lk9HHMEtP0vzx1CfSzrKhaL2N7eRj6ft1K/X/3qV3j27Jm17O33+/jxxx/R6XTw1VdfWW20ggH15La2tqzGu1AoRPZKIBpkEpcqQDbs4TgwEazValmXq9evX2MwGJhC1PJHMiU++5oTWDNRfdw39OKXhVR0ITx9+tQUaiKRwNbWlilgAqJXr15FqH81IHHVDUprk9l5KFm2MPw81r/r/GM8j2wB34sqC9b5ExjwvIlEwsIN7KWh3j/fFUEiPQi9Px7DKgBWvSjwUIqdwMCDAYYUyNCpESbADMlwOIw8IzsQAosEKW4cNhgMrFkS10ez2UQqlUKj0UClUokoTSZiegaJoMaHA3g/7Go4GAyMEeF845bnfI/s8sg8HX1/PqTC8+i51/J4xesedSLViCojwPesul4TDpkQzHkxm81wdHRkobFWq2U/2qzIs+gExD6hsVwuW/m65gooyOd1NUwfSti+D7l1nwHPCOhvfRgafjYh2dnZsWY8bHDz5MkTW2zcCIYGmTFvrbdPJBJG0WxvbxsI4HaoLHu7uLiI1IN3u118//336Pf7ODw8RKfTMYVA5K+sRqfTsfpsxoiYTarGmqWNvV4P2Ww2QvPP53PbB5v7IpD1CBn7kHi6SePM6k3phNP3EBJeU40PY9m8D74nAgHvLfK4uARGXRQ8PhS3fgjxizR0T34+hz737069z+l0ah45hQBW8zt8qSawGFNWbCig4xzVEtaLiwubq8lk0tYHlU4mk7Hv8d6YMzObzQzo6r3pfXBeqcFUOpXCf3Ou8JxkIry3r73fCaZ0bHROESQwUYvnpyLXkk7mFhDwsLpAY8TaYEnDmDofNC+DYaC1fDiiLDbni4K9kFOrSbHU1QT0/C7tVb/ftyowdSR9aI7zRhkDvUfV2/7+9TMfJljl+W8rtwID/K0xak/hc2ETCOzv76NSqeCzzz5DqVSyToCbm5vY3t42A03vmfHUdrttm+YQFKRSKWxvb6NcLuPly5d48eKFKdTZbGZNVX744QerFuh0OhgOh9aimAqRikOTmDh5jo6O0G63LYFqc3MT+/v75o1TwdLzPzs7s3bJij57vR4ODg7QarUskUopIEWpfiyBxYYuFF6X74JUtX6+TDTsoLXm7B1PQMAwAZNcWDKpiV8ALLPch3Q4R+hBc2wfWm6j0FVR+Hfjk9X0GHqfGkqhwWGyJ8EjAXMymTRQoKDU52doAp3meZDhIbBj6Ix5LqPRyMIUDGcxKXQ6vd5HYXd314y8ZjzTmyHToQmtqtxo5NnLQ3UC75VUKnvAs6qHLB9DJtzYiYaYQJt7h7A0kj1KWDI5mUwioFvnos59pW35vCzzoqg36Ksa1vK4xBtgZXe0Q6iGYZW11HWuIV6GjBWsj0YjXFxc4OLiAmdnZxgOh3Ysz+GBgGck+HcfbuVx/D/Xjh67ipG/q+N1610L+e+QR8ubpeei9Z2VSsW2Va1Wq9bbfz6fo1qtYj6fY3d31wa+UCgYDUgjlE6nsbe3h0qlgt3dXev9Tk+j0+lY5UG327VdC0lb8qX5OJI+I418MpnEYDCwGO5wOIwoRvYlYJ0qvQfSUkSX2jpWx0jHj7HVy8tLY0QSiQSGwyH6/b69XJ8kFnpHwHJU6Cesvk9OVi2l82WPvL4HMT8FiQu3hD7zx3kaW6lIfl89UhoZAOb5EgzwM584qO9JkxgJWnl+ggbG6rVHgvbO0H4G3G+A19UfznnmOagSo7HVRkE6V/msfE4A1lMEgIUHtEqFY61luEqNanKhen4Mc/D6/h2FxjLk0Kzlw5TQO/RAQY1vyLgmEglbI+l0OtKBVtcn16D31lfRh1wnXP/Mn9F15XX0+5A7dSD0hpQPQWVCg7+zs4MnT56gVqvhk08+QaVSwd7enpXEEd2z2129Xrcku4uLC3S7XZydndk1U6kUPvnkE9RqNQMDjHV3Oh0cHByg2+3ixx9/xOHhoXnWqiAVNSkVo0aQGdUArMpgc3PTmrcUi0W8fv0ax8fHaLfbOD09jZQgDQYDJBKJyEZHVM5sFqOGvdVqRRrY8HlbrRaazWak0czm5qZlvyqV61ElxU8oT+0z2YttZxn6SCQSODw8xPfff2+hFe78yO/5hBY1EDrWjwUs+PsKCeeC/55+3/+bXimNEIUsjjdoyjqol6rvjcmFBMdcX1rzrD+dTifyTMxLyOfzOD8/j4ALGl565+l02o7RZ+d1uK5ZdsWk2kKhgFqtZpsWaTIiATjHh7kSfIZ6vY5Op2M5Cqz68UmOnn1g6IqlXhy7YrGIq6sryxNS9o/n0d0YuV40KZSgRhkanmstj1dCwNznf1DobROAKqjm8YPBwKraFAxTR2v5INc4QYRKKBxBhoIVb9qXo1QqGdOgvTniwgPLwO5d5M5ggP/WByVCZwc2xp11K1P+X+kbDjQ7lgHXsZpWqxVRUOl0Gvv7+6hWq9ja2sLGxobFEWm8GRvU5kIhQxlCj3wmHkt6tVgsWh4BGYJer2e7r3mESGTJmKVvtEQvkddjeIGZ2jwXPXN6jb5tMdFkaDEoi7PsWQmCeK9kJZLJpDEsbM7EUA0Qpsofs4QYkZuODf09TuJYEv1/aFGrl6DnZ0hIgQLBhoYfaOCpVBSYMeatyo/vj/OGn1NR+efQ75Gd04YrTO7TxlyexWJGNRUnw1EMO2gDLx0Xfl9rtflcejzvXZ+BwEJDd7q2veLm3z1LoLpuLR+u+LWpLJY6B1wXmm+ijAAQXcfL1juvo/9Wx4JzmaBEk37vS5/eZt7euprAiz4UgYA2ZuFnITaBf+c5qtWqoXu+CDZj4fFMdmNZIT8HYMluwGIPA13YOgHiwhxqMDTL869//SvK5TKazSbK5bIhuEwmY3sfsA57Pp8bvU6lQ4WrpYC68QzHjCWSGlMGFn3o2cwlkUig3++jUqlgY2PDxl4nnBpqP9n5MxqN0G63jRlgudZ0OsXXX3+Nr776CkdHR+h0OmaIkslkkCajclZFy+veJvnlsYoHUirLwgtqIOO+qx4KAPOMgUVoydPz3giqF8+5xvN6o04DTZaJwNf/6P31ej0LkZGtOj09taoYJtySNajX6xGDPB6PLcOfIJ/3r1n9+Xw+Mr+0GoJC757fJyjR+aYsJhMQud4UJPj3oJ7lOnzw+CUulMbKETa1090ufW4Ow2DFYtESyvl9AJG9CshmMY+Kx8SFnVQfa6VNv983NgoANjc3I0CX96A5SPzxiYheVglXeLlTO2IvnhnQ7E2fXKY3qV5tIpGwHAIaQH5PlZ6neBOJhIEG3URC4416j0pdew/aH8dJw6RCUpOkNJPJpNH3TLKjctWWraq0OCb0dpSG5EunoqVyp0JNJBJWb8pMbeCaEp7P5wYGdOzojWlZIpUvEfBwOLQciMvLS0vePD09xenpKZrNJobDYYQK8++Tz8Zr6LUe2qu6j3tYBgRW/e6y+wh5pjfR0zqfqEDYLptAFoDRnWSSqMxo9MhKqWIKrRUyU0zQS6fT6Pf79luTTFnNwGsyAdDHRn2OAb0jVerqrXGu+RCMJlrpOCoQI1uQTCYjPQdCzIGeay2PXzyLo4CAjqXPfVIHCUCkvFsrC/Q7ykxp3koIWHp7RbtAtng8Hts1E4mEhbpYZePb7HtmIU6X3FXX3YkZ8MqePwwLcE8ANZp8MVQ4Po6jnpOK94RCXi/jmM+ePUMul7O9qdkykrXOmtQVonP0nvh5KGFkNptZDSo9cirbyWRiJV302pm74OOXvB+CCG6BzBALPTdOSk4KVeikgFVxxXmgKnwmbp9MxDyZTKz6gRsVkTHwClx/bqK2HhoQqCyj8pZ9xyubuHPoZwrIVlm8fo578Bv3N85NVgxwjuiWxrwO3zmZOw1ThK7F+/LvXxMcOaebzaZtdsQcE2bqs8cI1xnHRgFN6PoMK1BJh5IrQ2PKc/O5VO8QTOv69u+C//ee2Foel3jnUNepbw/MH78nDecvDTV1IfOntA8G1xZbaNO2aZ6Khs2AhdPHMDMdPpbGK5DnXO33+2g2m7bPiX9GXZt83reROzEDiqhUmbCCgDF97cbGwWKpkNbuenBBlE4FEXc/GoMsl8v49NNP0Wg0MBqNUCgUcHR0hNlsZjSRT/jQsICCAc2UV4WhoICNlPjsGqNkM6RarYatrS1TjvP5/I2tblnWxERK3fWQoklPVFoEEAoIdOLpc/pJw2O04kEXwPn5Oc7Pz3F0dITj42OjVWkwVMGG4m7eqPwUxIMBlbjnXQYe4o5VUKWsmJ5HKWytXgAWjYH4PZYOeg+f5/NsQOi+/Lv2bAQV4mw2w/n5OWazmVGhlUoFT58+jexxQIYBiPa9WAb29di48QuxYjqmfE5dWzwvFbn+5lpXILSWxyfLgLgaf+o3eucMGzDvjJ46AQGZWP7WXB2W8TLcRV1Kh0/vgZ9T/ytrXS6XDbRXKhVjg0ulku1jo0yaPnOcY3BXeaucATXmLAdk+RFfADcY0sWqVB6/r7/VSMc9nCa9MfOz3W7bv32ynXoycUjSe3CJRMJADttH1ut1yw+gx8LJxckEwJBjtVpFIpFAu92OUE+cJESJmrylySp6n3rvBAJK/1Pi2BQVBTY6Tr4k0icV8rw+fv3YZdlcetvz3ue1vDIDotuH8+8KYPU7BIfcmIWAXIGjB5Jxhi5u7nhGSA056VOyFOl02qoGCAZ4v5rVT/qeAFnBgD/ehwBCxlo/80DBP5NntvRegEUL5LU8TllmK3S+6I9685xXurcA54TqZs4HggGGxXQNAG+GR/W7ZFhZlg7ANjgiEFCGQX980jbPrc/6NnLnnAFdkKQ7uF1wOp02xNXpdFCr1fDs2TMzrmxCQiol5F3yGj6Dk5+zB0G73cbr168xHA5xdHSEwWCAk5MT6zPAJjCaiayejD6LzzPgRGDL3lqths8//xyNRsNeELdbns2umx7N53M8efLENjXK5/NWJgnA6CltgkGFzXvwzWWAaFxLk11IAyvQoUJe9i4VwBAJM67W6XTQbDYNXAGLnRCVHQh5c3HyGADDuwAEaoTu49whOtozDJqPw3lD5olJrGTnaISZjEqgoMm9ofen79e/Zx831fip1mOz0ydBAfcy4LzVBF1StZzfCoT4W+P8BA26hjwg9rHcuFCWz23ScedYrcHA4xZdI54B9awudR0dHlbGdLtd04nKdPP905DT0WNLcbbSV+DJNUDGjkwDQQfz3Jgfx1092V+HoVvtlKuh5WUM2V3lTk2HQkJDr4tZEzh85vltqGSPgvhiiK7Oz88xGAxwdnaG0WhkJX+6ucMyGjfu/zSsugWm9pSmUhoMBma09RkZF2IOAH/rFsZqYEN0vooq4RCluqqoN6Y/ZFu0Xza9pJtitKHx0789JBhYBgLuel8hWvqm6/p3ph5taO7pd5Tm1wRdzjseryW9Wtnjq31Y1cL3qnNwVUo8BBT0uXWOecbNjwVBAY/3Y62Ur3pzt9EjOu7Lnsk/nyY6r+XxyU3vlMdomEn/r4ytslP6w+8A0QoEXb/quftr8x41p4ChB/bC6ff7mM/nBgB0K3HPDISe723lzlsYh25K6yQBWB/0VCplVCERUpzSUYURokCI4prNJg4ODvD999/jX//1X9Hr9XB6emqDSOSlGch8GQpUNC/BTyjmBTARsF6vY2trC5ubm+aN9/t9bG5uYj5fJAIyGSWZTKJUKuHq6gq7u7sol8sW1uD4sZSKFBHBR6i1MJU3DQCBFj2vUCgkbmLqjm9Momm327axUqvVMlaFQIbX9aGMEO2r1wqN7Ycqfk76MVB2xr8DXxtPL0MNv+bhqNJRUXaNhpbzkXknnLf8dzKZtOQk9vlgrJLX1t+8b1Vkev/ey/ahDF6PLAUbE7HKgNfVHT/5rLPZzOYclTR7wmv4j2yAv+c4dpESYiHX8uGKt0WhEBZFjbGn4OlAapitXC5HcsmUeaXOJcug9kavpcI1QhtBrz+TyeD8/By5XM7uhx1w1TkL5c3cF1C9Vc7AKojEezJENnwgPkxoMd50fjVC3CCo1Wrh9evX1q1QN4dRL8MjOI+yQt6jZnySFdDGSVpFwHNqUx4qaFY7ANcJIxqmUKrTl6uohKhOpb/Uc1zmCfMYXyGhYQNFpXpdekh+DHVc4+SxKN4QSLrJw7/NeVUJhWKNnn5nOagmJylrREOv1+C80pJTNvAhrcl8AbIEBB68JsEADaoPRdxkRFXhhv6uTCENu64bzXehnkilUpEtkikewFOfEHh7wLls7i8DDmv56UgIJOq8Vh2mxtmHwEI5Ax5M6rwkw8Vr6m8gWumlDhWTfZmHACBSDun17W3GYVW5dc6AN7A0bNPpFK1WC1dXV7YFMFsucucnBQX0bvz544yZGkDt8U6PQRMHvcFT0UxsXk8z74GFsq3X63j+/Dk2NzextbVl2Z70ThKJhF0vmVzsGKfxWMabtra2rEkR46PemPouckqFeuGkY0e4kJfmvVWeW+lWPZYNOhjn1ZwO1t/y+3r/Sgv79/VY6NVlAGnV4zivNLGIxpTzKtRjg3NEs++1vEhj7UAUSALRJk4+WY/f5SZIXG/cB4TJTnrP1WoVpVIpEkLgMerVcP1wvWnfDTX2ev8sj9Xn432QgmVny/F4bPddKpWQSCQi+4iwrTffiVYiEcjw+zp2fI86TvxbnMQxWHEs6Foet3h2AFjoWI29c17TEANRg01w6nWyGvO45D7gzY63qov1O4lEwgAx5/OyXjkq750Z8MbTe6iTycQ2FWo0GrZINSau6GlZiCCEqPRa6rFeXV1FEi48GPBUp+9xELoO75l7IDQaDdTrdRSLxTeeg8JqCt3eVT0cdmPL5/ORGBDH7urqKrLfujYlUs9RlbXeh++REAJTfgL7vgsMOWgbZAARQ6IJXR6phhTqY/C++Hy3AQRA/CJToKe7mqVSKaPgOY/IYhFEKGtAT16NnE+Om80W+2p4YRiL102n0zZPq9Uq6vV65Nk5L7e3t9FoNCLXZaY/RenU4XBoTYPUS2EOgoa42ImQ6yybzaJSqWA+X+zoyPVKtoK7DyYSCZyenqLX60UAB8MbOnZXV1eWj6MASQHbsnDAMl3jj7uLR7aWhxH/vnVehHSf6l+fF0XwoE6Ot4Pe1ugxIQZN7RGPo67XBEVtvOXPFXrm+5ifd2465DPzAdgWwtqat1QqRSg+evH0TIA3PWH+nx44FeLV1ZV1w2OG/nw+x+bmpsV31Eh79gFAxABrrbMKmz8QBLDDH1FlMrnYAU7Hg8JnVeNJQEQwoPfDyUklR8aBLIoqNh0jKnhvmH2Guy4IT7XO59GENDUI3rtSJsjPhThmQN/vY5fQogqBKk0MZeMprgVvCOfzRUdNvx+HeuSk8jkX2NVSFQ3HlHOPzITmHjBcwPtTRaQG1ieisp5aAQuP4xxjhzSOCz/jOQkKtBEWmbT5fB5JulWGxCchemXOMeJxfP44ZtEzfcved9w71n8/NJhdy2riGVD+TUXZYs4psl9kBkIJ0wqqfQKhsk8KEELh3pAujDP0aldUx4bmrdddd5mzt2YG1DhoOdx0OsXZ2ZlRfqQH2ZKUxqfT6VictFQq2bmpLNSwDYdD21mNQIP5AWrUXrx4EakR5Q+pUt7zbLYoH+n1etYAgl46qf9Go4FSqYSnT59ib28vkhRIL0+rFDzq4+RS46513UoZA4sJWiwWjZZiMot/8QQwvNdMJoPpdBqJKy9TYD434Orq6o3mRRQFfQQzrC7QeyLo8VSYArsPBRAsEz4XS4uKxaLt1ufDBIzVA4tcASb3cbxVaNQ597e2trC/vx9RKLwGPeuQ8mOMsVwum7eviZ/z+TxisDm3WR5LD59JfgxB6Dz3FKzONa5hXXME8mS8qFR5fQ2R6DMRNGjCI4EL2UV9fl2XHhDz3Mve620/W8vjEnVMvFcNRPOlAFiDIf74klvPIHtWT9e6OpaaPxZiwHV9eJ0Z+jeAINjgM4XG4S5yZ2ZAL+g9F5b3kX5kbI/KQWkafp81+1pPye2D2S9/PB7j4ODAtmulYlQ6W++RmcykY+fz622BdR8DjZ2TjqVCBBaxeaJJduvjM3My6PepqDWW6ru86cTlbyo7bZWqDYpCk9x7VcvQoYYW6KUpc6BAT+OsfG8+FBFKMPspGP2Q6OLj+2RiqHrZ/LeCAc4BsmEh40cwoGECX32grb117fC3f/eqvABEQmj+R2OfqlB5Lu/B61zRMfIxUt5biGqn8da5pomXwMJL84ZfcwO84fe0bkg/rOWnJyEjGDKsqps1OZCOkrYS9jkHOu+5GR8bBdEx06ZF3lbyuzpvPVjhvyn+794O+OP9OVeVW/cZ8EqGxoXeLgCcnZ2h3+/bnuXs2sfSQt09CoB53N988w06nQ56vR6GwyFarRZOTk4wHo9xcXGB8XiM169fo9Pp2J7qmqlPhcP7q1areP78OarVKl6+fIlUKoXDw0OrPDg/P7cmP7PZoq87Df5oNML5+bkp9WQyaUmSHItKpYLt7W3M53Ocn58bQGHyJBFntVq1++JY6osio0LFVSgUbBMYTlRel/Fa0rI0MFTo/j3pZO/3+5acxQQtPhsXBxPDlCmg4lbDoeECnR86Ie8yKR+rcAGyFWm9Xsf+/r6FBfR5dQMSfl6r1ZDP5yNbo2pJYDJ5vZ8GwSSNN5UTW5fy3bCkie/DM1ak1zlnNMdEDebV1ZXliPBYDT/FjQW/q8L5w3ni54rOe01OJADixlwAjKnQcmUyiMPhMJLAqOwcv6uAmvcW8shueufrnIEPR5RV0v+rzUokFvvZ6Bqj3dFwAdcw5ynXbLFYtLybTqeDfr9v60ZD0P6+NLmWDiLnl+5yyznL9RgH4kPPr8zsbViCt97CGHgzYY0UNg0Ot/VVZkC9BXrVrVYLFxcXaLfbxgoQDLRaLYzHY5ydnZnCJCDQpCJgQcvymtxaNZPJWE2+Kka+BCI7Pk8ikYg04uEzkSpXLx5Y1KGS3meJl+YIAMubClGJc3KQvdD74TNq+csyr0fBAJtdEAErYPC0GsfRx89C7zxO7jIp71s8qn7b89BL54KmcVYjo3kYBAMsq6Py0dIl/VE2Rq9L4bsOJTzpM3pPXg1kHNOkv0PeiL8f/x0fQlLRYz1lq89GYKveFc+p4EK9LD1fnNLUcyp4Dc0L/p/dOddg4MOW0LrQ9QbgDbDMOeJZPOpEsn/ccpu/9Zqh+a1gQJ0sDRP6+15Fd3kde1um9tZ9BviAiuiBBYri4NLb7/f7yOfzRkvTuLLVIr3Vs7Mz/Lf/9t/w7bffGtLiMbw2Qwc0zv1+H5lMxlr9euFmKZ9++il++ctfolQqYXt7G/V63Ty78XhsOQiMoXe7XQwGA7vuZDLB4eGhGVR6LFTktVotkjSlZXkar/WhAf+y1JBoMpl2xqJSZ/iFgEhpZe+Rk40ZDoc4PDxEv9+3qgEq336/j3a7bTsU8j5Yr8794jn+BHR6396IPJawQdxiuitA4XuczWa2da/W8BMY6Dj4JD5v5LhFdCKRMMaLioOAV3MTABiwYDxeQTYBNpVOIpEwFsyHnpLJJCqVCgBYeSzBzHw+fyM/IeQpKxggoKXnQyVJMEsmTlkT/p86hHktOkZkE9lmnGuA40eQxXvjeamgPYuh4TJ2gVO9xvU+Go0wn8+xu7t7p/mylvcjanw5b6hDWSlFR475KkwCZj7beDy2fKCrqytUq9UIrc+5lcvlLAS9t7dn24TTUeXGeASSXDO+G6jqdAWnvoTRnyekV0OM8zsDA37A/U1xgElpc+HTeKhx08+GwyHa7Ta+/PJL/P3vfzdmgOenV0WFpvGddDptXrsfAHq9xWLRaEhWNwDXxo5UOV8cY5i5XM5qnjlRVIky2YRetnoaBAy6JSYQn6XsAQE9I6/I1YMvFAqRjWiUKvUeD4EO93Mg+OGz0nCwv4BSupooo7Svsif67H5ePCQjQLlPIMDvKvVOIQuwzPPV9+MBG7Do4qmsDMvnVBi+4bv3nTb53jXvQPNjPBjQqgbd1U/vjaJA0I8nr0WAqsyIGmpNsNJ5TVBL8KCifUpGo5F1MeQ9qhINJVrptQkuCIq73a6xZqqr6Hg8BlC7luXC+Q9EHRPqYp1/movC/B+GiOk4MWSl51RmgPOPHTa1iyaAiBPHeanJhxrC0/Or7gj9LJuLOga3nbO3BgP0UHizStXp7ndqxNvtNr766iuUy2W0Wi1Uq1UcHR2hVCpZPH04HKJSqWBzczNidJTKplH0nqdH/Lw37hSYTCbxP//n/8Tm5iaePn1qiVy7u7sYDofWMphKjuju/PzcJgj3PeDL0JaqGxsbZpSBaNtZZUOUctJ79YZTQxAArKZaFRwzrLV1sadKOblGo5GVYrIaQ/MBEokEer2e5Wv40kxey4dVKGrYfmriPXwANifm8zna7bbNc3q0vpEPvQaGfbRElZU1PL+yS9yOWCs5OJ95PA2p9qXgmlCQOptdtzfl5mE8J+eAp07jmB8qN0qo+kQVLO+f48Z7nc1mxmypZ+SZrfF4bLk9vV4v0sKVQJzPph6UDwPOZouNjsgwEAgQBIRCDPz+Ggx8GKIetjpkXE/qYQPXOq5cLgOAhaKn0yna7Tbm83kk6Zc2jnOI84dNsnQ/F849D0459xkm5FrVvDA1/D6kvso8fG/MAJGNetFc7H5BUanNZjN88cUXBgzq9boNws7ODn7/+99jPB6jXC5je3s7sr8Az+NpVVUY6skCsAFvt9tot9sYDof4H//jf2BjYwP/8i//gv39fTx58gQ7OzsWbtAX0Ov1bBMJKqKTkxNrusJJwsZK+/v7mM+v68mVLiZLwB/1wOJeHhWyGnxNwPLf1QYVpDj5m4qdmzi12238+OOPaLfbqFQqVvbJ8AvHiguGypz3QQaBypX3GwdwHhNACDFHNx3nj+VnnDMKmhgeGI1GpjzoEVORdLvdSEJnKrXoE6DGGMAb+1Po+1VmiyEE3wmQYECVCT0erWzxa4i/NU+G96a5KZqQpaLdF9Xz0twTBT2VSiXiHXl2azweGyvH3zxPPp83JosglnNTQTg7kzIExm6lDEXofPXltZpgvJbHLSEjqGCARlsdGbLFDM/xmFarhel0auuQ64wGnyDAh6593oFnU9VW0dFQAO8BgP/eXcZiVXmrMIFXnN5DpPGYzWbm5Q4GA6RSqQiyYtlfNpvF5uambZ07n18n0em1vCetA+UzulWxUEG0Wi1ThgwzkIZVDzuTyaDRaFit987ODnq9nvUpYOyy0+ng4uICk8kEtVrtjViPT5QMjZuOXcgTVWMbKuXyY8LvsVfB+fk5Tk9PLSZGJZlKpczDYnxWk9rUILHDo6LduPkR94wfuvDZ6ElynjA/hO9Xk9zUYA6HQwCLseF3aTyBqIdOkM3zEoBTMpkM+v3+G2CATAXZKobJWMXD3dH0mULPSsCimympKBD0Rp73rYwGDS7nLO+F5Vn+WTkvyQT4si92KmSejyphviOyOFyvzA/QHeH8GOg6pZe3lg9LfBgzFDYCYAwsbVQ+nze7xSR26sJkMmlzikDSg02CDt1uW6/PawJ4Q3f7MMBNa/S+5dZgQNGKUvf6gFxMTA6kpzQej60Uo9lsotlsIp/P44svvkC5XMbvfvc77O/vmweTTCYj5W8hocfkKUbN2MzlcqYcfvjhB5ydnaFYLKJYLGJ/fx+/+93vrJ87y0am0+td0TY2NnB8fGxA4rvvvkOr1cLp6SlarRZms5kBh42NDYutc6y06xrHRxNRVEKxZI6z311On18Vrcr5+TmOjo5wcnKCv/3tbxiNRuj1eoZmqRiZgKksQC6XswQtJngqNRv3LniPnr35qdCsNGL0Gpjbwd/T6dS8Y4bTuF7InGkFApkBD/L0XfL7VCy61mh8tYMkmQpmO9frdaRSqUjCbjqdNq+XHrGuG62hJhjweSlUkrr2eD+ag6MgCkDkGIbpGHYjSB0MBqZcaei1Sdh8Pke327VQHpsmESxp/TeTuVRBx+VyKNgAFr051vJhiMbcKZoDwveva4hdRGu1mjHKLDPsdDoAFgCROWDKQpGdol5QkMn56J01dQAoGp56CLkTMxC6WUVhPglCY//eI/VJbGwLWalUTOHqNan4qOhCyJ4vQbuslctli/8wmanf76Pf71vlAKkgIjR6XLlczlod6+YoRIXD4dAo9zg6McSihLz5EC2t4xsCRj6hU+lRxom1mZNmn9PjV+PlS9x4Tg0DhZ4p7rl/KkCAwvnFcSaDpXFBHysHYCEzvgN+rkZWPRodewCReakGWSl8vjuNWTKfQcMMNIgEdvR8FIByPnhR0Kr5BP5vIWWnz6iej44Zx9cnHqsu8Qqe468sHM/Buc5jdJ16ZkNjzupVruXDFs/U6jyl/dHeKrrGFdBTl3JuMRdFWXCdr3Gg0/8txFq8b2BwJ2bADyZFF5QqAlVobJJCIDCbzawr4N///ndUKhU0Gg384Q9/wKtXrzCfz9Hv99FsNnF1dd1iOJvNYmNjA41GA4PBwLwDKltSi4z17Ozs4J/+6Z/QaDTM4B8dHeHo6Ainp6f4+9//jmKxiGfPnqFQKFhcSanenZ0dlEolDIdDFItFNJtNu+b5+bnlR2hWeMhT03FRj8orHVX2nIT03Pk3jq3GnKbT67bQvV7Pnm8wGBgVxrEmSKAh0jCJMhGj0Qj9ft9iaHyPIQDjJ68aig9V9N6V/tbEoGQyaUyWxsvpbfN7nO/aU4D0uNbL87pqqPS3gjX1tvVzHk9vW/dPUHZKKw74vfl8bmyHUvueGfDshWe9PCjwipGsBXCdvAXAvLRutxthAtTjGg6Hb4QPPRBgQqGC2BBY9wmRFM32/pDn78cky/QNw9L84fok8CVDUK1W7XMAETaNDBRLTfmjoWEg2jJYwT0Q3TX3pnDA+2YK7tR06CbPMI7KJqL3tc5EVK1WC5PJBFtbW9jY2EC32zWPm4uSjXx0i9ZerwdgsbB5LSo10v2bm5tmTKl8mGk/Ho9Rr9cj90lJJpPW5bBcLluuAScCk+qIGNVj0/GIi7tSfE6BeidxHop+psqSNBfjWlTopLfIinCveX+PHEv1gEO5Ap698H9/DMzAXe8hjr1RA8SxVW+e71838eHxnDNUQHynoVwQb4zi2ANgUcZEgKAsETcvIhgNhaoI8H0cc5Vwlp4jBAj9XOD1ma8zm80sj8GHGHTO6drWcj9dJzT4mtDFRGSfhKVg1q8v7d6oGyKt5XGLX+v6jpUtAxBxqjgflR1QNoo5KyzRpoQMvnfUVPTe/DrzdvV96823ChPoAABv1tHrQLGUim1ESbEACyVyenqKdruN3d1dbG1toVgs4he/+AUGg4El53G/drY4Pjs7w8nJScR4JZNJCyMwa54Kg1u8Mono9PQUf/zjH63JRL1et2vo5GCmOL3sjY0NlMtly2tot9s4PT0FADx58gQbGxuWcJjJZKwsihUH3tDr+PEzKiNPZep7UDBycXGB0WiEo6Mj24iJTA0bMB0fH5vXxeTNUqlkxki9w263a0jYsxghEKAL66coyxB8yNAwHu/ftQcNPF7PFzK4HhT4z7zXP59fJ+X2ej2k02mUSqVIjbP2kGD4jjkjvrxwmSioUQUYxw5ubm5GtlomoGJzLHrzBAM06mTeCHI1nk8dotUECgyog5RpCL1TfS/aB2HNDjx+CYEAzkE6PtSL3IRL5wWwqJajjmYu0NXVlVVU6bV0fvmwQBy7p/qA59DfevyjZgY8CvLilRofhAvTZ1lyELno0+m0Gc58Po8nT55YuRbDBLlcDqVSCeVy2ahuXs/H+rWJSiKRMEah3W6bcfzb3/5myYRMAGNiFCcGwwfn5+cAYICBims4HKLZbJoSId1Eeoq0JZmJkIIJMQlU7P4zHTcmYjWbTQwGAzSbTavG4IRmH/tms4l2ux3pWMdn1GYzACybNsRYhN5x6Fk8Ev4QRRduHCvmE4Vue/6478Z9Fscycf4pGJhMJtaKW/ey4LxmSECBIJMjFdCTPfC0pwqP1fnt6U6WEDNJVbsNtlot9Pt9+54ab01g1RIvAg0mc2lsWEMHrCrgd0MxXV/NoYBtLY9fPGPG3wx3Mjk9kUhYFZuyBQyXMWRKhogVZ7RfuibigEBIb3sQr2sjjmF4X/JWYYKbJA4FaQ2wepwchNPTU3z55ZfWMhi4buwDRLdx7XQ6OD8/txJFKit2P9vZ2cHPfvYz7O7uWhtiJv+VSiVsbGxgZ2cHT58+RSKxKFWip8RWlKwHTyQS2NzcRDabxWeffWaG98cffzRmg8/GNpfFYtGoUE1i4uSiwtNx0HH2hph/Jw3abrdxfn6O8XiMdrttY5rP53F6eorz83O0Wi38+OOPxg4Q8NAT1DwEzYrVLWdDE/028+RDAwK6mEMexzJWZxnFHpK447wC8cDL35f/jF5Lv9+3BEJtm82mP4yXcpMVVjhwjtLzUdDNsfGVRLrOCUR5TL1etw1iyCYwjKVhFU/3c32z/wdLt/iZene+XbZWD/hEr1AyLJ9RE0DXYODDE127dJbI4mqvDQARQ+5BLBls6mfmUvE7Wp0QqhzwLHqIVQzpi4fQl28VDNNBWxYu4DFEYaRQLy8vzSApnfLq1SscHBzg2bNn+NWvfoVyuYwnT54Ya8CtjA8PD42K54Kn918ul/HZZ5/hP/yH/4CtrS08f/4cpVLJqFLuQz8YDPDzn//cvGAmynW7Xezu7kZa/mazWTx58sQSDBuNBr788ksru/vhhx/QbDYxHo9RLBZRLpdt4lGxMUxC5adgQJPI1CtSL4WfUSkeHR3hiy++iMSeSfW2Wi38+c9/RrPZxHfffWchlEQiYccwTMDrsYRS8w1U+et7WsVbfYyZ2Hrvy+hff5wPg3iP14dx1Nj4a4XO66/tvYhlElIymmnP+1bQSYYtk8mgUqkgk8mgWq2ax8T5yd0S6TWpwvTJh/59E1zmcjns7OygVqsZ+GT4CkAkdEaWUPc34WZlw+HQ5iabyWhPAe2loVU1mvSpeQghT1JDLrd9D2t5PKKsNOdFq9WyDqxez3odTJ1NI88mX9TnmtSq7ILXC/5+vNEPgfyHmG/3mhkTp9Ti/q5xOTV0VKI00JPJxIwXDRRpTZYhEqklk9dbF5dKJVQqFathpgH2eQX1eh3Pnz83JTOfz7GxsWFhCG4aw3sFrhXX5uamKZ9arYbhcGjUO/cAuLq6ssRD4DrJi2GG8XiMWq1moYg42tfTTPR4Op0OWq0WOp2OUaWsc+/3+5hMJjg7O0Or1bKwy3w+t1iZbrTEc3rFqmh2lYmt983PVjG670PiPPxVjvUSR+WFntX/29OHcR5D6Fr+3kLgQtdP3HlpsLVWWvMGGB/VihLOHc1xCF3LA6NUKmX9KvhdKl69Z137miTI8Bb7DQyHQwMHBMQ8dja77h6qCcoakvTgTO8zjs5dhwk+bFEnlAnUdBy98dZwtSYMUrcSIPpeMpqUG5cwuGx9PxbW9N6YgWVeole8HNxQ6RU/TyQSuLi4QK/XQz6fx/b2NrLZLEqlkm0gUa/XUavVsLe3h0QiYdnS7AXw4sULbG1t2UYSyWTSvKRMJmNJTHt7e0aLq9dQLpdRrVYjSUT0YLa2tiyU8P3335v3PZ1OcXBwgC+++ALb29vY39/HZDKxnRD//Oc/o9Vq4dNPP8X+/j42Njbw5MmTN5Kt1Avl+NFgj8djfPnll/jmm28iIIiMy//+3/8bBwcHODo6wvHxsSFcdnhkQiUVP8ttWKKpY6DvRg1TyJiG4sehOPOHJiHqPw7983jgzfEIhYGUVo9THHFry5cIapxc1xWFXo8yBgTG3GSl3W4jk8nYDpb1et36dPjdFAFYkqRuxsW1yJAAQ2sEHNrbQPffUAVMOpcNrwaDAb777jv0+310Oh3bHr3b7UbAiDKE/kcdj7h57MME/P2hzt2PUUKeNtusM1RaKpVszmoIl50HOceazSYAGDtGZ4oAlDaFjqaeD3gznHgXw/++wMKdwUDc4vAeSZzHu+wcPI/2eqZy0Va5TPKj4vO99OmNUAHyBVKoTEmTc6tVKhRej/fK6/CH4IT9B1gWxV0Yq9VqBDCRgj87OzP2Ip/Pv/HcIfGolUmLahho2JvNJs7OzowaA6JZstyOmIaDi4AelY5P6L2EZBVl+VAI+CZP/75llfCJ/t2HDG5iLeL+77/v2QdPd6u3rN/jemM+CecGAYcvS9TrawIeQ2vM4eEaoGgXR5bm6prnPKcC7/f76PV6VjZLQABEexl4b03ZDP8TGkNlA3iOtXy44j1/1X1AtOSXc4+6kDlYV1dXQVug/Qp0jntn4a466H3qrXthBoDbswPq7frGKX6jnfF4jJOTE+Tzeezu7qJer+PTTz/FJ598YnH4EO2TzWbR7XYtA5TZoPqy1EiSdVD6vN1uBxUHJ02r1bKqBdJQh4eHyOfzlr3MSdPr9fD69WscHBzYd3/961/jxYsXZsw5DhqXZtyTYYnLy0uUSiV8+umn6HQ6VpL5xRdfoNfr4dWrV+h2u5YwxlJMhlSy2axtXKRU8Xy+qPP2k9i31lUPy9Oo6uX60pqHlLcBBasAWCC8DjyNHgpXxBmcm2hEb8hDrIG+W74vVWjA4p2xjBRYtHHd2NiwjcQIkrUigQmIyeT1xkOaPMtzcYOgq6sr5PN5623BDp6Hh4fW84Mef7fbxcXFBb7++mv0ej188803VmnA+9MOjFTafFYd0xAQUvEAgWMa6nC6lg9HvBMHLFgo9gvgPGe1AcEmc1MSiYTpd857YLFleaVSwdXVlbW9183caMu0eiXEXHh5iDn31mAg7kFWVf7qLVBZKUDg/5kJzDDA1tYWPv30U9vVjIlK2rBES0G0CoEvjCVX7IuuL1lbuvJvwMLzUA9d8wlms+stkM/OzlAqlVCtVpHP51EqlSwjut1uW0Y19wVQpe2NK5+NE5hNj1i+yIzxV69eWY4AWwyTwSAIoBKmx0VqVhUq70cNjYI3b3y8hxv63mMAA8DbA4Kb2KxlscHQv/W8y4xV6F78cXFjHTqXr5/XEqn5fG7zh6A2nU5HSgD5QzDA/RYYTuDeH2yCxTnMuctSYeaotFotYwB0y9ler2flsM1m02rEtVsm564+r4Kh0LjxuBAbo0AcWIQv1/L4xbPSISaITg1pfmWNWbmiDBWPITMWCuH61uMAgiwEEE2yfUxA8972Jggp2WUP6inFOFpFS+vm80VSYbPZxMXFhSksTRRirI/NUxKJ61i6Gkdei0kfo9EIBwcHEQV3dnZmf9NWrvp83W7Xcg0oTHY6PDzEfD5Ho9HAz3/+cxSLRfzqV7+yrOpyuYytra03lA+fVycp2wdzK+WjoyNcXFzg6OjIPKfT01MrDeOmS7q99Hg8NvDBTWBCRmqZ0SNDoJ/7yR03Px5SbgIBcYb4JvZr2XlWva+7fK4MgIKwEHDz90zlpOV8pVLJ1hI36yoWi6hUKtjY2EA+nzeQUK1WUavVUCqV0Gg0kM1mLabK7zNkxbnLpEQmr6ZSKVxeXlpjK1YKEAycnZ3h9PQUFxcX1lKbiY7aP57PrHkCcePnwdIysKqfrcMEj1/iWGg/JzQU4HOjWELOH843YMFg044AbyYMM6yWy+UiuptrQcu2tfHVMnm0YQLv+QHxXoii7jjhoHLASffruXwyFBOILi4ucHFxgVKphM3NTcznc/P66anr3uREdswrUITHF91utyMv+8svv8S//du/RfIRfPc2MgsaetBqgn6/j6dPn0bAAPdNYFWCKhuPILV98GAwwMnJCdrtNv72t7/hhx9+wOHhoZUW8pnr9bolTbLZDJMDmX1N9OvravXdhChtbUyj7zr027/nhwIEtwEC93Gt+zje09Whz723o6EABdch8EZFRMBcLBaRTqexubmJUqmEYrFolTiNRsPKDKfTqZUgNhoNPH36NAIGOM7tdtvWEzthFotFy/dJpa53JmSScKvVsjJCdgY9ODjAxcWFle4S6HOeq8fl525orELjdpOh9wmHa3l8wvcTskkh+0RboEl+AMxJZJiLYEArXHzemYYgGGLWXATmHtA29fv9yGZHel/+Ofx9h577pmNuI/fGDFBChiXu/6T9+X8OBmPmHnQwhsfQwPHxMUqlkpUq9ft9i2lqNrNeX5WAJgkBC6+XyVJUQOwiyO8z6ZAJirolrSaRcOIMh0OcnZ1ZMhavRaXIJBUKt2NlslSn08Hr16/R6/Xw7bff2v/ZfpixWS0bJBtCRkBLakL176osl71j9Tp9GU3I89J3+5gosZDc1z3eRPffl2HR9xRXjcLrcd5ybeVyOUvuY9yTRj2fz1v/CfbL4BynMec8Y7mhr7Pm2mGDL2DRS4CZ3dzKnLFZMgOj0cjABFkFD2i8bggBAH320Lgvew9r4//himeX+TfmlcWFOhl60pyYZUCQc1BbW7OhGx1crhV+xpA1nU6f1KjnDYWx3qUOvTMY0Jtdxg7EncPTmQDeUFg0/AoQRqMROp2OJcmVSiXs7OyYl87Og7lcznZABGCeOxPxtAMVjXixWDTjzbg7AGtDrNQS908gigQQCVewuiCdTqPVauEf//iH5TqwRXK1WkUmk8FwODQlO5vNbP+Ak5MTHB0d4ezsDH//+9/R7XbxzTffWC01k1Z4H/V6PRLfarfbVnqluQ7ekHtvyb9rHkPPk5/5UlBdPKRub5oLH5LcBSys+p1lDEHc933PDIJl9XjUc9F5rnOQ/TYU0JbLZVQqFZTLZezs7FhlQCqVssZB2rCq3W5HPB2uN54bgHlDx8fHODk5Qa/Xw/n5ueW8TCYTnJ6eot/v4/DwED/++GOkFFFzWlRn8Bk9kI0LESxT8MqUPTSjtZbV5CZjDSz6AWhPCiDqFDJ3jI4eSwZD16BO1QTs0WiEdDodAc9ky1gOr9U5GjrQXDcCFgUC70OPvrPtuJYhGT4cH5jCheizgP35aNTYJIgggPFKZn4qve8VBSeBKhPd0pKiNag6iTiptAOVnpP3TOPMNsDs7qbtVFVZz+dzy6T2P71ez3IHCJC485uCG1+iBUR7OLwNZR6irb2CDXnGnkV4LLKKor+Jsg+J98xve0+rfj/EvtAj51wkkGbzLXbVZBWMbsZFg18oFGyTL+3Aqd0wNekwVFvNdUcAzc3BOp1OpERQwQA7ivLYELAJUcJxXv8yZmAtPz1ZtkZ1TYX0kc4VrhlNKvfgUpPGGabldfija4Tn4Rqi88c1wgo2dTqVNXjXDOud9yZYhlRCizV0Di2lYxtdYBHf5/e9ImDyEWnG77//3mhyMgO1Wg21Wi3iNbN8UMuiiO6SyaTtXKj9+8lOUFkxZ4DZ+Hyx9NCoOOnpM/b0+vVrJJPXTV0KhYI1VVFjSWRIWp/tkc/Pz22nQYIeZm1zUvF4xqV8SICemuY26NiuSkPpIvL12z6xRhfBYxCPtG8DBPT7q16L8q4Wr5ZL+S57NOa5XA61Ws0y/bV01r/76XRqiYF7e3t49uyZgQcF21y7l5eXaLfbABaZ0/zhGmMnzOFwiO+++86qA9gVk1UDx8fHGAwGeP36tW2kRaDsm/9oYyU/1jr+PgTJv8d9b50o+GGKt0XLQpz6mddPWjnGXLBKpRIpLyU4YLIsjyczwHOwiZdfn5yL3BdExffX0FwDsgjvsrLlTmGCVWQVDzMutqkUYOhcVEYsOex0OgBgMXzGZEjfsIxP26kSgGgylTdcGmPyZSWkdbgbllLlWraVTCYto5pxqclkglqtZuCDP8yY5g89J30WnpNbNGsZpPZrZ2tZX2seGvO493PT31dhBG463/uSZUAg7t9x57mN3IZNWHad0Hhy7ENhOzJGBI4aEuAc0u/RM6ESZK6AJuISDDApil6MsgSefWPSKkMCZLm0xJdbgLNZV7vdjoSyWN0ToveXjZN/18ve8xoIfLjytrrFMwbKuvqEcy1l5Z46oWRrZcyAaMkhEO1w6QEDy9UJCICorlXQE+fErercqbwVM3BXRQdEFyMpbb//OxUQr3l1dYVer4fRaBRpUMQqBBrcfr9v5SFMjCIzwK2HfQdDes1XV1c4OTlBt9vF0dGR7Y3A0ipel22Bc7kcOp2Otfql55VOpzEej3F6ehrxZIjw6Alp2KPZbNpYUNHSuHMi8j6m0+vNlKiYiR41/KJJkHHAK8QKrBJG8HStUlr6/pQ6W0u83LR4/VgD0eZOwDXArVarVhLYaDTMs9ffejw9eG5UtLW1ZTFTKiVSoMwVoOeSzWZtzfK+SO0zJ6DX6+Hw8BDj8RhnZ2cGnkejkbEETLBl5QHXGdenht6UcVRFqmOjSnuVkEIoLLmWn5aE9J13/rRhlTpaOn/4XQJtAgMAkdAvk7cJevmb32cn2EQiYfOcQvvBEDNtier2m5j50L9vkne6hfFNwocKdQ5U1EQlM5vNrAUvFQQNpnq/St8zw5519wQHjIWSMSAiu7q6wunpKbrdrmUz856AheIgjZlOp63XNfdP4L0R2WnyFstSfJ0/uywyN0HpIEWYrGElSFBmQqksemuhWla/KPy7Xeb9q9eniFnfI8dIOyk+pKxK798mDHCba9/H8cpq6N845lwv5XIZ9XrdwADfky99IpXPdbC7u2udKkulkjFNytLRUwIQAawKDElzNptNHBwcoNvt4uDgwJQjmQJ2Gzw6OsJ4PEaz2Yw0B9MeCCGgFJqXobm7jAUCwkmCa0Dw4YtfK5wrvpRaf6hnda2o3lddpsyANhyiw8pyWnbdJFtLYUhc98vQbpp0oujkMZ/gXc7NdxYmuEniFnKIatT4oNL0/n48laKd+0iD0oBrqRQVJMEAGwmpV01RI00FTCXY7/cjsVMabAUl3W7XPGbtV0BmgzkLrGTgM/K6NO6auLiMKo0zxt5L8kjTe1P+3FpLqz/AYoIrmFt3cXs7CQEVKjeW72UyGcuVKRaL5nnwPTDRlP/P5/MWOmAvAebDaFUC5ycZOO4iOJ/PrSyW98d4/8XFBc7Pz23TF3bfvLy8RKvVirR+5dr0c1XZR7/ONQwXx1L6BOVVFOlNYYi1PH5R3aZMpQ/fqg3SY5jYp/NKPXXqOjqh2k5f19Z8PkepVDKGgMf6Bkahe2SoW/MV/P3exODe1la/dQLhXYUPDiwMi3r3FB6jsW/1VkIDxXMSDPDcPA9RmE4I7RJFg67te9X4qQdOQzeZTIxJIFXK/tYMT2j2Kb9Hhcxnmc1mpiz9mAMLKsuPpf7m8TrGce9A32XIu/L34I/nxFYKS0MEjBv797qWhSwLEdz0Gen0Wq2GQqGAzc1NbGxsWPY/hQmy7C/AngIMbbEDYafTMaPPpitsHUyQymz/Xq+H4+Nju8ZsNrPwAFsLs1KA5bnD4dAaZymlz70N1DuLm48KaOJCATpmIRZsGSMQ+mwtj1+WvS/VRwo8Q3ND7QN/a+8WOoyc191uF7PZzBJ2mahOg55Opw30+koB5pSprubc1nJDvddlht7P59vo3AdjBlTUmGkMUA2NKoXQy1PPQP/tkVeIXlRv3n+Hx3iDqdSoei9aZqVlJYy/6rOpJ80JqF60HhPnuYfG8aa/hSSkdFf9ro61jq2ORQjErGX54l32mYI9VuKwkkXnuIaoFAxoqaBSp9qvQPuyz2YzdDodJBIJdDoda/Cl1SpXV1eWA8BdBQke2GSITJ2uNW/44wCQ9+J0LELHxOmLm97DWn6aEsdE+71g5vO5lWpzHpEJYz4Au2YydwtYOIcAIpQ/15p3ZH3IgmEA5uh4u7TKHA0xZKvKg+YMeCOtg0kD4mPePpmIhlUlhPD5wv2GJlqLT08qRI2rMZvP50bv8x58e1TeKxOwer1ecAzYupjeERUxx4ANV1T5KwhRBar7J3ivPzRBQsAiTgF7L8sbHD2e/yfNRUCwBgP3JwSkDEE1Gg2Uy2WLRVIymYztH7C5uWllgkpp6vpj4yqWvjL5iXkALKnVsj+28mafAHbsJDtwcXER2aNA8wEIgJUR8KIeExCdk7re1KMjgFlFdG2ogl7LhyHLdJsXdVAIdv071x0wmSx7fn5unTEZ4up0OpHGeMPhEKVSyZg5AFaNwEZETLpVptn3FOD89U7jbeW9hAnucqFl5wlRdTy/NmtYVfz5PDjQZAxPz+h9+bCFZyD8tVRY7qfUp/dgiAKpIDUZy28ZvGy8Q1TXbd9PnEe67Jr+2qF70GYZj13JPhRYuS2d51msUEtsjY2qd6IJrl4ReiPL7n/c3ZKVPKyk4bWn06n1DWAogWCAfQR00xYft/XPFFq/fp7FjVncXIwb98c+J9dye1m2lqhT1dnkPKZwXtDQc15za+1+v2/sF0PRzLXJZDLmAAEwsEtn0Rt+IBp69uzYqs91H3LvexPc9jxxRogKiUbbdwvkMfrdkFfraRj+m+fkDm3AAgzQE1KqVOOvwGITChWtTaVBDyWC8FqaQ6CAhNse8/k5GbV1pV7Te1ecjJoB60GWn3yhMdNx1XyA0JirwuZCoHfGWvfHUFnwGOWu64lJgRzfSqWCUqlkWc4ECZ4x0tIlggWWSbEVdrPZxPn5uYEBjV+qZ8X5yIYr5+fn1vSKoQOGBbiGeG3gzbbCGs+llxQqT/X6Q0GADw3eZjzX8mHKTYCac3Y4HKLT6Vh4QDvWqv7SHjbc0ZZMF3PLWB3Ac+mc57wmy1AqlaxCgccx/8A7m1yfXGOhUPkyuas+eWftiG+SuNi3p8DVCKmhiYsrxikBzzTwJVFhAgtlQOPN470Xw78rfaM0vmaeagtjvQ8quZCRpXLnGOjk86CCE5ohD37HJ52sEhLw78GLH9sQEODf9YelnR+zsvVANyRxc3nZOfn+6flr/TMNvwJoNbo+tDSfz43WPz09tS2x2QzLzyX2CpjP5zY3SaP6+Kgael4vFN4LGXMPHG4auxBrdRdZswYfhqy6tjSLn5tiUQf7ZHKW9DWbTfvN0Blr/lXnA3ij66vOX9oYAlwez/tXtpo2BECknPCmEO7byoPmDADLE6P0Jev/fZxQE5BC9xc6P5WXnpvn59/Vg1Ejy3/7sAJpUcaJSLNq3oM+gzfYFDau4HW045uWvPgffsbJFDLcce/uJgXqv6fX8gZlNBphNptZy+StrS28fPnyjUSd9yUPRf/fhywDBLrPAABjYSqVSmROE0Rzy2qdW5zTLKU9PDxEq9UySlRZKc2E5vfIFjAOSkUZAiD6PMtAu3pLChq0j4UHDnretyljXQOAD0/i1kjISRkOhza/uUGc7xqrYIDhL23wpufmeTn3yR5wS3Buixzax0DnttoTvZ9Qjhiv71mxt9Vzb7Vr4dtIiNZRBeC9yDij5oGDnxih+/WeshplAJG4KsuqtD6UL9MbN3ZAZCtgVaB6fr2uKk1gsUeBPo/uZuVBiXpgoYRJD6BuYgO8gg4dz+teXV1FaGgaHT53oVDA3t4e9vf38fnnn0eSN9+3vEtA8FAGhB06Wf46n1/vxU4WhuwUy2upyLitNY35aDTC8fExhsOh7Rugpbksi2Ls33v9PG42m1k/DT+vQ3PqJkaJ4st6dc0TDPgQoL/mbWQNCD48iVvfyioRDPDv3W43Mk81B4abvTEvQBleP7+070AikYiUlKujBCwqDrQBEYXnozPnndhHyQzcl8RR+f7zOA8iDhGGqPBVKXEqVVJHoYSPOMSmzELoe3H36WPsDAvwGE2+8l5Q6HzqefnnilN0cQDAo0//fQIkTuB0Oo3t7W2kUins7+9jZ2cH5XLZmi09lITGi7IKQFp23CqLcRUDs+weQ8KxV0VFNkoBGxUgG2AxKYqZ0ux+ORqNbN8ATbDyCjG0TkPVCf6547y323zu30UodOCBP/+2qpG/zbFreTyyytohi0WnTeepAmjtsBmXAO71JHPI2A6fe+Goc6vn0pwAz3YDi8ZyqvdX1Q93mcMPlkAIhBEW/08KJQ4QcFBDdAk/13tVQ67ni6PS9eVT6fL7+uPLH+kd02PxcVO9hr4w9Xp6vV7wWfU4fZabxvamiREHykJK1R9LY8RM9mw2i5///Oeo1WrY2dnBxsYG+v0+zs7O7lwic1+iYxGi24BwUup9GwZ/3mX3E3dfnGfD4TACWtnfXzesYgyf84WGnrtxjsdjHB8fGzDQ7pahOeyZOyYy6jF6r/536G9efOIrgDfA5DLGL+RA3CQaOw7dy1oepyyzR16Xsd176LOQjo7TWbp+OQ+ZTwBc68VisYjZbGY7HDIpUJNvGY5Qe6XOn4KFVRzbVcclJA9eWrjs/H6x30QB3gdqCoEdNSBx3ov/LA40LVNQPF6TRpad6zYSmvg3HRN3Hi4apcC4Kx635mSFBqm2TqfzYGDAe4dxrImX+wQCqygsf7/+b/7+1bugl89QFUsANeaviUtapULgoLkpPj/FA2e99zjv/r488WXXiDt+2XFr4/7xSBx7GnIUQ2DgNkL9RkPPLbyB670IQgZd7ZwyBgT7ug7vck+3kQdvOhT6rbR5iMrxHlUoW99LnPe77Pqq/DQG75mEUH7DTQorTrHepMhu+swLz6lhB96zT4BU9sHfp2dvWM/OjXFKpRKePHmCXC6HYrGIVCqFi4sL/PDDD2i32zg8PHyrxK53LXGsAfDuaOPbntcDGSYtAcDJyQk6nQ6azSYymUykvp/MAIUKhkqL3ogHnaE5H/cct3kmP87+Gf0xIeZkGZiPYwg826DnCCU78u/v4t2v5d1LnGN3m+PjjgudizpUw2zs7cFqHx92UCDgtwVXgBAKz61iK24jD5ozACyPw4bQW9yxccrlLtcOSUiBxL2MOEPvz+U/W6ZUV3nhq3j9q5xj2Xn4bAwLcLvccrlsm97we+zY1el0Hg0zcNNxcYbmXSHy0PWWGTv/d818pkJhXgDr/blJioJdeh2Mc8ZR+yFG4LZeyjI2zz/zXYHXqt9Zdi9x4YE1GFjLbYRJ4LoZkS9FV/HhBjIDagNvA2ruKnfem+C+bibkifH8iUQiWCIX8shXZQRuuhf+9i/DlwGGqgn8S7uNAQ8xFXeV0Pd9dQH/HZpsml2rzWuSyaRtBb25uYlisYhSqYRKpQIAlnnORh0EAsxcf5cU1zJRz/A2yP8hROeDN1zeU57P55bU12w2bU4mEolIYiG9/hDo8HPYrwH9mx4zn0dLa/0aDX1/FbZr2ffi7mkVww6Eu5kqy7JmAdZyH8J5xm27AVg5L+cwy4LZmZBsgA/PhRiBdyUPzgwA8UqCRlflNsb2tgtbjUYikYi8FA8G1Gvw5Ss3eddxKC/091Xp2Zuey/8sY110HGhg2O6WeQF7e3uo1+vI5XIoFAoYDoc4PT2139y5jpURq1Jw70LiAKeX+w7R3HSdVf+u11VAoDFFTRAE8EasEYjW5/t5vIxW1+vzWG+U9TwhNsHff9wz3hYM3Oe7YsnwWtbyNkIjrmHu6XQa6WnAZGtgsX+BJqKHqgvetTyKXQsfi/hafIKRUBmhVhiEaOhlSi90LH/fhcm4TQjAf99fm92yuBMeY15solGpVGwiX11dWetaggDuZc+ENJ+5+xASAkDLjr0JJK16zWUSAnres9Z718/1t4/rK2u2LJfFg4K4eww9S+jefHOuVc6vz7qKkV82DnESB2z0Oz4PyZdHrgHCWm4jGo5jGI4MAbBYm9ysSxvQaU8ZLef1nWffhTx4B8LHIt7Qz+dz84aBaLMIemJ+R8VVFWCcco1LYlpmxJaxB3GhB6+sOTl1h61sNmu5AKwUSCaTyOfzSKVS1ou+1Wrh/Pwcg8EAZ2dnGI/HkeY0pKI5jg8hofFexQsPjfuqgGAV1mqV3ICbvGzPSlF8lz6KB6E8TjukLaMo4wBF6JiQp+8ZqWWgxz9v6JqrzH3d6VTHR3/z3/y/fmcdPljLbYVsMnt+DIdDY54Y+uZeBQQEZBB8aa+vLntX8mhLCx9KqCCVigXeVKJkDOIMhsanQ4rcH8/f/vPbUMrLDJd2bAMWLS91Mxtub1sqlZDJZFAoFCyuxXONRiMAsPr0TqeDbrdrO9lpOEUV7m1Zj/sUP/6r5JfcNmQTZxBDchNDETK4oX8vuwc19J7C99cJGXcqJx2vkEe+DMwsAzKa+7CKcQ99dtOxccxGKDFQDb6O20PO27V82EJArV1lmVCore4JxH2HVublvK/E67fOGbhJSX4oElKKZAL4UgGY4aQXMZ8vyrJ4TJwnEuch6T0Ab7IUyyTO++f5eK/8zZLAfD5vBp873VWrVfsbJyYn83g8xuXlJVqtFi4vLy0vgFvV6j0nEosdG0Pe6fsWHxdfJVzgPdhVwgv+d+hvQLSJ1E3ARM/jmSOdH/4a2g3QPz//rcZaFROVkH/uZcBUwxN6j6F3H8pXuYkVCI1DnMeuNC0VbSj/IWTwOe/9Ns9rWcttRW0Dd8LluuLOotSVnHe0OawQClX7vCt5FAmEj03UO9I6z5DnpX+n4iFg0GN9Nv8q9DMpIpXbKCaCADXO/J3L5awzFpkAdsmiAtTmNPT8yQiw/3bcRkzeEDx0zsDbyCqA9yaP2/9e5Z5CBvKmc/vjQudbluTnqfu4/AlKXO5DHDNwm2f1f1/lGfW+PRhaNp7AmyBjHR5Yy9uK5pclEoswAHdLBBYAlJUFXGvqUL0PeSsw4Km+n5LoS1TvRb0OHsPubjSq9XodxWIx0rZYSyR9JQJfOGv0Q8wCx1i3nvXxXt2WWekljf/6xkJKZV1eXtr+9exOR4PPzWrYwIbZ6rxvBS7eowz19n6f4pPbQuINi/d2bwrL3GSwQ4ZslfGIO6+PbStbcJMBj2MvQp6yjoVv8b3sXm9iheLuMQ4U8Jz8vQqFHwoThPJy9FyhrZ3XzMBa7iLUifP53LqBKiCgk0Z9zQ3HmDyYTCYtLPuomYFQEtBPRRTkeBpWj1FDyuxQ7hpXqVQiFD0T6FR58/9EgEzg03iSKnj18FnypyEA/o3HaNmZ7rpI+ooJLgwBqOFnPoCGAnSrWs80hBLOdIwesvugGt64ueoNo4Y9/DmWXSfut//bKnHAmzxu4E1vNtRF8qZz+v/rOXX/j9DzL/PofRyef1+2rihxHpE30HGMm14jlCMQAgL+vv2Oc2tZy21EHUeCAoYK0um0Mao8TsPPqVTK8gtCzuG7kHthBn5qQuWgiXNeqfmdppLJpO0tX6vVUK1Wg8o7xCwQJTI+RNEe8b6dMGWZgQp56vw341Lc5Y4b19DgMzxA4x+asFTsvqGQGo9Qo5eHEO/phj5fBgj8XFfAGLqOBwIhgxOKx4fWU1xcPcQUaPLrTeOgfwtl1/vrh57R31NorEJePHMTKMsMd+i6et6454tjD0LsAEHFmhlYy30K5zj1OKsLAJgDyJwBz1BpE7H3IWswEBD/Qvg3YGHEaSwVubEvv3blA6IJgay7V7TovS4mLfod5LR/Nb+vuyZ6r5/XDtG/7AFAQBBKAvNJjxwT/1whr189wIf2rm7yhvk3/fHsUBwQWAYI4s7Nc8Sxa3rO0Gehc67yjMvGIsQ2qISeMXT9ULMUf38hAOFzbG4rCk55H6uAhlA+gWcGfoo6bi3vT6gj6fBxbjJvgD1b2IRIdW3c+n4Xsq4mCIhX/sCbXhCpHhqLyWSCVqtlfeCbzeYbNDPPzcnhDTaPIUjwLWUB2G/S/xqP5/k0oS/O49Td6Twg4fPrOISYDf4/bg6Env9DETX2+m8vcYDgttfQc/nPbjp3iPnx1/HH+/tX0ObBXVzlQyju7+dDIpGw+Ocy1uOmv+mz6A6M/JsPi+h9LAvJaHiFSbNaKfTQQHYtH6bE6Tu1F2RiyRAAsLyxRCKB4XBouQbvo7zwrXIGfopAwMe++WK8t6xxecbbX79+DQB49eqVoT6N9XsPynvtvtNZXOLdKqGCkIL2z7ZsDDzt78GA7r5F4+GV5kMmDap4g+qfxR8bMsghz51G5LbrII4F8Pe2DITwGA/gQiGFODYidGwodBGaC8qa6O8Qu5JKpW7dOGUZqOGaISunn/lzLBP9jg8PpFKpCDBYy1pWlZA3r+tUAQFzA1KplHV5pT3gluQ+JP2uZF1aGCOrKHj/sjUpSr129TK8YlYwwO/qxLmpHv2mZ9BreQZg2fPdZIT03yGj4T/7kGSZN34bj/2217gLsF72nkJ/D103xBTEvdNl93GXe7yNhM7zNuf2rIz+7afo6Kzl/clNutU7aGozQse9D0nMP0RtvZa1rGUta1nLWu5N1vzXWtaylrWsZS0fuazBwFrWspa1rGUtH7mswcBa1rKWtaxlLR+5rMHAWtaylrWsZS0fuazBwFrWspa1rGUtH7mswcBa1rKWtaxlLR+5rMHAWtaylrWsZS0fuazBwFrWspa1rGUtH7mswcBa1rKWtaxlLR+5/P8XtmzDbYv08QAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Plot 3 examples from the training set\n", + "check_data = first(train_loader)\n", + "fig, ax = plt.subplots(nrows=1, ncols=3)\n", + "for image_n in range(3):\n", + " ax[image_n].imshow(check_data[\"image\"][image_n, 0, :, :], cmap=\"gray\")\n", + " ax[image_n].axis(\"off\")" + ] + }, + { + "cell_type": "markdown", + "id": "d860d83a", + "metadata": {}, + "source": [ + "### Download Validation Data" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "ec954b77", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-03-11 22:42:02,011 - INFO - Verified 'MedNIST.tar.gz', md5: 0bc7306e7427e00ad1c5526a6677552d.\n", + "2023-03-11 22:42:02,012 - INFO - File exists: /tmp/tmp83zh5r1m/MedNIST.tar.gz, skipped downloading.\n", + "2023-03-11 22:42:02,012 - INFO - Non-empty folder exists in /tmp/tmp83zh5r1m/MedNIST, skipped extracting.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading dataset: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5895/5895 [00:03<00:00, 1812.52it/s]\n" + ] + } + ], + "source": [ + "val_transforms = transforms.Compose(\n", + " [\n", + " transforms.LoadImaged(keys=[\"image\"]),\n", + " transforms.EnsureChannelFirstd(keys=[\"image\"]),\n", + " transforms.ScaleIntensityRanged(keys=[\"image\"], a_min=0.0, a_max=255.0, b_min=0.0, b_max=1.0, clip=True),\n", + " ]\n", + ")\n", + "val_data = MedNISTDataset(root_dir=root_dir, section=\"validation\", download=True, seed=0, transform=val_transforms)\n", + "val_loader = DataLoader(val_data, batch_size=256, shuffle=False, num_workers=4, persistent_workers=True)" + ] + }, + { + "cell_type": "markdown", + "id": "09da3d54", + "metadata": {}, + "source": [ + "## Vector Quantized Variational Autoencoder\n", + "\n", + "The first step is to train a Vector Quantized Variation Autoencoder (VQ-VAE). This network is responsible for creating a compressed version of the inputted data. Once its training is done, we can use the encoder to obtain smaller and discrete representations of the 2D images to generate the inputs required for our autoregressive transformer.\n", + "\n", + "For its training, we will use the L1 loss, and we will update its codebook using a method based on Exponential Moving Average (EMA)." + ] + }, + { + "cell_type": "markdown", + "id": "2c7a91c3", + "metadata": {}, + "source": [ + "### Define network, optimizer and losses" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "757d00ff", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using cuda\n" + ] + }, + { + "data": { + "text/plain": [ + "VQVAE(\n", + " (encoder): Encoder(\n", + " (blocks): ModuleList(\n", + " (0): Convolution(\n", + " (conv): Conv2d(1, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1))\n", + " (adn): ADN(\n", + " (A): ReLU()\n", + " )\n", + " )\n", + " (1): VQVAEResidualUnit(\n", + " (conv1): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (adn): ADN(\n", + " (D): Dropout(p=0.0, inplace=False)\n", + " (A): ReLU()\n", + " )\n", + " )\n", + " (conv2): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " )\n", + " )\n", + " (2): VQVAEResidualUnit(\n", + " (conv1): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (adn): ADN(\n", + " (D): Dropout(p=0.0, inplace=False)\n", + " (A): ReLU()\n", + " )\n", + " )\n", + " (conv2): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " )\n", + " )\n", + " (3): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1))\n", + " (adn): ADN(\n", + " (D): Dropout(p=0.0, inplace=False)\n", + " (A): ReLU()\n", + " )\n", + " )\n", + " (4): VQVAEResidualUnit(\n", + " (conv1): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (adn): ADN(\n", + " (D): Dropout(p=0.0, inplace=False)\n", + " (A): ReLU()\n", + " )\n", + " )\n", + " (conv2): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " )\n", + " )\n", + " (5): VQVAEResidualUnit(\n", + " (conv1): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (adn): ADN(\n", + " (D): Dropout(p=0.0, inplace=False)\n", + " (A): ReLU()\n", + " )\n", + " )\n", + " (conv2): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " )\n", + " )\n", + " (6): Convolution(\n", + " (conv): Conv2d(256, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " )\n", + " )\n", + " )\n", + " (decoder): Decoder(\n", + " (blocks): ModuleList(\n", + " (0): Convolution(\n", + " (conv): Conv2d(64, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " )\n", + " (1): VQVAEResidualUnit(\n", + " (conv1): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (adn): ADN(\n", + " (D): Dropout(p=0.0, inplace=False)\n", + " (A): ReLU()\n", + " )\n", + " )\n", + " (conv2): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " )\n", + " )\n", + " (2): VQVAEResidualUnit(\n", + " (conv1): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (adn): ADN(\n", + " (D): Dropout(p=0.0, inplace=False)\n", + " (A): ReLU()\n", + " )\n", + " )\n", + " (conv2): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " )\n", + " )\n", + " (3): Convolution(\n", + " (conv): ConvTranspose2d(256, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1))\n", + " (adn): ADN(\n", + " (D): Dropout(p=0.0, inplace=False)\n", + " (A): ReLU()\n", + " )\n", + " )\n", + " (4): VQVAEResidualUnit(\n", + " (conv1): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (adn): ADN(\n", + " (D): Dropout(p=0.0, inplace=False)\n", + " (A): ReLU()\n", + " )\n", + " )\n", + " (conv2): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " )\n", + " )\n", + " (5): VQVAEResidualUnit(\n", + " (conv1): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (adn): ADN(\n", + " (D): Dropout(p=0.0, inplace=False)\n", + " (A): ReLU()\n", + " )\n", + " )\n", + " (conv2): Convolution(\n", + " (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " )\n", + " )\n", + " (6): Convolution(\n", + " (conv): ConvTranspose2d(256, 1, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1))\n", + " )\n", + " )\n", + " )\n", + " (quantizer): VectorQuantizer(\n", + " (quantizer): EMAQuantizer(\n", + " (embedding): Embedding(16, 64)\n", + " )\n", + " )\n", + ")" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "print(f\"Using {device}\")\n", + "\n", + "vqvae_model = VQVAE(\n", + " spatial_dims=2,\n", + " in_channels=1,\n", + " out_channels=1,\n", + " num_res_layers=2,\n", + " downsample_parameters=((2, 4, 1, 1), (2, 4, 1, 1)),\n", + " upsample_parameters=((2, 4, 1, 1, 0), (2, 4, 1, 1, 0)),\n", + " num_channels=(256, 256),\n", + " num_res_channels=(256, 256),\n", + " num_embeddings=16,\n", + " embedding_dim=64,\n", + ")\n", + "vqvae_model.to(device)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "7611f596", + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = torch.optim.Adam(params=vqvae_model.parameters(), lr=5e-4)\n", + "l1_loss = L1Loss()" + ] + }, + { + "cell_type": "markdown", + "id": "f1d81a89", + "metadata": {}, + "source": [ + "### VQ-VAE Model training\n", + "We will train our VQ-VAE for 50 epochs." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "fe7459e4", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Epoch 0: 100%|████████████████| 185/185 [02:50<00:00, 1.09it/s, recons_loss=0.152, quantization_loss=1.05e-5]\n", + "Epoch 1: 100%|███████████████| 185/185 [02:56<00:00, 1.05it/s, recons_loss=0.0358, quantization_loss=7.67e-6]\n", + "Epoch 2: 100%|███████████████| 185/185 [03:02<00:00, 1.01it/s, recons_loss=0.0296, quantization_loss=1.27e-5]\n", + "Epoch 3: 100%|███████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0273, quantization_loss=1.68e-5]\n", + "Epoch 4: 100%|███████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0274, quantization_loss=2.47e-5]\n", + "Epoch 5: 100%|███████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0269, quantization_loss=3.15e-5]\n", + "Epoch 6: 100%|███████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0261, quantization_loss=2.88e-5]\n", + "Epoch 7: 100%|███████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0247, quantization_loss=2.52e-5]\n", + "Epoch 8: 100%|███████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0246, quantization_loss=2.67e-5]\n", + "Epoch 9: 100%|███████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0248, quantization_loss=2.96e-5]\n", + "Epoch 10: 100%|██████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0247, quantization_loss=2.78e-5]\n", + "Epoch 11: 100%|██████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0245, quantization_loss=3.64e-5]\n", + "Epoch 12: 100%|██████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0234, quantization_loss=2.43e-5]\n", + "Epoch 13: 100%|██████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0231, quantization_loss=3.42e-5]\n", + "Epoch 14: 100%|██████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0229, quantization_loss=3.24e-5]\n", + "Epoch 15: 100%|███████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.023, quantization_loss=3.77e-5]\n", + "Epoch 16: 100%|██████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0227, quantization_loss=3.07e-5]\n", + "Epoch 17: 100%|███████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.022, quantization_loss=3.66e-5]\n", + "Epoch 18: 100%|██████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0223, quantization_loss=3.17e-5]\n", + "Epoch 19: 100%|██████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0218, quantization_loss=3.32e-5]\n", + "Epoch 20: 100%|██████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0218, quantization_loss=3.49e-5]\n", + "Epoch 21: 100%|██████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0218, quantization_loss=2.99e-5]\n", + "Epoch 22: 100%|██████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0217, quantization_loss=3.81e-5]\n", + "Epoch 23: 100%|██████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0212, quantization_loss=2.88e-5]\n", + "Epoch 24: 100%|██████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0216, quantization_loss=2.93e-5]\n", + "Epoch 25: 100%|██████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0216, quantization_loss=3.35e-5]\n", + "Epoch 26: 100%|██████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0212, quantization_loss=3.28e-5]\n", + "Epoch 27: 100%|███████████████| 185/185 [02:59<00:00, 1.03it/s, recons_loss=0.0213, quantization_loss=3.2e-5]\n", + "Epoch 28: 100%|██████████████| 185/185 [02:58<00:00, 1.04it/s, recons_loss=0.0209, quantization_loss=3.05e-5]\n", + "Epoch 29: 100%|██████████████| 185/185 [02:58<00:00, 1.03it/s, recons_loss=0.0205, quantization_loss=2.83e-5]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "train completed, total time: 5397.495220899582.\n" + ] + } + ], + "source": [ + "n_epochs = 30\n", + "val_interval = 10\n", + "epoch_losses = []\n", + "val_epoch_losses = []\n", + "\n", + "total_start = time.time()\n", + "for epoch in range(n_epochs):\n", + " vqvae_model.train()\n", + " epoch_loss = 0\n", + " progress_bar = tqdm(enumerate(train_loader), total=len(train_loader), ncols=110)\n", + " progress_bar.set_description(f\"Epoch {epoch}\")\n", + " for step, batch in progress_bar:\n", + " images = batch[\"image\"].to(device)\n", + " optimizer.zero_grad(set_to_none=True)\n", + "\n", + " # model outputs reconstruction and the quantization error\n", + " reconstruction, quantization_loss = vqvae_model(images=images)\n", + " recons_loss = l1_loss(reconstruction.float(), images.float())\n", + " loss = recons_loss + quantization_loss\n", + "\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " epoch_loss += recons_loss.item()\n", + "\n", + " progress_bar.set_postfix(\n", + " {\"recons_loss\": epoch_loss / (step + 1), \"quantization_loss\": quantization_loss.item() / (step + 1)}\n", + " )\n", + " epoch_losses.append(epoch_loss / (step + 1))\n", + "\n", + " if (epoch + 1) % val_interval == 0:\n", + " vqvae_model.eval()\n", + " val_loss = 0\n", + " with torch.no_grad():\n", + " for val_step, batch in enumerate(val_loader, start=1):\n", + " images = batch[\"image\"].to(device)\n", + " reconstruction, quantization_loss = vqvae_model(images=images)\n", + " recons_loss = l1_loss(reconstruction.float(), images.float())\n", + " val_loss += recons_loss.item()\n", + "\n", + " val_loss /= val_step\n", + " val_epoch_losses.append(val_loss)\n", + "\n", + "total_time = time.time() - total_start\n", + "print(f\"train completed, total time: {total_time}.\")" + ] + }, + { + "cell_type": "markdown", + "id": "8dfa3270", + "metadata": {}, + "source": [ + "### Plot reconstructions of final trained vqvae model" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "0789cfcc", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAELCAYAAABEYIWnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABJZUlEQVR4nO2de7hW47r/71KzUq1WdNJBs2YlHVRmipRSKjpQSmLtTbUWUeS8FsuW0N7tEFkkh7USykYhExHpgBCFkFLpJIcOqFD41Ry/P7b57O/znXM+833f5qnG93NdrusZ837HGM94xhhPw/297/spE0VRZEIIIYSILWVLugNCCCGEKFn0MSCEEELEHH0MCCGEEDFHHwNCCCFEzNHHgBBCCBFz9DEghBBCxBx9DAghhBAxRx8DQgghRMzRx4AQQggRc/QxEFPS09Nt2LBhJd0NIYRIiUWLFlmZMmVs0aJFJd2VQ4LYfQxMnz7dypQpY8uWLSvprpiZ2Z49e2zcuHF5PtBz5861cePGFXufkDJlythll11Won0QojSRM4fk/FeuXDmrV6+eDRs2zL766quS7l6hcv/999v06dNj34c4UK6kOxB39uzZY7fccouZmXXr1s2zzZ0716ZMmVLiHwRCiNzceuut1qhRI/vll1/s3XfftenTp9tbb71ln376qVWsWLGku1co3H///VajRo0S9SLm14dTTjnF9u7da2lpaSXTsUMMfQwIIUQKnHHGGda+fXszM/vLX/5iNWrUsIkTJ1pWVpYNGTKkhHtX/Pz8889WuXLlYjtf2bJlD5mPrtJA7GSCvBg2bJhVqVLFvvrqKxswYIBVqVLFatasaddee63t37/f/W7jxo1WpkwZu/POO+3uu++2hg0bWqVKlaxr16726aefesfs1q1brv/TzzlXenq6O17NmjXNzOyWW25xbsdx48bZsGHDbMqUKWZmnksyh+zsbJs8ebK1bNnSKlasaLVr17aRI0faDz/84J0viiIbP3681a9f3w4//HA79dRTbeXKlSmPVY5O9/TTT9stt9xi9erVs6pVq9rgwYNt165d9uuvv9qVV15ptWrVsipVqtjw4cPt119/9Y7xyCOPWPfu3a1WrVpWoUIFa9GihU2dOjXXubKzs23cuHFWt25d1/fPPvssz3iHnTt32pVXXmkNGjSwChUqWJMmTWzixImWnZ2d8rUKkQxdunQxM7MvvvjC/W316tU2ePBgO+KII6xixYrWvn17y8rKyrXvzp077aqrrrL09HSrUKGC1a9f3y644ALbsWOH+822bdvsz3/+s9WuXdsqVqxobdq0sUcffdQ7Ds5RDz30kGVkZFiFChXshBNOsPfff9/77bfffmvDhw+3+vXrW4UKFeyoo46ys846yzZu3Ghm/xtXtHLlSlu8eLGbf3LmtBypZPHixTZq1CirVauW1a9f38z8OQ4ZN26cN4flMGPGDOvQoYMdfvjhVr16dTvllFPs1VdfLbAP+cUMzJo1yzIzM61SpUpWo0YN+7d/+7dc8k2ic36ckGfgd/bv32+9e/e2jh072p133mnz58+3SZMmWUZGhl166aXebx977DH78ccfbfTo0fbLL7/YPffcY927d7dPPvnEateunfA5a9asaVOnTrVLL73UBg4caGeffbaZmR133HH2888/29dff22vvfaaPf7447n2HTlypE2fPt2GDx9uY8aMsQ0bNth9991nH374oS1ZssTKly9vZmZjx4618ePHW58+faxPnz72wQcfWK9evey33347gNEymzBhglWqVMmuv/56W7dund17771Wvnx5K1u2rP3www82btw45zpt1KiRjR071u07depUa9mypZ155plWrlw5e+GFF2zUqFGWnZ1to0ePdr+74YYb7Pbbb7f+/ftb7969bcWKFda7d2/75ZdfvL7s2bPHunbtal999ZWNHDnSjj76aHv77bfthhtusG+++cYmT558QNcqRCLk/CNavXp1MzNbuXKlnXzyyVavXj27/vrrrXLlyvb000/bgAED7JlnnrGBAweamdlPP/1kXbp0sVWrVtmIESPs+OOPtx07dlhWVpZt2bLFatSoYXv37rVu3brZunXr7LLLLrNGjRrZrFmzbNiwYbZz50674oorvL488cQT9uOPP9rIkSOtTJkydvvtt9vZZ59t69evd3PDoEGDbOXKlXb55Zdbenq6bdu2zV577TXbvHmzpaen2+TJk+3yyy+3KlWq2I033mhmlmt+GzVqlNWsWdPGjh1rP//8c9Jjdsstt9i4ceOsU6dOduutt1paWpotXbrUFixYYL169UqoD0jOnHjCCSfYhAkTbOvWrXbPPffYkiVL7MMPP7Q//vGP7rfJzPmxIIoZjzzySGRm0fvvv+/+duGFF0ZmFt16663eb9u1axdlZma67Q0bNkRmFlWqVCnasmWL+/vSpUsjM4uuuuoq97euXbtGXbt2zXX+Cy+8MGrYsKHb3r59e2Rm0c0335zrt6NHj47yukVvvvlmZGbRzJkzvb+/8sor3t+3bdsWpaWlRX379o2ys7Pd7/7+979HZhZdeOGFuY7NmFk0evRot71w4cLIzKJWrVpFv/32m/v7eeedF5UpUyY644wzvP1POukk73qjKIr27NmT6zy9e/eOGjdu7La//fbbqFy5ctGAAQO8340bNy5X32+77baocuXK0Zo1a7zfXn/99dFhhx0Wbd68ucDrFCJRcuaQ+fPnR9u3b4++/PLLaPbs2VHNmjWjChUqRF9++WUURVHUo0ePqHXr1tEvv/zi9s3Ozo46deoUNW3a1P1t7NixkZlFzz77bK5z5by3kydPjswsmjFjhrP99ttv0UknnRRVqVIl2r17dxRF/zdHHXnkkdH333/vfvv8889HZha98MILURRF0Q8//BCZWXTHHXcEr7Vly5Z5zmM5Y9C5c+do3759no3nuBxuvvlmbz5bu3ZtVLZs2WjgwIHR/v3787zuUB9y5qKFCxe68ahVq1bUqlWraO/eve53L774YmRm0dixY70+JjLnxwnJBMAll1zibXfp0sXWr1+f63cDBgywevXque0OHTpYx44dbe7cuUXeR7P/dYNVq1bNevbsaTt27HD/ZWZmWpUqVWzhwoVmZjZ//nz77bff7PLLL/fcc1deeeUB9+GCCy5w/4dhZtaxY0eLoshGjBjh/a5jx4725Zdf2r59+9zfKlWq5Nq7du2yHTt2WNeuXW39+vW2a9cuMzN7/fXXbd++fTZq1CjveJdffnmuvsyaNcu6dOli1atX98bjtNNOs/3799sbb7xxwNcrBHPaaadZzZo1rUGDBjZ48GCrXLmyZWVlWf369e3777+3BQsW2JAhQ+zHH390z+R3331nvXv3trVr1zrX9TPPPGNt2rRxngIk572dO3eu1alTx8477zxnK1++vI0ZM8Z++uknW7x4sbffueee6zwUZv8nYeTMZ5UqVbK0tDRbtGhRLmkxGS666CI77LDDUtp3zpw5lp2dbWPHjrWyZf1/ivKSEwpi2bJltm3bNhs1apQXS9C3b19r3ry5vfTSS7n2SXTOjwOSCX6nYsWKTr/PoXr16nm+KE2bNs31t2bNmtnTTz9dZP1D1q5da7t27bJatWrlad+2bZuZmW3atMnMcve3Zs2a3kSRCkcffbS3Xa1aNTMza9CgQa6/Z2dn265du+zII480M7MlS5bYzTffbO+8847t2bPH+/2uXbusWrVqru9NmjTx7EcccUSuvq9du9Y+/vjjXPcvh5zxEKIwmTJlijVr1sx27dpl06ZNszfeeMMqVKhgZmbr1q2zKIrspptusptuuinP/bdt22b16tWzL774wgYNGhQ816ZNm6xp06a5/tE89thjnR3h9zPnncmZzypUqGATJ060a665xmrXrm0nnnii9evXzy644AKrU6dOgiNg1qhRo4R/y3zxxRdWtmxZa9GiRcrHQHLG4Jhjjslla968ub311lve35KZ8+OAPgZ+J9Wv2/woU6aMRVGU6++FEZySnZ1ttWrVspkzZ+Zpz+8fxcIkv/HK7+85Y/HFF19Yjx49rHnz5nbXXXdZgwYNLC0tzebOnWt33313SgF/2dnZ1rNnT/vrX/+ap71Zs2ZJH1OIgujQoYPLJhgwYIB17tzZzj//fPv888/dc3zttdda796989yfP3QLk4LeQ7P/9RD279/f5syZY/PmzbObbrrJJkyYYAsWLLB27doldB708uWQ3//Vl7bAvMKe8w929DGQAmvXrs31tzVr1ngRtNWrV8/T3cRf8CF3WH62jIwMmz9/vp188sl5vow5NGzY0PW3cePG7u/bt28vsa/fF154wX799VfLysry/u8lR9rIIafv69at8/7v47vvvsvV94yMDPvpp5/stNNOK8KeC5E/hx12mE2YMMFOPfVUu++++5xcVr58+QKfy4yMjFzZSEzDhg3t448/tuzsbM87sHr1amdPhYyMDLvmmmvsmmuusbVr11rbtm1t0qRJNmPGDDNLzV1fvXp127lzZ66/89yXkZFh2dnZ9tlnn1nbtm3zPV6ifcgZg88//9y6d+/u2T7//POUxyguKGYgBebMmeOlqrz33nu2dOlSO+OMM9zfMjIybPXq1bZ9+3b3txUrVtiSJUu8Yx1++OFmZnm+PDk5u2wbMmSI7d+/32677bZc++zbt8/9/rTTTrPy5cvbvffe6/0fQUlG1+d8jWN/du3aZY888oj3ux49eli5cuVypRzed999uY45ZMgQe+edd2zevHm5bDt37vTiFYQoKrp162YdOnSwyZMn2x/+8Afr1q2bPfjgg/bNN9/k+i3OC4MGDbIVK1bYc889l+t3Oe9Jnz597Ntvv7WnnnrK2fbt22f33nuvValSxbp27ZpUX/fs2ZMrKycjI8OqVq3qpQJXrlw5z7kpREZGhu3atcs+/vhj97dvvvkm1/UNGDDAypYta7feemsujyDOD4n2oX379larVi174IEHvGt4+eWXbdWqVda3b9+kriNuyDOQAk2aNLHOnTvbpZdear/++qtNnjzZjjzySM9NPWLECLvrrrusd+/e9uc//9m2bdtmDzzwgLVs2dJ2797tflepUiVr0aKFPfXUU9asWTM74ogjrFWrVtaqVSvLzMw0M7MxY8ZY79697bDDDrOhQ4da165dbeTIkTZhwgT76KOPrFevXla+fHlbu3atzZo1y+655x4bPHiwy5udMGGC9evXz/r06WMffvihvfzyy1ajRo1iHzczs169ellaWpr179/fRo4caT/99JM9/PDDVqtWLW/SrF27tl1xxRU2adIkO/PMM+3000+3FStWuL7j/y1cd911lpWVZf369bNhw4ZZZmam/fzzz/bJJ5/Y7NmzbePGjSV2vSJeXHfddXbOOefY9OnTbcqUKda5c2dr3bq1XXTRRda4cWPbunWrvfPOO7ZlyxZbsWKF22f27Nl2zjnn2IgRIywzM9O+//57y8rKsgceeMDatGljF198sT344IM2bNgwW758uaWnp9vs2bNtyZIlNnnyZKtatWpS/VyzZo316NHDhgwZYi1atLBy5crZc889Z1u3brWhQ4e632VmZtrUqVNt/Pjx1qRJE6tVq1au/+tmhg4dan/7299s4MCBNmbMGNuzZ49NnTrVmjVrZh988IH7XZMmTezGG2+02267zbp06WJnn322VahQwd5//32rW7euTZgwIak+lC9f3iZOnGjDhw+3rl272nnnnedSC9PT0+2qq65KaoxiR4nlMZQQ+aUWVq5cOddvORUmJ23njjvuiCZNmhQ1aNAgqlChQtSlS5doxYoVufafMWNG1Lhx4ygtLS1q27ZtNG/evDzTbt5+++0oMzMzSktL89IM9+3bF11++eVRzZo1ozJlyuRKM3zooYeizMzMqFKlSlHVqlWj1q1bR3/961+jr7/+2v1m//790S233BIdddRRUaVKlaJu3bpFn376adSwYcMDSi2cNWtWgeOKY7h9+3b3t6ysrOi4446LKlasGKWnp0cTJ06Mpk2bFplZtGHDBve7ffv2RTfddFNUp06dqFKlSlH37t2jVatWRUceeWR0ySWXeOf58ccfoxtuuCFq0qRJlJaWFtWoUSPq1KlTdOedd3opkEIcKPk961H0v+9bRkZGlJGREe3bty/64osvogsuuCCqU6dOVL58+ahevXpRv379otmzZ3v7fffdd9Fll10W1atXL0pLS4vq168fXXjhhdGOHTvcb7Zu3RoNHz48qlGjRpSWlha1bt06euSRR7zj4BzF4NyyY8eOaPTo0VHz5s2jypUrR9WqVYs6duwYPf30094+3377bdS3b9+oatWqkZm5FL/QGERRFL366qtRq1atorS0tOiYY46JZsyYkWs+zWHatGlRu3btogoVKkTVq1ePunbtGr322msF9oFTC3N46qmn3PGOOOKI6E9/+pOXCh5Fic/5caJMFOUR5SbyZOPGjdaoUSO744477Nprry3p7sSSnTt3WvXq1W38+PGuCIkQQogDQzEDotSyd+/eXH/LiXfIq9SzEEKI1FDMgCi1PPXUUzZ9+nTr06ePValSxd566y37n//5H+vVq5edfPLJJd09IYQ4ZNDHgCi1HHfccVauXDm7/fbbbffu3S6ocPz48SXdNSGEOKRQzIAQQggRcxQzIIQQQsQcfQwIIYQQMUcfA0IIIUTMSTiAsG7dukXZDyFEAnz99dcl3YWkOeqooxL+LYYwpVIXX8SPQ/2Z4bA+vMaQjcmrLDYiz4AQQggRc/QxIIQQQsQc1RkQQhQrIbduKkt6i3jBrnFc0vlQIVHpozCv/dAbRSGEEEIkhT4GhBBCiJgjmUAIUawkKgWEiqOmGmFd3Da2Hwo2tpfEmObHoXIPE+0LcyCZFfIMCCGEEDFHHwNCCCFEzNHHgBBCCBFzFDMghCiVpJpmWJpspa0/siVvK239KYyYm7yQZ0AIIYSIOfoYEEIIIWKOZAIhRLFyqKUPlra0tEM9tVD3MH9bdna2aydbnVCeASGEECLm6GNACCGEiDn6GBBCCCFijmIGhBAlRmlNyyosW2nrj2zJ20pbf4pqFUN5BoQQQoiYo48BIYQQIuZIJhBClBoOpIKaKBlKUxpgqqsdljaKeiXEvJBnQAghhIg5+hgQQgghYo4+BoQQQoiYo5gBIUSRkox2majmmUw8geIQipaSSPUr7P2Kg4JKPId+Wxi2gpBnQAghhIg5+hgQQgghYo5kgt/Zt2+ft12unD80IfdlWlqaa+/fv9+z/frrr/mes2rVqq79888/ezZcfSrUF644ddhhh+X5OzOz//f//p+3jdeM+/G+hx9+eN4XYGa7d+/2tqtVq5bvb3fu3JnvMfH8PIbIL7/84m1XqFAh39+K0kEy6V6pponFWQpIxv1cFBRWql+q6YNFQVGt9ljUKLVQCCGEECmjjwEhhBAi5uhjQAghhIg5ihn4HdblWYtHDZ81bdSxQ1pX+fLlPdvevXvz7Q/GIXBfUOvHfvE221inx3gG7hvauJ/YN77eXbt2uXbFihU9G14H61kYM8Hji7EVoViD3377zcTBTaoaa2i/Qz2GoDiuLzSvJZPqVlSldAubVJ+nkn4OlVoohBBCiJTRx4AQQggRcyQT5AO7Wzj1DkG3NruCcD8+Jqb6oevdzHeps7s/5G4KueI5tRDd6qH0RU67RPd/lSpVPBuek89XqVKlfPfDMWR3P44hp2r+9NNPeR5fHJykmkJ2sKSllWZSddMXRWphUVXZS5SiSC0sjudCqYVCCCGESBl9DAghhBAxRx8DQgghRMxRzMDvsH4TKk/MaXio0/NxcJv1bjwOxySENHQ8H/cFj8Mpidw31P75/LgvH2fPnj2uzbEG6enprt20aVPPtn37dtdGrd/MHxsee46ZQDh9UZQ+DkR/Lur9igJ+Xgtj9cXCWvkxZEs1RTAZQucIzTl4vtAcWxwotVAIIYQQhyT6GBBCCCFijmSC32G3FLv0Q6sIoo2Pg9tcWY/TCZFQlUGUBtiG6XwsIYTkDZYi0KXFKXtYLZDH4phjjnHtlStXerYtW7bkeXwz370VWomQJQS+JlH6KOnVB4vbjVxYrvmQazx0jFRdxTiXFHSM0H1KNZ0ObZyWjDJmSIYprDTHEKU57fBAnnV5BoQQQoiYo48BIYQQIuboY0AIIYSIOYoZ+B3WvkMxA6z9o77FejfGBYRWGAzpbhxbgNuo33O/OV3w6KOP9rZ37Njh2j/++GO+fQtpktxvXMGxQYMGnm337t35ng+Pw+Mb6kvlypVdm8dCHHyUdGpWfoQ03YL6hXZ+J0Nps6Hz53d8s8RTnZPR+jkWCt/R0DWxLXT+UDliPF+q96KwyhiX5hUNlVoohBBCiJTRx4AQQggRcyQT/A67plk2qFq1qmuz2x6r6bE7DSvksW3v3r0J9Y37EnLTH3744a7dsGFDz9a6dWtve/ny5a7NqYU4Hlhx0MxPNWS31Pvvv5/n78zMfvjhB9cOVU4Mucw4tRDlBh5fcfBRkqsPhs6XTDobE1pJNL9jFnRctPFzn6ibnq8JJU5+P7lvOCeF5jGeV0PXFJJMUr0XhfU8HchqgAdKqs9Fsmj2FEIIIWKOPgaEEEKImKOPASGEECLmKGbgdziVkPXuGjVquHaVKlU8G6bocRlN1NZYz8HfhvSykA7EeiH289hjj/Vs33//vbe9fv161+brR62RdUcsOYwxCmZmq1atyvd8eI2sSYZWLcR7wasUFrWOLA6cwlq1sKjvbzLlYvP7nZnZH/7wB28b5wt+JxDW1xM9J88dOCewDd+fatWqebYjjjjCtf/4xz96NkwZNvPjiLjfGMcTmld4ruSVTBEci9CKhqnq6wXtV9xxAkhxpEuayTMghBBCxB59DAghhBAxp9TKBNWrV/e2N2zY4Nq1atXybOhy5ip06EbhFDl0cbMrfNSoUd42uvunTJni2Xbt2uXatWvX9mybN2/Os59mvpusbt26ng0r6x111FGeLZSGh+689PR0z8bXj+74E0880bNhKmXXrl09G7o9//nPf3o2vCaWEDB1iaUAHF92faGrkdM6cSz4+kKrQrKLEl2PvB/2ldOoWLYQuSmsVQuLI4UMCbnbEa602bZtW28bZcSQu53nIHxGeczwt/wM4nuH77GZP3fWrFnTs+E5cE4zyy0FoIzA7znOXTzn4RywdetWz/bFF1+49rfffuvZcCx4nEL9DFVjDKUrMkWxMmF+fUnmmKEqkskiz4AQQggRc/QxIIQQQsQcfQwIIYQQMafUxgyg1m7m61usQ23cuNG1WT9DLZ51YtTvxowZ49l69uzpbd94442uzfoSngNT68x8/ZC1H9TdWAdD3a158+aerXHjxq7N+vayZctcm1NwOCXywgsvdG3WMpcuXera7du392y4+uDdd9/t2bKyslz7vffe82wffPBBvudDOOUJdU/UX838NC5+LjguAbd53DBmgVdURDheBcdCHDjFvSJcotosv0v43J166qmejeNvPvroI9dmLR7T6Xh+QrhvmELMMUX4jPI7j9fEaX/fffeda/O7wxo+piHifGTmx1A0bdrUs9WpU8e1ea7esmWLa7/77rue7fXXX3ftL7/80rPhXMLHxLk6VLaZS77z9WMsQigugUk1PTVRm1ILhRBCCFFo6GNACCGEiDn6GBBCCCFiTqmNGeBymKiNsKaLNQlQyzLzdTDOQ8f8+euvv96zLVmyxNtGXez444/3bKeccopr79y507OtXr3atVknR52qRYsWni0zM9O1WUv89NNPXXvNmjWe7cMPP3RtLBtslltTx5oMmzZt8mwYQzFo0CDPhjokX9MJJ5zg2qxJrlixIs+2mdldd93l2jhmZn7MBGqOZr5eyno+Lqds5uuzHKOBWi6Xk8Vnatu2bZ6tJMuUHooURnnpZOoMhJYXRt2Y44TatWvn2j169PBsrJNjbj3HvGCuPfcN5zV+7rGeSUZGhuUHHxPfAX6WMS6A55xk6hVgX/mdxL7ye4bvOc6pZmZDhgxx7WeffdazPf/88679zTffeDa8vxwHgHMAxwxgzJaZP2485+FxC1r6GSmMZZFVZ0AIIYQQhYY+BoQQQoiYU2plAnbjojTw22+/eTZMs2HQpcNlde+77z7XRleeWe50nYsvvjjf82HK3Mcff+zZQqsWnnXWWa7NLn10PT355JOebd26da7dpEkTz4apNZiOY5Z7xbTjjjvOtf/1r395NnR1sgsLpRCWcxAeU5QQOnTo4NlwhcWZM2d6tkWLFrk2pxVh37ifgwcP9rbRJbtw4cJgXxFMeeK0Ii7FKg6MwliZMNVVEtmNjMfh93Po0KGuzfMKz0+tW7d2bS5nvX37dtdmVzzOM1zyGF3z7NJGeY4lVbzer776yrOh9LBgwQLPxteEpc5ZFsE5iPfDd5BtmFLM7yNKCJdffrln6969u2vPmjXLs+Hc8cMPP3g2lH74vWZQ0uAVbTG9OPQM8b9pRSEhhMovF7hvUr8WQgghxCGHPgaEEEKImKOPASGEECLmlNqYAUwJNDM78sgjXZs1FNTFWAdr2LCha/fr18+zcRlNhDU6TLXBpTbNzObPn+/anL6IsQask2O6CoMpKueff75nC+n0GE/AqXUnnXSSt43pg5zmg+k7AwcO9GyoCXKZ0ieeeMK1sdyxma8JcnpO586dXRtjC8zMFi9enGe/zMwef/xx1+a0zuXLl3vbeL9Z58XnDWNAzPznjZ8Lfk7FgZFqamGiSxiHYL23S5curn3VVVd5Nkz95bQ0Ph/GAvDyxrgvL9uONo5NCS2vjHMHzxUYs8DpivhO8tzE58Br4n5jKi6/55hayfEM+H5ySXLU2zlGAuc1Hl+Mm5o2bZpnw/ecr4/jGfD8HDOAcFxWaLlhJNU4F96voKWYQ8gzIIQQQsQcfQwIIYQQMafUygTo3jfzK0txZbtWrVq5Nq8Y1q1bN9fmSmEIp/y88cYb3jZWueIUkY4dO7o2pyDhdbCEgJX8OLUF01W4qhWmI/GqeXj+a665xrOxe61+/fquzS7DqVOnujaP93nnnefa7AZ888038+33ZZdd5tq8mhrC1c9QNsAURDM/BRSrGJrlTtXCyoYsJ2FFOZaPsOIjpzYqtbBwSTW1MFF5gaUAfEYGDBjg2a644grX5vQ5fCf4fPxO4vPDbmRMH2S3ObrK+R1kaQJBFzevzonn2LBhg2fDdEKW/7hvKBVy1Ve8Jl6lEVMLeT7E36IsbBZ2jeM2v4+9e/d2bVxd1szs9ttvd22e7/kcoUqGITkn9JyEVjsMEZIUDqSCpzwDQgghRMzRx4AQQggRc/QxIIQQQsScUhsz8PXXX3vbmM7Rt29fz3bTTTe5Nmt7mCLDJS5Rz+F0ESzNaWb20ksvuTZrNrjCIa9Khjr2OeecY/nB+tmjjz7q2suWLcv3txivYObr3azl8WqAEyZMcG3U1szMXn31VdfevHmzZ8OSyxgHYGZ255135ts3hLVT1NpC8QQcB9C/f3/X5pTPt99+29tG/ZRTcDDWgccNV1pjDRRXfhQHTqqaJ8bchFYf5DkAS1ZfdNFFng3vO8ex4HF49T/Wn+fMmeParMXjHMRzDj7r9erV82yoqXO6K6YTcqwBptvye/3JJ5+4NqcW8juJMQQcN4SxQpyyjPeCYytwDmZdPtXyvPjOY1loM7O///3vrs33l+8hzhecdojxExwnhc8lXxMeM5n4gdBYaNVCIYQQQqSMPgaEEEKImFNqZQJ256EbZ/369Z4N3d+cFoYuHE7HQRcOVrIzM3vooYe8bXS3scsMU/24Ct6LL77o2ueee65nw9XwJk2a5NkmT57s2qeeeqpnwzTAxx57zLOh+53TXEaPHu1tY2rRe++959mGDRvm2rVq1fJsuGIZrxKGfQul4LArHmGXHe7HbjFcefGSSy7xbCzZYEVGHht0r7FrNeRmlkxQuBRGamGIo48+2ts+/fTTXbtRo0b59oVlJXzPP/vsM8/2zDPPeNvvvvuua/M7ge5hTLsz893mfA58f7jKILrm2YW/ceNG1+Z0alzVj6+X5zxMzeUqnChpsKyH8zGv/hpKl8Rx4tTuROE0yzZt2rj23/72N8+GqexmfmVXngPw3xGWm0PXhPuFnvtkKmoqtVAIIYQQKaOPASGEECLm6GNACCGEiDkJxwwkoz+EVk5C7Yf1FNSCWFtD7WvNmjWeDbXvUNoHg7rQzJkzPdvnn3/ubbNOhOD1skaGeiHHDOB1cFocpg6xho2pjKyvY/wCpzJijIKZr5uHtG/WqFDr4ngCTEni+AnU7Fhbw3Hj9ByEnws8Zvv27T3b2LFjvW1MI+PnAseYxxvHlLVTjFFhnRHHNDSGofeANV+RN4mmnvGzheWsWUPH3/Jzh885pqWa5Y6/4XcUwWeNnwM8B+vkmHrNsTHYVy5zjs8TjxPOB5zqzKmGWJabUysxZZBjg3BMQ2l4fE343vG/LxgjwdebaHwBp0Hjiq5mZtddd51r8/yAcxmPE/57l4z2j7/ltEO8fqUWCiGEEKLQ0MeAEEIIEXOKJLXwQNIbcmjevLm3jdWxOJWmT58+rs3uZ0yXYbfUv/71L9fmlMSlS5d62+hCxNXEzHy3XMiGVf3MfOmBxwlX5+MKYyiL8PnQ3c6yAFfgQvclu9OmT5/u2n/60588W0jOef31112bpQ98LlhOwcqRmPJj5t9vlmvQZcZuVl7hsFOnTq7NrtyQDIT3np8vlALYhpIVuxZDUgj+NvQ78X/gsxVa1Y7vA6biomRg5j8TLHmh/Ld48WLPxtIOvtv8nvEzk9/52f2Nzz1X80T4fImudsjPHe+H14jpimZ+5UZO5cT3nldCxOOgNMfn45REnB/5HoYISUtckTUrK8u1582b59l4DkzkfAWB/WFZRKmFQgghhCgS9DEghBBCxBx9DAghhBAxJ+WYgUS1iUTTJ3ibNTo8DmtNqC81adLEs1WvXt21Oe0DtfAHH3zQs/FqW7gv62motXGKHmptrEvjNsdB4DWxJoj7cYocjhunBz388MPeNqblcSlWXNGQ9UJMWWQNduXKla7N9wm1NR5DvPd4z8z8uIiQHsw2jqc488wzXRtLE5v595BTebCvPKao5XIaFd4bHifUclkPxu1QLEOcSUYPxbHmZxLL6nJsDr5LHEP05JNP5nkMPh/3ld8lfNa4XC4+B/y+4PMUmkdD48RpsqE0R55ncJvfF0xR5NggfEd5BVAcRyz/y/3hFW1xm1etbdmyZZ7H4L4w/C5369bNtTlGJHQcfrfzg8cwdN9Cqx0qtVAIIYQQKaOPASGEECLmFEpqYTJSQMiG2+wmwop8XPFqxIgRrt2/f3/Phu6dqVOnejZMrVu0aJFnY/csygbsbqpcubJr8wpe6CbitBesssVuKXS1ffvtt54N0yXZvY9uMkzHNMudLoOph+wGRbiyHo4Npw/iMbkaJLr/WQpAtyNWMDPzZRLeD12bnKbFKTldunRxbZZlNm3a5NrsrkVXJ0s2+Mzy9eI95Cp0/FsEzx+q5nmwkEzltUQJVV5jG449ywRvvfWWa+N7ZeZLfuwaxneL7xHfW9xm1zz2LVQ9j+UpfO5DrmIeezwmzzns0kd4lc+aNWu6NkuzOM/xvUCpkGUCnGd4TENV91DC5cqFuM0piTxuIRvOsyzZhFK0QysTJnp+3i/ROSHZ90yeASGEECLm6GNACCGEiDn6GBBCCCFiTpGXI04G1EY4LQzTfNiGMQT/+Mc/PBuWAF6xYoVnwzgE1pvr1auX7/lxxS4zX4tiDR/LaLImidoPp1JiugzrUKjt8eqKbdu2de2rr77as3F5XtS+uMQm6pyss6IOxroX2rAsqZkfW8EaJI4Tjy/eb773eN+4LChra3i/OSYEj8vHCZUHxmeWU1fxfKzH4v3lGBTUlXnsD0YKI0aASUZ/xfeHYzcWLFjg2h988EG+5+D7gPeIV+7k+7l+/XrX5piTUN/wOQzFSIRirzilFZ/JDh06eDbU+jEmwCz3/IgxN1xyGOdOviZ8JzkWivuaH1zuGeccnmNDKzji2HOcB88P+N7z3IXPRjJp9qE4CHyeQ+WIDySVkJFnQAghhIg5+hgQQgghYk6RpBYm6roIpUxwKg265ZJJCUG3UWZmpmfD1f8uvfRSzzZ8+HBvG91Gd999t2ebNm2aa7M7DdPZ2BWFaYjLli3zbOjCql27tmdD9xq76C677DLX5pUYGXSb9ejRw7NhKiXfJ3T91a9f37OdcMIJrh1KwWH3YaiiGtpCLmd2p3GKIFY4Y1kmPT3dtdklvGrVKtdmlzC6LHF1Q7PEKwmGXN6HQmphUZDMcxD6Ld5rdj8jXIEPXersbj/55JO9bXzuPvvsM8+2fPly1+bKejhfhN4JlhHx+tmG80WrVq08G767BUkfOKYheY7nAHwn6tat69kwXZMlA3wn+JrQxlJAaJxwDuLrC1UO5H9jcDvk7udjJlq5MFR1NYRWLRRCCCFEUuhjQAghhIg5+hgQQgghYk6RpxammvoQKo3J2juWDa1Tp45nw9Q7XMHKzOyUU05x7aFDhwbPj8flmAG03XjjjZ4NtemePXt6ttGjR7v2p59+6tmef/5512Z9nY+DoAbKGnZIF+NVGnGMOX4D4yJCqXac9hhKHQ2VKka9na8JNTnWdRlcUZHLljZr1sy1uVTx2rVrXZt1v9DzjWPI8QvYV9ZccT/WOQ81komXSFQrTVVj5fuQX7/M/FK+7du392zHH3+8t92uXTvX5niC0047zbXnz5/v2RYuXOjaHOOC/eG+4XzB7yemDGKMgJkfT8CxT6zh4/PM5crx/Bwrg889pjma+TFOnAaOxwm9czzH4bzG8yi+WwWVzN68ebNr83yIxwmVIw6VyQ6lhyZTevtAYozkGRBCCCFijj4GhBBCiJhT5KmFyVTOCoFV6Vq0aOHZMCWH0/AwBYirCmIlKXaTs9yAbjJ2fWFaItuysrJc+/333/dsr732mmsPHDjQ8oNT3dANySuGtWnTxrXZZcTuxJD7FNN8uCIgrgTJLnVcGZHTJTHNM+SS5Up+KAOxG47dmQiPG66wyHIDbjds2NCzoWuVK8ihW5KlgNDKhNhvvD7uC7t5DwVC8wOSTOVC/G1ozuF3IFTlL+SKx9Q7Tu9l0B3Mx0HX+KmnnurZMPWZqyNiv/ldCklL+CzzqoWhd4mPic82vxPoUudqfTh3s0xw4oknujY/91jFkd8rrGzKczy+n8lIbiwpoGzB8yqeg/uNzxRLLXicZFaeDBFa7bDAfZP6tRBCCCEOOfQxIIQQQsQcfQwIIYQQMSfhmAHWwlEnYt0Yf8s6Ceo9rP2gbh9KEeH9UKdh3RZLhe7atcuzob7C6TFr1qzxtjt37uzaqF+Z+brQxRdf7Nmw5Oc111zj2XC1xRdeeMGzYZlf7hvGBaDOxhSkue7evdu1+Xrfeecd12a9EscYYwTMzBo0aODarKWiBso6OZZsRQ3QzB/DZFJpVq9e7W1Pnz7dtVk/xPgC1kBxm/uG5w+VbQ6thMjg+BbmqmQlRTIrDIauF+9ZKOYklCbG++HcEepn06ZNPRvOKwXFiuBx+bnD94DTbVH/5tRjfH5C6a58TIyhSiYehd9XPAem7Jr5KcUch4BxCtw3jMPgFExcjXXjxo2eDdPAOQ4B31eeO/Be8Bhu3brV2/7www9dO1SOmMH5gZ+9oni3U401MJNnQAghhIg9+hgQQgghYo4+BoQQQoiYk3DMAOd+45LCrCehxsolYlHfYRuW3AzlxLMNawLgssBmZuPHj3ftRx55xLOhnsSlirEGgJnZunXrXHvx4sWeDc/JpUlvuOEG1+YlSlG/4xKXqFlxaWTUxTi2AmFtjbVwrHswc+ZMz4bxDMccc4xnw3Oy9s15xQjn7yMZGRmuzc8F3vvQMqCsCT733HPeNt5Drs+Ax8XYBrPwEqUhXS6k3yVqOxSXMMZr4vFEXZX1dXxf+RnB94dzxEOxM7hsMceDYM0Sfq+xPgDfI45NSrTuCr/nGCfFY4HvHc8B+A7yOKEuH8q7D+Xym/kaOsdQ4RLr+O+EmT9X8jiF4glwG2slmIXnABwnHgt85/nflDlz5njbGLfFMSK4Lz97+DyHYlmS0fdTLctdEPIMCCGEEDFHHwNCCCFEzElYJmA3CrqN2E0VSo3ClLlGjRp5tqVLl7o2p3agixlXkeO+cSpL8+bNXRtdZGZmb775pmuzCx3dgGZmjz32mGsvX77cs/3Hf/yHa2PJW+4bjxO6zNLT0/Pdj11m+NtQehCXZsa0PzN/THkVP+wbXxOmGvbq1cuzhdxiob7i2LAUkN/vGE6/wlXfzPznklMy0bW5atUqz4buNnYDIiF3PxNy/x9IetDBQGhlQrxHnIqFciCm+pr50s6SJUvyPTePJ7rKOUW6e/furo3ziJn/HGKKbl7gNaIsYeY/6yy3Yn9YQkAZj98JLGfNEl/dunVdm8ciVJqZXfooQfKKiuiOZ/lz+/btlh+hFNBQyV8cX547cL9QeXYuFf/ss89623hvWG7AvvI5QvMFkoxLP1HZKVnkGRBCCCFijj4GhBBCiJijjwEhhBAi5iQcM8A6DepknIaC6SSc6ofa1/Dhwz1bz549XXvs2LGeDXVcjjVAOJ4A9f2zzz7bs82YMcO1WXvB5Xb5t6y9Yynb4447zrOh9jR06FDPhksD8zLBqJ1yWV9ebhlB7b9GjRqejdMAUetDDdLMbNCgQa6N5UXN/NLBIfiZweeEtUTWaxHUKzlVCcf+6quv9mys/aPWx9oljg0/CyG9EjXBZJbjTTStKNn0oNIIX18otTAE6vunnHKKZ8P4F1xu1syfq/h8+Dzxe433mtOCMU2V36tQzAtryPhbfrbxecXUWzP/+Q3Nv7xMMc6/nCKH2jfHIbAWjql2fH68Jk5JxGeB4wLw3oTSHkPPE8e2oY2PiXFCzzzzjGfj+43jwWOD/eF5LZT2GCplHnovQvNFKB6nIOQZEEIIIWKOPgaEEEKImJOwTMCuIHQxsUsDXb6cLoNpapyyhul8p59+umdDt9Tzzz/v2Tp27OjaLCH885//dO3Zs2d7NnQtsuuFUxvxGjnNCCvdXXnllZ4N093YDYipf+zeatasmWvzGL733nuujaunmeVOe0F45Ul0dWKapZnvfmdXH/aNCaXZ4DMTkhB4P0ytZNv999/v2u+++65n42qImHaKaaxm4ZU2QxXGQiuWJWoLrZZ3KFQgDEkkTMhVjemDfP/OPPNM177kkks825NPPunanEIbquaJlfU2b97s2fAaOLWOwVQ/XlUP3wlOZ0aZ74QTTvBs6I7mdEV0h3NqYag6Ib6DPBY4H5j5zyi/Lzhf8H7YN+536H1B+J3AvvL7ifMM3ydcKXbBggXBc+JY8TyK7y9fE45p6PpCKYKpph0mizwDQgghRMzRx4AQQggRc/QxIIQQQsSchGMGOF0G9W/W1zFNjbXhTZs2ufbcuXM927//+7+7dv/+/T0bpsRwChCmKN5xxx2e7ZVXXnFt1A7N/PgC7JeZ2auvvuptoy7F6YOYusQpkajLczwBllfl1EJMneLyqhhrwTEDHJcQYsqUKa7NWiqmD3IKEJbyDa1mybZQnElIg0S9jp+ZSZMmuTbHNoS0No7DwL5xChJqhDxOfM78SEb3C6UjHQqgrsk6aqIr6fGKlCeddJJrn3feefnacLU9Mz/9lOMCUGPm9wrnI54buTwxzh2soWPcEGvRGOPCqbcYt8RpwTjnchpyKKYH7wW/85yGiKXdsTy5mT8eHCOB18Fpj/kdg/vKttDqg5iWzOmDs2bNcu3QfGDmv4dc8h7jBELlrpMhFFOEFGa5cnkGhBBCiJijjwEhhBAi5iQsE7BrBN0moepxnKKC7pg33njDs2EKzj333OPZMJ3u1ltv9WyYLrNs2TLPhmlp//3f/+3ZMLUQ5QSz3FX2MjMzXfvll1/2bOi+xHQkM7Pzzz/ftTntBVf+euKJJzwbygtt2rTxbCiTcLoMjje7Ntl9yfcU+eSTT1y7VatWng1Xc+Nz4FiwDaum8SpsKAWwK3Xq1KmuzeMUSvvjbXxmOa0KK7xhGquZ/1zyfqm68EIrEx5qqxYW1ops6CrG6nFmZg8++KBr/+Mf//Bs+Py2aNHCs+FqdQ8//LBnw2eZXfjo3meXMlehQ2mJn5+QJITPIbv08bfsbkfpim2hVUVDK4JyNdPWrVu79meffebZ+N4goVUpsT+cyo5zFUsBKBOsWbPGsy1atMi1X3zxRc+GK9XyfMTg2FSvXt2zhe5TqDpiSC5L9b0/kLlDngEhhBAi5uhjQAghhIg5+hgQQgghYk7CMQOcIoGlQVGX599yehlqT6wZYVzA7bff7tmwrDDrMtiX3r17ezYsQcxaD5YwZT2dVwnDa3rqqac8G2rjrP289tprrs1pNpiSU79+fc+G6Sq8wh5qoKF4DS6NyWlGqK2OGDHCs2HMxFVXXeXZ6tat69qh1EKmZs2ars2plI8++qhrv/76654NNUlO7evUqZNrc5ofp7VivAqvpomlYEOpsqwthsoFpxoXEFrN7FAjVKqYbXg/eT7CWKHp06d7tjFjxrg2a/8YC8Src2J6MT9L2LeWLVt6Nn7v8PnBVDf+LcbpmPmxUKH4F55z8B3kNDx87nllVpxLeOx5vLEk+cCBAz3bwoUL890P4yk4LgCvkffDdGOOScD5As9t5s9jofR4vl6eK/G3POdiTBHP1aH7FFq5NFS6OLQqplYtFEIIIUTK6GNACCGEiDkJywTs0meXd35w2g27hhB04XEaCFaPOuusszwbVhK87777PBtWNZw8ebJnq1evnmtfd911ni3k/h48eLBnw9XyOO0Q3TbokjTzKxnyaovoqmYJY+XKla591FFHeTaUGwpK6cIKiJiaZea7rY4++mjPhm57XJXRzHcfspsK7/3bb7/t2dC1y+5LfC64+iTLQgi7EzGVk1eIQzfwli1bPBumkWE6Eu/Hrk189nkM0a3NKZ74zLB7+lAjmdTMUNohbqPkZOa7dQcMGODZ8B3glQGx0ie7bTHdlyU+djGja57lMZQGMJ3YzOzrr792bXYNY4oipw/i3BV6XnkOx2vkY4ZW62zatKlnQ9mYrwn7ze8LHhNd72a+1JKVleXZFi9e7NqcEo6yHl8Tvncsb/KKp2jnFEx8TtgWqviI8H54v0PvSGiOV2qhEEIIIZJCHwNCCCFEzNHHgBBCCBFzEo4ZKGn+67/+y7U59Qz1X9aMUM9n/QpX32NCqR3dunXztlH7Z10KS5xyKVLUxjmtadCgQa7NehKm13Fp5h49erj2ueee69k4DgJTOVHrN/O1edbQUe9evny5Z0OtL1TulI+JcRGsg+GYsr6OujyfD8fCzF+l8aWXXvJsqOmzdhzS91H35NLQTZo0cW1eiRH1WtYSMd2MV1OLE8nooTiGHGODMQS8Oineo1DsBqeU4jPJqYV8HHwn0tPTPRvq66FUO46j4ZLdCKZQcyojjimvhorvJJdt5tikRGO/MJ3YzH9fOM0Tn3uOdcA4gaVLl3o2jMng9GLU3tmGKy9yzADHuiFcbhpJVd/n2CDc5rFOtGR3ssgzIIQQQsQcfQwIIYQQMafUygTsUsc0sQULFng2TOfjynLohuOVt9BNhatbmeWWAtDNy+4tdBm2b9/es02cONG10U1t5rseueIhuvrefPNNz4buLU6fw4qLa9eu9WzsvsSx4nE7/vjj8+yLmdkZZ5zh2uhmNfNTp3g/dGGxyw7HlF3q6Frk5wLdgOzKZVkIz9+2bVvP1qFDB9dmtxy6U9mdiOdn1yK6Vnk1TXT1sWsP3dOc4nQoEHKXhqqyIWzD+8AS3+bNm117xowZng3TDvnZQlmL5w7cZpc6PwehqpwoMfC8gqu6fvzxx54NU+04RQ/lBX6WMWU5NB+wvMDvOb6/fI5QFUlMu2SpA+/hvHnzPNvq1atdm9MOQ+m3OP+zhItppQWl4eF4cJonjj8/e6HUQux3SN7g84VWnjyQiqXyDAghhBAxRx8DQgghRMzRx4AQQggRc0ptzACnVKH2xCvXoS7EaT6YTsdph6jhcJnQ0MpQIVgv7Nq1q2tzGU3Ul7CkpZmv/b/zzjue7eKLL3Zt1PbN/HTB9evXezYem379+rk2luo181MdWWvD8Wc9FHXA999/37PhNeKKjWZ+ug6XGMZyp1ii1cxfJYw112nTpnnbPXv2dO3u3bt7NtRgn376ac/WsGFD1+ZxQn2W7wU+C7ySHeqsnDqKz/qhEDPAunEoNSpRzTMUZ8GEbJgOGlpVju8favjt2rXzbKFSvqzv49zFJbLxGWFdHuNqktGUcT7kMcTj8PsZeu5DZbg51gJTlvm5x3NgyXUzf17hfmOsB99DjFHgvmC/eXx5TLkcc37wftjX0EqQoVgoTgnH+8tjn2jMTV7IMyCEEELEHH0MCCGEEDGn1MoE7ELC1AtOSUEXD7tNEHbho8uK3fTsmu/cuXO+50cXd6tWrTzbHXfc4dpc2Q5XUWRXEKZDcRUvTHPBlRfN/Opr7GrjaoFYORFX5uP+oAvdzGzDhg2uzZUEX3nlFddm1xv2tVevXvnaeNU3lDu4yh+m73HVslGjRnnbmEa2atUqz4YyEVc4Q3cbp0tifzA90cx/ptgNiG5Hfi7xeQpVmjtYKYzUwoJW5ETQdRyqXMi2kLyAFQE5hZdXA8QUNpbVcJ7juQvlB66qiO8gS6r4bLG8gdfIKbxo477w/Ijn5OqIKJPgO2fmp0LzSrgoP/K7jHDf8N1iWa1u3br52lBWDFUBNfPTANmGz0noueR+43hzeiQeJyRRJCOXFYQ8A0IIIUTM0ceAEEIIEXP0MSCEEELEnFIbM8ApGqGys6irctlO1GVYT0GdmLXZSy+91NvGVL9jjjnGs2FZT9Z3UCPkY3bp0sW1cVVGM1/TzszM9GyYToe6v5mf3sZ6Pmv4V199tWtj/ISZr6exJojjxlomllXG6zPz9UMsqWzma/FYMtXMX2kupIHyMVmjwxgK1llxFUNOQUKtkccCU0c5tuPVV1/N93yo13I8QUifPNQ5kHKqqRDSeEMpXBjDxO8Zr4iJ7wSXwcZYHda08bngVFh8lzmFF58ZLjmM26HYK45j4XkN3wOOd8Jtnh/wnDymOMeHbBxThHEXnNaJ8wqX/MV/DzgWiVcmxP7wM4plnDkOArfZhvMhz784B/J+ofLHobTdgpBnQAghhIg5+hgQQgghYk6plQmwMpeZ7yrmSnPoHuYUDXSxcPocrjD4wAMPeLZPP/3U237++edd+8orr/Rs6LbhlMiQmxfTEO+55x7Phisxsmt6xYoVrn300Ud7tjPPPNO12f382GOPedvoFsQUPTM/tYfTitA1FVrBkV1vH374oWuj1GFm1qxZM9du3ry5Z8PKbOzqw+qA7DLj9NCPPvrItXlsMEWH5RSsjFa/fn3PhlUdWaJCVx+nAOEzwxICPvu838EIu1VTTS1Mtboa/jZ0TO4XbvM9whQ5vkcsY+JzwDbcZpkApSSWoPr37+/aLAXgu8vzEbq/OX0P5yqet7jqK9rZpY7vIUsRPK/ntx+7xkP3Cd3tnLqJYxqq1MjXEEof53HD37KcgvMj3/uQxPjJJ5+4dmjuCK1aKJlACCGEEEmhjwEhhBAi5uhjQAghhIg5pTZmgHVy1IYZ1J85ZgDTxDDuwMzs/vvvd23WiTm97IknnnDtQYMGeTaMWeCUHNSmOc0GNTLWjFBD57KhmMoyc+ZMz4alUDnlifUl7CuvUoZ6E6fy4DbrWbjaYceOHfM9P+t3GAvAcRDYT9bkUMvFc5uZzZ8/39tGPZF/i+PG+iGmko4cOdKzrVmzxrUxzsPMfxa5LCvqvLyaJpLq6pkHK6mmFiYaa5DMufHZ5ucO42j4PWMNP1RqG2Ns+LnH+WHv3r2eDXVjfrbwt3xNOI9y/Atq4ayLc0lytLO+j/3m2Bz8LcczIKF7xsfEbR5ftPF7jXEQ33zzjWfjfyvQzteL8zOfH+8px0LhnLNkyRLPhv/+pbqyZ7LIMyCEEELEHH0MCCGEEDGn1MoE7G7n1B6EU3IQXOGPf4cV4tgdyylzuErYf/7nf3o2dAPi+czMOnXq5NqcnoPuJb7e5557zrVRMjDzV+JiG650hi5ss9yuMEyJYZcdVhFr0qSJZ8MVB1mWQRcpSy1cWRBB1yLLIugG5cpkWKnxtdde82zsFsR7zK5cdMvx6pLo3nvzzTc927vvvuvanDqEsAyF25wOhW7ekCv1UKQw0geTsSXaF75/+CxxGhqnuKKLnecAfCdxrjDz031x5UMzP7WRU2jxvWeXOs6BjRo18mw4B/Fzx/MTvqMsoaCN31c8Ls5jZuE5IFRJFtMVQ+8gSx+4MiyvYsqyCEoDfH6UOLkCIt43TtGeN2+ea4cqq/K/RSxT5LdfssgzIIQQQsQcfQwIIYQQMUcfA0IIIUTMKRMlKDKwvnOowZoYp8yh3nTsscd6NtRwOE0MU3k4XRK1RtaasMwtg31jXT49Pd21u3Xr5tmwrK6ZvxIjat9mvobFK62dddZZro3xEmZ+bAfrrKjhh8pohsBSzGb+CnGsXXLpU4zn4HQsXGGR9dJly5a5Nsdd8HNT1Hz99dfFer7CgEtdJ1smNa/9imN1w0TTFXmuCJU1ZlDTZ30fV9zj5wy1cdab8Xysb+NxON4H53i+Z7wy4eLFi10bVwPl/vD8dOKJJ7r2SSed5NnwmniuxPHn9xPHiccaY704hgptXDqdYxYw9oE1fLxvPKY4blyCHfvK14T/piTzPCH8zPLcxcgzIIQQQsQcfQwIIYQQMUcfA0IIIUTMiVcScwDWt1kHxFxw1pdQC2f9DnPrWdPGEsSc54qaNmuJWMaSc04ff/xx18Yle83M/vKXv3jbqP3zNb333nuuzWWUv/zyS9cOLcnKOcaYY815/g0aNHBtHosZM2a49vr16z0bllQNxXmYmfXt29e1scSwmVlWVpZrc2wFLn3NGqwomGRiBFKNCyiMeIJU6xoUVC421LdQjA2WHedzoDbN7xLOF6F6ATyvYO0CfB/NzI477jhvu1+/fnn2xcxf/p3rt4TOj3Mezyu4H9ecwXmF4xcwt59rwCAcB8C1P3Au4zkA5+BQCXieD5HQM8TnCz17B7LkuWY2IYQQIuboY0AIIYSIOUot/B12r7D7Hd1W7PrC8rHsMkMXD7vs0A3IqW7opuJyp+jCy8jI8GzoJvv44489W+PGjb3tFi1auHabNm08G6YMLl++3LOhW653796eDVOSWHrAUqh8zDlz5rg2p8Dg2IdSQNu3b+/Z2NW5du1a1+ZUylA6Ft5fLD1aEhyMqYWH+txRECG3LpKMDZ9RnrvQHR1KfeXnHOc1fs75XcIUQS7zjjIbz10oDWRmZnq2kEyAaXksFaKkyW56TBHk+Rfn2IJSy3GscN428+WHUEoiywSJuvSTkZ1CNqUWCiGEECKIPgaEEEKImKOPASGEECLmKGbgd1i/4TQf1LC4NCdqbcksR4s6FGt0qLeznoVpMBzbgOfna2A9C5ci5uPgvpx2g6k9nNaEj1NIh/v88889G+qArBdiCiaXGEadcdOmTZ4Nlyg1Cy9hjGPDrwTqnDxOHD9S1ByMMQPJlCNOVQ9N1VbSJJpClugxzPzrZZ06dL7QMXnuwjLDuMS3mV8emGMGQkvR47vM7xnOh6x9Y+ozz3nYb74G3A4tMW7mzw+huIBU02GZRFNXQ6mFbFPMgBBCCCGC6GNACCGEiDmSCfIhVDkr5IoKuZ/ZZReSEPD8ofQcdMkx7Cbiil9YoY9TItENyKk06OqrVauWZ8Mqi+zux2viSo2YnoQV//iY7OrCvvD4sksy5NJH9xrfCxxHvGfc7+LgUJcJQhysUkBxE3Ijpzr2PKb4roVS9kLvJEsB+J6zNIm/DckboSp/fA0haZD7hoTOHxrv4n4ulVoohBBCiKTQx4AQQggRc/QxIIQQQsQcxQz8DutArINxnACCuhD/LqTfoYYf0tA5RQ/1LNbz8ThYUtgsd+wBHpf1dFyJEVdJNPPjIvj8OI68ShimCLLWjloi9xP7xnEPeJzQ9Zn5cQGcyok6J67eZuankvKY4jgVBwdjzEAyc0dxpxYWtS0ve1GTaPnjUL8Kii1AbZ7nDpw7+RwYF8BzbihFL1R+Gc/HWj8eh4+Z6sqTIZIZ00TPz+MUSh8MnV8xA0IIIYQIoo8BIYQQIuZIJvgdThljNzK63zl9EN04XJ0Qf8vuHkxt4RRBdFWzZIF949uHKTlbt271bOzux9RCThFEuYMrbmEaIlb/MvMrBLILC4/DNk5tRPAa2SWJ/eRj8j3EfXlFRdyXxwndl7xfaFW4ouBQlwnEwUcy6YqJptql6qYv6Dip9KWgY6bat6JGqYVCCCGESAp9DAghhBAxRx8DQgghRMxRzIAQBxEHY8xAaV61sLhXQiyOtMNUV7xL9JjJ7pvKMZM5X6pxAaF0xWRWkEx1TEOkeg+VWiiEEEKIlNHHgBBCCBFzyhX8EyGEKB6SSTcrrbYQxVGNMFEZprCOWVjHKeq+FZa7PdXzJ0Oq43QgqyTKMyCEEELEHH0MCCGEEDFHHwNCCCFEzFHMgBBCxITiTi0srJiM0rTyZGGla4YoitTCgpBnQAghhIg5+hgQQgghYo5kAiGEiAnFnVpYGClyJW070H1TQamFQgghhCh29DEghBBCxBx9DAghhBAxRzEDQogSoyhWwxOFT2GtMFjUxywsimN1yRBKLRRCCCFEsaOPASGEECLmSCYQQhQrheECDbmNC8vFXBzV+hJ16xZHRbxU9yuM6ysqUn1mUj0Hk+o5E00RLMzxlWdACCGEiDn6GBBCCCFijj4GhBBCiJijmAEhRImRnZ3tbZct+3//f1Ka0tmKasW7RPcrLIp6hb2Cjnsg5XITOV9RpSsWxj0srP2UWiiEEEKIIkEfA0IIIUTMkUwghChSQq5MlAXYxshWOmyJ2IuzP7Ilb8sLeQaEEEKImKOPASGEECLm6GNACCGEiDllopKuFymEEEKIEkWeASGEECLm6GNACCGEiDn6GBBCCCFijj4GhBBCiJijjwEhhBAi5uhjQAghhIg5+hgQQgghYo4+BoQQQoiYo48BIYQQIub8f9rlpkK3+6ZoAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "fig, ax = plt.subplots(nrows=1, ncols=2)\n", + "ax[0].imshow(images[0, 0].detach().cpu(), vmin=0, vmax=1, cmap=\"gray\")\n", + "ax[0].axis(\"off\")\n", + "ax[0].title.set_text(\"Inputted Image\")\n", + "ax[1].imshow(reconstruction[0, 0].detach().cpu(), vmin=0, vmax=1, cmap=\"gray\")\n", + "ax[1].axis(\"off\")\n", + "ax[1].title.set_text(\"Reconstruction\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "773f5f43", + "metadata": {}, + "source": [ + "# Autoregressive Transformer\n", + "\n", + "Now that our VQ-VAE model has been trained, we can use this model to encode the data into its discrete latent representations. Then, to be able to input it into the autoregressive Transformer, it is necessary to transform this 2D latent representation into a 1D sequence.\n", + "\n", + "In order to train it in an autoregressive manner, we will use the CrossEntropy Loss as the Transformer will try to predict the next token value for each position of the sequence.\n", + "\n", + "Here we will use the MONAI's `VQVAETransformerInferer` class to help with the forward pass and to get the predicted likelihood from the VQ-VAE + Transformer models." + ] + }, + { + "cell_type": "markdown", + "id": "83352d19", + "metadata": {}, + "source": [ + "### Datasets\n", + "To train the transformer, we only use the `HeadCT` class." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "2b3c3a82", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading dataset: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 47164/47164 [00:14<00:00, 3317.48it/s]\n", + "Loading dataset: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5895/5895 [00:01<00:00, 3413.76it/s]\n" + ] + } + ], + "source": [ + "train_data = MedNISTDataset(root_dir=root_dir, section=\"training\", seed=0)\n", + "train_datalist = [{\"image\": item[\"image\"]} for item in train_data.data if item[\"class_name\"] == \"HeadCT\"]\n", + "train_transforms = transforms.Compose(\n", + " [\n", + " transforms.LoadImaged(keys=[\"image\"]),\n", + " transforms.EnsureChannelFirstd(keys=[\"image\"]),\n", + " transforms.ScaleIntensityRanged(keys=[\"image\"], a_min=0.0, a_max=255.0, b_min=0.0, b_max=1.0, clip=True),\n", + " transforms.RandAffined(\n", + " keys=[\"image\"],\n", + " rotate_range=[(-np.pi / 36, np.pi / 36), (-np.pi / 36, np.pi / 36)],\n", + " translate_range=[(-1, 1), (-1, 1)],\n", + " scale_range=[(-0.01, 0.01), (-0.01, 0.01)],\n", + " spatial_size=[64, 64],\n", + " padding_mode=\"zeros\",\n", + " prob=0.5,\n", + " ),\n", + " ]\n", + ")\n", + "train_ds = Dataset(data=train_datalist, transform=train_transforms)\n", + "train_loader = DataLoader(train_ds, batch_size=32, shuffle=True, num_workers=4, persistent_workers=True)\n", + "\n", + "val_data = MedNISTDataset(root_dir=root_dir, section=\"validation\", seed=0)\n", + "val_datalist = [{\"image\": item[\"image\"]} for item in val_data.data if item[\"class_name\"] == \"HeadCT\"]\n", + "val_transforms = transforms.Compose(\n", + " [\n", + " transforms.LoadImaged(keys=[\"image\"]),\n", + " transforms.EnsureChannelFirstd(keys=[\"image\"]),\n", + " transforms.ScaleIntensityRanged(keys=[\"image\"], a_min=0.0, a_max=255.0, b_min=0.0, b_max=1.0, clip=True),\n", + " ]\n", + ")\n", + "val_ds = Dataset(data=val_datalist, transform=val_transforms)\n", + "val_loader = DataLoader(val_ds, batch_size=32, shuffle=False, num_workers=4, persistent_workers=True)" + ] + }, + { + "cell_type": "markdown", + "id": "b0f5a3cd", + "metadata": {}, + "source": [ + "### 2D latent representation -> 1D sequence\n", + "We need to define an ordering of which we convert our 2D latent space into a 1D sequence. For this we will use a simple raster scan." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "efab0cc5", + "metadata": {}, + "outputs": [], + "source": [ + "spatial_shape = next(iter(train_loader))[\"image\"].shape[2:]" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "f91086e3", + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "# Get spatial dimensions of data\n", + "# We divide the spatial shape by 4 as the vqvae downsamples the image by a factor of 4 along each dimension\n", + "spatial_shape = next(iter(train_loader))[\"image\"].shape[2:]\n", + "spatial_shape = (int(spatial_shape[0] / 4), int(spatial_shape[1] / 4))\n", + "\n", + "ordering = Ordering(ordering_type=OrderingType.RASTER_SCAN.value, spatial_dims=2, dimensions=(1,) + spatial_shape)" + ] + }, + { + "cell_type": "markdown", + "id": "ace09890", + "metadata": {}, + "source": [ + "### Define network, inferer, optimizer and loss function" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "aab1891a", + "metadata": {}, + "outputs": [], + "source": [ + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "\n", + "transformer_model = DecoderOnlyTransformer(\n", + " num_tokens=16 + 1,\n", + " max_seq_len=spatial_shape[0] * spatial_shape[1],\n", + " attn_layers_dim=128,\n", + " attn_layers_depth=16,\n", + " attn_layers_heads=12,\n", + ")\n", + "transformer_model.to(device)\n", + "\n", + "inferer = VQVAETransformerInferer()" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "fa3cd231", + "metadata": {}, + "outputs": [], + "source": [ + "optimizer = torch.optim.Adam(params=transformer_model.parameters(), lr=1e-4)\n", + "ce_loss = CrossEntropyLoss()" + ] + }, + { + "cell_type": "markdown", + "id": "0921fcfb", + "metadata": {}, + "source": [ + "### Transformer Training\n", + "We will train the Transformer for 5 epochs." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "9c32f0a9", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Epoch 0: 100%|███████████████████████████████████████████████| 250/250 [00:52<00:00, 4.78it/s, ce_loss=0.222]\n", + "Epoch 1: 100%|█████████████████████████████████████████████| 250/250 [00:52<00:00, 4.72it/s, ce_loss=0.00988]\n", + "Epoch 2: 100%|█████████████████████████████████████████████| 250/250 [00:53<00:00, 4.70it/s, ce_loss=0.00582]\n", + "Epoch 3: 100%|█████████████████████████████████████████████| 250/250 [00:53<00:00, 4.71it/s, ce_loss=0.00385]\n", + "Epoch 4: 100%|█████████████████████████████████████████████| 250/250 [00:53<00:00, 4.70it/s, ce_loss=0.00271]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "train completed, total time: 270.7773208618164.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "n_epochs = 5\n", + "val_interval = 2\n", + "epoch_losses = []\n", + "val_epoch_losses = []\n", + "vqvae_model.eval()\n", + "\n", + "total_start = time.time()\n", + "for epoch in range(n_epochs):\n", + " transformer_model.train()\n", + " epoch_loss = 0\n", + " progress_bar = tqdm(enumerate(train_loader), total=len(train_loader), ncols=110)\n", + " progress_bar.set_description(f\"Epoch {epoch}\")\n", + " for step, batch in progress_bar:\n", + "\n", + " images = batch[\"image\"].to(device)\n", + "\n", + " optimizer.zero_grad(set_to_none=True)\n", + "\n", + " logits, quantizations_target, _ = inferer(images, vqvae_model, transformer_model, ordering, return_latent=True)\n", + " logits = logits.transpose(1, 2)\n", + "\n", + " loss = ce_loss(logits, quantizations_target)\n", + "\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " epoch_loss += loss.item()\n", + "\n", + " progress_bar.set_postfix({\"ce_loss\": epoch_loss / (step + 1)})\n", + " epoch_losses.append(epoch_loss / (step + 1))\n", + "\n", + " if (epoch + 1) % val_interval == 0:\n", + " transformer_model.eval()\n", + " val_loss = 0\n", + " with torch.no_grad():\n", + " for val_step, batch in enumerate(val_loader, start=1):\n", + "\n", + " images = batch[\"image\"].to(device)\n", + "\n", + " logits, quantizations_target, _ = inferer(\n", + " images, vqvae_model, transformer_model, ordering, return_latent=True\n", + " )\n", + " logits = logits.transpose(1, 2)\n", + "\n", + " loss = ce_loss(logits, quantizations_target)\n", + "\n", + " val_loss += loss.item()\n", + "\n", + " val_loss /= val_step\n", + " val_epoch_losses.append(val_loss)\n", + "\n", + "total_time = time.time() - total_start\n", + "print(f\"train completed, total time: {total_time}.\")" + ] + }, + { + "cell_type": "markdown", + "id": "29a35d4b", + "metadata": {}, + "source": [ + "## Image-wise anomaly detection\n", + "\n", + "To verify the performance of the VQ-VAE + Transformer performing unsupervised anomaly detection, we will use the images from the test set of the MedNIST dataset. We will consider images from the `HeadCT` class as in-distribution images." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "aa3938fe", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-03-12 00:16:51,478 - INFO - Verified 'MedNIST.tar.gz', md5: 0bc7306e7427e00ad1c5526a6677552d.\n", + "2023-03-12 00:16:51,479 - INFO - File exists: /tmp/tmp83zh5r1m/MedNIST.tar.gz, skipped downloading.\n", + "2023-03-12 00:16:51,480 - INFO - Non-empty folder exists in /tmp/tmp83zh5r1m/MedNIST, skipped extracting.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading dataset: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 5895/5895 [00:01<00:00, 3258.71it/s]\n", + "In-distribution data: 100%|███████████████████████████████████████████████████| 17/17 [00:05<00:00, 3.22it/s]\n" + ] + } + ], + "source": [ + "vqvae_model.eval()\n", + "transformer_model.eval()\n", + "\n", + "test_data = MedNISTDataset(root_dir=root_dir, section=\"test\", download=True, seed=0)\n", + "\n", + "in_distribution_datalist = [{\"image\": item[\"image\"]} for item in test_data.data if item[\"class_name\"] == \"HeadCT\"]\n", + "in_distribution_ds = Dataset(data=in_distribution_datalist, transform=val_transforms)\n", + "in_distribution_loader = DataLoader(\n", + " in_distribution_ds, batch_size=64, shuffle=False, num_workers=4, persistent_workers=True\n", + ")\n", + "\n", + "in_likelihoods = []\n", + "\n", + "progress_bar = tqdm(enumerate(in_distribution_loader), total=len(in_distribution_loader), ncols=110)\n", + "progress_bar.set_description(f\"In-distribution data\")\n", + "for step, batch in progress_bar:\n", + " images = batch[\"image\"].to(device)\n", + "\n", + " log_likelihood = inferer.get_likelihood(\n", + " inputs=images, vqvae_model=vqvae_model, transformer_model=transformer_model, ordering=ordering\n", + " )\n", + " in_likelihoods.append(log_likelihood.sum(dim=(1, 2)).cpu().numpy())\n", + "\n", + "in_likelihoods = np.concatenate(in_likelihoods)" + ] + }, + { + "cell_type": "markdown", + "id": "19541717", + "metadata": {}, + "source": [ + "We will use the \"ChestCT\" class of the dataset for the out-of-distribution examples." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "f3e714ee", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "out-of-distribution data: 100%|███████████████████████████████████████████████| 16/16 [00:05<00:00, 3.15it/s]\n" + ] + } + ], + "source": [ + "ood_datalist = [{\"image\": item[\"image\"]} for item in test_data.data if item[\"class_name\"] == \"ChestCT\"]\n", + "ood_ds = Dataset(data=ood_datalist, transform=val_transforms)\n", + "ood_loader = DataLoader(ood_ds, batch_size=64, shuffle=False, num_workers=4, persistent_workers=True)\n", + "\n", + "ood_likelihoods = []\n", + "\n", + "progress_bar = tqdm(enumerate(ood_loader), total=len(ood_loader), ncols=110)\n", + "progress_bar.set_description(f\"out-of-distribution data\")\n", + "for step, batch in progress_bar:\n", + " images = batch[\"image\"].to(device)\n", + "\n", + " log_likelihood = inferer.get_likelihood(\n", + " inputs=images, vqvae_model=vqvae_model, transformer_model=transformer_model, ordering=ordering\n", + " )\n", + " ood_likelihoods.append(log_likelihood.sum(dim=(1, 2)).cpu().numpy())\n", + "\n", + "ood_likelihoods = np.concatenate(ood_likelihoods)" + ] + }, + { + "cell_type": "markdown", + "id": "5aa92638", + "metadata": {}, + "source": [ + "## Log-likehood plot\n", + "\n", + "Here, we plot the log-likelihood of the images. In this case, the lower the log-likelihood, the more unlikely the image belongs to the training set." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "cd456a7c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Text(0.5, 0, 'Log-likelihood')" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAGxCAYAAACeKZf2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABX4UlEQVR4nO3deViVdf7/8edhF5RNZNFwN5dcMBVDW7Qwbd/z1zS5ZDZN2jK0OpW2mVZq9jXLpjKn3WmmbaZGM0YzjbI0zXLJHVM2NxBUQLh/f9ycAygiHA7cZ3k9rutc5+Zs9xuPcF58VpthGAYiIiIiXsLP6gJEREREXEnhRkRERLyKwo2IiIh4FYUbERER8SoKNyIiIuJVFG5ERETEqyjciIiIiFdRuBERERGvEmB1AU2tvLycvXv30qJFC2w2m9XliIiISB0YhsHhw4dp3bo1fn61t834XLjZu3cviYmJVpchIiIiTti9ezdnnHFGrY9xi3Azd+5cnn/+ebKzs+nTpw9z5swhOTm5xscuWLCAsWPHVrstODiYY8eO1elcLVq0AMx/nPDw8IYVLiIiIk2ioKCAxMREx+d4bSwPNwsXLiQtLY158+YxcOBAZs+ezfDhw9m8eTOxsbE1Pic8PJzNmzc7vq5P95L9seHh4Qo3IiIiHqYun/mWDyieNWsW48ePZ+zYsfTo0YN58+YRGhrK/PnzT/kcm81GfHy84xIXF9eEFYuIiIg7szTclJSUsHr1alJTUx23+fn5kZqaSkZGximfV1hYSLt27UhMTOSqq67i119/PeVji4uLKSgoqHYRERER72VpuNm3bx9lZWUntbzExcWRnZ1d43O6du3K/Pnz+fTTT3nnnXcoLy9n0KBB/P777zU+ftq0aURERDguGkwsIiLi3Swfc1NfKSkppKSkOL4eNGgQ3bt359VXX+Wpp5466fGTJk0iLS3N8bV9QNLplJWVUVpa6pqixacEBgbi7+9vdRkiIj7L0nATExODv78/OTk51W7PyckhPj6+Tq8RGBhI37592bp1a433BwcHExwcXOeaDMMgOzubQ4cO1fk5IieKjIwkPj5eaymJiFjA0nATFBREv379SE9P5+qrrwbMRfbS09OZOHFinV6jrKyM9evXc+mll7qkJnuwiY2NJTQ0VB9OUi+GYXDkyBFyc3MBSEhIsLgiERHfY3m3VFpaGqNHj6Z///4kJycze/ZsioqKHGvZjBo1ijZt2jBt2jQAnnzySc455xw6d+7MoUOHeP7559m1axe33XZbg2spKytzBJuWLVs2+PXENzVr1gyA3NxcYmNj1UUlItLELA83I0eOJC8vj8mTJ5OdnU1SUhKLFi1yDDLOzMystszywYMHGT9+PNnZ2URFRdGvXz++/fZbevTo0eBa7GNsQkNDG/xa4tvs/4dKS0sVbkREmpjNMAzD6iKaUkFBAREREeTn55+0iN+xY8fYsWMHHTp0ICQkxKIKxRvo/5KIiGvV9vl9IssX8RMRERFxJYUbERER8SoKN3JKO3fuxGazsXbtWgCWLVuGzWZrlGnyNpuNTz75pMbzNua5RETE+yjceIkxY8Y4ptM3lkGDBpGVlUVERMRpH1vfIJSVlcUll1zSwAqre/zxx0lKSmqSc4mIiPtQuJE6CwoKcvnCdCUlJQDEx8fXa7HFhmjKc4mInOhIKfxvB2TmW12J91K4OQ3DMP8jWnFxdh7bkCFDuPvuu3nwwQeJjo4mPj6exx9//LTPW7VqFX379iUkJIT+/fvz008/Vbv/xNaYXbt2ccUVVxAVFUVYWBhnnXUWX3zxBTt37mTo0KEAREVFYbPZGDNmjKO2iRMncu+99xITE8Pw4cOBmruKNm3axKBBgwgJCaFnz558/fXXjvsWLFhAZGRktcd/8sknjuC1YMECnnjiCdatW4fNZsNms7FgwYIaz7V+/XouvPBCmjVrRsuWLbn99tspLCx03G9vFZsxYwYJCQm0bNmSCRMmaHsOEam33CIY8S6M/QyGvgWfb7G6Iu9k+To37u7ocej+sjXn3ngnhAY699y///3vpKWl8f3335ORkcGYMWMYPHgww4YNq/HxhYWFXH755QwbNox33nmHHTt2cM8999R6jgkTJlBSUsLy5csJCwtjw4YNNG/enMTERP71r39x3XXXsXnzZsLDwx0L29lr+/Of/8zKlStrff0HHniA2bNn06NHD2bNmsUVV1zBjh076rTA4siRI/nll19YtGgRX331FUCN3WlFRUUMHz6clJQUfvjhB3Jzc7ntttuYOHGiIwwBLF26lISEBJYuXcrWrVsZOXIkSUlJjB8//rS1iIjYPboUdlW02BwvhweWwMA2EKPl1VxKLTdeqnfv3kyZMoUuXbowatQo+vfvT3p6+ikf/95771FeXs4bb7zBWWedxeWXX84DDzxQ6zkyMzMZPHgwvXr1omPHjlx++eWcf/75+Pv7Ex0dDUBsbCzx8fHVgkWXLl147rnn6Nq1K127dj3l60+cOJHrrruO7t2788orrxAREcEbb7xRp++/WbNmNG/enICAAOLj44mPj68WsKp+38eOHeOtt96iZ8+eXHjhhbz00ku8/fbb1fY8i4qK4qWXXqJbt25cfvnlXHbZZbX+e4qInGjTPli8DWzAf/8AvWOhqBReX2N1Zd5HLTen0SzAbEGx6tzO6t27d7WvExISHPsd3XHHHbzzzjuO+woLC9m4cSO9e/eutuBc1d3Xa3L33Xfz5z//mS+//JLU1FSuu+66k85bk379+tXpe6h6/oCAAPr378/GjRvr9Ny62rhxI3369CEsLMxx2+DBgykvL2fz5s2OlbLPOuusaisNJyQksH79epfWIiLebeGv5vWIztCjFUxMhtv/A//cCPelQKAWM3cZtdychs1mdg1ZcWnIuN3AwOr9WTabjfLycsDcn2vt2rWOi7Nuu+02tm/fzi233ML69evp378/c+bMOe3zqgYJZ/n5+XHi4tqNOQamtn9PEZHTMQz4T8X4mhsqdgu6sD1EN4O8I/BjlmWleSWFGx8UGxtL586dHReA7t278/PPP3Ps2DHH47777rvTvlZiYiJ33HEHH330Effddx+vvfYaYM6sAnMzUmdVPf/x48dZvXo13bt3B6BVq1YcPnyYoqIix2NODGpBQUGnPX/37t1Zt25dtddZuXIlfn5+tXaZiYjUx2/7zcHEIQFwbqJ5W6A/nN/WPF6+y7ravJHCjQDwhz/8AZvNxvjx49mwYQNffPEFM2bMqPU59957L4sXL2bHjh2sWbOGpUuXOsJHu3btsNls/Oc//yEvL6/a7KO6mjt3Lh9//DGbNm1iwoQJHDx4kFtvvRWAgQMHEhoayl//+le2bdvGe++9V20AMED79u3ZsWMHa9euZd++fRQXF590jptvvpmQkBBGjx7NL7/8wtKlS7nrrru45ZZbHF1SIiINtTzTvB7YBoKrDDm4oJ15/bXCjUsp3AgAzZs359///jfr16+nb9++PPLIIzz77LO1PqesrIwJEybQvXt3RowYwZlnnsnLL5tTy9q0acMTTzzBww8/TFxcHBMnTqx3TdOnT2f69On06dOHFStW8NlnnxETEwNAdHQ077zzDl988QW9evXi/fffP2m6+3XXXceIESMYOnQorVq14v333z/pHKGhoSxevJgDBw4wYMAArr/+ei666CJeeumletcrInIq31SEG3tLjd15FV//mgd5RYiLaFfwKrSTs7iK/i+JiN3xcuj5irm0yOKboVtM9ftHvAsb98Erl8KlXayp0RNoV3ARERE3sfWAGWyaB8GZNSzTlRRvXv+cc/J94hyFGxERkUZkDy09Y8GvhlmwfSqG9/2c23Q1eTuFGxERkUZkDy29Y2u+v3dFuFmfA+U+NVCk8SjciIiINKL1FS03vU8xAfPMaAj2h4IS2HWoycryago3IiIijaS0zBwsDKduuQn0N1csBlivrimXULgRERFpJDsOQXEZhAVC25P37nXoXjGD6rcDTVKW11O4ERERaSRbK8JK5+jat9TpYu41zJb9jV+TL1C4ERERaSRbKsKNPbyciiPcqOXGJRRuREREGsnWuoabivVvdh6C4uONWpJPULjxIrt37+bWW2+ldevWBAUF0a5dO+655x7276/ezvnrr79y44030qpVK4KDgznzzDOZPHkyR44cqfa49u3bY7PZsNlsNGvWjPbt23PjjTfyv//9rym/LRERj1XXlpu4MGgRBGWGGXCkYRRuvMT27dvp378/W7Zs4f3332fr1q3MmzeP9PR0UlJSOHDA/An77rvvGDhwICUlJXz++ef89ttvTJ06lQULFjBs2DBKSkqqve6TTz5JVlYWmzdv5q233iIyMpLU1FSmTp1qxbcpIuIxysph+0HzuEsNKxNXZbNVPkZdUw0XcPqHiCeYMGECQUFBfPnllzRr1gyAtm3b0rdvXzp16sQjjzzCyy+/zLhx4+jevTsfffQRfn5mtm3Xrh1nnnkmffv25YUXXuChhx5yvG6LFi2Ij493vN75559PQkICkydP5vrrr6dr165N/82KiHiA3wvMmVLB/tCmxekf3ykK1mTBtoONX5u3U8vN6RgGFJVac6njnqYHDhxg8eLF3HnnnY5gYxcfH8/NN9/MwoULWbt2LRs2bCAtLc0RbOz69OlDampqjTtnn+iee+7BMAw+/fTTuv87ioj4mF355nW7SPCvw6dth8jqzxPnqeXmdI4ch/avWnPunX8yF0c4jS1btmAYBt27d6/x/u7du3Pw4EF+++03x9enetyKFStOe77o6GhiY2PZuXPnaR8rIuKrMitCStvaN7B2aFexDo7G3DScWm68iFHHlp66Pu50r2GrbdEGEREfl1lgXte2eF9V7SLNa23B0HBquTmd0ACzBcWqc9dB586dsdlsbNy4kWuuueak+zdu3EhUVBRnnnmm4+u+ffvW+Dj7Y2qzf/9+8vLy6NChQ53qExHxRY6Wm7qGm4rH7TsKhSXQPKhx6vIFark5HZvN7Bqy4lLHlpGWLVsybNgwXn75ZY4ePVrtvuzsbN59911GjhxJUlIS3bp144UXXqC8vLza49atW8dXX33FTTfddNrzvfjii/j5+XH11VfX+Z9RRMTX1LdbKjwYoiuGTWrcTcMo3HiJl156ieLiYoYPH87y5cvZvXs3ixYtYtiwYbRp04apU6dis9l444032LBhA9dddx2rVq0iMzOTDz/8kCuuuIKUlBTuvffeaq97+PBhsrOz2b17N8uXL+f222/n6aefZurUqXTu3Nmab1ZExM0ZRmW4Saxjyw1o3I2rKNx4iS5duvDjjz/SsWNHbrzxRjp16sTtt9/O0KFDycjIIDraXEFq0KBBfPfdd/j7+3PJJZfQuXNnJk2axOjRo1myZAnBwcHVXnfy5MkkJCTQuXNnbrnlFvLz80lPT682XVxERKrLL4bDFcuGJdax5QagfaR5rXE3DaMxN16kXbt2LFiw4LSP69WrF//85z9P+zjNhhIRcY691SY2DJqdftKrg70L6/cC19fkS9RyIyIi4mL1HW9j17pisb89h11bj69RuBEREXGxzCoL+NVHG4Ubl1C4ERERcTHHGjf1bLlpU/H4vYfrvEi91EDhRkRExMXqu8aNnb1bqqgUCopdW5MvUbipgStW8BXfpv9DIr5tT0XLTZt6ttyEBEDLirVu1DXlPIWbKgIDzSHtR44csbgS8XT2/0P2/1Mi4jsMA7IKzeOE5vV/vr31Zq/CjdM0FbwKf39/IiMjyc3NBSA0NFT7J0m9GIbBkSNHyM3NJTIyEn9/f6tLEpEmdvAYFJeZx3Fh9X9+6xawPhd+V7hxmsLNCeLj4wEcAUfEGZGRkY7/SyLiW+wtLq1CIdiJT9kz1HLTYAo3J7DZbCQkJBAbG0tpaanV5YgHCgwMVIuNiA/LruiSineiSwrULeUKCjen4O/vrw8oERGpN3sosYeU+tJCfg2nAcUiIiIu1NCWmzZquWkwhRsREREX2lsRblo7G24qpo/nFEJJmWtq8jUKNyIiIi6U1cCWm5bNINgfDCpbgaR+FG5ERERcKKuiOynByTE3NltlMFK4cY7CjYiIiIsYRmUgcWYBPzt7uMkpanhNvkjhRkRExEUOHG3YAn529ufmqOXGKQo3IiIiLmIfb+PsAn52sfZwo5YbpyjciIiIuEhDBxPbxalbqkEUbkRERFzEMZi4oeFG3VINonAjIiLiIvaWlrgGhhsNKG4YhRsREREXya0II7ENGEwMVVpuiswZWFI/CjciIiIu4gg3oQ17HXs4OlIKhSUNey1fpHAjIiLiIrlHzOuGttyEBkJ4kHmsrqn6U7gRERFxkTwXdUsBxGrcjdMUbkRERFzgeDnsc1HLDWjGVEMo3IiIiLjA/iPmZpd+NnPzy4aK00J+TlO4ERERcQH7YOKYUPB3waerpoM7T+FGRETEBVw1DdwuVt1STnOLcDN37lzat29PSEgIAwcOZNWqVXV63gcffIDNZuPqq69u3AJFREROwzFTqoHTwO3ULeU8y8PNwoULSUtLY8qUKaxZs4Y+ffowfPhwcnNza33ezp07uf/++znvvPOaqFIREZFTa7SWG4WberM83MyaNYvx48czduxYevTowbx58wgNDWX+/PmnfE5ZWRk333wzTzzxBB07dmzCakVERGrm6nDTqqIFaN8RrVJcX5aGm5KSElavXk1qaqrjNj8/P1JTU8nIyDjl85588kliY2MZN27cac9RXFxMQUFBtYuIiIiruTrcxFSEm2PHoajUNa/pKywNN/v27aOsrIy4uLhqt8fFxZGdnV3jc1asWMEbb7zBa6+9VqdzTJs2jYiICMclMTGxwXWLiIicyNXhJizIXKkYKtfPkbqxvFuqPg4fPswtt9zCa6+9RkxMTJ2eM2nSJPLz8x2X3bt3N3KVIiLii1wdbqCy9SZP4aZeAqw8eUxMDP7+/uTk5FS7PScnh/j4+JMev23bNnbu3MkVV1zhuK28vByAgIAANm/eTKdOnao9Jzg4mODg4EaoXkRExGQYlQHEVbOlwAw3mflquakvS1tugoKC6NevH+np6Y7bysvLSU9PJyUl5aTHd+vWjfXr17N27VrH5corr2To0KGsXbtWXU4iImKJg8eg1Pxbm1YubLmxDyrO04yperG05QYgLS2N0aNH079/f5KTk5k9ezZFRUWMHTsWgFGjRtGmTRumTZtGSEgIPXv2rPb8yMhIgJNuFxERaSr2lpXIEAjyd93rxjSr/vpSN5aHm5EjR5KXl8fkyZPJzs4mKSmJRYsWOQYZZ2Zm4ufnUUODRETEx9jDhyv2lKrK3gqkcFM/locbgIkTJzJx4sQa71u2bFmtz12wYIHrCxIREakHe/ho5cLxNlA5oHjfUde+rrdTk4iIiEgD7a8IHy0bKdxotlT9KNyIiIg0kD18xDRWuNGA4npRuBEREWmg/Y005ia2yhYMUncKNyIiIg3U2GNujh6HohLXvrY3U7gRERFpoH2N1C0VFgTNAqqfQ05P4UZERKSB9jXSgGLQoGJnKNyIiIg0gGE0XrcUVJkOrnBTZwo3IiIiDXCkFI4dN49dPaAYKjfiVMtN3SnciIiINIC9RaVZgDlGxtW0BUP9KdyIiIg0QGOOtwF1SzlD4UZERKQBGnO8DWhAsTMUbkRERBqgsTbNtFPLTf0p3IiIiDRAY61xY9dKA4rrTeFGRESkARwtN40VbtRyU28KNyIiIg1g3xG8scfcHCk1L3J6CjciIiIN0NjdUmGBEOxvHtuDlNRO4UZERKQBGntAsc0G0RWvfUBdU3WicCMiItIAjd1yA5XhRi03daNwIyIi4qSSMsgvNo8bM9zYX/uAwk2dKNyIiIg4yR42/G0QGdJ451HLTf0o3IiIiDjJvvZMdDPwszXeeRRu6kfhRkRExEn7m2C8DVQOVtaA4rpRuBEREXGSvVuqsWZK2anlpn4UbkRERJxkDxvRjRxuHC03Cjd1onAjIiLipINNFW4qur3UclM3CjciIiJOauqWG4WbulG4ERERcdKBJgo39tc/UgrHjjfuubyBwo2IiIiTmqrlpkUQBFZ8Yu/XjKnTUrgRERFx0sEmmi1VbX8pdU2dlsKNiIiIk5qq5QY07qY+FG5EREScUFplX6kmCTeaMVVnCjciIiJOOHTMvLYBUY24r5SdFvKrO4UbERERJ9jHvkSGgH8TfJpqC4a6U7gRERFxQlOOt6l6HrXcnJ7CjYiIiBOaao0bO23BUHcKNyIiIk5o6nCjlpu6U7gRERFxQlPtCG6n2VJ1p3AjIiLiBHvIiFK3lNtRuBEREXFCk7fcVJynsASKtb9UrRRuREREnNDUY27CgyHAr/q5pWYKNyIiIk5o6pYbm61yscB9Cje1UrgRERFxQlOPuQGNu6krhRsREZF6Mgw4WLH9QlO13IBmTNWVwo2IiEg9FZTA8XLzuKnG3IC2YKgrhRsREZF6soeLsEAICWi680arW6pOFG5ERETqyYrxNlAZbjSguHYKNyIiIvVkxXibqudTy03tFG5ERETqaX9Ft1RTjrepej4NKK6dwo2IiEg9NfUaN3b22VIaUFw7hRsREZF6snrMzYFjTXteT6NwIyIiUk9Wj7kpKIbSsqY9tydRuBEREaknq8bcRASDn808VuvNqSnciIiI1FNTb5pp5+8HkRX7S2nczakp3IiIiNSTvdWkqcNN1XNqxtSpKdyIiIjUk1Wzpaqe86C6pU5J4UZERKQejh2HI6XmsVpu3JPCjYiISD3YBxMH+kGLoKY/f7TG3JyWwo2IiEg92MfbRDUDm63pzx9dsZCfWm5OTeFGRESkHqwcb1P1vBpzc2oKNyIiIvWw36Jp4HYac3N6CjciIiL1cNDicKOdwU9P4UZERKQerG65se9npQHFp+YW4Wbu3Lm0b9+ekJAQBg4cyKpVq0752I8++oj+/fsTGRlJWFgYSUlJvP32201YrYiI+DJ3GnNTblhTg7uzPNwsXLiQtLQ0pkyZwpo1a+jTpw/Dhw8nNze3xsdHR0fzyCOPkJGRwc8//8zYsWMZO3YsixcvbuLKRUTEF9nDTVSINee3n7fMMDfQlJNZHm5mzZrF+PHjGTt2LD169GDevHmEhoYyf/78Gh8/ZMgQrrnmGrp3706nTp2455576N27NytWrGjiykVExBc5Wm5CrTl/cEDl+joaVFwzS8NNSUkJq1evJjU11XGbn58fqampZGRknPb5hmGQnp7O5s2bOf/882t8THFxMQUFBdUuIiIizrJq08yqNO6mdpaGm3379lFWVkZcXFy12+Pi4sjOzj7l8/Lz82nevDlBQUFcdtllzJkzh2HDhtX42GnTphEREeG4JCYmuvR7EBER37Lf4jE3Vc+tlpuaWd4t5YwWLVqwdu1afvjhB6ZOnUpaWhrLli2r8bGTJk0iPz/fcdm9e3fTFisiIl7jeDkcsq9QbNGYG6hsNdJCfjULsPLkMTEx+Pv7k5OTU+32nJwc4uPjT/k8Pz8/OnfuDEBSUhIbN25k2rRpDBky5KTHBgcHExwc7NK6RUTENx2qEiaiLGy5cSzkp26pGlnachMUFES/fv1IT0933FZeXk56ejopKSl1fp3y8nKKizVkXEREGpd9vE1kCARY+AmqhfxqZ2nLDUBaWhqjR4+mf//+JCcnM3v2bIqKihg7diwAo0aNok2bNkybNg0wx9D079+fTp06UVxczBdffMHbb7/NK6+8YuW3ISIiPsAxmNjCLimobDXSmJuaWR5uRo4cSV5eHpMnTyY7O5ukpCQWLVrkGGScmZmJn19lPC4qKuLOO+/k999/p1mzZnTr1o133nmHkSNHWvUtiIiIj7B6dWI7bZ5ZO5thGD61vmFBQQERERHk5+cTHh5udTkiIuJB3v4ZHl0KwzvB3y63ro70HXDrZ9AzFj6/ybo6mlJ9Pr89craUiIiIFaxendiupda5qZXCjYiISB1Zva+UXXSVMTe+1f9SNwo3IiIideRouXGTcFNcBkdKra3FHSnciIiI1JG7tNyEBUKwv3l8QIOKT+JUuNm+fbur6xAREXF79iBhdcuNzVbZeqNxNydzKtx07tyZoUOH8s4773DsmCKjiIj4BndpuQGtdVMbp8LNmjVr6N27N2lpacTHx/OnP/2JVatWubo2ERERt2EY7rEjuJ1WKT41p8JNUlISL774Inv37mX+/PlkZWVx7rnn0rNnT2bNmkVeXp6r6xQREbFUUSmUlJnH7hBuHN1S6kA5SYMGFAcEBHDttdfy4Ycf8uyzz7J161buv/9+EhMTGTVqFFlZWa6qU0RExFL2FpKQAAgNtLYW0Fo3tWlQuPnxxx+58847SUhIYNasWdx///1s27aNJUuWsHfvXq666ipX1SkiImIpd9lXyi5aY25Oyam9pWbNmsWbb77J5s2bufTSS3nrrbe49NJLHXtAdejQgQULFtC+fXtX1ioiImIZx75SodbWYRetMTen5FS4eeWVV7j11lsZM2YMCQkJNT4mNjaWN954o0HFiYiIuAu13HgOp8LNkiVLaNu2bbXdugEMw2D37t20bduWoKAgRo8e7ZIiRURErOZOM6Wgys7gCjcncWrMTadOndi3b99Jtx84cIAOHTo0uCgRERF3405r3IC6pWrjVLgxTrFLV2FhISEhbtJeJyIi4kLusq+UnT3cFJRUTlEXU726pdLS0gCw2WxMnjyZ0NDKUVVlZWV8//33JCUlubRAERERd+BuLTeRIeBng3LD7JqKa251Re6jXuHmp59+AsyWm/Xr1xMUFOS4LygoiD59+nD//fe7tkIRERE34C77Stn52SAqxBxQfEDhppp6hZulS5cCMHbsWF588UXCw8MbpSgRERF3Y18sz11absDsmtp/VDOmTuTUbKk333zT1XWIiIi4NXebLQUaVHwqdQ431157LQsWLCA8PJxrr7221sd+9NFHDS5MRETEXZSWmQN3wT3DjVpuqqtzuImIiMBmszmORUREfIV9vI2fDSKCra2lKq11U7M6h5uqXVHqlhIREV9iDw+RIeDfoF0ZXUstNzVz6i06evQoR45UbkO6a9cuZs+ezZdffumywkRERNzFfjccbwMac3MqToWbq666irfeeguAQ4cOkZyczMyZM7nqqqt45ZVXXFqgiIiI1Q662b5Sdgo3NXMq3KxZs4bzzjsPgH/+85/Ex8eza9cu3nrrLf7v//7PpQWKiIhYzV1bblqqW6pGToWbI0eO0KJFCwC+/PJLrr32Wvz8/DjnnHPYtWuXSwsUERGxmrutTmwXrQHFNXIq3HTu3JlPPvmE3bt3s3jxYi6++GIAcnNztbCfiIh4HXfbV8rOMVvqmLkNg5icCjeTJ0/m/vvvp3379gwcOJCUlBTAbMXp27evSwsUERGxmru23ERWjAEqMyD/mLW1uBOnVii+/vrrOffcc8nKyqJPnz6O2y+66CKuueYalxUnIiLiDty15SY4AFoEweESc9yNu9VnFafCDUB8fDzx8fHVbktOTm5wQSIiIu7GXVtuwBx3c7hE426qcircFBUVMX36dNLT08nNzaW8vLza/du3b3dJcSIiIu5gv5u23IAZbnbla8ZUVU6Fm9tuu42vv/6aW265hYSEBMe2DCIiIt7GMMwBu+CeLTcttdbNSZwKN//973/5/PPPGTx4sKvrERERcSsFJXC8ooPC3da5gcrWJLXcVHJqtlRUVBTR0dGurkVERMTt2MeyhAZCiNMjVRuPWm5O5lS4eeqpp5g8eXK1/aVERES8kbuuTmynLRhO5lQGnTlzJtu2bSMuLo727dsTGBhY7f41a9a4pDgRERGrueu+UnZquTmZU+Hm6quvdnEZIiIi7sndW2405uZkToWbKVOmuLoOERERt+RY4ybU2jpORS03J3NqzA3AoUOHeP3115k0aRIHDhwAzO6oPXv2uKw4ERERqzlWJ3bTbqmqY24M7S8FONly8/PPP5OamkpERAQ7d+5k/PjxREdH89FHH5GZmclbb73l6jpFREQs4c6rE0NlXcVlcKQUwoKsrccdONVyk5aWxpgxY9iyZQshIZVR9tJLL2X58uUuK05ERMRq7rqvlF1oIAT7m8cad2NyKtz88MMP/OlPfzrp9jZt2pCdnd3gokRERNyFu7fc2GyaDn4ip8JNcHAwBQUFJ93+22+/0apVqwYXJSIi4i7ceV8pO4Wb6pwKN1deeSVPPvkkpaWlANhsNjIzM3nooYe47rrrXFqgiIiIldx5Xyk7zZiqzqlwM3PmTAoLC2nVqhVHjx7lggsuoHPnzrRo0YKpU6e6ukYRERFLHDsOhSXmsbtOBYfKlhuNuTE5NVsqIiKCJUuWsHLlStatW0dhYSFnn302qamprq5PRETEMvsrdhkK9INwN56FpG6p6uodbsrLy1mwYAEfffQRO3fuxGaz0aFDB+Lj4zEMA5vN1hh1ioiINLn9VRbwc+ePN7XcVFevbinDMLjyyiu57bbb2LNnD7169eKss85i165djBkzhmuuuaax6hQREWly+ypabtx5vA1U1ndQ4QaoZ8vNggULWL58Oenp6QwdOrTaff/73/+4+uqreeuttxg1apRLixQREbGCvSUkxo3H24Babk5Ur5ab999/n7/+9a8nBRuACy+8kIcffph3333XZcWJiIhYab+HtNxozE119Qo3P//8MyNGjDjl/Zdccgnr1q1rcFEiIiLuYJ+bb5ppp3BTXb3CzYEDB4iLizvl/XFxcRw8eLDBRYmIiLgDe8tNjJu33Nhblg6XQEmZtbW4g3qFm7KyMgICTj1Mx9/fn+PHjze4KBEREXew30NabiJCwK9iNpcGFddzQLFhGIwZM4bg4OAa7y8uLnZJUSIiIu7APlvK3QcU+9kgKsQMY/uPQlxzqyuyVr3CzejRo0/7GM2UEhERb+GYLeXm3VJgjrvZf1TjbqCe4ebNN99srDpERETcimFUmS3l5i03YI672YKmg4OTe0uJiIh4u4ISKC03j6M9pOUG1HIDCjciIiI1srfatAiCEKd2YmxaCjeVFG5ERERq4CkL+Nkp3FRSuBEREamBpyzgZ6ctGCop3IiIiNRgv4dMA7fT5pmVFG5ERERq4FjAz8O6pdRy4ybhZu7cubRv356QkBAGDhzIqlWrTvnY1157jfPOO4+oqCiioqJITU2t9fEiIiLO2OdB08ChMoQp3LhBuFm4cCFpaWlMmTKFNWvW0KdPH4YPH05ubm6Nj1+2bBk33XQTS5cuJSMjg8TERC6++GL27NnTxJWLiIg387SWG3v32cGjcLzc2lqsZnm4mTVrFuPHj2fs2LH06NGDefPmERoayvz582t8/Lvvvsudd95JUlIS3bp14/XXX6e8vJz09PQmrlxERLyZp425iW5mbsNgoNYbS8NNSUkJq1evJjU11XGbn58fqampZGRk1Ok1jhw5QmlpKdHR0TXeX1xcTEFBQbWLiIjI6ezzsJYbf7/KWvOKrK3FapaGm3379lFWVkZcXFy12+Pi4sjOzq7Tazz00EO0bt26WkCqatq0aURERDguiYmJDa5bRES8n6e13EBlrfbxQr7K8m6phpg+fToffPABH3/8MSEhITU+ZtKkSeTn5zsuu3fvbuIqRUTE05SWwcFj5rGntNyAwo2dpQtKx8TE4O/vT05OTrXbc3JyiI+Pr/W5M2bMYPr06Xz11Vf07t37lI8LDg4mODjYJfWKiIhvOFARbPxsEFnz385uqVVFuMnz8XBjactNUFAQ/fr1qzYY2D44OCUl5ZTPe+6553jqqadYtGgR/fv3b4pSRUTEh9i7pKJDzLEsniJG4QawuOUGIC0tjdGjR9O/f3+Sk5OZPXs2RUVFjB07FoBRo0bRpk0bpk2bBsCzzz7L5MmTee+992jfvr1jbE7z5s1p3ry5Zd+HiIh4j/0etsaNnbqlTJaHm5EjR5KXl8fkyZPJzs4mKSmJRYsWOQYZZ2Zm4udXGZtfeeUVSkpKuP7666u9zpQpU3j88cebsnQREfFSnjZTyq5VmHmtcOMGJk6cyMSJE2u8b9myZdW+3rlzZ+MXJCIiPs0TZ0oBtLJPBffxcONBPYkiIiJNY7+H7Qhup24pk8KNiIjICezhIMZDu6UOHjWns/sqhRsREZETeGrLTVRI5RYMB3x4CwaFGxERkRPYty+I9bBwU20LBh/umlK4EREROUFuRTCwd/N4klYad6NwIyIiUlW5URkMWnlYyw1oIT9QuBEREanm0DE4Xm4ee9qYG9CMKVC4ERERqcY+3iYqBIL8ra3FGfauNLXciIiICFAZCmI9cLwNqOUGFG5ERESqyfXg8TZQZcxNkbV1WEnhRkREpIrcilDgiTOloHL6+j6tcyMiIiJQ2eLh6S036pYSERERoHLMjae23NjDzQEf3oJB4UZERKQKT12d2M6+BQP47hYMCjciIiJVeHrLjbZgULgRERGpJs/DZ0uBtmBQuBEREalQfNxcoRg8d50b0BYMCjciIiIV7C0dQf4QEWxtLQ1h71JTy42IiIiPq9olZbNZW0tDqOVGREREAO8YbwOV9fvqKsUKNyIiIhXyPHx1Yru4ivpzFW5ERER8m7e03MQ1N69zFG5ERER8W66Hb71gZ2+5ySkCw7C2Fiso3IiIiFTw9AX87OzT2I+UQmGJtbVYQeFGRESkgr3lxpPXuAEIDYTwIPM42we7phRuREREKthbbmI8vFsKINY+7qbQ2jqsoHAjIiKCOTbF3nIT7+EtN1D5PfjijCmFGxEREcwdtEvKwIbnd0uBb8+YUrgRERGhcmxKTCgE+ltbiytUnTHlaxRuREREgOyKsSn2Fg9PZ2990pgbERERH2UPAXFe0CUFarkRERHxefaWG28YTAwacyMiIuLz7GNuvKVbqur+Ur62SnGA1QWIiDSZ4jL43y5YsQfWZMPeIjhUDM38IboZdI2Gs+NgWDvz2GazumJpQo6WGy8JN/YtJErK4NAxiGpmbT1NSeFGRLzfvqPw8k/w7gY4cOzk+4+Uwv5jsOUg/GcbPPktdIuG0T3h/3WD5kFNX7M0uRwvCzfBAWZmP3DU7JpSuBER8QalZfDyWpj9IxSWmrclhMGIjnBOArSPgMhg80/b7CLYsB+W/w4rfodNB2DScpixCiaeDeN6QzP9yvRm2V60gJ9dXJgZbrILoVuM1dU0Hf2kioh3+u0ATPgK1uaaX/duBQ8kQ2o7CKhhuGG3ljCkLdzZF/KL4R+b4LWfYUc+PPEtvLkenj4PLunYtN+HNIljx82uG/Celhsww83Gfb43qFgDikXE+/x7Kwz70Aw2EcHwUiosuRFGdKg52JwoIhjG94Fvb4YXLzRbezIPw6gv4E+LKz8FxWvYu6RCAiA82NpaXMlXZ0wp3IiI9zAMmPkD3LrIHEdz3hmw4iYY2Q38nBgcHOAHf+hhhpy7zgZ/G3y0Bc5/H77e7fr6xTJVu6S8aRx5nI8u5KdwIyLeodyAh5fD9O/Nr+/oA/+40jV9DM2DYPIg+Pw66BgBWUVw/afw6DfmeB3xeN42U8ouzkc3z1S4ERHPV1YOd6fD/PXmrofTzoenzqtbF1R99IuH//0/GNvT/PrVdXDNJ5V/9ovH8ratF+zs30+2Wm5ERDxIuQFpS2HhJrPb6JVhcFvvxjtfWCA8NwTevgxaBMGqLLhoIXyf1XjnlEbnbdPA7ezfT5bCjYiIhzAMeGwFvLfRHFPzysVwXdemOfeIDuYg5e7RkHsErv4YFvzSNOcWl7MPuPWWfaXsWleEm7wjUHzc2lqaksKNiHiuOWvgb+vM47mpcE2Xpj1/p0j47w3meY+XwwPL4JFvzG4y8SjeOuYmuhkE+5vHvjRjSuFGRDzTexvgqQzz+Klz4fomarE5UVggvHox/PUc8+u/rYNbPofCEmvqEadke2nLjc0GrVuYx3sOW1tLU1K4ERHPs3iHOc4GzCnadyRZWg42G/ylP7w+AkL8YckuuPxf8LsPfZp4MMOonE3kbS03UBlusnzov6PCjYh4lh+zYfxiKDPM9WseS7G6okpXdYZPrzV3LPx1P1z8IazJsboqOY39R80Z/TYg1stabqBy3I1abkRE3NG2Q/DH/8DR4+Y2Ci8Mdb8V186Ogy9vgB4tzVGcV39stjSJ29pb8aHfKgyC/K2tpTGo5UZExF3lHoH/95m5e3dSLLw2HALd9JPojBbmgn8XtTOD2Ogv4J0NVlclp2Bv0WjTwto6GovG3IiIuKOiUrj5P7CzANqHw7uXm6sGu7PmQfD2pfD/upldaH/5n7k1hGFYXZmcYK+PhJu9PrTWjcKNiLi34+Vw2yJzE8zoEPjgCogNtbqqugn0h/+7yBxsDObWEA8vNxceFLexp8C8bu3l4UbdUiIi7sAwzLVjvtplzkJ65zLoFGV1VfVjs5nTxKefb45Ynb/ebMXRWjhuw95d4+3h5nAJFBRbW0tTUbgREfdkGPB0hjlWxc8GfxsOAxKsrsp543rD3GHm9/LeRpjwldkqJZbz9m6p0ECIDDGPfaX1RuFGRNyPYZhdOP+3xvx6+vlwSUdra3KFG7qaA6ED/OBfv5ndbdpV3HKOcBNubR2NydemgyvciIj7mfEDzPrRPH7qXBjby9p6XOnKzrDgEgjyg8+3w51L1EVloWPHYd9R89hbW26gyqBihRsREQvM/AGeW2UePzHY+tWHG8PwDvDWZRDoB59uNccVaRaVJewf9qGBEBFsbS2NKcHHZkwp3Ih4K8PwrFk55QY8+o3ZHQUweRDc2dfamhrTRe1g3sXmGJy3q+yTJU1qb5XBxO62HqQrtfGxlpsAqwsQERfYdgi+3g3f7YUtB809jQpLzQGrgX4QGQyJ4ZDYAtqGQ/eWcHYsdIg0P1ytVlgCE78yu2nAbLHx5mBjd2VnKCgxZ0/NWWO+T3f3s7oqn+LtC/jZ+dp0cIUbEU9VWgYfboa3foXVtexfVFoOeUfNy4n7HEUEQ99YOPcMuOAM6NUK/Ju4QXfzARj7XzOUBfrBnIvgOot2+LbCH3tA/jF4/Fuz9aZ9hBl6pEnY17jx+nDjYwOKFW5EPI1hwL+3mR+EO/PN2wL8IKU1DG4DZ8VAu3CzFSDQH4qPw4FjkFkAuw/DjnxYnwc/50F+MSzbbV6exnzO4DZwQaJ56RDReG31ZeXwylqzG6q4DOLD4I0RkOzB072dNeFsyC6Ceevgrq+gU6T5Pkqj2+vla9zY2b+/7EKzB9gdGmwbk8KNiCfJPQIPLqvsvmnVDP7c19wdu7ZVe9u0MFtlqiotg40H4Pu98M3vsGIPHCo2X9v++u3CYUhF0DnvjMrFMhrCMCA9E55caZ4fYGhbeCnVc1YebgxTBsOmA2bQHPU5fHkjtGxmdVVe73cf6ZaKa27+DVRaDjmFlQOMvZXCjYinWJNjbsCYXWT+lrq3H0zo6/weS4H+0LuVeRnfxxyfszYXlv8OX2fCD9mwqwD+/qt58bOZ43SGtDW7sfq0qt+5Dx4zQ9PrP8Ov+8zbIoPND/Wbu3v3aM66CPAzFyq8+EOzRW7cIvjwSvfdHNRL7K5o/EyMsLaOxhbgZ7beZOabjbgKN41s7ty5PP/882RnZ9OnTx/mzJlDcnJyjY/99ddfmTx5MqtXr2bXrl288MIL3HvvvU1bsIgVPtliDrgtLoMzo+DV4dDTxd0WAX7QP968pPU3B/l+uxeWZZqDlX87CD/mmJcZP5hbCXSJgj6xZk1tw829n5oHme3eRaWQVWg+b3UO/JBlbiAJ0CwAxvY091xyRWuQt4gKMTfbHPFPWLkHnvgWnj7P6qq8VmlZ5dTotl4ebsD8Ec3MNy8D21hdTeOyNNwsXLiQtLQ05s2bx8CBA5k9ezbDhw9n8+bNxMbGnvT4I0eO0LFjR2644Qb+8pe/WFCxiAXe2wB/WWoGhhEd4OVh0KIJdsRuHgQXtzcvYI5EtI/PWZVlfir8dtC81FWPlnB9V3MQbZRCTY26tYRXhsGoL+DVdXBhW7iwndVVeaW9h80fq2B/3+gRbRsB7K5srfJmloabWbNmMX78eMaOHQvAvHnz+Pzzz5k/fz4PP/zwSY8fMGAAAwYMAKjxfhGv8/5GuOd/5vGos+D5IdaNBGzTAm7uYV7AHP+zLte87Mg3By8cKjZbfPz8IDQA4sKgfTgkxUJKG3OAspzeJR1hXC94Yz3clQ7Lb9L4m0aQWaVLyhd6RRMrtpfILLC2jqZgWbgpKSlh9erVTJo0yXGbn58fqampZGS4bjGr4uJiiosrt0EtKPCBd1W8w+Id5vonAON7w9Tz3Os3cGwoDGtvXsT1pgw2B3r/dhDSlppbNrjT++8F7B/ybb14T6mq7F1vmT7QcmPZCsX79u2jrKyMuLi4arfHxcWRnZ3tsvNMmzaNiIgIxyUxMdFlry3SaH7KMTdVLDPg/3Vzv2Ajja9ZgLmCcaAffLEd3t1odUVex/4h7wvjbaDy+9ztA3/je/32C5MmTSI/P99x2b17t9UlidQuu9Acb3GsDIa1gxcuVLDxVb1awV/PMY8f+cbs/hOXsX/I+0q4sXdL5RbB0VJra2lsloWbmJgY/P39ycmpvmJqTk4O8fHxLjtPcHAw4eHh1S4ibqukDMb815zu3TXanBUV4PV/g0ht7uwL57aBI6XmGkfaYNNlfK3lJjKkci6Ct7feWPZbMygoiH79+pGenu64rby8nPT0dFJSUqwqS8Ra074zp01HBsPblzXNrChxb342mDnUnNKzbDd89JvVFXkNR7jxkb95bbbK9XwUbhpRWloar732Gn//+9/ZuHEjf/7znykqKnLMnho1alS1AcclJSWsXbuWtWvXUlJSwp49e1i7di1bt2616lsQcZ2lmfDST+bx/12kmUVSqWOkuSYQwGMr4NAxS8vxBvnF5gXgDB8JN1AZ5Lx9ULGlU8FHjhxJXl4ekydPJjs7m6SkJBYtWuQYZJyZmYmfX2X+2rt3L337Vu4UPGPGDGbMmMEFF1zAsmXLmrp8EdfJOwITvjKPx/Y0pwKLVDXxbPjXb+YGo09nwIyhVlfk0exrvcQ0gzAfaiD1lZYby1conjhxIhMnTqzxvhMDS/v27THU3yzexjDg7nQz4HSLhifOtboicUfB/jBjCFz1sbkdxshuMMAHNxl1kapr3PgSe8vNrkOWltHoNFJRxGqv/wxf7TI/vF692JwCLFKTQW3gpu7m8f3LzP3AxCk7DpnX7SOtrKLpdYwyr+3fv7dSuBGxUmaB2cUA8Phg6OHi/aLE+0wZBFHBsGE/vLPB6mo81o6KXUPsH/a+okOkeZ2Z793ZWOFGxCqGAQ99DUeOQ0prc7l9kdNp2QweqNhc+NnvoaC49sdLjbYfMq87RlpZRdNLaGE2EpeWw+9ePO5G4UbEKp9uNbujAv3MsRRaqE/qakxP6BwJ+47C7NVWV+OR7N0yHXys5cbPVtl6s70ee956GoUbESvkF5srzgLc0w/OjLa2HvEsgf7wxGDz+NW1sMuL/wRvBIeOwYGj5rH9g96XdPCBcTcKNyJWeDrD3FW7U6QZbkTqa1h7OP8MKCmHp761uhqPYm+xSGgOoYHW1mIFe1ecWm5ExHVWZcGCX8zjmUMgRLOjxAk2Gzx5LtgwuzhXZVldkcdwdElFWlmFdXxhxpTCjUhTKimD+5aaxzd1h8FnWFuPeLazYuDmHubx4yu171QdbfPRmVJ2jm4ptdyIiEvM/Qk2HYCWIfD4IKurEW/w0EBzbaQfsuHLnVZX4xF8dRq4nb1bam+h9+4OrnAj0lS2H4KZP5jHT50H0c0sLUe8RHwYjO9tHk/NgDIvXrzERXy9WyqqmblDOMDOQ5aW0mgUbkSagmHAA8uguAyGJML1Z1pdkXiTu86GiGDYeAA+2mJ1NW6t3KgMN77acgOVwW6rl3ZNKdyINIUPN8Py3yHEH54bojVtxLUiQ2BixabCz35vju2SGv1eAMeOmwvZJfrQbuAnOrOlef3bfmvraCwKNyKNbf9RmLzCPE4bAB18bKc+aRrj+0BsqLnmzdu/Wl2N29pc8WHeORr8ffgTsKvCjYg0yOMrYf8xc8fvCX2trka8VVgg3DfAPJ75IxR56UjRBtq8z7y2t1z4Kvu6oQo3IlJ/K3+HDzaZxzOHQpC/tfWId/tjD2gXDnlH4LV1VlfjluwtN119PdxU7NG7M9/spvM2CjcijeXYcbhvmXk8pickJ1hZjfiCIH94eKB5PGcNHDxmbT1uyN5S4estN7Gh5lCtcqNy3R9vonAj0lheXA3bDpm/RR5Nsboa8RXXngk9WkJBiRlwxKG0rPKD3Ndbbmy2yq6pzV7YNaVwI9IYfjtghhuAZ843p+mKNAU/G/z1HPP49Z8hu9DaetzIjkNQWm4OT2rTwupqrOeYMbXP2joag8KNiKuVG2Z3VGk5DGsHV3ayuiLxNRe3hwHxcPQ4zPjB6mrcRtUuKa3GUCXcHLC2jsagcCPiau9ugO/2QmgAPHuBfotK07PZ4LGK7T3e2eCdgyqcoMHE1dn/HTar5UZEapV7BJ741jx+eKBvrxIm1kppbbYclhkw7Xurq3ELmyo+xBVuTN0qZkz9fhjyvWzsucKNiCs98g3kF0OvVuaiaiJWeiQFbMCnW2FdrtXVWG59xT9Bz1hr63AXkSFwRsXfX7/kWVuLqynciLjKoh3wyRZzQOcLQyFAP15isbNi4Pqu5vHTGdbWYrF9RyCr0Mx6PVpZXY376FUR9NZ7WfbVb18RVygoNjfGBHMV4j7601DcxEMDIdAPlu2G5butrsYy9g/vjlHQPMjaWtyJPdz8qnAjIid54lvILjL3jXog2epqRCq1C4fRPc3jpzLMHep90C/qkqpRT7XciEiNVvwOb1VsVPjChdAswNp6RE6U1h9CA2FtLvx7m9XVWML+4d1L4aYa+7/HjkNwuNjSUlxK4UakIY6Uwl+WmsdjesLgNtbWI1KTVqFwZ5J5/Mx3cLzc0nKs8IvCTY2im1UuaPirFw0qVrgRaYgnvjV3nksIg8mDrK5G5NTu7AstQ8wtQd7dYHU1TerAUdhz2Dw+S4OJT2LvmvrFi7qmFG5EnLU0E+avN4//7yJooVGK4sZaBMFfBpjHz35vDoL3ET/nmNcdIqGFdkI5ib01a22OtXW4ksKNiDMOHYO7083jW3vBkLbW1iNSF2N7QqdIyDsKs360upom82OWeX12grV1uKt+Ff8uP+61tg5XUrgRccZDX5uzozpFwhR1R4mHCPKHp841j/+2zuyi8gE/VHxoD2htbR3uKike/G3mOkB7CqyuxjUUbkTq6+Mt8NEW87fBy8PMWSginmJYe7ionbmx65QVVlfT6ErLYG22edxf4aZGoYGV425+8JLWG4UbkfrYVQD3V8yOurc/nB1nbT0iznjqXHMF7cU74X+7rK6mUf2aB8eOm1sNdIqyuhr31d/eNZVlbR2uonAjUlclZTB+ERSUQP84uK+/1RWJOKdLFNzWyzx+bIXZvOGl7C0R/RPMnVGkZvZWrdVquRHxMU9+Cz/lQmQw/G04BPpbXZGI8+5PNqeG/3YQ5q2zuppGo/E2dWMPNxv3ecdEOoUbkbr473Z4teIDYE4qJIZbW49IQ0UEw5TB5vFz33vl4OJyo3IGUD+Fm1rFhkG7CDCAH/ZYXU3DKdyInE5mAdxVMe37jj4wooO19Yi4yv/rBkMS4VgZ3LfU6/ad2pAH+49CWCD00fC40zo30bz+xgv2V1W4EalNYQmM+gLyi83Bw49p2rd4EZsNZgyF0ABYuQfe8a6Vi7+uGCudcoY5C15qd14783q5F4wxV7gROZVyAyZ+Bb/ug1bN4I0R+g0p3qddODw80Dx+fCVkF1pbjwvZP6TPb2dtHZ5iUKK5wsW2g/C7h693o3AjcirPrYLPt0OQHyy4FM5oYXVFIo3j9j7QN9acCfjg117RPVVYUjmt+QKFmzqJCDYX9AP4JtPaWhpK4UakJp9sgZk/mMczhkKy1m0XL+bvBy9caK59898d8Pdfra6owTJ+Nzc/bxsB7SOtrsZzXOAlXVMKNyIn+j6rct+oO5Pgpu6WliPSJM6KgcdSzONHvzG7Yz3Ysp3m9fna9q1e7F1432RC8XFra2kIhRuRqn7ZB3/4Nxw9Dhe3h8kaQCw+5I4kGNYOistg/GKzb8cDlZXDoq3m8bCO1tbiafrEQVwYHC6BFR7cNaVwI2K3/RDc+Jk57mBgArw23GyuF/EVfjZzHaeEMNhyEB5ebnVFTvl+D+w7ao4hGZxodTWexc8Gl3Q2j7/Yam0tDaHf3CJgbod7w6eQd8Rsnn/3cm2IKb6pZTOYd7H5KbdwE7y/0eqK6u3zLeb18E5aSNwZl3Uxr7/cbu4644kUbkT2HIZrPoHMw9AhAv5xpfknn4ivGtQGHkw2j+9fCqs8ZzfFql1S9g9pqZ9+CdAq1NyGwVO7phRuxLdtOwiXf2QuPX9GC/jnVRAbanVVItb7S3+4rCOUlMPoL2CXZyx8snK3uqQayt8PLq0Ihh9vsrYWZynciO/6ZR9c8RH8fhg6R8J/roW22jNKBDC7peYOg54xZlq4saLb1s29u968vrqbuqQa4oYe5vWibbDf/d/2kyjciG9a8Ttc/THkHTV/eX92LbTRIn0i1YQFwnuXQ2IL2J4PN/3bnEbjpnKKYMl28/jmntbW4ul6xULvWHPMzT89b9iVwo34GMOAeWvh+k/N/aKSE+CTa8wOZhE5WUJz+PBKaBkC6/LMFhw3DTj/+BXKDOifAF1jrK7G8/2hl3n9/i+et2i1wo34jiOl8Ocl8NgK8zfgjV3NMTYaPCxSu05R8I+rIDIYfswxA86Bo1ZXVU3x8couqZt7WVuLt7jyTGgeBDsOwf92Wl1N/SjciG/4OQ8u+Sf86zdzZ7hnzoOXUqFZgNWViXiG3q3gX1dXBpxL/wU7862uyuHDDeaKDnFhlYNhpWHCguCPFUFxzirPar1RuBHvduw4PJ0BF/8DNuyHmGbmL+jxfcBms7o6Ec/SuxX8+1pzZuG2Q+YfDGtyrK6KkjJ4+Ufz+M/9IUR/s7jM+LPNf8+fsj1rM02FG/Fey3fD0A/gxdVmN9SVneHrm2BwG6srE/Fc3VrCf6+DXq3MWVRXfwyfWbuU7Qe/mMtVxYbBTRpI7FIxoZXdfDMyoNxDWm8UbsT7fLfX/IV73aew9ZC5bs2CS+CNEVrDRsQV4pvDZ9fAhW3NfdjGLYIHlpnHTSyvCJ7PMI8nDlCrTWO4o5859mZdjhkkPYHCjXiH4+Xw5Q5zb6grPoKVeyDID8b1gpV/gMs6WV2hiHdpHmRuU3LX2ebXC34xu3837m/SMp5ZYa6ke1YrDSRuLLFhcN855vH0lZ6x7o3NMDxpiFDDFRQUEBERQX5+PuHhWrDNoxmG2TLzj03wwSbILjJvD/CDP3Q3V1g9Q2vXiDS6ZZlw51fmIn8h/nBvf5jQt9GbUf67Fe74HGzAJyMhKb5RT+fTjpfDFR/Ahjy4qAO8foW5zmNTqs/nt8KNeJZDx+D7LEjPhP/tqr4kfMsQuLEb3NoL2kdYV6OIL8o7AnelQ/ou8+v2ETDtfEht1yin237Q/LAtLIHbz4ZHzmuU00gVv+bBNQuhuAweGgR3Dmja8yvc1ELhxkOUlkFmgTkjY8tBc/Gwtbmw44Spp4F+cN4ZcHMPGNEBgrTeuohlDAM+3gKTV0BORd/Fxe3NTTj7xLrsNDlFMPKf5vorA9vAe9eaDbbS+N7/BR5ON1vLXhwBV3VtunMr3NRC4caNGAbsLTQDzPZ8cxPLbYfMy64Cc4ZTTTpEwJC2cFFbc+ZT86AmLFpETutwCcxYBa+uq/w5HtYO7hsA/RrWd5RTCDd9ZP66OKMFfDTSXNtGmoZhwKNL4Z315pJhL46AK85smnMr3NRC4cYCxWWw/ZDZArPlIGw9CL8dNMfLHCk99fOaBUDHSOgYAT1bQd9YSIqFqJAmKlxEGmTrQZj1o7l4pn0OcXIC/LGHuTRDWGC9Xu6HvfDnz80esNbNYeH10FY90E2u3ID7l8C/KvacuisZ/jLQ3E28MSnc1ELhphEdOApbDlUGGHuY2VVw6sUR/G1m33zHCOgUaS7z3inSvMSHNf2INRFxve2HzPWm/rHZHJkK0DwQrukCl3aCc9vUOvi4oBhe+A4WrDN/lXRtaQ5oVbCxTlk5TF0Bb/xkft07FqZeCL3jGu+cHhdu5s6dy/PPP092djZ9+vRhzpw5JCcnn/LxH374IY899hg7d+6kS5cuPPvss1x66aV1OpfCTQMVFMOeQnM8zJaK1hd7mDlw7NTPaxEEXaKgcyScGW0ed4mCduEaJyPiK7KLYOEmeGdD9a0bQgPggkQY2hb6x0P3lhDgR14RvLkW3v4ZCir26rymG0wdam4NINb7aBNMWVr5/lzYHsYmweBE17fkeFS4WbhwIaNGjWLevHkMHDiQ2bNn8+GHH7J582ZiY08egPbtt99y/vnnM23aNC6//HLee+89nn32WdasWUPPnqdfmtLnw41hQEm5ucvc0eNwrMzcouDYcThyHA4eM0PKwSqXrCJz+c89haffDfiMFmaAsYeXLlHQOQriQrXdgYiYDAO+3QOfbIXFO8zfMVWFBkBSLD9FtuSTski2RUdh6xjB7ZeGcd6ZWqXP3eQWmesNfbq5spG+UxQs+aNrA45HhZuBAwcyYMAAXnrpJQDKy8tJTEzkrrvu4uGHHz7p8SNHjqSoqIj//Oc/jtvOOecckpKSmDdv3mnP12jh5sBRc+G4cswfXMMw32WD6tf24xPvL7c/p+K+snIzhBwvNzdOKa24Pl5xe2mV20rLKy5l1e+z328PL8VlZqBp6DseGVwRYqKgS6QZYOzdSfXsQxcRH2cYsH4fLNlpri6+Oqf2P6Iigs0u66gQaBFoTihoUXEJCzSnTQX6Vb+2H/vZzIsN84+tPrHmBAVxiR0HYf5aM+SkdoRZF7v29evz+W1pBC4pKWH16tVMmjTJcZufnx+pqalkZGTU+JyMjAzS0tKq3TZ8+HA++eSTGh9fXFxMcXGx4+v8fLMptKCgoMbHO21dNoz51LWv2dhsmIN2gwMg2B+aBUJUEESGmAHGfh0bBglh0DrMXHb9VLOTyo5CwdEm/RZExAu0D4bxXc1LuWFOhVqbZ86c3JFvLmqz+7D5B9rBo3DwkGvOO/U8+EMP17yW0NIfHugHd/WBohJw9ces/XO7Lm0yloabffv2UVZWRlxc9RFIcXFxbNq0qcbnZGdn1/j47OzsGh8/bdo0nnjiiZNuT0xMdLJqERHxCn+uuIhHOXz4MBERtbe4eX3n5aRJk6q19JSXl3PgwAFatmyJzUVjQAoKCkhMTGT37t2+OY7Hjem9cU96X9yT3hf3pffGbLE5fPgwrVu3Pu1jLQ03MTEx+Pv7k5OTU+32nJwc4uNrXugpPj6+Xo8PDg4mODi42m2RkZHOF12L8PBwn/1P5+703rgnvS/uSe+L+/L19+Z0LTZ2li5YHRQURL9+/UhPT3fcVl5eTnp6OikpKTU+JyUlpdrjAZYsWXLKx4uIiIhvsbxbKi0tjdGjR9O/f3+Sk5OZPXs2RUVFjB07FoBRo0bRpk0bpk2bBsA999zDBRdcwMyZM7nsssv44IMP+PHHH/nb3/5m5bchIiIibsLycDNy5Ejy8vKYPHky2dnZJCUlsWjRIseg4czMTPz8KhuYBg0axHvvvcejjz7KX//6V7p06cInn3xSpzVuGktwcDBTpkw5qftLrKf3xj3pfXFPel/cl96b+rF8nRsRERERV9Im8SIiIuJVFG5ERETEqyjciIiIiFdRuBERERGvonDTADt37mTcuHF06NCBZs2a0alTJ6ZMmUJJSfVN337++WfOO+88QkJCSExM5LnnnrOoYt8xdepUBg0aRGho6CkXbbTZbCddPvjgg6Yt1AfV5b3JzMzksssuIzQ0lNjYWB544AGOHz/etIUK7du3P+lnZPr06VaX5XPmzp1L+/btCQkJYeDAgaxatcrqktye5VPBPdmmTZsoLy/n1VdfpXPnzvzyyy+MHz+eoqIiZsyYAZhLZl988cWkpqYyb9481q9fz6233kpkZCS33367xd+B9yopKeGGG24gJSWFN95445SPe/PNNxkxYoTj68ZavVoqne69KSsr47LLLiM+Pp5vv/2WrKwsRo0aRWBgIM8884wFFfu2J598kvHjxzu+btGihYXV+J6FCxeSlpbGvHnzGDhwILNnz2b48OFs3ryZ2NhYq8tzX4a41HPPPWd06NDB8fXLL79sREVFGcXFxY7bHnroIaNr165WlOdz3nzzTSMiIqLG+wDj448/btJ6pNKp3psvvvjC8PPzM7Kzsx23vfLKK0Z4eHi1nyNpfO3atTNeeOEFq8vwacnJycaECRMcX5eVlRmtW7c2pk2bZmFV7k/dUi6Wn59PdHS04+uMjAzOP/98goKCHLfZU/fBgwetKFGqmDBhAjExMSQnJzN//nwMLftkuYyMDHr16uVYyBPMn5mCggJ+/fVXCyvzTdOnT6dly5b07duX559/Xt2DTaikpITVq1eTmprquM3Pz4/U1FQyMjIsrMz9qVvKhbZu3cqcOXMcXVIA2dnZdOjQodrj7L+0s7OziYqKatIapdKTTz7JhRdeSGhoKF9++SV33nknhYWF3H333VaX5tOys7OrBRuo/jMjTefuu+/m7LPPJjo6mm+//ZZJkyaRlZXFrFmzrC7NJ+zbt4+ysrIafx42bdpkUVWeQS03NXj44YdrHGxa9XLif6w9e/YwYsQIbrjhhmr90+I6zrwvtXnssccYPHgwffv25aGHHuLBBx/k+eefb8TvwHu5+r2RxlOf9yotLY0hQ4bQu3dv7rjjDmbOnMmcOXMoLi62+LsQqZ1abmpw3333MWbMmFof07FjR8fx3r17GTp0KIMGDTppA8/4+HhycnKq3Wb/Oj4+3jUF+4j6vi/1NXDgQJ566imKi4u1f0s9ufK9iY+PP2k2iH5mXKch79XAgQM5fvw4O3fupGvXro1QnVQVExODv79/jZ8h+lmoncJNDVq1akWrVq3q9Ng9e/YwdOhQ+vXrx5tvvlltk0+AlJQUHnnkEUpLSwkMDARgyZIldO3aVV1S9VSf98UZa9euJSoqSsHGCa58b1JSUpg6dSq5ubmO2SBLliwhPDycHj16uOQcvqwh79XatWvx8/PTLJ0mEhQURL9+/UhPT+fqq68GoLy8nPT0dCZOnGhtcW5O4aYB9uzZw5AhQ2jXrh0zZswgLy/PcZ89Vf/hD3/giSeeYNy4cTz00EP88ssvvPjii7zwwgtWle0TMjMzOXDgAJmZmZSVlbF27VoAOnfuTPPmzfn3v/9NTk4O55xzDiEhISxZsoRnnnmG+++/39rCfcDp3puLL76YHj16cMstt/Dcc8+RnZ3No48+yoQJExQ8m1BGRgbff/89Q4cOpUWLFmRkZPCXv/yFP/7xj/rDrAmlpaUxevRo+vfvT3JyMrNnz6aoqIixY8daXZp7s3q6lid78803DaDGS1Xr1q0zzj33XCM4ONho06aNMX36dIsq9h2jR4+u8X1ZunSpYRiG8d///tdISkoymjdvboSFhRl9+vQx5s2bZ5SVlVlbuA843XtjGIaxc+dO45JLLjGaNWtmxMTEGPfdd59RWlpqXdE+aPXq1cbAgQONiIgIIyQkxOjevbvxzDPPGMeOHbO6NJ8zZ84co23btkZQUJCRnJxsfPfdd1aX5PZshqG5ryIiIuI9NFtKREREvIrCjYiIiHgVhRsRERHxKgo3IiIi4lUUbkRERMSrKNyIiIiIV1G4EREREa+icCMiIiJeReFGRNyGzWbjk08+AWDnzp3YbDbH9gzLli3DZrNx6NAhp177dK+3YMECIiMjG1S/s8aMGePYO0hEGk57S4lIjcaMGcOhQ4ccYaOpJSYmkpWVRUxMTKO8/qBBg8jKyiIiIqJRXl9ErKNwIyJuyd/f37EBbWMICgpq1NcXEeuoW0pE6u3rr78mOTmZ4OBgEhISePjhhzl+/Ljj/sOHD3PzzTcTFhZGQkICL7zwAkOGDOHee++t8zlO7EY60ZEjR7jkkksYPHiwo2vp9ddfp3v37oSEhNCtWzdefvnlU77+qbq5Fi9eTPfu3WnevDkjRowgKyvLcV95eTlPPvkkZ5xxBsHBwSQlJbFo0aJqz1+/fj0XXnghzZo1o2XLltx+++0UFhY67i8rKyMtLY3IyEhatmzJgw8+iLb4E3EthRsRqZc9e/Zw6aWXMmDAANatW8crr7zCG2+8wdNPP+14TFpaGitXruSzzz5jyZIlfPPNN6xZs8ZlNRw6dIhhw4ZRXl7OkiVLiIyM5N1332Xy5MlMnTqVjRs38swzz/DYY4/x97//vc6ve+TIEWbMmMHbb7/N8uXLyczM5P7773fc/+KLLzJz5kxmzJjBzz//zPDhw7nyyivZsmULAEVFRQwfPpyoqCh++OEHPvzwQ7766ismTpzoeI2ZM2eyYMEC5s+fz4oVKzhw4AAff/yxy/5tRASweFdyEXFTo0ePNq666qqTbv/rX/9qdO3a1SgvL3fcNnfuXKN58+ZGWVmZUVBQYAQGBhoffvih4/5Dhw4ZoaGhxj333FPrOQHj448/NgzDMHbs2GEAxk8//WQYhmEsXbrUAIyNGzcavXv3Nq677jqjuLjY8dxOnToZ7733XrXXe+qpp4yUlJRaX+/gwYOGYRjGm2++aQDG1q1bq31fcXFxjq9bt25tTJ06tdo5BgwYYNx5552GYRjG3/72NyMqKsooLCx03P/5558bfn5+RnZ2tmEYhpGQkGA899xzjvtLS0uNM844o8Z/axFxjlpuRKReNm7cSEpKCjabzXHb4MGDKSws5Pfff2f79u2UlpaSnJzsuD8iIoKuXbs6vn7mmWdo3ry545KZmVnn8w8bNozOnTuzcOFCgoKCALPFZNu2bYwbN67a6z799NNs27atzq8dGhpKp06dHF8nJCSQm5sLQEFBAXv37mXw4MHVnjN48GA2btzo+Lfp06cPYWFh1e4vLy9n8+bN5Ofnk5WVxcCBAx33BwQE0L9//zrXKCKnpwHFItLk7rjjDm688UbH161bt67zcy+77DL+9a9/sWHDBnr16gXgGNPy2muvVQsOYA5MrqvAwMBqX9tsNo2HEfFAarkRkXrp3r07GRkZ1T70V65cSYsWLTjjjDPo2LEjgYGB/PDDD4778/Pz+e233xxfR0dH07lzZ8clIKDuf2dNnz6d0aNHc9FFF7FhwwYA4uLiaN26Ndu3b6/2up07d6ZDhw4u+K4hPDyc1q1bs3Llymq3r1y5kh49egDmv826desoKiqqdr+fnx9du3YlIiKChIQEvv/+e8f9x48fZ/Xq1S6pUURMarkRkVPKz88/abbS7bffzuzZs7nrrruYOHEimzdvZsqUKaSlpeHn50eLFi0YPXo0DzzwANHR0cTGxjJlyhT8/PyqdWU1xIwZMygrK+PCCy9k2bJldOvWjSeeeIK7776biIgIRowYQXFxMT/++CMHDx4kLS3NJed94IEHmDJlCp06dSIpKYk333yTtWvX8u677wJw8803M2XKFEaPHs3jjz9OXl4ed911F7fccgtxcXEA3HPPPUyfPp0uXbrQrVs3Zs2a5fTChCJSM4UbETmlZcuW0bdv32q3jRs3ji+++IIHHniAPn36EB0dzbhx43j00Ucdj5k1axZ33HEHl19+OeHh4Tz44IPs3r2bkJAQl9X2wgsvVAs4t912G6GhoTz//PM88MADhIWF0atXr3pNPz+du+++m/z8fO677z5yc3Pp0aMHn332GV26dAHMMTuLFy/mnnvuYcCAAYSGhnLdddcxa9Ysx2vcd999ZGVlMXr0aPz8/Lj11lu55ppryM/Pd1mdIr7OZqhDWUQaWVFREW3atGHmzJmMGzfO6nJExMup5UZEXO6nn35i06ZNJCcnk5+fz5NPPgnAVVddZXFlIuILFG5EpFHMmDGDzZs3ExQURL9+/fjmm28abZ8oEZGq1C0lIiIiXkVTwUVERMSrKNyIiIiIV1G4EREREa+icCMiIiJeReFGREREvIrCjYiIiHgVhRsRERHxKgo3IiIi4lX+P5fu60YIA10lAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sns.kdeplot(in_likelihoods, color=\"dodgerblue\", bw_adjust=50, label=\"In-distribution\")\n", + "sns.kdeplot(ood_likelihoods, color=\"deeppink\", bw_adjust=1, label=\"OOD\")\n", + "plt.legend()\n", + "plt.xlabel(\"Log-likelihood\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "89c3dc99", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "jupytext": { + "formats": "py:percent,ipynb" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tutorials/generative/anomaly_detection/anomaly_detection_with_transformers.py b/tutorials/generative/anomaly_detection/anomaly_detection_with_transformers.py new file mode 100644 index 00000000..a436cab9 --- /dev/null +++ b/tutorials/generative/anomaly_detection/anomaly_detection_with_transformers.py @@ -0,0 +1,437 @@ +# --- +# jupyter: +# jupytext: +# formats: py:percent,ipynb +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# jupytext_version: 1.14.4 +# kernelspec: +# display_name: Python 3 (ipykernel) +# language: python +# name: python3 +# --- + +# %% [markdown] +# # Anomaly Detection with Transformers +# +# This tutorial illustrates how to use MONAI to perform image-wise anomaly detection with transformers based on the method proposed in Pinaya et al.[1]. +# +# Here, we will work with the [MedNIST dataset](https://docs.monai.io/en/stable/apps.html#monai.apps.MedNISTDataset) available on MONAI, and similar to "Experiment 2 – image-wise anomaly detection on 2D synthetic data" from [1], we will train a general-purpose VQ-VAE (using all MEDNIST classes), and then a generative models (i.e., Transformer) on `HeadCT` images. +# +# Finally, we will compute the log-likelihood of images from the same class (in-distribution class) and images from other classes (out-of-distribution). +# +# [1] - [Pinaya et al. "Unsupervised brain imaging 3D anomaly detection and segmentation with transformers"](https://doi.org/10.1016/j.media.2022.102475) + +# %% [markdown] +# ### Setup environment + +# %% +# !python -c "import seaborn" || pip install -q seaborn +# %matplotlib inline + +# %% [markdown] +# ### Setup imports + +# %% +# Copyright 2020 MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import tempfile +import time + +import matplotlib.pyplot as plt +import numpy as np +import seaborn as sns +import torch +from monai import transforms +from monai.apps import MedNISTDataset +from monai.config import print_config +from monai.data import DataLoader, Dataset +from monai.utils import first, set_determinism +from torch.nn import CrossEntropyLoss, L1Loss +from tqdm import tqdm + +from generative.inferers import VQVAETransformerInferer +from generative.networks.nets import VQVAE, DecoderOnlyTransformer +from generative.utils.enums import OrderingType +from generative.utils.ordering import Ordering + +print_config() + +# %% +# for reproducibility purposes set a seed +set_determinism(42) + +# %% [markdown] +# ### Setup a data directory and download dataset +# +# Specify a `MONAI_DATA_DIRECTORY` variable, where the data will be downloaded. If not +# specified a temporary directory will be used. + +# %% +directory = os.environ.get("MONAI_DATA_DIRECTORY") +root_dir = tempfile.mkdtemp() if directory is None else directory +print(root_dir) + +# %% [markdown] +# ### Download training data + +# %% +train_transforms = transforms.Compose( + [ + transforms.LoadImaged(keys=["image"]), + transforms.EnsureChannelFirstd(keys=["image"]), + transforms.ScaleIntensityRanged(keys=["image"], a_min=0.0, a_max=255.0, b_min=0.0, b_max=1.0, clip=True), + transforms.RandAffined( + keys=["image"], + rotate_range=[(-np.pi / 36, np.pi / 36), (-np.pi / 36, np.pi / 36)], + translate_range=[(-1, 1), (-1, 1)], + scale_range=[(-0.01, 0.01), (-0.01, 0.01)], + spatial_size=[64, 64], + padding_mode="zeros", + prob=0.5, + ), + ] +) +train_data = MedNISTDataset(root_dir=root_dir, section="training", download=True, seed=0, transform=train_transforms) +train_loader = DataLoader(train_data, batch_size=256, shuffle=True, num_workers=4, persistent_workers=True) + +# %% [markdown] +# ### Visualise some examples from the dataset + +# %% +# Plot 3 examples from the training set +check_data = first(train_loader) +fig, ax = plt.subplots(nrows=1, ncols=3) +for image_n in range(3): + ax[image_n].imshow(check_data["image"][image_n, 0, :, :], cmap="gray") + ax[image_n].axis("off") + +# %% [markdown] +# ### Download Validation Data + +# %% +val_transforms = transforms.Compose( + [ + transforms.LoadImaged(keys=["image"]), + transforms.EnsureChannelFirstd(keys=["image"]), + transforms.ScaleIntensityRanged(keys=["image"], a_min=0.0, a_max=255.0, b_min=0.0, b_max=1.0, clip=True), + ] +) +val_data = MedNISTDataset(root_dir=root_dir, section="validation", download=True, seed=0, transform=val_transforms) +val_loader = DataLoader(val_data, batch_size=256, shuffle=False, num_workers=4, persistent_workers=True) + +# %% [markdown] +# ## Vector Quantized Variational Autoencoder +# +# The first step is to train a Vector Quantized Variation Autoencoder (VQ-VAE). This network is responsible for creating a compressed version of the inputted data. Once its training is done, we can use the encoder to obtain smaller and discrete representations of the 2D images to generate the inputs required for our autoregressive transformer. +# +# For its training, we will use the L1 loss, and we will update its codebook using a method based on Exponential Moving Average (EMA). + +# %% [markdown] +# ### Define network, optimizer and losses + +# %% +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +print(f"Using {device}") + +vqvae_model = VQVAE( + spatial_dims=2, + in_channels=1, + out_channels=1, + num_res_layers=2, + downsample_parameters=((2, 4, 1, 1), (2, 4, 1, 1)), + upsample_parameters=((2, 4, 1, 1, 0), (2, 4, 1, 1, 0)), + num_channels=(256, 256), + num_res_channels=(256, 256), + num_embeddings=16, + embedding_dim=64, +) +vqvae_model.to(device) + +# %% +optimizer = torch.optim.Adam(params=vqvae_model.parameters(), lr=5e-4) +l1_loss = L1Loss() + +# %% [markdown] +# ### VQ-VAE Model training +# We will train our VQ-VAE for 50 epochs. + +# %% +n_epochs = 30 +val_interval = 10 +epoch_losses = [] +val_epoch_losses = [] + +total_start = time.time() +for epoch in range(n_epochs): + vqvae_model.train() + epoch_loss = 0 + progress_bar = tqdm(enumerate(train_loader), total=len(train_loader), ncols=110) + progress_bar.set_description(f"Epoch {epoch}") + for step, batch in progress_bar: + images = batch["image"].to(device) + optimizer.zero_grad(set_to_none=True) + + # model outputs reconstruction and the quantization error + reconstruction, quantization_loss = vqvae_model(images=images) + recons_loss = l1_loss(reconstruction.float(), images.float()) + loss = recons_loss + quantization_loss + + loss.backward() + optimizer.step() + + epoch_loss += recons_loss.item() + + progress_bar.set_postfix( + {"recons_loss": epoch_loss / (step + 1), "quantization_loss": quantization_loss.item() / (step + 1)} + ) + epoch_losses.append(epoch_loss / (step + 1)) + + if (epoch + 1) % val_interval == 0: + vqvae_model.eval() + val_loss = 0 + with torch.no_grad(): + for val_step, batch in enumerate(val_loader, start=1): + images = batch["image"].to(device) + reconstruction, quantization_loss = vqvae_model(images=images) + recons_loss = l1_loss(reconstruction.float(), images.float()) + val_loss += recons_loss.item() + + val_loss /= val_step + val_epoch_losses.append(val_loss) + +total_time = time.time() - total_start +print(f"train completed, total time: {total_time}.") + +# %% [markdown] +# ### Plot reconstructions of final trained vqvae model + +# %% +fig, ax = plt.subplots(nrows=1, ncols=2) +ax[0].imshow(images[0, 0].detach().cpu(), vmin=0, vmax=1, cmap="gray") +ax[0].axis("off") +ax[0].title.set_text("Inputted Image") +ax[1].imshow(reconstruction[0, 0].detach().cpu(), vmin=0, vmax=1, cmap="gray") +ax[1].axis("off") +ax[1].title.set_text("Reconstruction") +plt.show() + +# %% [markdown] +# # Autoregressive Transformer +# +# Now that our VQ-VAE model has been trained, we can use this model to encode the data into its discrete latent representations. Then, to be able to input it into the autoregressive Transformer, it is necessary to transform this 2D latent representation into a 1D sequence. +# +# In order to train it in an autoregressive manner, we will use the CrossEntropy Loss as the Transformer will try to predict the next token value for each position of the sequence. +# +# Here we will use the MONAI's `VQVAETransformerInferer` class to help with the forward pass and to get the predicted likelihood from the VQ-VAE + Transformer models. + +# %% [markdown] +# ### Datasets +# To train the transformer, we only use the `HeadCT` class. + +# %% +train_data = MedNISTDataset(root_dir=root_dir, section="training", seed=0) +train_datalist = [{"image": item["image"]} for item in train_data.data if item["class_name"] == "HeadCT"] +train_transforms = transforms.Compose( + [ + transforms.LoadImaged(keys=["image"]), + transforms.EnsureChannelFirstd(keys=["image"]), + transforms.ScaleIntensityRanged(keys=["image"], a_min=0.0, a_max=255.0, b_min=0.0, b_max=1.0, clip=True), + transforms.RandAffined( + keys=["image"], + rotate_range=[(-np.pi / 36, np.pi / 36), (-np.pi / 36, np.pi / 36)], + translate_range=[(-1, 1), (-1, 1)], + scale_range=[(-0.01, 0.01), (-0.01, 0.01)], + spatial_size=[64, 64], + padding_mode="zeros", + prob=0.5, + ), + ] +) +train_ds = Dataset(data=train_datalist, transform=train_transforms) +train_loader = DataLoader(train_ds, batch_size=32, shuffle=True, num_workers=4, persistent_workers=True) + +val_data = MedNISTDataset(root_dir=root_dir, section="validation", seed=0) +val_datalist = [{"image": item["image"]} for item in val_data.data if item["class_name"] == "HeadCT"] +val_transforms = transforms.Compose( + [ + transforms.LoadImaged(keys=["image"]), + transforms.EnsureChannelFirstd(keys=["image"]), + transforms.ScaleIntensityRanged(keys=["image"], a_min=0.0, a_max=255.0, b_min=0.0, b_max=1.0, clip=True), + ] +) +val_ds = Dataset(data=val_datalist, transform=val_transforms) +val_loader = DataLoader(val_ds, batch_size=32, shuffle=False, num_workers=4, persistent_workers=True) + +# %% [markdown] +# ### 2D latent representation -> 1D sequence +# We need to define an ordering of which we convert our 2D latent space into a 1D sequence. For this we will use a simple raster scan. + +# %% +spatial_shape = next(iter(train_loader))["image"].shape[2:] + +# %% +# Get spatial dimensions of data +# We divide the spatial shape by 4 as the vqvae downsamples the image by a factor of 4 along each dimension +spatial_shape = next(iter(train_loader))["image"].shape[2:] +spatial_shape = (int(spatial_shape[0] / 4), int(spatial_shape[1] / 4)) + +ordering = Ordering(ordering_type=OrderingType.RASTER_SCAN.value, spatial_dims=2, dimensions=(1,) + spatial_shape) + + +# %% [markdown] +# ### Define network, inferer, optimizer and loss function + +# %% +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +transformer_model = DecoderOnlyTransformer( + num_tokens=16 + 1, + max_seq_len=spatial_shape[0] * spatial_shape[1], + attn_layers_dim=128, + attn_layers_depth=16, + attn_layers_heads=12, +) +transformer_model.to(device) + +inferer = VQVAETransformerInferer() + +# %% +optimizer = torch.optim.Adam(params=transformer_model.parameters(), lr=1e-4) +ce_loss = CrossEntropyLoss() + +# %% [markdown] +# ### Transformer Training +# We will train the Transformer for 5 epochs. + +# %% +n_epochs = 5 +val_interval = 2 +epoch_losses = [] +val_epoch_losses = [] +vqvae_model.eval() + +total_start = time.time() +for epoch in range(n_epochs): + transformer_model.train() + epoch_loss = 0 + progress_bar = tqdm(enumerate(train_loader), total=len(train_loader), ncols=110) + progress_bar.set_description(f"Epoch {epoch}") + for step, batch in progress_bar: + + images = batch["image"].to(device) + + optimizer.zero_grad(set_to_none=True) + + logits, quantizations_target, _ = inferer(images, vqvae_model, transformer_model, ordering, return_latent=True) + logits = logits.transpose(1, 2) + + loss = ce_loss(logits, quantizations_target) + + loss.backward() + optimizer.step() + + epoch_loss += loss.item() + + progress_bar.set_postfix({"ce_loss": epoch_loss / (step + 1)}) + epoch_losses.append(epoch_loss / (step + 1)) + + if (epoch + 1) % val_interval == 0: + transformer_model.eval() + val_loss = 0 + with torch.no_grad(): + for val_step, batch in enumerate(val_loader, start=1): + + images = batch["image"].to(device) + + logits, quantizations_target, _ = inferer( + images, vqvae_model, transformer_model, ordering, return_latent=True + ) + logits = logits.transpose(1, 2) + + loss = ce_loss(logits, quantizations_target) + + val_loss += loss.item() + + val_loss /= val_step + val_epoch_losses.append(val_loss) + +total_time = time.time() - total_start +print(f"train completed, total time: {total_time}.") + +# %% [markdown] +# ## Image-wise anomaly detection +# +# To verify the performance of the VQ-VAE + Transformer performing unsupervised anomaly detection, we will use the images from the test set of the MedNIST dataset. We will consider images from the `HeadCT` class as in-distribution images. + +# %% +vqvae_model.eval() +transformer_model.eval() + +test_data = MedNISTDataset(root_dir=root_dir, section="test", download=True, seed=0) + +in_distribution_datalist = [{"image": item["image"]} for item in test_data.data if item["class_name"] == "HeadCT"] +in_distribution_ds = Dataset(data=in_distribution_datalist, transform=val_transforms) +in_distribution_loader = DataLoader( + in_distribution_ds, batch_size=64, shuffle=False, num_workers=4, persistent_workers=True +) + +in_likelihoods = [] + +progress_bar = tqdm(enumerate(in_distribution_loader), total=len(in_distribution_loader), ncols=110) +progress_bar.set_description(f"In-distribution data") +for step, batch in progress_bar: + images = batch["image"].to(device) + + log_likelihood = inferer.get_likelihood( + inputs=images, vqvae_model=vqvae_model, transformer_model=transformer_model, ordering=ordering + ) + in_likelihoods.append(log_likelihood.sum(dim=(1, 2)).cpu().numpy()) + +in_likelihoods = np.concatenate(in_likelihoods) + +# %% [markdown] +# We will use the "ChestCT" class of the dataset for the out-of-distribution examples. + +# %% +ood_datalist = [{"image": item["image"]} for item in test_data.data if item["class_name"] == "ChestCT"] +ood_ds = Dataset(data=ood_datalist, transform=val_transforms) +ood_loader = DataLoader(ood_ds, batch_size=64, shuffle=False, num_workers=4, persistent_workers=True) + +ood_likelihoods = [] + +progress_bar = tqdm(enumerate(ood_loader), total=len(ood_loader), ncols=110) +progress_bar.set_description(f"out-of-distribution data") +for step, batch in progress_bar: + images = batch["image"].to(device) + + log_likelihood = inferer.get_likelihood( + inputs=images, vqvae_model=vqvae_model, transformer_model=transformer_model, ordering=ordering + ) + ood_likelihoods.append(log_likelihood.sum(dim=(1, 2)).cpu().numpy()) + +ood_likelihoods = np.concatenate(ood_likelihoods) + +# %% [markdown] +# ## Log-likehood plot +# +# Here, we plot the log-likelihood of the images. In this case, the lower the log-likelihood, the more unlikely the image belongs to the training set. + +# %% +sns.kdeplot(in_likelihoods, color="dodgerblue", bw_adjust=50, label="In-distribution") +sns.kdeplot(ood_likelihoods, color="deeppink", bw_adjust=1, label="OOD") +plt.legend() +plt.xlabel("Log-likelihood") + +# %%