moocs/fast.ai/deep-learning-2018/lesson4-prudential-life-insurance-assessment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prudential Life Insurance Assessment\n",
    "\n",
    "An example of the structured data lessons from Lesson 4 on another dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "import os\n",
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "from fastai import structured\n",
    "from fastai.column_data import ColumnarModelData\n",
    "from fastai.dataset import get_cv_idxs\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "from ml_metrics import quadratic_weighted_kappa\n",
    "from torch.nn.init import kaiming_uniform, kaiming_normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path('./data/prudential')\n",
    "PATH.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: Looks like you're using an outdated API Version, please consider updating (server 1.4.4 / client 1.4.3)\n",
      "Downloading sample_submission.csv.zip to data/prudential\n",
      "  0%|                                               | 0.00/24.9k [00:00<?, ?B/s]\n",
      "100%|███████████████████████████████████████| 24.9k/24.9k [00:00<00:00, 948kB/s]\n",
      "Downloading test.csv.zip to data/prudential\n",
      " 98%|█████████████████████████████████████ | 1.00M/1.02M [00:00<00:00, 1.40MB/s]\n",
      "100%|██████████████████████████████████████| 1.02M/1.02M [00:00<00:00, 1.40MB/s]\n",
      "Downloading train.csv.zip to data/prudential\n",
      " 97%|████████████████████████████████████▋ | 3.00M/3.10M [00:02<00:00, 1.46MB/s]\n",
      "100%|██████████████████████████████████████| 3.10M/3.10M [00:02<00:00, 1.47MB/s]\n"
     ]
    }
   ],
   "source": [
    "!kaggle competitions download -c prudential-life-insurance-assessment --path={PATH}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in os.listdir(PATH):\n",
    "    if not file.endswith('zip'):\n",
    "        continue\n",
    "        \n",
    "    !unzip -q -d {PATH} {PATH}/{file}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(PATH/'train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Product_Info_1</th>\n",
       "      <th>Product_Info_2</th>\n",
       "      <th>Product_Info_3</th>\n",
       "      <th>Product_Info_4</th>\n",
       "      <th>Product_Info_5</th>\n",
       "      <th>Product_Info_6</th>\n",
       "      <th>Product_Info_7</th>\n",
       "      <th>Ins_Age</th>\n",
       "      <th>Ht</th>\n",
       "      <th>...</th>\n",
       "      <th>Medical_Keyword_40</th>\n",
       "      <th>Medical_Keyword_41</th>\n",
       "      <th>Medical_Keyword_42</th>\n",
       "      <th>Medical_Keyword_43</th>\n",
       "      <th>Medical_Keyword_44</th>\n",
       "      <th>Medical_Keyword_45</th>\n",
       "      <th>Medical_Keyword_46</th>\n",
       "      <th>Medical_Keyword_47</th>\n",
       "      <th>Medical_Keyword_48</th>\n",
       "      <th>Response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>D3</td>\n",
       "      <td>10</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.641791</td>\n",
       "      <td>0.581818</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>A1</td>\n",
       "      <td>26</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.059701</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>E1</td>\n",
       "      <td>26</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.029851</td>\n",
       "      <td>0.745455</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>D4</td>\n",
       "      <td>10</td>\n",
       "      <td>0.487179</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.164179</td>\n",
       "      <td>0.672727</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>D2</td>\n",
       "      <td>26</td>\n",
       "      <td>0.230769</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.417910</td>\n",
       "      <td>0.654545</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 128 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  Product_Info_1 Product_Info_2  Product_Info_3  Product_Info_4  \\\n",
       "0   2               1             D3              10        0.076923   \n",
       "1   5               1             A1              26        0.076923   \n",
       "2   6               1             E1              26        0.076923   \n",
       "3   7               1             D4              10        0.487179   \n",
       "4   8               1             D2              26        0.230769   \n",
       "\n",
       "   Product_Info_5  Product_Info_6  Product_Info_7   Ins_Age        Ht  \\\n",
       "0               2               1               1  0.641791  0.581818   \n",
       "1               2               3               1  0.059701  0.600000   \n",
       "2               2               3               1  0.029851  0.745455   \n",
       "3               2               3               1  0.164179  0.672727   \n",
       "4               2               3               1  0.417910  0.654545   \n",
       "\n",
       "     ...     Medical_Keyword_40  Medical_Keyword_41  Medical_Keyword_42  \\\n",
       "0    ...                      0                   0                   0   \n",
       "1    ...                      0                   0                   0   \n",
       "2    ...                      0                   0                   0   \n",
       "3    ...                      0                   0                   0   \n",
       "4    ...                      0                   0                   0   \n",
       "\n",
       "   Medical_Keyword_43  Medical_Keyword_44  Medical_Keyword_45  \\\n",
       "0                   0                   0                   0   \n",
       "1                   0                   0                   0   \n",
       "2                   0                   0                   0   \n",
       "3                   0                   0                   0   \n",
       "4                   0                   0                   0   \n",
       "\n",
       "   Medical_Keyword_46  Medical_Keyword_47  Medical_Keyword_48  Response  \n",
       "0                   0                   0                   0         8  \n",
       "1                   0                   0                   0         4  \n",
       "2                   0                   0                   0         8  \n",
       "3                   0                   0                   0         8  \n",
       "4                   0                   0                   0         8  \n",
       "\n",
       "[5 rows x 128 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extra feature engineering taken from the forum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df['Product_Info_2_char'] = train_df.Product_Info_2.str[0]\n",
    "train_df['Product_Info_2_num'] = train_df.Product_Info_2.str[1]\n",
    "train_df['BMI_Age'] = train_df['BMI'] * train_df['Ins_Age']\n",
    "med_keyword_columns = train_df.columns[train_df.columns.str.startswith('Medical_Keyword_')]\n",
    "train_df['Med_Keywords_Count'] = train_df[med_keyword_columns].sum(axis=1)\n",
    "train_df['num_na'] = train_df.apply(lambda x: sum(x.isnull()), 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_columns = 'Product_Info_1, Product_Info_2, Product_Info_3, Product_Info_5, Product_Info_6, Product_Info_7, Employment_Info_2, Employment_Info_3, Employment_Info_5, InsuredInfo_1, InsuredInfo_2, InsuredInfo_3, InsuredInfo_4, InsuredInfo_5, InsuredInfo_6, InsuredInfo_7, Insurance_History_1, Insurance_History_2, Insurance_History_3, Insurance_History_4, Insurance_History_7, Insurance_History_8, Insurance_History_9, Family_Hist_1, Medical_History_2, Medical_History_3, Medical_History_4, Medical_History_5, Medical_History_6, Medical_History_7, Medical_History_8, Medical_History_9, Medical_History_11, Medical_History_12, Medical_History_13, Medical_History_14, Medical_History_16, Medical_History_17, Medical_History_18, Medical_History_19, Medical_History_20, Medical_History_21, Medical_History_22, Medical_History_23, Medical_History_25, Medical_History_26, Medical_History_27, Medical_History_28, Medical_History_29, Medical_History_30, Medical_History_31, Medical_History_33, Medical_History_34, Medical_History_35, Medical_History_36, Medical_History_37, Medical_History_38, Medical_History_39, Medical_History_40, Medical_History_41'.split(', ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_columns += ['Product_Info_2_char', 'Product_Info_2_num']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "cont_columns = 'Product_Info_4, Ins_Age, Ht, Wt, BMI, Employment_Info_1, Employment_Info_4, Employment_Info_6, Insurance_History_5, Family_Hist_2, Family_Hist_3, Family_Hist_4, Family_Hist_5, Medical_History_1, Medical_History_10, Medical_History_15, Medical_History_24, Medical_History_32'.split(', ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "cont_columns += [c for c in train_df.columns if c.startswith('Medical_Keyword_')] + ['BMI_Age', 'Med_Keywords_Count', 'num_na']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Product_Info_1</th>\n",
       "      <th>Product_Info_2</th>\n",
       "      <th>Product_Info_3</th>\n",
       "      <th>Product_Info_5</th>\n",
       "      <th>Product_Info_6</th>\n",
       "      <th>Product_Info_7</th>\n",
       "      <th>Employment_Info_2</th>\n",
       "      <th>Employment_Info_3</th>\n",
       "      <th>Employment_Info_5</th>\n",
       "      <th>InsuredInfo_1</th>\n",
       "      <th>...</th>\n",
       "      <th>Medical_History_34</th>\n",
       "      <th>Medical_History_35</th>\n",
       "      <th>Medical_History_36</th>\n",
       "      <th>Medical_History_37</th>\n",
       "      <th>Medical_History_38</th>\n",
       "      <th>Medical_History_39</th>\n",
       "      <th>Medical_History_40</th>\n",
       "      <th>Medical_History_41</th>\n",
       "      <th>Product_Info_2_char</th>\n",
       "      <th>Product_Info_2_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>D3</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>D</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>A1</td>\n",
       "      <td>26</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>A</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>E1</td>\n",
       "      <td>26</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>E</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>D4</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>D</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>D2</td>\n",
       "      <td>26</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>D</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 62 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Product_Info_1 Product_Info_2  Product_Info_3  Product_Info_5  \\\n",
       "0               1             D3              10               2   \n",
       "1               1             A1              26               2   \n",
       "2               1             E1              26               2   \n",
       "3               1             D4              10               2   \n",
       "4               1             D2              26               2   \n",
       "\n",
       "   Product_Info_6  Product_Info_7  Employment_Info_2  Employment_Info_3  \\\n",
       "0               1               1                 12                  1   \n",
       "1               3               1                  1                  3   \n",
       "2               3               1                  9                  1   \n",
       "3               3               1                  9                  1   \n",
       "4               3               1                  9                  1   \n",
       "\n",
       "   Employment_Info_5  InsuredInfo_1         ...          Medical_History_34  \\\n",
       "0                  3              1         ...                           3   \n",
       "1                  2              1         ...                           1   \n",
       "2                  2              1         ...                           3   \n",
       "3                  3              2         ...                           3   \n",
       "4                  2              1         ...                           3   \n",
       "\n",
       "   Medical_History_35  Medical_History_36  Medical_History_37  \\\n",
       "0                   1                   2                   2   \n",
       "1                   1                   2                   2   \n",
       "2                   1                   3                   2   \n",
       "3                   1                   2                   2   \n",
       "4                   1                   3                   2   \n",
       "\n",
       "   Medical_History_38  Medical_History_39  Medical_History_40  \\\n",
       "0                   1                   3                   3   \n",
       "1                   1                   3                   3   \n",
       "2                   1                   3                   3   \n",
       "3                   1                   3                   3   \n",
       "4                   1                   3                   3   \n",
       "\n",
       "   Medical_History_41  Product_Info_2_char  Product_Info_2_num  \n",
       "0                   3                    D                   3  \n",
       "1                   1                    A                   1  \n",
       "2                   1                    E                   1  \n",
       "3                   1                    D                   4  \n",
       "4                   1                    D                   2  \n",
       "\n",
       "[5 rows x 62 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df[categorical_columns].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Product_Info_4</th>\n",
       "      <th>Ins_Age</th>\n",
       "      <th>Ht</th>\n",
       "      <th>Wt</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Employment_Info_1</th>\n",
       "      <th>Employment_Info_4</th>\n",
       "      <th>Employment_Info_6</th>\n",
       "      <th>Insurance_History_5</th>\n",
       "      <th>Family_Hist_2</th>\n",
       "      <th>...</th>\n",
       "      <th>Medical_Keyword_42</th>\n",
       "      <th>Medical_Keyword_43</th>\n",
       "      <th>Medical_Keyword_44</th>\n",
       "      <th>Medical_Keyword_45</th>\n",
       "      <th>Medical_Keyword_46</th>\n",
       "      <th>Medical_Keyword_47</th>\n",
       "      <th>Medical_Keyword_48</th>\n",
       "      <th>BMI_Age</th>\n",
       "      <th>Med_Keywords_Count</th>\n",
       "      <th>num_na</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.641791</td>\n",
       "      <td>0.581818</td>\n",
       "      <td>0.148536</td>\n",
       "      <td>0.323008</td>\n",
       "      <td>0.028</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000667</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.207304</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.059701</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.131799</td>\n",
       "      <td>0.272288</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0018</td>\n",
       "      <td>0.000133</td>\n",
       "      <td>0.188406</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.016256</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.029851</td>\n",
       "      <td>0.745455</td>\n",
       "      <td>0.288703</td>\n",
       "      <td>0.428780</td>\n",
       "      <td>0.030</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0300</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.304348</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.012799</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.487179</td>\n",
       "      <td>0.164179</td>\n",
       "      <td>0.672727</td>\n",
       "      <td>0.205021</td>\n",
       "      <td>0.352438</td>\n",
       "      <td>0.042</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.2000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.420290</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.057863</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.230769</td>\n",
       "      <td>0.417910</td>\n",
       "      <td>0.654545</td>\n",
       "      <td>0.234310</td>\n",
       "      <td>0.424046</td>\n",
       "      <td>0.027</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.463768</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.177213</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 69 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Product_Info_4   Ins_Age        Ht        Wt       BMI  Employment_Info_1  \\\n",
       "0        0.076923  0.641791  0.581818  0.148536  0.323008              0.028   \n",
       "1        0.076923  0.059701  0.600000  0.131799  0.272288              0.000   \n",
       "2        0.076923  0.029851  0.745455  0.288703  0.428780              0.030   \n",
       "3        0.487179  0.164179  0.672727  0.205021  0.352438              0.042   \n",
       "4        0.230769  0.417910  0.654545  0.234310  0.424046              0.027   \n",
       "\n",
       "   Employment_Info_4  Employment_Info_6  Insurance_History_5  Family_Hist_2  \\\n",
       "0                0.0                NaN             0.000667            NaN   \n",
       "1                0.0             0.0018             0.000133       0.188406   \n",
       "2                0.0             0.0300                  NaN       0.304348   \n",
       "3                0.0             0.2000                  NaN       0.420290   \n",
       "4                0.0             0.0500                  NaN       0.463768   \n",
       "\n",
       "    ...    Medical_Keyword_42  Medical_Keyword_43  Medical_Keyword_44  \\\n",
       "0   ...                     0                   0                   0   \n",
       "1   ...                     0                   0                   0   \n",
       "2   ...                     0                   0                   0   \n",
       "3   ...                     0                   0                   0   \n",
       "4   ...                     0                   0                   0   \n",
       "\n",
       "   Medical_Keyword_45  Medical_Keyword_46  Medical_Keyword_47  \\\n",
       "0                   0                   0                   0   \n",
       "1                   0                   0                   0   \n",
       "2                   0                   0                   0   \n",
       "3                   0                   0                   0   \n",
       "4                   0                   0                   0   \n",
       "\n",
       "   Medical_Keyword_48   BMI_Age  Med_Keywords_Count  num_na  \n",
       "0                   0  0.207304                   0       6  \n",
       "1                   0  0.016256                   0       5  \n",
       "2                   0  0.012799                   0       7  \n",
       "3                   0  0.057863                   1       7  \n",
       "4                   0  0.177213                   0       8  \n",
       "\n",
       "[5 rows x 69 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df[cont_columns].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = train_df[categorical_columns + cont_columns + ['Response']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "132"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_df.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert to categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in categorical_columns:\n",
    "    train_df[col] = train_df[col].astype('category').cat.as_ordered()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CategoricalDtype(categories=[1, 2], ordered=True)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['Product_Info_1'].dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(59381, 132)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Numericalise and process DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df, y, nas, mapper = structured.proc_df(train_df, 'Response', do_scale=True)\n",
    "y = y.astype('float')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_targets = len(set(y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create ColumnData object (instead of ImageClassifierData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv_idx = get_cv_idxs(len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([41952, 28556,  7984, ..., 39401,  9440, 31775])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_data = ColumnarModelData.from_data_frame(\n",
    "    PATH, cv_idx, df, y, cat_flds=categorical_columns, is_reg=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "144"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_data.trn_ds[0][0].shape[0] + model_data.trn_ds[0][1].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(82,)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_data.trn_ds[0][1].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get embedding sizes\n",
    "\n",
    "The formula Jeremy uses for getting embedding sizes is: cardinality / 2 (maxed out at 50).\n",
    "\n",
    "We reproduce that below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical_column_sizes = [\n",
    "    (c, len(train_df[c].cat.categories) + 1) for c in categorical_columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Product_Info_1', 3),\n",
       " ('Product_Info_2', 20),\n",
       " ('Product_Info_3', 35),\n",
       " ('Product_Info_5', 3),\n",
       " ('Product_Info_6', 3)]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categorical_column_sizes[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_sizes = [(c, min(50, (c+1)//2)) for _, c in categorical_column_sizes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(3, 2), (20, 10), (35, 18), (3, 2), (3, 2)]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_sizes[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "def emb_init(x):\n",
    "    x = x.weight.data\n",
    "    sc = 2/(x.size(1)+1)\n",
    "    x.uniform_(-sc,sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MixedInputModel(nn.Module):\n",
    "    def __init__(self, emb_sizes, num_cont):\n",
    "        super().__init__()\n",
    "        \n",
    "        embedding_layers = []\n",
    "        for size, dim in emb_sizes:\n",
    "            embedding_layers.append(\n",
    "                nn.Embedding(\n",
    "                    num_embeddings=size, embedding_dim=dim))\n",
    "        \n",
    "        self.embeddings = nn.ModuleList(embedding_layers)\n",
    "        for emb in self.embeddings: emb_init(emb)\n",
    "        self.embedding_dropout = nn.Dropout(0.04)\n",
    "        \n",
    "        self.batch_norm_cont = nn.BatchNorm1d(num_cont)\n",
    "        \n",
    "        num_emb = sum(e.embedding_dim for e in self.embeddings)\n",
    "        \n",
    "        self.fc1 = nn.Linear(\n",
    "            in_features=num_emb + num_cont,\n",
    "            out_features=1000)\n",
    "        kaiming_normal(self.fc1.weight.data)\n",
    "        self.dropout_fc1 = nn.Dropout(p=0.01)\n",
    "        self.batch_norm_fc1 = nn.BatchNorm1d(1000)\n",
    "        \n",
    "        self.fc2 = nn.Linear(\n",
    "            in_features=1000,\n",
    "            out_features=500)\n",
    "        kaiming_normal(self.fc2.weight.data)\n",
    "        self.dropout_fc2 = nn.Dropout(p=0.01)\n",
    "        self.batch_norm_fc2 = nn.BatchNorm1d(500)\n",
    "        \n",
    "        self.output_fc = nn.Linear(\n",
    "            in_features=500,\n",
    "            out_features=1\n",
    "        )\n",
    "        kaiming_normal(self.output_fc.weight.data)\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "        \n",
    "        \n",
    "    def forward(self, categorical_input, continuous_input):\n",
    "        # Add categorical embeddings together\n",
    "        categorical_embeddings = [e(categorical_input[:,i]) for i, e in enumerate(self.embeddings)]\n",
    "        categorical_embeddings = torch.cat(categorical_embeddings, 1)\n",
    "        categorical_embeddings_dropout = self.embedding_dropout(categorical_embeddings)\n",
    "             \n",
    "        # Batch normalise continuos vars\n",
    "        continuous_input_batch_norm = self.batch_norm_cont(continuous_input)\n",
    "        \n",
    "        # Create a single vector\n",
    "        x = torch.cat([\n",
    "            categorical_embeddings_dropout, continuous_input_batch_norm\n",
    "        ], dim=1)\n",
    "        \n",
    "        # Fully-connected layer 1\n",
    "        fc1_output = self.fc1(x)\n",
    "        fc1_relu_output = F.relu(fc1_output)\n",
    "        fc1_dropout_output = self.dropout_fc1(fc1_relu_output)\n",
    "        fc1_batch_norm = self.batch_norm_fc1(fc1_dropout_output)\n",
    "        \n",
    "        # Fully-connected layer 2\n",
    "        fc2_output = self.fc2(fc1_batch_norm)\n",
    "        fc2_relu_output = F.relu(fc2_output)\n",
    "        fc2_batch_norm = self.batch_norm_fc2(fc2_relu_output)\n",
    "        fc2_dropout_output = self.dropout_fc2(fc2_batch_norm)\n",
    "        \n",
    "        output = self.output_fc(fc2_dropout_output)\n",
    "        output = self.sigmoid(output)\n",
    "        \n",
    "        output = output * 7\n",
    "        output = output + 1\n",
    "\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_cont = len(df.columns) - len(categorical_columns)\n",
    "\n",
    "model = MixedInputModel(\n",
    "    embedding_sizes,\n",
    "    num_cont\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MixedInputModel(\n",
       "  (embeddings): ModuleList(\n",
       "    (0): Embedding(3, 2)\n",
       "    (1): Embedding(20, 10)\n",
       "    (2): Embedding(35, 18)\n",
       "    (3): Embedding(3, 2)\n",
       "    (4): Embedding(3, 2)\n",
       "    (5): Embedding(4, 2)\n",
       "    (6): Embedding(37, 19)\n",
       "    (7): Embedding(3, 2)\n",
       "    (8): Embedding(3, 2)\n",
       "    (9): Embedding(4, 2)\n",
       "    (10): Embedding(3, 2)\n",
       "    (11): Embedding(12, 6)\n",
       "    (12): Embedding(3, 2)\n",
       "    (13): Embedding(3, 2)\n",
       "    (14): Embedding(3, 2)\n",
       "    (15): Embedding(3, 2)\n",
       "    (16): Embedding(3, 2)\n",
       "    (17): Embedding(4, 2)\n",
       "    (18): Embedding(4, 2)\n",
       "    (19): Embedding(4, 2)\n",
       "    (20): Embedding(4, 2)\n",
       "    (21): Embedding(4, 2)\n",
       "    (22): Embedding(4, 2)\n",
       "    (23): Embedding(4, 2)\n",
       "    (24): Embedding(580, 50)\n",
       "    (25): Embedding(4, 2)\n",
       "    (26): Embedding(3, 2)\n",
       "    (27): Embedding(4, 2)\n",
       "    (28): Embedding(4, 2)\n",
       "    (29): Embedding(4, 2)\n",
       "    (30): Embedding(4, 2)\n",
       "    (31): Embedding(4, 2)\n",
       "    (32): Embedding(4, 2)\n",
       "    (33): Embedding(4, 2)\n",
       "    (34): Embedding(4, 2)\n",
       "    (35): Embedding(4, 2)\n",
       "    (36): Embedding(4, 2)\n",
       "    (37): Embedding(4, 2)\n",
       "    (38): Embedding(4, 2)\n",
       "    (39): Embedding(4, 2)\n",
       "    (40): Embedding(4, 2)\n",
       "    (41): Embedding(4, 2)\n",
       "    (42): Embedding(3, 2)\n",
       "    (43): Embedding(4, 2)\n",
       "    (44): Embedding(4, 2)\n",
       "    (45): Embedding(4, 2)\n",
       "    (46): Embedding(4, 2)\n",
       "    (47): Embedding(4, 2)\n",
       "    (48): Embedding(4, 2)\n",
       "    (49): Embedding(4, 2)\n",
       "    (50): Embedding(4, 2)\n",
       "    (51): Embedding(3, 2)\n",
       "    (52): Embedding(4, 2)\n",
       "    (53): Embedding(4, 2)\n",
       "    (54): Embedding(4, 2)\n",
       "    (55): Embedding(4, 2)\n",
       "    (56): Embedding(3, 2)\n",
       "    (57): Embedding(4, 2)\n",
       "    (58): Embedding(4, 2)\n",
       "    (59): Embedding(4, 2)\n",
       "    (60): Embedding(6, 3)\n",
       "    (61): Embedding(9, 5)\n",
       "  )\n",
       "  (embedding_dropout): Dropout(p=0.04)\n",
       "  (batch_norm_cont): BatchNorm1d(82, eps=1e-05, momentum=0.1, affine=True)\n",
       "  (fc1): Linear(in_features=303, out_features=1000, bias=True)\n",
       "  (dropout_fc1): Dropout(p=0.01)\n",
       "  (batch_norm_fc1): BatchNorm1d(1000, eps=1e-05, momentum=0.1, affine=True)\n",
       "  (fc2): Linear(in_features=1000, out_features=500, bias=True)\n",
       "  (dropout_fc2): Dropout(p=0.01)\n",
       "  (batch_norm_fc2): BatchNorm1d(500, eps=1e-05, momentum=0.1, affine=True)\n",
       "  (output_fc): Linear(in_features=500, out_features=1, bias=True)\n",
       "  (sigmoid): Sigmoid()\n",
       ")"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.column_data import StructuredLearner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weighted_kappa_metric(probs, y):\n",
    "    return quadratic_weighted_kappa(probs[:,0], y[:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner = StructuredLearner.from_model_data(model, model_data, metrics=[weighted_kappa_metric])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78ee5188c2ca40218a5fd9605fcb2aaf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   weighted_kappa_metric      \n",
      "    0      10.12092   9.828057   0.257332  \n"
     ]
    }
   ],
   "source": [
    "learner.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEOCAYAAACEiBAqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4VGXax/HvnU4gCSQESAghhB6aQJAiKNjWhqJi18Xu6mtf911d19XV9bWtu2tX7A11sa6IBZGigEBCJ4SaQIAACakkIW2e948ZsgGSMAkzc6bcn+uaK3POnDnn9zBh7pz2PGKMQSmlVOAKsjqAUkopa2khUEqpAKeFQCmlApwWAqWUCnBaCJRSKsBpIVBKqQCnhUAppQKcFgKllApwWgiUUirAaSFQSqkAF2J1AGd07tzZpKSkWB1DKaV8SmZmZqExJv5Yy/lEIUhJSSEjI8PqGEop5VNEZLszy+mhIaWUCnBaCJRSKsBpIVBKqQCnhUAppQKcFgKllApwWgiUUirAaSFQSqkW7C6poriixuoYbuUT9xEopZQnfb5iJ8/P3Uzu/koAzkjryuu/Tbc4lftoIVBKqUZKKmu4//O11NTZGuYt2brfwkTup4eGlFKqkefmbqau3sZdp/Xlx3tP5r4z+3Gguo4D1XVWR3MbLQRKKeVgsxm+Xp3Pib1iueeMfvTpEkW/rlEAbNpbbnE699FCoJRSDu//up3CA9WkxndomNe/m6MQ7NFCoJRSfu+n7H0A3HpK74Z5PTpFEhkWzIb8MqtiuZ0WAqWUwn5YaFVeCZel96BHbGTD/KAgYVBiNGt3lVqYzr20ECilFLBpXzmlVbWkp3Q66rXB3WPIyi+jrt7WxDt9nxYCpZQCvlmTjwiM79v5qNeGdI/hYK2NrQUVFiRzPy0ESqmAV28zfJq5k5P7xpMQ0+6o14cmdQRgydZCT0fzCC0ESqmA98uWQvJLD3Jpeo8mX+8d355BidF8vDwPY4yH07mfFgKlVEDbVVLFtLeWERYSxOlpXZpcRkS4ekxPsveU8/L8rR5O6H5aCJRSAe3TjJ0A/HZMT8JDgptd7tL0HpzcL55X529lX9lBT8XzCC0ESqmA9eS32fzzx00kxkTwp3MGtrhscJDw8OQ0qutt/GPOJg8l9AwtBEqpgFRaVcurC+yHecb27kxQkBzzPb3jO3De0AS+W7/Hr84VaCFQSgWkxVvsVwBdPSaZx6YMcvp9Q7rHUFJZS8GBandF8zgtBEqpgGOM4bm5m2kXGszDkwcRGeZ8j/z9HZ3Q+VPX1FoIlFIB58tVu8jeU87o1FhCg1v3NZieEkufLh14bcE2N6XzPC0ESqmAYozho6V5ADx50dBWvz8sJIgrTkwmK7+M//lwBTab758r0EKglAooK/NKWJZbxMOT0+gWE9GmdYzvY++G4pu1+aT+aTb/9PGriNxWCETkLRHZJyLrGs17RkSyRWSNiHwhIh3dtX2llGrK9+v3EBosXDwyqc3r6Nulw2HTz83dfLyxLOXOPYJ3gLOOmDcHGGyMGQpsAh5w4/aVUuoo63aVMjAhmuiI0DavIyhI2Pz42Tw2ZTAA4SFBHKytd1VEj3NbITDGLASKjpj3gzHm0MCfvwJtL8lKKdVKxhg25JczsFv0ca8rNDiIa8b05L3rT6S6zubTJ4+tPEdwPfCthdtXSgWY7D3lFFXUMKKn645Kn9wvnlMHdOG9Jbk+u1dgSSEQkQeBOuDDFpa5WUQyRCSjoKDAc+GUUn5rWY79IMX4vvEuXe/1J/Vif0UNP27Y69L1eorHC4GITAPOA64yLdyjbYyZboxJN8akx8e79kNTSgWmnMIKOoSHkNjGq4WaM7Z3HNERISzc5Jt/tDp/O50LiMhZwB+BU4wxlZ7ctlJK5RRWkNI5EpFj9yvUGsFBwom9Ylmxo8Sl6/UUd14++hGwBOgvIjtF5AbgRSAKmCMiq0TkVXdtXymlGjtYW8/KHcUuOVHclLTEGLYVHKCqxvfOE7htj8AYc0UTs9901/aUUqols9fmU3awjotGuOdixUGJ0dgMbNhTxojkTm7ZhrvoncVKqYDww/q9dO/YjjGpsW5Z/6BE+57G+t1lblm/O2khUEoFhG2FBxiYEO3y8wOHdO/Yjtj2YWTkFh17YS+jhUAp5fdsNkPu/kp6dY502zZEhDMGdmXuhn0+dz+BFgKllN+bmZlHTZ2Nwd1j3Lqdc4cmcKC6jkdnZbl1O66mhUAp5dfmb9zHHz9bS3xUOOcNTXTrtsb1jiMlLpLZa/N9aihLLQRKKb/2Q5b9bt9/XXYCwU6MS3w8QoKDmDYuhZLKWr5ek+/WbbmSFgKllN+y2Qzzsvdx+sCunOQYQ8DdBiXaDz/d+8kqj2zPFbQQKKX81sa95eSXHuTswd08ts1RKZ04qU8cIlBd5xsnjbUQKKX81o4ie082/RwDznuCiHDj+FRq6w3zN/pG30NaCJRSfmtncRUA3Tu18+h2J/TtTFRECG/9kuMTYxprIVBK+a2dxZVEhgXTKbLto5G1RUhwEBhYmlPEvzPyPLrtttBCoJTyW1sLKujVub3b7iZuyZ/PGwjAvI37PL7t1tJCoJTyS3X1NlZuL6a/B88PNHbZqGTOH5bI9+v3MifLuwes0UKglPJLry7YSnl1HeM8dNloU5Ic5yZuei/DsgzO0EKglPI7NXU23l2ynfF9OjN1pHu6nXbGlaOTG54frK2ntKqWqa8sZuWOYssyNUULgVLK7yzLKaKgvJpp41IszZHUKZKPbhoDwMzMndw3czUZ24t55vuNluY6kkeHqlRKKU/IK7bfP5CW6J7RyFpjTGosAxOieejLdQ3zcgsrKD9YS1SEZ69mao7uESil/MoTszfwwOdrAegaFW5xGvsNZrdP6tMw/cENo8kvO8hv31rmNcNaaiFQSvmNHfsreW3htobpkGDv+Io7Z0g3bp3Ym79NGcz4vp15fMoQVu4o4dUFW62OBuihIaWUH1mw+b9dOtw0oZeFSQ4nIvzxrAEN01eOTua9Jbms8JKTxloIlFJ+I2t3GTHtQln1lzMsuYmsNdJTOvHvjJ3kFVXSI9Z9I6c5wzv2m5RSygWydpcyKNF94xK70g3jU6mps3nFncdaCJRSfqGu3kb2nnLSEqy/UsgZKXGRdIoMJWt3mdVRtBAopfxD7v4KqutsXnHJqDNEhGE9OvLrtv2WD2uphUAp5RcOdTndM87a4+2tcfrAruTur2RrQYWlObQQKKX8wr6yagC6REVYnMR5h4bPXLJtv6U5tBAopfzC3rKDAHSJtv4mMmelxEXSMy6S5+du5mCtdTeXaSFQSvm8gvJqnp2zCYDwkGCL0zhPRHj0gsEUlFfzg6Or6orqOjbvLedAdZ3Hcuh9BEopn/Zj1l5udHTzPCqlk8VpWm9Cn84kxkRw50cr+W5dPtn55WwrrODUAV1469pRHsmgewRKKZ9VbzMNRQDgtol9WljaOwUFCY9fOITzhyUye+0ethXaTxz/lL2PzXvLPZPBI1tRSik3yMgtOmw63gs6mWuLSQO68PwVw/nDb/qTEhfJN3eOJ0jgP6t3e2T7emhIKeWzluYcXgh86URxU/5nUh9um9gbEaF3fAc25HvmZjPdI1BK+axft+0nLSGaL24bx6XpSXRu79uFAGjoHmNI9xiW5hRRXFHj9m1qIVBK+aTPV+xk8db9jEmNY3hyJ56eOoygIO/vY8hZN5+SijGwameJ27elh4aUUj6nsqaOe/+9GoApwxMtTuMeA7pFs+SBUz0yitkx9whEpL2IBDme9xOR80XEO8ZXU0oFpFfm2wd0+ev5gxia1NHiNO7jqaEsnTk0tBCIEJHuwFzgOuAdd4ZSSqnmFFXU8Mr8rVw4vLvlg9P7C2cKgRhjKoGLgBeMMRcCae6NpZRSTVu8tZA6m+HqMT2tjuI3nCoEIjIWuAr4xjHvmOcWROQtEdknIusazYsVkTkistnx0/duA1RKWcZmM/zrx83ER4UzNCnG6jh+w5lCcDfwAPCFMWa9iKQC85x43zvAWUfMux+Ya4zpi/0w0/2tyKqUCnBvL85ly74D3H/WAEK9ZGB6f3DMv+yNMQuABQCOk8aFxpg7nXjfQhFJOWL2BcBEx/N3gfnAH51Oq5QKWAXl1Tz1XTYn9orl3KEJVsfxK85cNTRDRKJFpD2QBWwUkT+0cXtdjTH5AI6fXdq4HqWUn6m3GV6Zv5VdJVWHzTfG8P6v2xn1+I/U1Nn487kDiQj1nR5GfYEz+1ZpxpgyYAowG0gGrnFrKkBEbhaRDBHJKCgocPfmlFIWW7erlKe+y+akJ38ir6iSA9V1PPvDRqa8tIiHvmw41chAHxmT2Jc4c0NZqOO+gSnAi8aYWhFp6wCbe0UkwRiTLyIJwL7mFjTGTAemA6Snp1s7oKdSyu2yGvWrM+Hpw09DJsREcOOEVAZ2i9JzA27gTCF4DcgFVgMLRaQn0NaekP4DTAOedPz8qo3rUUr5uLKDtSzeUshvBnVDRFizs5So8BC6xkSwZd+BhuU+uGE0Q3vEEO2hm6sCkTMni58Hnm80a7uITDrW+0TkI+wnhjuLyE7gYewF4N8icgOwA7ikLaGVUr5tV0kVZ/xjAZU19bx05QjOGtyNnzcXMDo1lnvO6MfHy/K49qQUOncIJ6adFgB3E2NaPuoiIjHYv8RPdsxaADxqjCl1c7YG6enpJiMj49gLKqW82qa95Tz93UZ+3LC3ydefv2I45w/zz76DrCAimcaY9GMt58zBtreAcuBSx6MMePv44imlAtGtH2Q2FIG49mHcckpqw2vRESFM1stCLeHMOYLexpiLG03/VURWuSuQUso/VdXUk1f030tDMx86A4BpY1O4fcYK7jitb0Nf/MqznCkEVSIy3hjzC4CInARUHeM9SinVwBjD+Kd+oqbexu/P6MdvBndreC2xYzs+v+0kC9MpZwrBrcC7jnMFAhQB17ozlFLKv+QVVbG/oob4qHBuP7WP/uXvZZy5amgVMExEoh3TnhlEUynlN5Y5Bpl//4YTtQh4oWYLgYjc28x8AIwx/3BTJqWUn1meU0RMu1D6dYmyOopqQkt7BPqJKaVcYvn2ItJ7dvKrMYX9SbOFwBjzV08GUUr5p6zdZWwrqODS9B5WR1HN0E47lFJuU1lTx2XTl9ApMpSzG10ppLyLM1cNKaVUq/yUvZevV+czomcnyg/W8fa1o+gZ197qWKoZWgiUUi732oJtLM0p4ouVuwAY0VNHpfVmzow9HA5cDKQ0Xt4Y86j7YimlfFVtvY0NjbqU/sNv+mvHcV7OmT2Cr4BSIBOodm8cpZSve+GnLZQdrOPG8b2wGbh2XIrVkdQxOFMIkowxRw5Cr5RSR6mps/HJ8h1M7B/Pn89LszqOcpIzVw0tFpEhbk+ilPJpBeXV9Pvzt+wtq9YrhHyMM3sE44FrRSQH+6EhAYwxZqhbkymlfMq/M/IASI6N5NyhOqaAL3GmEJzt9hRKKZ9msxk+Xr6D0b1i+eSWsVbHUa10zENDxpjtQEdgsuPR0TFPKaUA+DVnP3lFVVw5OtnqKKoNjlkIROQu4EOgi+PxgYjc4e5gSinfUFpVy5WvLwXgtIFdLU6j2sKZQ0M3AKONMRUAIvIUsAR4wZ3BlFLez2Yz3PpBZsN0h3C9R9UXOfOpCVDfaLreMU8pFeAueW0JmduLmdQ/nt+f2d/qOKqNnCkEbwNLReQLx/QU4E33RVJK+YJ52fvI3F5M+7BgXr5qJO3Cgq2OpNrImRHK/iEi87FfRirAdcaYle4OppTyXsUVNdzyfiadO4Tzyx8nERGqRcCXtTRCWbQxpkxEYoFcx+PQa7HGmCL3x1NKeRObzfDrtv2UV9dRU2/jL5PTtAj4gZb2CGYA52HvY8g0mi+O6VQ35lJKeaE3f8nh8dkbGqZPHdDFwjTKVVoaoew8x89enoujlPJmi7YWNjyfckKiXiXkJ5zphnquMea0Y81TSvm/3MIKzh2SwJMXD6F9mBYBf9HsDWUiEuE4P9BZRDqJSKzjkQJoRyJKBZiaOht5xVWkdI4kKiJUB6L3Iy2V9FuAu7F/6Wfy33sHyoCX3JxLKeVlNu4pp95mGJgQbXUU5WLN7hEYY55znB+4zxiTaozp5XgMM8a86MGMx+3Rr7O4b+ZqSitrrY6ilE85WFvPj1l7sdkMc7P3AjC0e0eLUylXc+Y+ghdEZDCQBkQ0mv+eO4O5Qk2djf0V1by1KAeATzN3ctOEXlya3oNthRWcmdYVEd29Vao5f/h0DV+v3s2tE3vz3uJcTukXT4/YdlbHUi7mzMnih4GJ2AvBbOzdUv8CeH0heOjLdXzi6CP9kNd/zuH1n+2F4YMbRjO+b2croinl9eZk7eXr1bsBeGX+VgDuOLWP/vHkh5wZoWwqcBqwxxhzHTAMCHdrKhdJ6BjR5PxDA2kvy9nvyThK+ZQPl24ntn0Yi+4/lfOHJfLncweSnhJrdSzlBs5c/1VljLGJSJ2IRAP78JGbyRJi/lsIsh87i1V5JbQPC2FIUgyXvbaEN3/J4aoxPeka3XTBUCpQ7SqpYsGmAm6f1IfuHdvx/BXDrY6k3MiZPYIMEekIvI796qEVwDK3pnKRTpFhAEzo25mI0GDGpMYxJCkGgKenDqWm3sZFLy9my75yK2Mq5XW+XLkLY+DS9B5WR1Ee4MwIZbcZY0qMMa8CZwDTHIeIvF7PuPYAnJF29GAZPePa89a1oygor+aJ2dmejqaUV7LZDPU2w+KthQxMiKZHbKTVkZQHtNTp3IiWXjPGrHBPJNfp3y2KRfefSmJM04d+JvSN567T+/LM9xvZuKec/t2iPJxQKe/y4rwt/GPOJgCuP0l7lwkULe0RPOt4vAQsBaZjPzy0FHje/dFco3vHdi1e5XD5KPuu72/+tZAt+w54KpZSXidrd1lDEQAY2zvOwjTKk1q6oWySMWYSsB0YYYxJN8aMBIYDWzwV0N3iOoQ37DFc9PIibDZzjHco5Z/eXZwLwIm9Yrl9Uh8m6KXVAcOZk8UDjDFrD00YY9YBJxzPRkXkHhFZLyLrROQjEbH0sp2XrhpBr87tKTtYR8b2YiujKGWJBZsK+D5rDxGhQbw5LZ37ftNfxxkIIM4Ugg0i8oaITBSRU0TkdWDDMd/VDBHpDtwJpBtjBgPBwOVtXZ8rDE/uxKe/GwvAqjwtBCqwLN5SyLS3llFSWcvzlw8nKiLU6kjKw5wpBNcB64G7sHdCl+WYdzxCgHYiEgJEAruPc33HLa5DOHHtw/i/2dn8sH6P1XGU8pj3f90OwD8uHcaZg7pZnEZZwZnLRw8aY/5pjLnQ8finMeZgWzdojNkF/B3YAeQDpcaYH9q6Ple6eGQSAG84uqBQyt/tP1DN3Ox9TBvbk4tGJFkdR1mkpctH/22MuVRE1nL4UJUAGGOGtmWDItIJuADoBZQAM0XkamPMB0csdzNwM0BycnJbNtVq953Zn69W7aKyts4j21PKKtV19Vz8ymL2lB6kps7GFaM9839MeaeWupi4y/HzPBdv83QgxxhTACAinwPjgMMKgTFmOvZLVklPT/fIpTxhIUFccWIyz83dTNbuMtIStd915X9sNsNLP21h3a6yhnn9u+o9NIGspctH8x0/tzf1OI5t7gDGiEik2C/wP43jOPnsateN60V0RCiXvLqYvKJKq+Mo5XKvLtzK8z/ZrwAf0C2KB84eoD2KBriWhqosF5GyJh7lIlLW3PuOxRizFPgUe59Fax0Zprd1fa4WExnK2YO7UVFTz4Sn57Fkq/ZQqvzHe0tyeWGuvQj8bcpgvrv7ZG45pbe1oZTlmj00ZIxx276iMeZh4GF3rf943XxyKl+v3k1FTT0fLduhd1gqv7CzuJK/fLUegFl3jGdw9xiLEylv4czlowCISBcRST70cGcoq6XGd2D9o2cxdWQS8zfuo7beZnUkpY7bgk0FAMz9/SlaBNRhjlkIROR8EdkM5AALgFzgWzfn8gqnDehC2cE6MvVuY+XjZq3ZzVu/5NA+LJjUzu2tjqO8jDN7BI8BY4BNjsHsTwMWuTWVl5jQL56w4CDmbthrdRSl2iy/tIrbZ6xka0EFg7vH6IlhdRRnCkGtMWY/ECQiQcaYeRxnX0O+okN4CKNTY5m9do92Rqe8WmllbbO/oxvy7dd2nDqgC09cNMSTsZSPcKYQlIhIB2Ah8KGIPAcEzB1X5w5JYFdJFROensdnmTv5KXsvB2vrrY6lVIPKmjqGPfoDj87KOuq1OVl7+ds3GwgS+NflJ5Aa38GChMrbOVMILgAqgXuA74CtwGR3hvIml43qwV/OS6Oooobfz1zN9e9kcOsHmVbHUqrB6rxSAN5ZnHvYXkHm9iJuei+DbQUV3HJKb6K1MznVDGcGr78ZmGmM2Qm86+Y8XkdEuH58L7p3asct79sLwLyNBTz7w0YuG9WDpE46lJ+y1uKthQ3Pv1+/h7CQIPJLDzYMtPTsJcOYMry7VfGUDxBjWj72LSIPA5cCRcDHwKfGGI+ePU1PTzcZGRme3ORRjDHkFFbQISKEEx+f2zD/i9vGMTy5k4XJVCDbW3aQs5/7md7x7Sk/WEf2nvLDXj8zrSvTf5tuUTplNRHJNMYc8xfAmd5H/2qMGQT8D5AILBCRH12Q0aeICKnxHegSFcFNE3rRvWM7AB78Yp3FyVSg2lpwgAlPz6OoooZHzh/Ew5MHHbXMtHEpng+mfI4zh4YO2QfsAfYDXdwTxzc8eG4aD56bxruLc3n4P+v5dm0+Zw9JsDqWCjD3f7aGmjobk4clMijRfoPY6F6xLM0p4uf/nURlTT39u2lncurYnLmh7FYRmQ/MBToDN7W1C2p/c9GI7gzr0ZHbZqzglflbqdM7kJUHlB2s5eJXFrM8t5iHJ6fxwhXDG157+aoRzLhpND1iI7UIKKc5c9VQT+BuY8wgY8zDxpijr1ELUFERoXxy8xhO6NGRp77LbjiZrJQ7fbVyF5nbi5nYP57LRvU47LW4DuGM662DzqvWceYcwf3GmFWeCOOLIkKDmX5NOpemJzE3ex8ZuUVWR1J+bvHW/STHRvLOdScSGdaao7tKNc3pTudU8+Kjwnnk/EF07hDG5dN/5atVu6yOpPxYTmEFfbvojWHKdbQQuEhkWAgf3jiG5NhI7v33albllVgdSfkhYwzb91eSoh3HKRfSQuBC/btF8cVtJ9E+LJjXF26zOo7yQ1+t2k1VbT2DdBhV5UJaCFwsJjKUK05M5tt1+ewpPWh1HOVHjDE88vV6ukVHcI5erqxcSAuBG1x+YjI2A2OemMtP2dqFtXKNvKIqSiprufO0vkSEBlsdR/kRLQRu0Ktze04f2BWA69/J4P0luZbmUb6vqKKGk5+ZB8Dg7npYSLmWFgI3efHK4Vw+qgfBQcJDX61nydb9VkdSPuxfP25qeH7oLmKlXEULgZtEhAbz5MVDWfmXM+gSFc4Vr/9KTmGF1bGUD/plcyHvLdkOwKMXDCI4SEcYU66lhcDNoiNCeei8NADeXZxrbRjlc/aVH+Tat5cB8MM9J/PbsSnWBlJ+SQuBB0welshFI7ozY+kOdpVUWR1H+ZAfs/ZRZzO8ds1I+nXVvoOUe2gh8JDfn9mfemN4ad4Wq6MoH/JT9j66RUdwZlpXq6MoP6aFwEO6d2zH1aOTmbF0B6c+O59lOdonkWrZrDW7+Sl7LxeO6I6InhdQ7qOFwIOucRzf3VZQwdVvLCVXTx6rJnyauZO7Pl7J7TNWMrh7DLdN7G11JOXntBB4UJ8uHfj5fyex5IFTEYG3FuVYHUl5mU17y7lv5mq+WrWbqSOT+PjmMUTpoPPKzbQPWw/rEWsf7P6MtK58vCyPkT07ccEJOrC4spu+cBshQcJ3d0+gTxc9Oaw8Q/cILPLHswZQU2/jro9XsXhrodVxlBeoq7fxzZp8LklP0iKgPEoLgUV6xEby90uGAXDNm8tYvEWLQaBbv7uMqtp6xqTGWR1FBRg9NGShqSOTiI8KZ9pby/jHnE2M66NDDAai79fvIb+kivd/3U6H8BBO0t8D5WFaCCx2Sr94/njWAJ76Lpsd+ytJjou0OpLyoO/W5fO7D1Y0TL905Qg6dwi3MJEKRHpoyAtMHmbvW/7rNbstTqI8xWYzLMsp4u1FuYC9R9HHLhjEuUN1nAHlebpH4AWSOkUyNjWOtxflcs3YnkTr5YJ+76vVu7jnk9UATBvbk79eMNjiRCqQ6R6Bl3jgnAHsr6jmhbmbrY6i3Kiu3sbKHcW87+hN9MResVw9pqfFqVSg0z0CLzE0qSMXj0ji3SXbueWU3nqc2E+9tnAbz3y/EYD/u3AIV45OtjiRUrpH4FWuP6kXdfU2Tnl6Ho9+nYUxxupIysVmrckH4M7T+nLFiT0sTqOUnRYCL5KWGM1nt45jYv8uvLUoh4Wb9d4Cf5JXVMmG/DL+fO5A7j2jn3Ykp7yGFgIvMzy5E/+87AS6RIXzzzmbKK2qtTqScpE1O0sBGNtbbxhT3sWSQiAiHUXkUxHJFpENIjLWihzeKiwkiAfOGcDaXaVc+NIiVuwotjqScoFv19kPC/Xq3N7iJEodzqo9gueA74wxA4BhwAaLcnitC4cn8eGNo9lZXMVFLy/mG8exZeV7Sqtque7tZQ3nByLD9BoN5V08XghEJBo4GXgTwBhTY4wp8XQOXzAmNY45955MVEQI981cTUlljdWRVCst3lLIje8uZ97GAi5L78GMG0dbHUmpo1ixR5AKFABvi8hKEXlDRI7aVxaRm0UkQ0QyCgoKPJ/SS/SMa8/M342lqrZeryTyEcYY/v79Rqa+spgr31jK8txiosJDeGrqUO1PSnklKwpBCDACeMUYMxyoAO4/ciFjzHRjTLoxJj0+Pt7TGb3KgG7RXD0mmc9X7iLtL9/z6NdZ1Nu0IHirzfsO8OK8LazfXcapA7rwtymDeeXqkVbHUqpZVhys3AnsNMYsdUx/ShOFQB3u0fMHE9s+nI17ynhrUQ57yqp4Zuow2ofr8WZvc82b9l/tr24/iX5ddVwB5f08/i1ijNkjInki0t8YsxE4DcjydA5fExQk3HtGPwBeW7CVJ744MD7/AAAQXUlEQVTNZltBBU9dPJRhPTpanE4dsjy3iL1l1QCk6tVBykdYddXQHcCHIrIGOAH4P4ty+KRbTunNOUO6kb2nnAteWkRVTb3VkRQwJ2svl7y6BIDnLj+BkGC9TUf5Bkt+U40xqxzH/4caY6YYY/RC+VZ6ePIgejrGLrjh3eXU1tssTqTeXZwLwG/H9uTcIdqdtPId+ieLj+oaHcGCP0zi2UuGsXjrfvo++C3XvLmU/NIqq6MFpB/W7+GXLYXceWofHr1gsO4NKJ+iZxp93MUjkwgJFmYs3cHSbUVc8+YyLhmZxE0TUgkK0r5sPOXRWVmkJURzyym9rY6iVKvpny1+4IITuvPJLWN557pRbN9fwRPfZvP5yl1WxwoYRRU17Cyu4sLh3fUqLuWTtBD4kXF9OrP2kd/QuUM4981czesLt+kNaG5kjOGleVt4b0kuAAMToi3No1Rb6Z8vfiYiNJhv7hzPw1+t5/HZG3h89gaevWQYF49MsjqaX1m7s5QLXvqFQ/f1JcREkJ7SydpQSrWR7hH4oa7REbxy9QjuPK0vAL+fuZrHZmVRU2fDpnckH5e6ehvXv7OcyS/+QmRYCO1Cg0mOjeTlq0YQERpsdTyl2kR84dBBenq6ycjIsDqGTyqqqOGJ2RuYmbkTgJS4SD65ZSxdoyMsTuabPsvcye9n2gedf+O36Zye1tXiREo1T0QyjTHpx1pO9wj8XGz7MJ65ZBivXj2CKSckkru/kjs+WkluYYXV0XzSjGU7SImLZPH9p2oRUH5DC0GAOGtwAv+6fDj/vGwYK3cU85t/LeS1BVup0xvRnLZpbzmZ24u5anRPEju2szqOUi6jhSDAXDg8ia/vGM/g7jE88W02N72XocXASW8vyiEsOEhPvCu/o4UgAA3oFs1nt47jz+cOZN7GgobzB6p563eX8vHyPK4ak0xs+zCr4yjlUloIAtgN43uRlhDNK/O3siqvhF0lVXrfQTO+WLGL0OAg7j69n9VRlHI5vY8ggIkI/3tWf659ezlTXlrUMP/FK4dz3tBEC5N5l7yiSt5dkkt6z1hi2oVaHUcpl9M9ggA3sX8XXr16RMN0RGgQ936ymkVbCi1M5V1+P3M1tfWGK0cnWx1FKbfQPQLFWYMTyH3yXAAKyqs5+7mf+eecTYzuFcuqvBKmL9xG1+gIHjovjbCQwPrbYdGWQpblFHH6wC5MHqZ7Sco/aSFQh4mPCufWib15bFYWA//yHbX1/z1nkNAxgtsm9rEwnWet3VnKrR9kkhgTweMXDrE6jlJuo4VAHeW6cSkkxkTwzdp8MnKLeXrqUGYs3cHT321kYLdoJg3o0rBsXb3Nb/ve/9MXa4kIDeb1ael6J7bya1oI1FGCgoSzhyRwdqNRtkanxjLxmflc985yJvWPZ3RqHD9l7yO3sILPbh1Hj9hICxO73pysvazdVcqD5wxkUGKM1XGUcistBMop4SHBvHTVCL5dm887i3OZt7Gg4bUrXv+VG8f34tJRPQBYuKmAk/vFExnmu79e0xdupWNkKJek681jyv/57v9U5XEjkjsxIrkTt03sw7bCA3SKDGPd7jL+9PlaHvk6i5mZO6mps7F53wEuOCGRy0clM7Z3nNWxW+2bNfkszy3mllNS6RipN48p/6e9j6rjZrMZPl+5iye/zSYsWNhTdrChn/7EmAjOSOvKA+cM9Ppumksqa7jolcVsK6hgRHJHpv82nc4dwq2OpVSbOdv7qBYC5TKHfpdW5pVw0cuLD3ttQLcoZt0x3mUnlssO1nLvJ6tIT4nl3CEJx3WOoqqmnlV5JXy2YiefZu5kaFIMH988xqcPbSkFWgiUxWw2w4GaOvKKKvnbrA0s2baf4CDhmalDSYhpx/Dkjk7tIazYUczmveWcNzSR2nobN76bQcb2YhJiIsgvPQhAcmwkN07oRV5RJfvKq3nq4qHHXHfm9mKy95Rx0fAkznpuIdv3VwJw1qBuvHrNyOP/B1DKC2ghUF7DZjM89NU6Ply6o2He4O7R/G3KEPp17cD+AzVH/UX/4dLtPPjFumbXeVKfOM5M68b+A9U8/9OWw1675eRUHjhn4FHvyS+tYl52AZMGxDP2iZ8Oe+2KE5Pp26UD14ztSaifXg6rAo8WAuV1Nu4pZ2nOflbnlTJv4z6KKmoaXotpF8qwHh0Z2C2KTu3DePLb7IbXzh2SwKq8EgAuHpnE3af1JShIGl7fVnCA2npDvc3w9qIcPl+5i/evP5EhSTHYDIQECdV1Nv7y1TpmrclveN+43nHkFVcyNKkjL1w+/LB1KuUPtBAor7arpIpnf9hI5vbihsMyXaLC2VdeDdgLw9vXjSItIbpVJ5l3lVRx6atL2FVS1eTrPWLbkRjTjkGJMTx03kBE9Mtf+S8tBMpnLM8tIjk2ki5R4Xy/fi9zsvZy7bgUhiS17Uau4ooa/vXjJrYUHCA5NpLNew+wp+wg6T07ccdpfekd38HFLVDKO2khUEqpAKeD1yullHKKFgKllApwWgiUUirAaSFQSqkAp4VAKaUCnBYCpZQKcFoIlFIqwGkhUEqpAOf1/eyKyGSgUES2O2bFAKUtPO8MFB7HJhuvs7XLNDX/yHktTftrexrPO572ONOWlpbzxfa0tIy2x7/a447vgp5OLWWM8eoHML256aaeAxmu3F5rlmlqfkv5A6U9R8xrc3ucaYu/taelZbQ9/tUed38XtPTwhUNDX7cw3dxzV26vNcs0Nb+l/EdO+2t7PNmWlpbzxfa0tIy2x7/a4+7vgmb5RF9DrSEiGcaJvjV8hbbHu2l7vJs/tcedbfGFPYLWmm51ABfT9ng3bY9386f2uK0tfrdHoJRSqnX8cY9AKaVUK2ghUEqpAKeFQCmlAlxAFQIRmSgiP4vIqyIy0eo8riAi7UUkU0TOszrL8RCRgY7P5VMRudXqPK4gIlNE5HUR+UpEzrQ6z/EQkVQReVNEPrU6S1s5/q+86/hMrrI6z/Fy5WfiM4VARN4SkX0isu6I+WeJyEYR2SIi9x9jNQY4AEQAO92V1Rkuag/AH4F/uyelc1zRFmPMBmPM74BLAcsv93NRm740xtwEXAtc5sa4LXJRW7YZY25wb9LWa2XbLgI+dXwm53s8rBNa0x6XfibuulPN1Q/gZGAEsK7RvGBgK5AKhAGrgTRgCDDriEcXIMjxvq7Ah37QntOBy7F/0Zzny21xvOd8YDFwpT/8vjV637PACD9py6dWfzbH0bYHgBMcy8ywOvvxtseVn4nX9zV0iDFmoYikHDH7RGCLMWYbgIh8DFxgjHkCaOlQSTEQ7o6cznJFe0RkEtAe+y95lYjMNsbY3Bq8Ca76bIwx/wH+IyLfADPcl/jYXPT5CPAk8K0xZoV7EzfPxf93vEpr2ob9KEASsAovPRrSyvZkuWq7XvmP0QrdgbxG0zsd85okIheJyGvA+8CLbs7WFq1qjzHmQWPM3di/NF+3ogi0oLWfzUQRed7x+cx2d7g2alWbgDuw77VNFZHfuTNYG7T284kTkVeB4SLygLvDHafm2vY5cLGIvIIHum1woSbb48rPxGf2CJohTcxr9g45Y8zn2H8ZvFWr2tOwgDHvuD7KcWvtZzMfmO+uMC7S2jY9DzzvvjjHpbVt2Q94WzFrTpNtM8ZUANd5OowLNNcel30mvr5HsBPo0Wg6CdhtURZX8Kf2+FNbDvGnNvlTW47kb21ze3t8vRAsB/qKSC8RCcN+4vQ/Fmc6Hv7UHn9qyyH+1CZ/asuR/K1t7m+P1WfJW3E2/SMgH6jFXiFvcMw/B9iE/az6g1bnDMT2+FNb/LFN/tQWf2+bVe3RTueUUirA+fqhIaWUUsdJC4FSSgU4LQRKKRXgtBAopVSA00KglFIBTguBUkoFOC0EyuVE5IAHtnG+k910u3KbE0VkXBveN1xE3nA8v1ZEvKKfKxFJObK74yaWiReR7zyVSVlDC4HyWiIS3Nxrxpj/GGOedMM2W+p/ayLQ6kIA/Al4oU2BLGaMKQDyReQkq7Mo99FCoNxKRP4gIstFZI2I/LXR/C/FPrLaehG5udH8AyLyqIgsBcaKSK6I/FVEVojIWhEZ4Fiu4S9rEXnH0XPpYhHZJiJTHfODRORlxzZmicjsQ68dkXG+iPyfiCwA7hKRySKyVERWisiPItLV0TXw74B7RGSViExw/LX8maN9y5v6shSRKGCoMWZ1E6/1FJG5jn+buSKS7JjfW0R+dazz0ab2sMQ+2tY3IrJaRNaJyGWO+aMc/w6rRWSZiEQ5/vL/2fFvuKKpvRoRCRaRZxp9Vrc0evlLwOdH9FItsPqWan343wM44Ph5JjAde++JQdgHOTnZ8Vqs42c7YB0Q55g2wKWN1pUL3OF4fhvwhuP5tcCLjufvADMd20jD3nc7wFTsXVoHAd2wj0MxtYm884GXG013goa77m8EnnU8fwS4r9FyM4DxjufJwIYm1j0J+KzRdOPcXwPTHM+vB750PJ8FXOF4/rtD/55HrPdi7F2PH5qOwT5oyTZglGNeNPYehiOBCMe8vkCG43kKjgFQgJuBPzuehwMZQC/HdHdgrdW/V/pw38PXu6FW3u1Mx2OlY7oD9i+ihcCdInKhY34Px/z9QD3w2RHrOdR1eCb24Qab8qWxj8eQJSJdHfPGAzMd8/eIyLwWsn7S6HkS8ImIJGD/cs1p5j2nA2kiDb0ER4tIlDGmvNEyCUBBM+8f26g97wNPN5o/xfF8BvD3Jt67Fvi7iDwFzDLG/CwiQ4B8Y8xyAGNMGdj3HoAXReQE7P++/ZpY35nA0EZ7TDHYP5McYB+Q2EwblB/QQqDcSYAnjDGvHTZTZCL2L9GxxphKEZmPfRxpgIPGmPoj1lPt+FlP87+z1Y2eyxE/nVHR6PkLwD+MMf9xZH2kmfcEYW9DVQvrreK/bTsWpzv+MsZsEpGR2Dsje0JEfsB+CKepddwD7AWGOTIfbGIZwb7n9X0Tr0Vgb4fyU3qOQLnT98D1ItIBQES6i0gX7H9tFjuKwABgjJu2/wv2EamCHHsJE518Xwywy/F8WqP55UBUo+kfgNsPTTj+4j7SBqBPM9tZjL1LYbAfg//F8fxX7Id+aPT6YUQkEag0xnyAfY9hBJANJIrIKMcyUY6T3zHY9xRswDXYx8A90vfArSIS6nhvP8eeBNj3IFq8ukj5Ni0Eym2MMT9gP7SxRETWAp9i/yL9DggRkTXAY9i/+NzhM+xd+a4DXgOWAqVOvO8RYKaI/AwUNpr/NXDhoZPFwJ1AuuPkahZNjBZljMkGYhwnjY90J3Cd49/hGuAux/y7gXtFZBn2Q0tNZR4CLBORVcCDwN+MMTXAZcALIrIamIP9r/mXgWki8iv2L/WKJtb3BvYxcFc4Lil9jf/ufU0CvmniPcpPaDfUyq+JSAdjzAERiQOWAScZY/Z4OMM9QLkx5g0nl48EqowxRkQux37i+AK3hmw5z0LsA9sXW5VBuZeeI1D+bpaIdMR+0vcxTxcBh1eAS1qx/EjsJ3cFKMF+RZElRCQe+/kSLQJ+TPcIlFIqwOk5AqWUCnBaCJRSKsBpIVBKqQCnhUAppQKcFgKllApwWgiUUirA/T981acrPQAeLgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit() warning: use_wd_sched is set to True, but weight decay(s) passed are 0. Use wds to pass weight decay values.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80278fae5fdf4ff5b371b289cf2acb2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=3), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   weighted_kappa_metric       \n",
      "    0      4.045112   4.093557   0.500641  \n",
      "    1      3.890376   3.969434   0.511264                   \n",
      "    2      3.697106   3.902796   0.517778                    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([3.9028]), 0.5177778103949219]"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit(0.0001, 3, use_wd_sched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit() warning: use_wd_sched is set to True, but weight decay(s) passed are 0. Use wds to pass weight decay values.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc24b55a5cdf41cc9f2dfd0476747df8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=31), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   weighted_kappa_metric      \n",
      "    0      3.669837   3.878638   0.525141  \n",
      "    1      3.753951   3.853548   0.531628                   \n",
      "    2      3.693147   3.823528   0.537189                   \n",
      "    3      3.769119   3.787117   0.532412                   \n",
      "    4      3.518593   3.804554   0.537648                   \n",
      "    5      3.534063   3.783307   0.542196                   \n",
      "    6      3.596886   3.779415   0.542739                   \n",
      "    7      3.571018   3.778712   0.541825                    \n",
      "    8      3.641947   3.760956   0.54653                    \n",
      "    9      3.546584   3.752363   0.554844                    \n",
      "    10     3.656892   3.735377   0.551656                   \n",
      "    11     3.458163   3.745283   0.554712                    \n",
      "    12     3.429208   3.739722   0.550354                    \n",
      "    13     3.492735   3.718385   0.550943                    \n",
      "    14     3.367725   3.742327   0.551551                   \n",
      "    15     3.435102   3.764339   0.549016                   \n",
      "    16     3.39208    3.745095   0.555498                   \n",
      "    17     3.369494   3.778072   0.555524                   \n",
      "    18     3.557012   3.739735   0.547905                   \n",
      "    19     3.377399   3.736502   0.557275                    \n",
      "    20     3.438189   3.801713   0.55953                    \n",
      "    21     3.318981   3.77109    0.553916                   \n",
      "    22     3.308797   3.784635   0.55882                    \n",
      "    23     3.191291   3.75078    0.567412                   \n",
      "    24     3.154368   3.761348   0.556422                    \n",
      "    25     3.196123   3.768691   0.563026                    \n",
      "    26     3.196808   3.73859    0.561961                   \n",
      "    27     3.242004   3.723597   0.564393                   \n",
      "    28     3.186991   3.774825   0.556722                   \n",
      "    29     3.182807   3.746873   0.559449                    \n",
      "    30     3.108987   3.75402    0.559776                   \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([3.75402]), 0.5597761107909427]"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit(0.0001, 5, cycle_len=1, cycle_mult=2, use_wd_sched=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fit() warning: use_wd_sched is set to True, but weight decay(s) passed are 0. Use wds to pass weight decay values.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a18e6c2296a74208a853571638b3fa0d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   weighted_kappa_metric      \n",
      "    0      3.139123   3.736645   0.561148  \n",
      "    1      3.069922   3.761296   0.5581                     \n",
      "    2      3.16154    3.75898    0.556315                    \n",
      "    3      3.167846   3.765681   0.552248                   \n",
      "    4      3.088502   3.771049   0.557628                   \n",
      "    5      3.270517   3.763619   0.561866                    \n",
      "    6      3.176511   3.758319   0.557705                   \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([3.75832]), 0.5577047550521828]"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit(0.00001, 3, cycle_len=1, cycle_mult=2, use_wd_sched=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There's either a bug in my implementation, or a NN doesn't do that well at this problem."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}