L5 - Decision Tree.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict the Sun Hours by Decision Tree"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.tree import DecisionTreeClassifier, export_graphviz\n",
    "from IPython.display import Image\n",
    "from sklearn.externals.six import StringIO\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "RANDOM_SEED = 9"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## Import Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below is the [Daily Weather Observations of Sydney, New South Wales between Aug 2015 and Aug 2016](http://www.bom.gov.au/climate/dwo/IDCJDW0210.shtml)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(397, 24)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Date</th>\n",
       "      <th>Day</th>\n",
       "      <th>Temps_min</th>\n",
       "      <th>Temps_max</th>\n",
       "      <th>Rain</th>\n",
       "      <th>Evap</th>\n",
       "      <th>Sun_hours</th>\n",
       "      <th>Max_wind_dir</th>\n",
       "      <th>...</th>\n",
       "      <th>CLD_at_9am</th>\n",
       "      <th>Wind_dir_at_9am</th>\n",
       "      <th>Wind_spd_at_9am</th>\n",
       "      <th>MSLP_at_9am</th>\n",
       "      <th>Temp_at_3pm</th>\n",
       "      <th>RH_at_3pm</th>\n",
       "      <th>CLD_at_3pm</th>\n",
       "      <th>Wind_dir_at_3pm</th>\n",
       "      <th>Wind_spd_at_3pm</th>\n",
       "      <th>MSLP_at_3pm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2015</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>Sa</td>\n",
       "      <td>9.4</td>\n",
       "      <td>22.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>9.2</td>\n",
       "      <td>WNW</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NW</td>\n",
       "      <td>15</td>\n",
       "      <td>1018.6</td>\n",
       "      <td>20.7</td>\n",
       "      <td>28</td>\n",
       "      <td>6</td>\n",
       "      <td>NW</td>\n",
       "      <td>28</td>\n",
       "      <td>1015.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2015</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>Su</td>\n",
       "      <td>14.3</td>\n",
       "      <td>24.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.4</td>\n",
       "      <td>7.8</td>\n",
       "      <td>WNW</td>\n",
       "      <td>...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>WNW</td>\n",
       "      <td>9</td>\n",
       "      <td>1015.1</td>\n",
       "      <td>23.8</td>\n",
       "      <td>30</td>\n",
       "      <td>5</td>\n",
       "      <td>NW</td>\n",
       "      <td>20</td>\n",
       "      <td>1010.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2015</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>Mo</td>\n",
       "      <td>8.4</td>\n",
       "      <td>19.4</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.4</td>\n",
       "      <td>10.1</td>\n",
       "      <td>WSW</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>WNW</td>\n",
       "      <td>20</td>\n",
       "      <td>1015.2</td>\n",
       "      <td>19.0</td>\n",
       "      <td>29</td>\n",
       "      <td>1</td>\n",
       "      <td>WSW</td>\n",
       "      <td>17</td>\n",
       "      <td>1013.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2015</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>Tu</td>\n",
       "      <td>8.3</td>\n",
       "      <td>15.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.8</td>\n",
       "      <td>4.6</td>\n",
       "      <td>WSW</td>\n",
       "      <td>...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>W</td>\n",
       "      <td>19</td>\n",
       "      <td>1023.0</td>\n",
       "      <td>14.4</td>\n",
       "      <td>42</td>\n",
       "      <td>4</td>\n",
       "      <td>SSE</td>\n",
       "      <td>2</td>\n",
       "      <td>1018.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2015</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>We</td>\n",
       "      <td>5.0</td>\n",
       "      <td>14.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.6</td>\n",
       "      <td>4.8</td>\n",
       "      <td>W</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>WNW</td>\n",
       "      <td>20</td>\n",
       "      <td>1014.5</td>\n",
       "      <td>13.0</td>\n",
       "      <td>46</td>\n",
       "      <td>6</td>\n",
       "      <td>WNW</td>\n",
       "      <td>19</td>\n",
       "      <td>1010.3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year  Month  Date Day  Temps_min  Temps_max  Rain  Evap  Sun_hours  \\\n",
       "0  2015      8     1  Sa        9.4       22.3   0.0   3.4        9.2   \n",
       "1  2015      8     2  Su       14.3       24.7   0.0   5.4        7.8   \n",
       "2  2015      8     3  Mo        8.4       19.4   0.0   5.4       10.1   \n",
       "3  2015      8     4  Tu        8.3       15.1   0.0   3.8        4.6   \n",
       "4  2015      8     5  We        5.0       14.2   0.0   0.6        4.8   \n",
       "\n",
       "  Max_wind_dir     ...       CLD_at_9am Wind_dir_at_9am  Wind_spd_at_9am  \\\n",
       "0          WNW     ...              6.0              NW               15   \n",
       "1          WNW     ...              5.0             WNW                9   \n",
       "2          WSW     ...              1.0             WNW               20   \n",
       "3          WSW     ...              7.0               W               19   \n",
       "4            W     ...              6.0             WNW               20   \n",
       "\n",
       "   MSLP_at_9am  Temp_at_3pm RH_at_3pm  CLD_at_3pm  Wind_dir_at_3pm  \\\n",
       "0       1018.6         20.7        28           6               NW   \n",
       "1       1015.1         23.8        30           5               NW   \n",
       "2       1015.2         19.0        29           1              WSW   \n",
       "3       1023.0         14.4        42           4              SSE   \n",
       "4       1014.5         13.0        46           6              WNW   \n",
       "\n",
       "   Wind_spd_at_3pm  MSLP_at_3pm  \n",
       "0               28       1015.6  \n",
       "1               20       1010.4  \n",
       "2               17       1013.7  \n",
       "3                2       1018.9  \n",
       "4               19       1010.3  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('./asset/Daily_Weather_Observations.csv', sep=',')\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Column Meanings\n",
    "| Heading         | Meaning                                                  | Units               |\n",
    "|-----------------|----------------------------------------------------------|---------------------|\n",
    "| Day             | Day of the week                                          | first two letters   |\n",
    "| Temps_min       | Minimum temperature in the 24 hours to 9am.              | degrees Celsius     |\n",
    "| Temps_max       | Maximum temperature in the 24 hours from 9am.            | degrees Celsius     |\n",
    "| Rain            | Precipitation (rainfall) in the 24 hours to 9am.         | millimetres         |\n",
    "| Evap            | Class A pan evaporation in the 24 hours to 9am           | millimetres         |\n",
    "| Sun_hours       | Bright sunshine in the 24 hours to midnight              | hours               |\n",
    "| Max_wind_dir    | Direction of strongest gust in the 24 hours to midnight  | 16 compass points   |\n",
    "| Max_wind_spd    | Speed of strongest wind gust in the 24 hours to midnight | kilometres per hour |\n",
    "| Max_wind_time   | Time of strongest wind gust                              | local time hh:mm    |\n",
    "| Temp_at_9am     | Temperature at 9 am                                      | degrees Celsius     |\n",
    "| RH_at_9am       | Relative humidity at 9 am                                | percent             |\n",
    "| CLD_at_9am      | Fraction of sky obscured by cloud at 9 am                | eighths             |\n",
    "| Wind_dir_at_9am | Wind direction averaged over 10 minutes prior to 9 am    | compass points      |\n",
    "| Wind_spd_at_9am | Wind speed averaged over 10 minutes prior to 9 am        | kilometres per hour |\n",
    "| MSLP_at_9am     | Atmospheric pressure reduced to mean sea level at 9 am   | hectopascals        |\n",
    "| Temp_at_3pm     | Temperature at 3 pm                                      | degrees Celsius     |\n",
    "| RH_at_3pm       | Relative humidity at 3 pm                                | percent             |\n",
    "| CLD_at_3pm      | Fraction of sky obscured by cloud at 3 pm                | eighths             |\n",
    "| Wind_dir_at_3pm | Wind direction averaged over 10 minutes prior to 3 pm    | compass points      |\n",
    "| Wind_spd_at_3pm | Wind speed averaged over 10 minutes prior to 3 pm        | kilometres per hour |\n",
    "| MSLP_at_3pm     | Atmospheric pressure reduced to mean sea level at 3 pm   | hectopascals        |"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have noticed that the *Sun_hours* values are missing in some rows. We want to predict these missing *Sun_hours* values.\n",
    "\n",
    "We firstly seperate those rows with missing Sun_hours from the original dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Date</th>\n",
       "      <th>Day</th>\n",
       "      <th>Temps_min</th>\n",
       "      <th>Temps_max</th>\n",
       "      <th>Rain</th>\n",
       "      <th>Evap</th>\n",
       "      <th>Sun_hours</th>\n",
       "      <th>Max_wind_dir</th>\n",
       "      <th>...</th>\n",
       "      <th>CLD_at_9am</th>\n",
       "      <th>Wind_dir_at_9am</th>\n",
       "      <th>Wind_spd_at_9am</th>\n",
       "      <th>MSLP_at_9am</th>\n",
       "      <th>Temp_at_3pm</th>\n",
       "      <th>RH_at_3pm</th>\n",
       "      <th>CLD_at_3pm</th>\n",
       "      <th>Wind_dir_at_3pm</th>\n",
       "      <th>Wind_spd_at_3pm</th>\n",
       "      <th>MSLP_at_3pm</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>2015</td>\n",
       "      <td>11</td>\n",
       "      <td>25</td>\n",
       "      <td>We</td>\n",
       "      <td>17.3</td>\n",
       "      <td>32.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NNE</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>W</td>\n",
       "      <td>9</td>\n",
       "      <td>1013.1</td>\n",
       "      <td>28.7</td>\n",
       "      <td>44</td>\n",
       "      <td>0</td>\n",
       "      <td>NNE</td>\n",
       "      <td>24</td>\n",
       "      <td>1007.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>2015</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>Sa</td>\n",
       "      <td>15.5</td>\n",
       "      <td>24.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NE</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>W</td>\n",
       "      <td>7</td>\n",
       "      <td>1022.6</td>\n",
       "      <td>24.1</td>\n",
       "      <td>52</td>\n",
       "      <td>1</td>\n",
       "      <td>NE</td>\n",
       "      <td>20</td>\n",
       "      <td>1018.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>2015</td>\n",
       "      <td>12</td>\n",
       "      <td>20</td>\n",
       "      <td>Su</td>\n",
       "      <td>19.9</td>\n",
       "      <td>30.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NE</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>E</td>\n",
       "      <td>11</td>\n",
       "      <td>1013.4</td>\n",
       "      <td>28.5</td>\n",
       "      <td>52</td>\n",
       "      <td>1</td>\n",
       "      <td>NNE</td>\n",
       "      <td>31</td>\n",
       "      <td>1008.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166</th>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>Th</td>\n",
       "      <td>22.7</td>\n",
       "      <td>39.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>SE</td>\n",
       "      <td>6</td>\n",
       "      <td>1011.7</td>\n",
       "      <td>32.4</td>\n",
       "      <td>43</td>\n",
       "      <td>7</td>\n",
       "      <td>ENE</td>\n",
       "      <td>15</td>\n",
       "      <td>1008.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>334</th>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>30</td>\n",
       "      <td>Th</td>\n",
       "      <td>5.4</td>\n",
       "      <td>16.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NW</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>W</td>\n",
       "      <td>11</td>\n",
       "      <td>1021.2</td>\n",
       "      <td>14.5</td>\n",
       "      <td>33</td>\n",
       "      <td>7</td>\n",
       "      <td>NNW</td>\n",
       "      <td>24</td>\n",
       "      <td>1015.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>335</th>\n",
       "      <td>2016</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>Fr</td>\n",
       "      <td>7.7</td>\n",
       "      <td>16.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NW</td>\n",
       "      <td>...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>WNW</td>\n",
       "      <td>13</td>\n",
       "      <td>1015.7</td>\n",
       "      <td>16.4</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "      <td>WSW</td>\n",
       "      <td>20</td>\n",
       "      <td>1016.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>369</th>\n",
       "      <td>2016</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>Th</td>\n",
       "      <td>10.8</td>\n",
       "      <td>18.1</td>\n",
       "      <td>61.0</td>\n",
       "      <td>9.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>SE</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>W</td>\n",
       "      <td>15</td>\n",
       "      <td>1024.3</td>\n",
       "      <td>15.5</td>\n",
       "      <td>60</td>\n",
       "      <td>7</td>\n",
       "      <td>SSW</td>\n",
       "      <td>19</td>\n",
       "      <td>1025.1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Year  Month  Date Day  Temps_min  Temps_max  Rain  Evap  Sun_hours  \\\n",
       "116  2015     11    25  We       17.3       32.7   0.0   9.0        NaN   \n",
       "126  2015     12     5  Sa       15.5       24.9   0.0   8.0        NaN   \n",
       "141  2015     12    20  Su       19.9       30.3   0.0  13.0        NaN   \n",
       "166  2016      1    14  Th       22.7       39.2   0.0   8.4        NaN   \n",
       "334  2016      6    30  Th        5.4       16.5   0.0   1.8        NaN   \n",
       "335  2016      7     1  Fr        7.7       16.7   0.0   4.0        NaN   \n",
       "369  2016      8     4  Th       10.8       18.1  61.0   9.4        NaN   \n",
       "\n",
       "    Max_wind_dir     ...       CLD_at_9am Wind_dir_at_9am  Wind_spd_at_9am  \\\n",
       "116          NNE     ...              1.0               W                9   \n",
       "126           NE     ...              1.0               W                7   \n",
       "141           NE     ...              1.0               E               11   \n",
       "166            S     ...              1.0              SE                6   \n",
       "334           NW     ...              1.0               W               11   \n",
       "335           NW     ...              7.0             WNW               13   \n",
       "369           SE     ...              6.0               W               15   \n",
       "\n",
       "     MSLP_at_9am  Temp_at_3pm RH_at_3pm  CLD_at_3pm  Wind_dir_at_3pm  \\\n",
       "116       1013.1         28.7        44           0              NNE   \n",
       "126       1022.6         24.1        52           1               NE   \n",
       "141       1013.4         28.5        52           1              NNE   \n",
       "166       1011.7         32.4        43           7              ENE   \n",
       "334       1021.2         14.5        33           7              NNW   \n",
       "335       1015.7         16.4        33           1              WSW   \n",
       "369       1024.3         15.5        60           7              SSW   \n",
       "\n",
       "     Wind_spd_at_3pm  MSLP_at_3pm  \n",
       "116               24       1007.4  \n",
       "126               20       1018.9  \n",
       "141               31       1008.7  \n",
       "166               15       1008.1  \n",
       "334               24       1015.1  \n",
       "335               20       1016.2  \n",
       "369               19       1025.1  \n",
       "\n",
       "[7 rows x 24 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_missing_sun_hours = data[pd.isnull(data['Sun_hours'])]\n",
    "data_missing_sun_hours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(390, 24)\n"
     ]
    }
   ],
   "source": [
    "data = data[pd.notnull(data['Sun_hours'])]\n",
    "print(data.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We firstly category Sun_hours into three levels: High(>10), Med(>5 and <=10), and Low(<=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sun_hours</th>\n",
       "      <th>Sun_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9.2</td>\n",
       "      <td>Med</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7.8</td>\n",
       "      <td>Med</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10.1</td>\n",
       "      <td>High</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>Low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.8</td>\n",
       "      <td>Low</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Sun_hours Sun_level\n",
       "0        9.2       Med\n",
       "1        7.8       Med\n",
       "2       10.1      High\n",
       "3        4.6       Low\n",
       "4        4.8       Low"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = ['Low','Med','High']\n",
    "data['Sun_level'] = pd.cut(data.Sun_hours, [-1,5,10,25], labels=labels)\n",
    "data[['Sun_hours','Sun_level']].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing (Handling Missing Values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From time to time, observations will not be available, for a variety of reasons. We need to handle missing values before training a classifier. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Date</th>\n",
       "      <th>Day</th>\n",
       "      <th>Temps_min</th>\n",
       "      <th>Temps_max</th>\n",
       "      <th>Rain</th>\n",
       "      <th>Evap</th>\n",
       "      <th>Sun_hours</th>\n",
       "      <th>Max_wind_dir</th>\n",
       "      <th>...</th>\n",
       "      <th>Wind_dir_at_9am</th>\n",
       "      <th>Wind_spd_at_9am</th>\n",
       "      <th>MSLP_at_9am</th>\n",
       "      <th>Temp_at_3pm</th>\n",
       "      <th>RH_at_3pm</th>\n",
       "      <th>CLD_at_3pm</th>\n",
       "      <th>Wind_dir_at_3pm</th>\n",
       "      <th>Wind_spd_at_3pm</th>\n",
       "      <th>MSLP_at_3pm</th>\n",
       "      <th>Sun_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>2015</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>We</td>\n",
       "      <td>9.8</td>\n",
       "      <td>20.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>10.9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>W</td>\n",
       "      <td>20</td>\n",
       "      <td>1023.1</td>\n",
       "      <td>18.9</td>\n",
       "      <td>39</td>\n",
       "      <td>1</td>\n",
       "      <td>SSE</td>\n",
       "      <td>17</td>\n",
       "      <td>1022.0</td>\n",
       "      <td>High</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>2015</td>\n",
       "      <td>9</td>\n",
       "      <td>15</td>\n",
       "      <td>Tu</td>\n",
       "      <td>15.8</td>\n",
       "      <td>29.8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.4</td>\n",
       "      <td>11.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NNE</td>\n",
       "      <td>9</td>\n",
       "      <td>1015.4</td>\n",
       "      <td>29.2</td>\n",
       "      <td>25</td>\n",
       "      <td>3</td>\n",
       "      <td>NNW</td>\n",
       "      <td>35</td>\n",
       "      <td>1009.0</td>\n",
       "      <td>High</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>2015</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "      <td>Th</td>\n",
       "      <td>18.3</td>\n",
       "      <td>25.5</td>\n",
       "      <td>2.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.2</td>\n",
       "      <td>NNE</td>\n",
       "      <td>...</td>\n",
       "      <td>NNE</td>\n",
       "      <td>17</td>\n",
       "      <td>1020.4</td>\n",
       "      <td>22.2</td>\n",
       "      <td>69</td>\n",
       "      <td>8</td>\n",
       "      <td>ENE</td>\n",
       "      <td>17</td>\n",
       "      <td>1018.0</td>\n",
       "      <td>Low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>2015</td>\n",
       "      <td>12</td>\n",
       "      <td>17</td>\n",
       "      <td>Th</td>\n",
       "      <td>16.6</td>\n",
       "      <td>26.1</td>\n",
       "      <td>9.8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.0</td>\n",
       "      <td>E</td>\n",
       "      <td>...</td>\n",
       "      <td>WNW</td>\n",
       "      <td>9</td>\n",
       "      <td>1018.5</td>\n",
       "      <td>24.6</td>\n",
       "      <td>60</td>\n",
       "      <td>1</td>\n",
       "      <td>E</td>\n",
       "      <td>24</td>\n",
       "      <td>1016.9</td>\n",
       "      <td>High</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>2015</td>\n",
       "      <td>12</td>\n",
       "      <td>31</td>\n",
       "      <td>Th</td>\n",
       "      <td>17.4</td>\n",
       "      <td>26.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>12.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>WNW</td>\n",
       "      <td>9</td>\n",
       "      <td>1019.4</td>\n",
       "      <td>24.2</td>\n",
       "      <td>52</td>\n",
       "      <td>5</td>\n",
       "      <td>E</td>\n",
       "      <td>24</td>\n",
       "      <td>1016.8</td>\n",
       "      <td>High</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159</th>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>Th</td>\n",
       "      <td>16.5</td>\n",
       "      <td>24.4</td>\n",
       "      <td>41.4</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>SSW</td>\n",
       "      <td>22</td>\n",
       "      <td>1014.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>66</td>\n",
       "      <td>7</td>\n",
       "      <td>SSW</td>\n",
       "      <td>24</td>\n",
       "      <td>1015.8</td>\n",
       "      <td>Med</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>Fr</td>\n",
       "      <td>15.5</td>\n",
       "      <td>25.4</td>\n",
       "      <td>0.4</td>\n",
       "      <td>6.6</td>\n",
       "      <td>12.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NW</td>\n",
       "      <td>4</td>\n",
       "      <td>1020.1</td>\n",
       "      <td>24.1</td>\n",
       "      <td>53</td>\n",
       "      <td>1</td>\n",
       "      <td>E</td>\n",
       "      <td>19</td>\n",
       "      <td>1019.0</td>\n",
       "      <td>High</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "      <td>23</td>\n",
       "      <td>Sa</td>\n",
       "      <td>22.3</td>\n",
       "      <td>26.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>9.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>S</td>\n",
       "      <td>17</td>\n",
       "      <td>1006.7</td>\n",
       "      <td>25.5</td>\n",
       "      <td>59</td>\n",
       "      <td>6</td>\n",
       "      <td>S</td>\n",
       "      <td>33</td>\n",
       "      <td>1008.6</td>\n",
       "      <td>Med</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>Mo</td>\n",
       "      <td>21.6</td>\n",
       "      <td>25.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>E</td>\n",
       "      <td>13</td>\n",
       "      <td>1014.5</td>\n",
       "      <td>24.7</td>\n",
       "      <td>64</td>\n",
       "      <td>8</td>\n",
       "      <td>E</td>\n",
       "      <td>13</td>\n",
       "      <td>1013.3</td>\n",
       "      <td>Low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178</th>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>Tu</td>\n",
       "      <td>21.6</td>\n",
       "      <td>27.8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.4</td>\n",
       "      <td>8.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>E</td>\n",
       "      <td>11</td>\n",
       "      <td>1017.7</td>\n",
       "      <td>26.4</td>\n",
       "      <td>54</td>\n",
       "      <td>4</td>\n",
       "      <td>ENE</td>\n",
       "      <td>19</td>\n",
       "      <td>1017.3</td>\n",
       "      <td>Med</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>2016</td>\n",
       "      <td>3</td>\n",
       "      <td>17</td>\n",
       "      <td>Th</td>\n",
       "      <td>19.0</td>\n",
       "      <td>26.1</td>\n",
       "      <td>13.6</td>\n",
       "      <td>5.2</td>\n",
       "      <td>7.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NW</td>\n",
       "      <td>4</td>\n",
       "      <td>1017.0</td>\n",
       "      <td>25.5</td>\n",
       "      <td>59</td>\n",
       "      <td>4</td>\n",
       "      <td>E</td>\n",
       "      <td>15</td>\n",
       "      <td>1012.9</td>\n",
       "      <td>Med</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>Tu</td>\n",
       "      <td>18.6</td>\n",
       "      <td>26.2</td>\n",
       "      <td>6.4</td>\n",
       "      <td>3.0</td>\n",
       "      <td>10.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NNE</td>\n",
       "      <td>9</td>\n",
       "      <td>1020.1</td>\n",
       "      <td>25.6</td>\n",
       "      <td>58</td>\n",
       "      <td>3</td>\n",
       "      <td>NNE</td>\n",
       "      <td>24</td>\n",
       "      <td>1015.2</td>\n",
       "      <td>High</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253</th>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>Su</td>\n",
       "      <td>16.1</td>\n",
       "      <td>27.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.8</td>\n",
       "      <td>9.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>WNW</td>\n",
       "      <td>19</td>\n",
       "      <td>1019.8</td>\n",
       "      <td>23.7</td>\n",
       "      <td>51</td>\n",
       "      <td>3</td>\n",
       "      <td>E</td>\n",
       "      <td>19</td>\n",
       "      <td>1015.5</td>\n",
       "      <td>Med</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>275</th>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>Mo</td>\n",
       "      <td>13.9</td>\n",
       "      <td>21.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.8</td>\n",
       "      <td>6.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NW</td>\n",
       "      <td>7</td>\n",
       "      <td>1017.3</td>\n",
       "      <td>21.2</td>\n",
       "      <td>60</td>\n",
       "      <td>7</td>\n",
       "      <td>NE</td>\n",
       "      <td>9</td>\n",
       "      <td>1012.2</td>\n",
       "      <td>Med</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279</th>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>Fr</td>\n",
       "      <td>14.0</td>\n",
       "      <td>26.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>9.9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>W</td>\n",
       "      <td>19</td>\n",
       "      <td>1019.6</td>\n",
       "      <td>23.2</td>\n",
       "      <td>42</td>\n",
       "      <td>1</td>\n",
       "      <td>ESE</td>\n",
       "      <td>13</td>\n",
       "      <td>1017.2</td>\n",
       "      <td>Med</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>2016</td>\n",
       "      <td>5</td>\n",
       "      <td>20</td>\n",
       "      <td>Fr</td>\n",
       "      <td>12.8</td>\n",
       "      <td>24.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9.2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>WSW</td>\n",
       "      <td>19</td>\n",
       "      <td>1020.3</td>\n",
       "      <td>22.2</td>\n",
       "      <td>51</td>\n",
       "      <td>1</td>\n",
       "      <td>SE</td>\n",
       "      <td>17</td>\n",
       "      <td>1019.3</td>\n",
       "      <td>Med</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>306</th>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>Th</td>\n",
       "      <td>11.1</td>\n",
       "      <td>19.1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.4</td>\n",
       "      <td>8.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>W</td>\n",
       "      <td>19</td>\n",
       "      <td>1027.0</td>\n",
       "      <td>18.6</td>\n",
       "      <td>60</td>\n",
       "      <td>7</td>\n",
       "      <td>SSE</td>\n",
       "      <td>15</td>\n",
       "      <td>1024.3</td>\n",
       "      <td>Med</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308</th>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>Sa</td>\n",
       "      <td>13.3</td>\n",
       "      <td>21.1</td>\n",
       "      <td>64.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>ENE</td>\n",
       "      <td>...</td>\n",
       "      <td>NNE</td>\n",
       "      <td>13</td>\n",
       "      <td>1015.4</td>\n",
       "      <td>18.8</td>\n",
       "      <td>82</td>\n",
       "      <td>8</td>\n",
       "      <td>NE</td>\n",
       "      <td>30</td>\n",
       "      <td>1009.5</td>\n",
       "      <td>Low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309</th>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>Su</td>\n",
       "      <td>16.3</td>\n",
       "      <td>19.7</td>\n",
       "      <td>93.8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>ENE</td>\n",
       "      <td>...</td>\n",
       "      <td>NE</td>\n",
       "      <td>31</td>\n",
       "      <td>1002.5</td>\n",
       "      <td>19.7</td>\n",
       "      <td>85</td>\n",
       "      <td>8</td>\n",
       "      <td>ENE</td>\n",
       "      <td>57</td>\n",
       "      <td>995.9</td>\n",
       "      <td>Low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>318</th>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "      <td>Tu</td>\n",
       "      <td>9.9</td>\n",
       "      <td>21.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.2</td>\n",
       "      <td>9.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>W</td>\n",
       "      <td>15</td>\n",
       "      <td>1035.8</td>\n",
       "      <td>20.9</td>\n",
       "      <td>42</td>\n",
       "      <td>4</td>\n",
       "      <td>WNW</td>\n",
       "      <td>6</td>\n",
       "      <td>1031.6</td>\n",
       "      <td>Med</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>342</th>\n",
       "      <td>2016</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>Fr</td>\n",
       "      <td>12.4</td>\n",
       "      <td>15.6</td>\n",
       "      <td>22.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>W</td>\n",
       "      <td>...</td>\n",
       "      <td>W</td>\n",
       "      <td>17</td>\n",
       "      <td>1019.8</td>\n",
       "      <td>15.5</td>\n",
       "      <td>80</td>\n",
       "      <td>8</td>\n",
       "      <td>SSW</td>\n",
       "      <td>7</td>\n",
       "      <td>1018.4</td>\n",
       "      <td>Low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384</th>\n",
       "      <td>2016</td>\n",
       "      <td>8</td>\n",
       "      <td>19</td>\n",
       "      <td>Fr</td>\n",
       "      <td>11.2</td>\n",
       "      <td>24.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.0</td>\n",
       "      <td>NNE</td>\n",
       "      <td>...</td>\n",
       "      <td>W</td>\n",
       "      <td>11</td>\n",
       "      <td>1020.1</td>\n",
       "      <td>22.4</td>\n",
       "      <td>37</td>\n",
       "      <td>6</td>\n",
       "      <td>NNW</td>\n",
       "      <td>24</td>\n",
       "      <td>1013.9</td>\n",
       "      <td>Med</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>2016</td>\n",
       "      <td>8</td>\n",
       "      <td>31</td>\n",
       "      <td>We</td>\n",
       "      <td>13.6</td>\n",
       "      <td>23.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.3</td>\n",
       "      <td>NNW</td>\n",
       "      <td>...</td>\n",
       "      <td>N</td>\n",
       "      <td>20</td>\n",
       "      <td>1018.7</td>\n",
       "      <td>21.2</td>\n",
       "      <td>53</td>\n",
       "      <td>7</td>\n",
       "      <td>NNW</td>\n",
       "      <td>39</td>\n",
       "      <td>1013.9</td>\n",
       "      <td>Low</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>23 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Year  Month  Date Day  Temps_min  Temps_max  Rain  Evap  Sun_hours  \\\n",
       "39   2015      9     9  We        9.8       20.2   0.0   5.0       10.9   \n",
       "45   2015      9    15  Tu       15.8       29.8   0.0   6.4       11.0   \n",
       "103  2015     11    12  Th       18.3       25.5   2.4   NaN        4.2   \n",
       "138  2015     12    17  Th       16.6       26.1   9.8   NaN       12.0   \n",
       "152  2015     12    31  Th       17.4       26.7   0.0   8.0       12.7   \n",
       "159  2016      1     7  Th       16.5       24.4  41.4   2.0        7.6   \n",
       "160  2016      1     8  Fr       15.5       25.4   0.4   6.6       12.1   \n",
       "175  2016      1    23  Sa       22.3       26.0  13.0   3.4        9.5   \n",
       "177  2016      1    25  Mo       21.6       25.2   0.0   4.6        0.0   \n",
       "178  2016      1    26  Tu       21.6       27.8   0.0   2.4        8.7   \n",
       "229  2016      3    17  Th       19.0       26.1  13.6   5.2        7.5   \n",
       "248  2016      4     5  Tu       18.6       26.2   6.4   3.0       10.4   \n",
       "253  2016      4    10  Su       16.1       27.3   0.0   3.8        9.6   \n",
       "275  2016      5     2  Mo       13.9       21.3   0.0   4.8        6.1   \n",
       "279  2016      5     6  Fr       14.0       26.7   0.0   3.4        9.9   \n",
       "293  2016      5    20  Fr       12.8       24.9   0.0   4.0        9.2   \n",
       "306  2016      6     2  Th       11.1       19.1   1.0   2.4        8.7   \n",
       "308  2016      6     4  Sa       13.3       21.1  64.4   NaN        0.0   \n",
       "309  2016      6     5  Su       16.3       19.7  93.8   NaN        0.0   \n",
       "318  2016      6    14  Tu        9.9       21.2   0.0   2.2        9.6   \n",
       "342  2016      7     8  Fr       12.4       15.6  22.4   NaN        0.0   \n",
       "384  2016      8    19  Fr       11.2       24.3   0.0   NaN       10.0   \n",
       "396  2016      8    31  We       13.6       23.6   0.0   NaN        2.3   \n",
       "\n",
       "    Max_wind_dir    ...      Wind_dir_at_9am Wind_spd_at_9am  MSLP_at_9am  \\\n",
       "39           NaN    ...                    W              20       1023.1   \n",
       "45           NaN    ...                  NNE               9       1015.4   \n",
       "103          NNE    ...                  NNE              17       1020.4   \n",
       "138            E    ...                  WNW               9       1018.5   \n",
       "152          NaN    ...                  WNW               9       1019.4   \n",
       "159          NaN    ...                  SSW              22       1014.0   \n",
       "160          NaN    ...                   NW               4       1020.1   \n",
       "175          NaN    ...                    S              17       1006.7   \n",
       "177          NaN    ...                    E              13       1014.5   \n",
       "178          NaN    ...                    E              11       1017.7   \n",
       "229          NaN    ...                   NW               4       1017.0   \n",
       "248          NaN    ...                  NNE               9       1020.1   \n",
       "253          NaN    ...                  WNW              19       1019.8   \n",
       "275          NaN    ...                   NW               7       1017.3   \n",
       "279          NaN    ...                    W              19       1019.6   \n",
       "293          NaN    ...                  WSW              19       1020.3   \n",
       "306          NaN    ...                    W              19       1027.0   \n",
       "308          ENE    ...                  NNE              13       1015.4   \n",
       "309          ENE    ...                   NE              31       1002.5   \n",
       "318          NaN    ...                    W              15       1035.8   \n",
       "342            W    ...                    W              17       1019.8   \n",
       "384          NNE    ...                    W              11       1020.1   \n",
       "396          NNW    ...                    N              20       1018.7   \n",
       "\n",
       "     Temp_at_3pm  RH_at_3pm CLD_at_3pm  Wind_dir_at_3pm  Wind_spd_at_3pm  \\\n",
       "39          18.9         39          1              SSE               17   \n",
       "45          29.2         25          3              NNW               35   \n",
       "103         22.2         69          8              ENE               17   \n",
       "138         24.6         60          1                E               24   \n",
       "152         24.2         52          5                E               24   \n",
       "159         21.0         66          7              SSW               24   \n",
       "160         24.1         53          1                E               19   \n",
       "175         25.5         59          6                S               33   \n",
       "177         24.7         64          8                E               13   \n",
       "178         26.4         54          4              ENE               19   \n",
       "229         25.5         59          4                E               15   \n",
       "248         25.6         58          3              NNE               24   \n",
       "253         23.7         51          3                E               19   \n",
       "275         21.2         60          7               NE                9   \n",
       "279         23.2         42          1              ESE               13   \n",
       "293         22.2         51          1               SE               17   \n",
       "306         18.6         60          7              SSE               15   \n",
       "308         18.8         82          8               NE               30   \n",
       "309         19.7         85          8              ENE               57   \n",
       "318         20.9         42          4              WNW                6   \n",
       "342         15.5         80          8              SSW                7   \n",
       "384         22.4         37          6              NNW               24   \n",
       "396         21.2         53          7              NNW               39   \n",
       "\n",
       "     MSLP_at_3pm  Sun_level  \n",
       "39        1022.0       High  \n",
       "45        1009.0       High  \n",
       "103       1018.0        Low  \n",
       "138       1016.9       High  \n",
       "152       1016.8       High  \n",
       "159       1015.8        Med  \n",
       "160       1019.0       High  \n",
       "175       1008.6        Med  \n",
       "177       1013.3        Low  \n",
       "178       1017.3        Med  \n",
       "229       1012.9        Med  \n",
       "248       1015.2       High  \n",
       "253       1015.5        Med  \n",
       "275       1012.2        Med  \n",
       "279       1017.2        Med  \n",
       "293       1019.3        Med  \n",
       "306       1024.3        Med  \n",
       "308       1009.5        Low  \n",
       "309        995.9        Low  \n",
       "318       1031.6        Med  \n",
       "342       1018.4        Low  \n",
       "384       1013.9        Med  \n",
       "396       1013.9        Low  \n",
       "\n",
       "[23 rows x 25 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# output all rows with missing values\n",
    "data[data.isnull().any(axis=1)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# output the number of missing values with in each column\n",
    "pd.isnull(data).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For *CLD_at_9am*, *Max_wind_dir*, *Max_wind_spd* and *Max_wind_dir*, we simply drop the rows with missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(374, 25)\n"
     ]
    }
   ],
   "source": [
    "data = data.dropna(subset = ['CLD_at_9am', 'Max_wind_dir', 'Max_wind_spd', 'Max_wind_dir'])\n",
    "print(data.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "According to the Bureau of Meteorology (http://www.bom.gov.au/climate/dwo/IDCJDW0000.shtml), sometimes when the evaporation are missing, the next value given has been accumulated over several days rather than the normal one day. Therefore, we will not only drop the rows with missing *Evap* data, but also the rows below them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(362, 25)\n"
     ]
    }
   ],
   "source": [
    "bitmap1 = data.Evap.notnull()\n",
    "bitmap2 = bitmap1.shift(1)\n",
    "bitmap2[0] = True\n",
    "data = data[bitmap1 & bitmap2]\n",
    "print(data.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Select Feature(s)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "We firstly need to decide which feature(s) to use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def corr(data):\n",
    "    c = data.corr()\n",
    "    df = c.Sun_level_num.to_frame()\n",
    "    df['abs'] = abs(df['Sun_level_num'])\n",
    "    df.sort_values(by = 'abs', ascending=False)\n",
    "    print(df.sort_values(by = 'abs', ascending=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 Sun_level_num       abs\n",
      "Sun_level_num         1.000000  1.000000\n",
      "Sun_hours             0.903667  0.903667\n",
      "CLD_at_3pm           -0.695504  0.695504\n",
      "CLD_at_9am           -0.667313  0.667313\n",
      "RH_at_3pm            -0.461339  0.461339\n",
      "RH_at_9am            -0.432428  0.432428\n",
      "Temp_at_3pm           0.356488  0.356488\n",
      "Temps_max             0.333498  0.333498\n",
      "Rain                 -0.314637  0.314637\n",
      "Evap                  0.178738  0.178738\n",
      "Wind_spd_at_3pm       0.161778  0.161778\n",
      "Temp_at_9am           0.156106  0.156106\n",
      "Max_wind_spd         -0.134165  0.134165\n",
      "Wind_spd_at_9am      -0.115718  0.115718\n",
      "Year                 -0.053103  0.053103\n",
      "Date                 -0.045060  0.045060\n",
      "MSLP_at_9am           0.042240  0.042240\n",
      "Month                -0.028115  0.028115\n",
      "Temps_min            -0.005813  0.005813\n",
      "MSLP_at_3pm          -0.001053  0.001053\n"
     ]
    }
   ],
   "source": [
    "# we need to convert labels (string) into numeric to get the correlation\n",
    "labels = [0,1,2]\n",
    "data['Sun_level_num'] = pd.cut(data.Sun_hours, [-1,5,10,25], labels=labels)\n",
    "data[['Sun_level_num']] = data[['Sun_level_num']].apply(pd.to_numeric)\n",
    "corr(data)\n",
    "# c = data.corr()\n",
    "# c.sort_values(by='Sun_level_num', ascending=True)['Sun_level_num']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you have domain knowledge, you will know that the difference between max and min temperature might be highly correlated to sun hours. Let us add such column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 Sun_level_num       abs\n",
      "Sun_level_num         1.000000  1.000000\n",
      "Sun_hours             0.903667  0.903667\n",
      "CLD_at_3pm           -0.695504  0.695504\n",
      "CLD_at_9am           -0.667313  0.667313\n",
      "Temps_diff            0.513831  0.513831\n",
      "RH_at_3pm            -0.461339  0.461339\n",
      "RH_at_9am            -0.432428  0.432428\n",
      "Temp_at_3pm           0.356488  0.356488\n",
      "Temps_max             0.333498  0.333498\n",
      "Rain                 -0.314637  0.314637\n",
      "Evap                  0.178738  0.178738\n",
      "Wind_spd_at_3pm       0.161778  0.161778\n",
      "Temp_at_9am           0.156106  0.156106\n",
      "Max_wind_spd         -0.134165  0.134165\n",
      "Wind_spd_at_9am      -0.115718  0.115718\n",
      "Year                 -0.053103  0.053103\n",
      "Date                 -0.045060  0.045060\n",
      "MSLP_at_9am           0.042240  0.042240\n",
      "Month                -0.028115  0.028115\n",
      "Temps_min            -0.005813  0.005813\n",
      "MSLP_at_3pm          -0.001053  0.001053\n"
     ]
    }
   ],
   "source": [
    "data['Temps_diff'] = data['Temps_max'] - data['Temps_min']\n",
    "corr(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's try the features with higher correlations to *Sun_level*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_list = ['CLD_at_9am', 'CLD_at_3pm', 'RH_at_3pm', 'RH_at_9am', 'Temps_min', 'Temps_diff', 'Month']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Decision Tree"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We preserve 80% of the data as training data, and the rest will be used to tune the classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train, test = train_test_split(data, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we generate features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CLD_at_9am</th>\n",
       "      <th>CLD_at_3pm</th>\n",
       "      <th>RH_at_3pm</th>\n",
       "      <th>RH_at_9am</th>\n",
       "      <th>Temps_min</th>\n",
       "      <th>Temps_diff</th>\n",
       "      <th>Month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>260</th>\n",
       "      <td>7.0</td>\n",
       "      <td>7</td>\n",
       "      <td>77</td>\n",
       "      <td>73</td>\n",
       "      <td>19.1</td>\n",
       "      <td>4.1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>348</th>\n",
       "      <td>7.0</td>\n",
       "      <td>2</td>\n",
       "      <td>27</td>\n",
       "      <td>54</td>\n",
       "      <td>6.7</td>\n",
       "      <td>10.3</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>315</th>\n",
       "      <td>1.0</td>\n",
       "      <td>5</td>\n",
       "      <td>34</td>\n",
       "      <td>53</td>\n",
       "      <td>10.1</td>\n",
       "      <td>9.7</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>7.0</td>\n",
       "      <td>7</td>\n",
       "      <td>62</td>\n",
       "      <td>63</td>\n",
       "      <td>8.3</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>387</th>\n",
       "      <td>7.0</td>\n",
       "      <td>7</td>\n",
       "      <td>67</td>\n",
       "      <td>66</td>\n",
       "      <td>9.5</td>\n",
       "      <td>4.3</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     CLD_at_9am  CLD_at_3pm  RH_at_3pm  RH_at_9am  Temps_min  Temps_diff  \\\n",
       "260         7.0           7         77         73       19.1         4.1   \n",
       "348         7.0           2         27         54        6.7        10.3   \n",
       "315         1.0           5         34         53       10.1         9.7   \n",
       "54          7.0           7         62         63        8.3         8.0   \n",
       "387         7.0           7         67         66        9.5         4.3   \n",
       "\n",
       "     Month  \n",
       "260      4  \n",
       "348      7  \n",
       "315      6  \n",
       "54       9  \n",
       "387      8  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = train[feature_list]\n",
    "X_test = test[feature_list]\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And we use *Sun_level* as labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_train = train.Sun_level\n",
    "y_test = test.Sun_level"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With features and labels, we can then train our decision tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clf = DecisionTreeClassifier(criterion = \"gini\")\n",
    "dtree = clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluations and Investigations of the Trained Decision Tree"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Firstly let's see the accuracy of the decision tree on the training dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtree.score(X_train,y_train) # output the accuracy of the trained decision tree"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's see the accuracy on the testing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.61643835616438358"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtree.score(X_test,y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can also get the importance of each feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>importance</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.288737</td>\n",
       "      <td>CLD_at_3pm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.185660</td>\n",
       "      <td>Temps_diff</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.167558</td>\n",
       "      <td>Month</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.125743</td>\n",
       "      <td>CLD_at_9am</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.104030</td>\n",
       "      <td>RH_at_9am</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.077647</td>\n",
       "      <td>Temps_min</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.050624</td>\n",
       "      <td>RH_at_3pm</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   importance        name\n",
       "1    0.288737  CLD_at_3pm\n",
       "5    0.185660  Temps_diff\n",
       "6    0.167558       Month\n",
       "0    0.125743  CLD_at_9am\n",
       "3    0.104030   RH_at_9am\n",
       "4    0.077647   Temps_min\n",
       "2    0.050624   RH_at_3pm"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature = pd.DataFrame({'name': pd.Series(feature_list),\n",
    "                        'importance': pd.Series(dtree.feature_importances_)})\n",
    "feature.sort_values(by = 'importance', ascending = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apparently the above tree is overfitting. One way to deal with it is to change the maximum height of the decision tree."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def experiment(train, test, features, depth=5):\n",
    "    X_train = train[features]\n",
    "    y_train = train.Sun_level\n",
    "    clf = DecisionTreeClassifier(criterion = \"gini\", max_depth = depth, random_state = RANDOM_SEED)\n",
    "    dtree = clf.fit(X_train, y_train)\n",
    "    err_training = dtree.score(X_train,y_train)\n",
    "    X_test = test[features]\n",
    "    y_test = test.Sun_level\n",
    "    err_testing = dtree.score(X_test,y_test)\n",
    "    err_diff = err_training - err_testing\n",
    "    print('{}, {}, {}'.format(err_training, err_testing, err_diff))\n",
    "    return err_training, err_testing\n",
    "        \n",
    "def evaluate(features, repeat_times = 10, depth = 5):\n",
    "    print('features: {}'.format(features))\n",
    "    print('max_depth: {}\\n'.format(depth))\n",
    "    total_err_training = 0\n",
    "    total_err_testing = 0\n",
    "    for i in range(repeat_times):\n",
    "        train, test = train_test_split(data, test_size = 0.2, random_state = RANDOM_SEED + i)\n",
    "        err_training, err_testing = experiment(train, test, features, depth)\n",
    "        total_err_training += err_training\n",
    "        total_err_testing += err_testing\n",
    "        print('==============')\n",
    "    print('avg. training error:\\t{}'.format(total_err_training/repeat_times))\n",
    "    print('avg. testing error:\\t{}'.format(total_err_testing/repeat_times))\n",
    "    print('avg. difference:\\t{}'.format((total_err_training - total_err_testing)/repeat_times))\n",
    "    print('============================')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features: ['CLD_at_9am', 'CLD_at_3pm', 'RH_at_3pm', 'RH_at_9am', 'Temps_min', 'Temps_diff', 'Month']\n",
      "max_depth: 6\n",
      "\n",
      "0.9065743944636678, 0.6575342465753424, 0.2490401478883254\n",
      "==============\n",
      "0.8961937716262975, 0.7397260273972602, 0.1564677442290373\n",
      "==============\n",
      "0.8996539792387543, 0.6164383561643836, 0.28321562307437076\n",
      "==============\n",
      "0.9065743944636678, 0.6986301369863014, 0.20794425747736645\n",
      "==============\n",
      "0.8961937716262975, 0.6438356164383562, 0.25235815518794136\n",
      "==============\n",
      "0.9100346020761245, 0.6575342465753424, 0.2525003555007821\n",
      "==============\n",
      "0.889273356401384, 0.7397260273972602, 0.14954732900412382\n",
      "==============\n",
      "0.9065743944636678, 0.6301369863013698, 0.276437408162298\n",
      "==============\n",
      "0.8442906574394463, 0.7671232876712328, 0.07716736976821348\n",
      "==============\n",
      "0.8788927335640139, 0.726027397260274, 0.15286533630373988\n",
      "==============\n",
      "avg. training error:\t0.8934256055363322\n",
      "avg. testing error:\t0.6876712328767123\n",
      "avg. difference:\t0.20575437265962\n",
      "============================\n"
     ]
    }
   ],
   "source": [
    "feature_list = ['CLD_at_9am', 'CLD_at_3pm', 'RH_at_3pm', 'RH_at_9am', 'Temps_min', 'Temps_diff', 'Month']\n",
    "evaluate(feature_list, 10, 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features: ['CLD_at_9am', 'CLD_at_3pm', 'RH_at_3pm', 'RH_at_9am', 'Temps_min', 'Temps_diff', 'Month']\n",
      "max_depth: 5\n",
      "\n",
      "0.8719723183391004, 0.6712328767123288, 0.2007394416267716\n",
      "==============\n",
      "0.8546712802768166, 0.7945205479452054, 0.06015073233161117\n",
      "==============\n",
      "0.8650519031141869, 0.6986301369863014, 0.1664217661278855\n",
      "==============\n",
      "0.8719723183391004, 0.7123287671232876, 0.15964355121581275\n",
      "==============\n",
      "0.8615916955017301, 0.7671232876712328, 0.09446840783049726\n",
      "==============\n",
      "0.8788927335640139, 0.684931506849315, 0.19396122671469884\n",
      "==============\n",
      "0.8477508650519031, 0.7671232876712328, 0.08062757738067028\n",
      "==============\n",
      "0.8650519031141869, 0.589041095890411, 0.2760108072237759\n",
      "==============\n",
      "0.8408304498269896, 0.7671232876712328, 0.07370716215575679\n",
      "==============\n",
      "0.8477508650519031, 0.7534246575342466, 0.09432620751765652\n",
      "==============\n",
      "avg. training error:\t0.860553633217993\n",
      "avg. testing error:\t0.7205479452054794\n",
      "avg. difference:\t0.14000568801251365\n",
      "============================\n"
     ]
    }
   ],
   "source": [
    "evaluate(feature_list, 10, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features: ['CLD_at_9am', 'CLD_at_3pm', 'RH_at_3pm', 'RH_at_9am', 'Temps_min', 'Temps_diff', 'Month']\n",
      "max_depth: 4\n",
      "\n",
      "0.8166089965397924, 0.726027397260274, 0.09058159927951837\n",
      "==============\n",
      "0.8131487889273357, 0.7671232876712328, 0.04602550125610283\n",
      "==============\n",
      "0.8235294117647058, 0.7123287671232876, 0.11120064464141821\n",
      "==============\n",
      "0.8235294117647058, 0.7397260273972602, 0.08380338436744561\n",
      "==============\n",
      "0.8166089965397924, 0.7808219178082192, 0.03578707873157316\n",
      "==============\n",
      "0.8166089965397924, 0.6986301369863014, 0.11797885955349097\n",
      "==============\n",
      "0.8096885813148789, 0.7397260273972602, 0.06996255391761863\n",
      "==============\n",
      "0.8269896193771626, 0.6301369863013698, 0.19685263307579282\n",
      "==============\n",
      "0.7993079584775087, 0.726027397260274, 0.0732805612172347\n",
      "==============\n",
      "0.7923875432525952, 0.7945205479452054, -0.002133004692610241\n",
      "==============\n",
      "avg. training error:\t0.813840830449827\n",
      "avg. testing error:\t0.7315068493150684\n",
      "avg. difference:\t0.08233398113475862\n",
      "============================\n"
     ]
    }
   ],
   "source": [
    "evaluate(feature_list, 10, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features: ['CLD_at_9am', 'CLD_at_3pm', 'RH_at_3pm', 'RH_at_9am', 'Temps_min', 'Temps_diff', 'Month']\n",
      "max_depth: 3\n",
      "\n",
      "0.7197231833910035, 0.6027397260273972, 0.11698345736360627\n",
      "==============\n",
      "0.740484429065744, 0.6575342465753424, 0.08295018249040154\n",
      "==============\n",
      "0.740484429065744, 0.5753424657534246, 0.16514196331231934\n",
      "==============\n",
      "0.7439446366782007, 0.6027397260273972, 0.14120491065080343\n",
      "==============\n",
      "0.726643598615917, 0.7123287671232876, 0.014314831492629354\n",
      "==============\n",
      "0.7162629757785467, 0.6301369863013698, 0.08612598947717687\n",
      "==============\n",
      "0.7335640138408305, 0.6575342465753424, 0.07602976726548805\n",
      "==============\n",
      "0.7335640138408305, 0.547945205479452, 0.18561880836137845\n",
      "==============\n",
      "0.7093425605536332, 0.6712328767123288, 0.03810968384130442\n",
      "==============\n",
      "0.7335640138408305, 0.6164383561643836, 0.11712565767644689\n",
      "==============\n",
      "avg. training error:\t0.7297577854671279\n",
      "avg. testing error:\t0.6273972602739727\n",
      "avg. difference:\t0.1023605251931552\n",
      "============================\n"
     ]
    }
   ],
   "source": [
    "evaluate(feature_list, 10, 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predict the missing *Sun_level* value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = data[feature_list]\n",
    "y = data.Sun_level\n",
    "clf = DecisionTreeClassifier(criterion = \"gini\", max_depth = 4, random_state = RANDOM_SEED)\n",
    "dtree = clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_missing_sun_hours['Temps_diff'] = data_missing_sun_hours['Temps_max'] - data_missing_sun_hours['Temps_min']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Date</th>\n",
       "      <th>Day</th>\n",
       "      <th>Temps_min</th>\n",
       "      <th>Temps_max</th>\n",
       "      <th>Rain</th>\n",
       "      <th>Evap</th>\n",
       "      <th>Sun_hours</th>\n",
       "      <th>Max_wind_dir</th>\n",
       "      <th>...</th>\n",
       "      <th>Wind_spd_at_9am</th>\n",
       "      <th>MSLP_at_9am</th>\n",
       "      <th>Temp_at_3pm</th>\n",
       "      <th>RH_at_3pm</th>\n",
       "      <th>CLD_at_3pm</th>\n",
       "      <th>Wind_dir_at_3pm</th>\n",
       "      <th>Wind_spd_at_3pm</th>\n",
       "      <th>MSLP_at_3pm</th>\n",
       "      <th>Temps_diff</th>\n",
       "      <th>Sun_level_pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>2015</td>\n",
       "      <td>11</td>\n",
       "      <td>25</td>\n",
       "      <td>We</td>\n",
       "      <td>17.3</td>\n",
       "      <td>32.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NNE</td>\n",
       "      <td>...</td>\n",
       "      <td>9</td>\n",
       "      <td>1013.1</td>\n",
       "      <td>28.7</td>\n",
       "      <td>44</td>\n",
       "      <td>0</td>\n",
       "      <td>NNE</td>\n",
       "      <td>24</td>\n",
       "      <td>1007.4</td>\n",
       "      <td>15.4</td>\n",
       "      <td>High</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>2015</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>Sa</td>\n",
       "      <td>15.5</td>\n",
       "      <td>24.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NE</td>\n",
       "      <td>...</td>\n",
       "      <td>7</td>\n",
       "      <td>1022.6</td>\n",
       "      <td>24.1</td>\n",
       "      <td>52</td>\n",
       "      <td>1</td>\n",
       "      <td>NE</td>\n",
       "      <td>20</td>\n",
       "      <td>1018.9</td>\n",
       "      <td>9.4</td>\n",
       "      <td>High</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>2015</td>\n",
       "      <td>12</td>\n",
       "      <td>20</td>\n",
       "      <td>Su</td>\n",
       "      <td>19.9</td>\n",
       "      <td>30.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NE</td>\n",
       "      <td>...</td>\n",
       "      <td>11</td>\n",
       "      <td>1013.4</td>\n",
       "      <td>28.5</td>\n",
       "      <td>52</td>\n",
       "      <td>1</td>\n",
       "      <td>NNE</td>\n",
       "      <td>31</td>\n",
       "      <td>1008.7</td>\n",
       "      <td>10.4</td>\n",
       "      <td>High</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166</th>\n",
       "      <td>2016</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>Th</td>\n",
       "      <td>22.7</td>\n",
       "      <td>39.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>1011.7</td>\n",
       "      <td>32.4</td>\n",
       "      <td>43</td>\n",
       "      <td>7</td>\n",
       "      <td>ENE</td>\n",
       "      <td>15</td>\n",
       "      <td>1008.1</td>\n",
       "      <td>16.5</td>\n",
       "      <td>Med</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>334</th>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>30</td>\n",
       "      <td>Th</td>\n",
       "      <td>5.4</td>\n",
       "      <td>16.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NW</td>\n",
       "      <td>...</td>\n",
       "      <td>11</td>\n",
       "      <td>1021.2</td>\n",
       "      <td>14.5</td>\n",
       "      <td>33</td>\n",
       "      <td>7</td>\n",
       "      <td>NNW</td>\n",
       "      <td>24</td>\n",
       "      <td>1015.1</td>\n",
       "      <td>11.1</td>\n",
       "      <td>Med</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>335</th>\n",
       "      <td>2016</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>Fr</td>\n",
       "      <td>7.7</td>\n",
       "      <td>16.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NW</td>\n",
       "      <td>...</td>\n",
       "      <td>13</td>\n",
       "      <td>1015.7</td>\n",
       "      <td>16.4</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "      <td>WSW</td>\n",
       "      <td>20</td>\n",
       "      <td>1016.2</td>\n",
       "      <td>9.0</td>\n",
       "      <td>High</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>369</th>\n",
       "      <td>2016</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>Th</td>\n",
       "      <td>10.8</td>\n",
       "      <td>18.1</td>\n",
       "      <td>61.0</td>\n",
       "      <td>9.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>SE</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>1024.3</td>\n",
       "      <td>15.5</td>\n",
       "      <td>60</td>\n",
       "      <td>7</td>\n",
       "      <td>SSW</td>\n",
       "      <td>19</td>\n",
       "      <td>1025.1</td>\n",
       "      <td>7.3</td>\n",
       "      <td>Low</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Year  Month  Date Day  Temps_min  Temps_max  Rain  Evap  Sun_hours  \\\n",
       "116  2015     11    25  We       17.3       32.7   0.0   9.0        NaN   \n",
       "126  2015     12     5  Sa       15.5       24.9   0.0   8.0        NaN   \n",
       "141  2015     12    20  Su       19.9       30.3   0.0  13.0        NaN   \n",
       "166  2016      1    14  Th       22.7       39.2   0.0   8.4        NaN   \n",
       "334  2016      6    30  Th        5.4       16.5   0.0   1.8        NaN   \n",
       "335  2016      7     1  Fr        7.7       16.7   0.0   4.0        NaN   \n",
       "369  2016      8     4  Th       10.8       18.1  61.0   9.4        NaN   \n",
       "\n",
       "    Max_wind_dir       ...        Wind_spd_at_9am MSLP_at_9am  Temp_at_3pm  \\\n",
       "116          NNE       ...                      9      1013.1         28.7   \n",
       "126           NE       ...                      7      1022.6         24.1   \n",
       "141           NE       ...                     11      1013.4         28.5   \n",
       "166            S       ...                      6      1011.7         32.4   \n",
       "334           NW       ...                     11      1021.2         14.5   \n",
       "335           NW       ...                     13      1015.7         16.4   \n",
       "369           SE       ...                     15      1024.3         15.5   \n",
       "\n",
       "     RH_at_3pm  CLD_at_3pm Wind_dir_at_3pm  Wind_spd_at_3pm  MSLP_at_3pm  \\\n",
       "116         44           0             NNE               24       1007.4   \n",
       "126         52           1              NE               20       1018.9   \n",
       "141         52           1             NNE               31       1008.7   \n",
       "166         43           7             ENE               15       1008.1   \n",
       "334         33           7             NNW               24       1015.1   \n",
       "335         33           1             WSW               20       1016.2   \n",
       "369         60           7             SSW               19       1025.1   \n",
       "\n",
       "     Temps_diff  Sun_level_pred  \n",
       "116        15.4            High  \n",
       "126         9.4            High  \n",
       "141        10.4            High  \n",
       "166        16.5             Med  \n",
       "334        11.1             Med  \n",
       "335         9.0            High  \n",
       "369         7.3             Low  \n",
       "\n",
       "[7 rows x 26 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_pred = data_missing_sun_hours[feature_list]\n",
    "data_missing_sun_hours['Sun_level_pred'] = dtree.predict(X_pred)\n",
    "data_missing_sun_hours"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Although we cannot remember what happened on these 7 days (can you?), it seems the predictions are probably correct. E.g., \n",
    "* the first instance is predicted as 'high' and it is a summer day, without rain or even cloud; \n",
    "* the last instance is predicted as 'low' as there was heavy rainfall, as `Rain = 61.0 (mm)`. Note that the model didn't use the `rain` feature at all! "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize of the Decision Tree"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You need to install pydotplus (use ```pip install pydotplus```) and [graphviz](http://www.graphviz.org/) on your computer to get the following code run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "InvocationException",
     "evalue": "GraphViz's executables not found",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mInvocationException\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-28-2cd2ed50a8ac>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[0mexport_graphviz\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtree\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout_file\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdotfile\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeature_names\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX_train\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[0mgraph\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpydotplus\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgraph_from_dot_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdotfile\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetvalue\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m \u001b[0mImage\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgraph\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcreate_png\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\pydotplus\\graphviz.py\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m(f, prog)\u001b[0m\n\u001b[0;32m   1795\u001b[0m             self.__setattr__(\n\u001b[0;32m   1796\u001b[0m                 \u001b[1;34m'create_'\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mfrmt\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1797\u001b[1;33m                 \u001b[1;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfrmt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprog\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprog\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcreate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprog\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mprog\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1798\u001b[0m             )\n\u001b[0;32m   1799\u001b[0m             \u001b[0mf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__dict__\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'create_'\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mfrmt\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Anaconda3\\lib\\site-packages\\pydotplus\\graphviz.py\u001b[0m in \u001b[0;36mcreate\u001b[1;34m(self, prog, format)\u001b[0m\n\u001b[0;32m   1958\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprogs\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1959\u001b[0m                 raise InvocationException(\n\u001b[1;32m-> 1960\u001b[1;33m                     'GraphViz\\'s executables not found')\n\u001b[0m\u001b[0;32m   1961\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1962\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mprog\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprogs\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mInvocationException\u001b[0m: GraphViz's executables not found"
     ]
    }
   ],
   "source": [
    "import pydotplus\n",
    "import sys\n",
    "\n",
    "# sys.path.append(\"C:\\\\Program Files (x86)\\\\Graphviz2.38\\bin\")\n",
    "\n",
    "dotfile = StringIO()\n",
    "export_graphviz(dtree, out_file = dotfile, feature_names = X_train.columns)\n",
    "graph = pydotplus.graph_from_dot_data(dotfile.getvalue())\n",
    "Image(graph.create_png())  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can generate a pdf file if you want"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "graph.write_pdf(\"./asset/dtree.pdf\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}