Spam dectection using NLP and ML classification models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Importing the required modules/packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import re\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.model_selection import KFold, cross_val_score\n",
    "from sklearn.metrics import precision_recall_fscore_support as score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.ensemble import GradientBoostingClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading file and looking into the dimensions of data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there g...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 87121 to receive ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives around here though</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  label  \\\n",
       "0   ham   \n",
       "1   ham   \n",
       "2  spam   \n",
       "3   ham   \n",
       "4   ham   \n",
       "\n",
       "                                                                                                  text  \n",
       "0  Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there g...  \n",
       "1                                                                        Ok lar... Joking wif u oni...  \n",
       "2  Free entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 87121 to receive ...  \n",
       "3                                                    U dun say so early hor... U c already then say...  \n",
       "4                                        Nah I don't think he goes to usf, he lives around here though  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_data = pd.read_csv(\"C:\\\\MSA\\\\Spring_Semester\\\\7152\\\\nlp\\\\SMSSpamCollection.tsv\",sep='\\t',names=['label','text'])\n",
    "pd.set_option('display.max_colwidth',100)\n",
    "raw_data.head()\n",
    "\n",
    "#print(raw_data.shape)\n",
    "#print(pd.crosstab(raw_data['label'],columns = 'label'))\n",
    "#pd.crosstab(raw_data['label'],columns = 'label',normalize=True)\n",
    "#raw_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5572, 4)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>col_0</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ham</th>\n",
       "      <td>0.865937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spam</th>\n",
       "      <td>0.134063</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "col_0     label\n",
       "label          \n",
       "ham    0.865937\n",
       "spam   0.134063"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(raw_data.shape)\n",
    "#print(pd.crosstab(raw_data['label'],columns = 'label'))\n",
    "pd.crosstab(raw_data['label'],columns = 'label',normalize=True)\n",
    "#raw_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating new features and cleaning function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopwords = nltk.corpus.stopwords.words('english')\n",
    "ps = nltk.PorterStemmer()\n",
    "\n",
    "def punct_pc(text):\n",
    "    punct_count = sum([1 for char in text if char in string.punctuation])\n",
    "    return (punct_count/(len(text) - text.count(' ')))*100\n",
    "\n",
    "raw_data['text_length'] = raw_data['text'].apply(lambda x : len(x)-x.count(' '))\n",
    "raw_data['Punct_pc'] = raw_data['text'].apply(lambda x : punct_pc(x))\n",
    "\n",
    "def clean_data(text):\n",
    "    punct = \"\".join([word.lower() for word in text if word not in string.punctuation])\n",
    "    splt = re.split('\\W+',punct)\n",
    "    txt = [ps.stem(word) for word in splt if word not in stopwords]\n",
    "    return txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating test train splits "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,Y_train,Y_test = train_test_split(raw_data[['text','text_length','Punct_pc']],raw_data['label'],test_size=0.2,random_state=123)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "col_0     label\n",
      "label          \n",
      "ham    0.866726\n",
      "spam   0.133274\n",
      "col_0    label\n",
      "label         \n",
      "ham    0.86278\n",
      "spam   0.13722\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>text_length</th>\n",
       "      <th>Punct_pc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>385</th>\n",
       "      <td>Double mins and txts 4 6months FREE Bluetooth on Orange. Available on Sony, Nokia Motorola phone...</td>\n",
       "      <td>128</td>\n",
       "      <td>3.125000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4003</th>\n",
       "      <td>Did you get any gift? This year i didnt get anything. So bad</td>\n",
       "      <td>48</td>\n",
       "      <td>4.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1283</th>\n",
       "      <td>Ever green quote ever told by Jerry in cartoon \"A Person Who Irritates u Always Is the one Who L...</td>\n",
       "      <td>128</td>\n",
       "      <td>11.718750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2327</th>\n",
       "      <td>The Xmas story is peace.. The Xmas msg is love.. The Xmas miracle is jesus.. Hav a blessed month...</td>\n",
       "      <td>105</td>\n",
       "      <td>10.476190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1103</th>\n",
       "      <td>Black shirt n blue jeans... I thk i c ü...</td>\n",
       "      <td>33</td>\n",
       "      <td>18.181818</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                     text  \\\n",
       "385   Double mins and txts 4 6months FREE Bluetooth on Orange. Available on Sony, Nokia Motorola phone...   \n",
       "4003                                         Did you get any gift? This year i didnt get anything. So bad   \n",
       "1283  Ever green quote ever told by Jerry in cartoon \"A Person Who Irritates u Always Is the one Who L...   \n",
       "2327  The Xmas story is peace.. The Xmas msg is love.. The Xmas miracle is jesus.. Hav a blessed month...   \n",
       "1103                                                           Black shirt n blue jeans... I thk i c ü...   \n",
       "\n",
       "      text_length   Punct_pc  \n",
       "385           128   3.125000  \n",
       "4003           48   4.166667  \n",
       "1283          128  11.718750  \n",
       "2327          105  10.476190  \n",
       "1103           33  18.181818  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(pd.crosstab(Y_train,columns = 'label',normalize=True))\n",
    "print(pd.crosstab(Y_test,columns = 'label',normalize=True))\n",
    "X_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vectorizing using TFIDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_length</th>\n",
       "      <th>Punct_pc</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>...</th>\n",
       "      <th>7075</th>\n",
       "      <th>7076</th>\n",
       "      <th>7077</th>\n",
       "      <th>7078</th>\n",
       "      <th>7079</th>\n",
       "      <th>7080</th>\n",
       "      <th>7081</th>\n",
       "      <th>7082</th>\n",
       "      <th>7083</th>\n",
       "      <th>7084</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>128</td>\n",
       "      <td>3.125000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>48</td>\n",
       "      <td>4.166667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>128</td>\n",
       "      <td>11.718750</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>105</td>\n",
       "      <td>10.476190</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33</td>\n",
       "      <td>18.181818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.252939</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 7087 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   text_length   Punct_pc    0    1    2    3    4    5    6    7  ...   7075  \\\n",
       "0          128   3.125000  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0   \n",
       "1           48   4.166667  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0   \n",
       "2          128  11.718750  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0   \n",
       "3          105  10.476190  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0   \n",
       "4           33  18.181818  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0   \n",
       "\n",
       "   7076  7077  7078  7079  7080  7081  7082      7083  7084  \n",
       "0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  0.000000   0.0  \n",
       "1   0.0   0.0   0.0   0.0   0.0   0.0   0.0  0.000000   0.0  \n",
       "2   0.0   0.0   0.0   0.0   0.0   0.0   0.0  0.000000   0.0  \n",
       "3   0.0   0.0   0.0   0.0   0.0   0.0   0.0  0.000000   0.0  \n",
       "4   0.0   0.0   0.0   0.0   0.0   0.0   0.0  0.252939   0.0  \n",
       "\n",
       "[5 rows x 7087 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Tfidf_Vect = TfidfVectorizer(analyzer=clean_data)\n",
    "Tfidf_vect_fit = Tfidf_Vect.fit(X_train['text'])\n",
    "\n",
    "X_train_Tfidf_vect = Tfidf_vect_fit.transform(X_train['text'])\n",
    "X_test_Tfidf_vect = Tfidf_vect_fit.transform(X_test['text'])\n",
    "\n",
    "X_train_vect = pd.concat([X_train[['text_length','Punct_pc']].reset_index(drop=True) ,\n",
    "                         pd.DataFrame(X_train_Tfidf_vect.toarray())],axis=1)\n",
    "\n",
    "\n",
    "X_test_vect = pd.concat([X_test[['text_length','Punct_pc']].reset_index(drop=True) , \n",
    "                        pd.DataFrame(X_test_Tfidf_vect.toarray())],axis=1)\n",
    "X_train_vect.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_length</th>\n",
       "      <th>Punct_pc</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>...</th>\n",
       "      <th>7075</th>\n",
       "      <th>7076</th>\n",
       "      <th>7077</th>\n",
       "      <th>7078</th>\n",
       "      <th>7079</th>\n",
       "      <th>7080</th>\n",
       "      <th>7081</th>\n",
       "      <th>7082</th>\n",
       "      <th>7083</th>\n",
       "      <th>7084</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>73</td>\n",
       "      <td>4.109589</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65</td>\n",
       "      <td>4.615385</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>55</td>\n",
       "      <td>3.636364</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23</td>\n",
       "      <td>4.347826</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>124</td>\n",
       "      <td>7.258065</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 7087 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   text_length  Punct_pc    0    1    2    3    4    5    6    7  ...   7075  \\\n",
       "0           73  4.109589  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0   \n",
       "1           65  4.615385  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0   \n",
       "2           55  3.636364  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0   \n",
       "3           23  4.347826  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0   \n",
       "4          124  7.258065  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  ...    0.0   \n",
       "\n",
       "   7076  7077  7078  7079  7080  7081  7082  7083  7084  \n",
       "0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "1   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "2   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "3   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "4   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0   0.0  \n",
       "\n",
       "[5 rows x 7087 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_vect.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text_length</th>\n",
       "      <th>Punct_pc</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>...</th>\n",
       "      <th>7075</th>\n",
       "      <th>7076</th>\n",
       "      <th>7077</th>\n",
       "      <th>7078</th>\n",
       "      <th>7079</th>\n",
       "      <th>7080</th>\n",
       "      <th>7081</th>\n",
       "      <th>7082</th>\n",
       "      <th>7083</th>\n",
       "      <th>7084</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>128</td>\n",
       "      <td>3.125000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>48</td>\n",
       "      <td>4.166667</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>128</td>\n",
       "      <td>11.718750</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>105</td>\n",
       "      <td>10.476190</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33</td>\n",
       "      <td>18.181818</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 7087 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   text_length   Punct_pc  0  1  2  3  4  5  6  7  ...   7075  7076  7077  \\\n",
       "0          128   3.125000  0  0  0  0  0  0  0  0  ...      0     0     0   \n",
       "1           48   4.166667  0  0  0  0  0  0  0  0  ...      0     0     0   \n",
       "2          128  11.718750  0  0  0  0  0  0  0  0  ...      0     0     0   \n",
       "3          105  10.476190  0  0  0  0  0  0  0  0  ...      0     0     0   \n",
       "4           33  18.181818  0  0  0  0  0  0  0  0  ...      0     0     0   \n",
       "\n",
       "   7078  7079  7080  7081  7082  7083  7084  \n",
       "0     0     0     0     0     0     0     0  \n",
       "1     0     0     0     0     0     0     0  \n",
       "2     0     0     0     0     0     0     0  \n",
       "3     0     0     0     0     0     0     0  \n",
       "4     0     0     0     0     0     1     0  \n",
       "\n",
       "[5 rows x 7087 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Count_Vect = CountVectorizer(analyzer=clean_data)\n",
    "Count_vect_fit = Count_Vect.fit(X_train['text'])\n",
    "\n",
    "X_train_Count_vect = Count_vect_fit.transform(X_train['text'])\n",
    "X_test_Count_vect = Count_vect_fit.transform(X_test['text'])\n",
    "\n",
    "X_train_Count_vect = pd.concat([X_train[['text_length','Punct_pc']].reset_index(drop=True) ,\n",
    "                         pd.DataFrame(X_train_Count_vect.toarray())],axis=1)\n",
    "\n",
    "\n",
    "X_test_Count_vect = pd.concat([X_test[['text_length','Punct_pc']].reset_index(drop=True) , \n",
    "                        pd.DataFrame(X_test_Count_vect.toarray())],axis=1)\n",
    "X_train_Count_vect.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RF - Tuning the hyperparameters for best model with TFIDF vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = RandomForestClassifier(random_state=123,n_jobs=3)\n",
    "param = {'n_estimators' : [10,25,50,100,300], 'max_depth' : [10, 20, 50,100, None],'max_features' : [10,50,'auto']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_max_features</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>params</th>\n",
       "      <th>rank_test_score</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>...</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split2_train_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split3_train_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>split4_train_score</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>std_train_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>26.992298</td>\n",
       "      <td>0.621765</td>\n",
       "      <td>0.976442</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>None</td>\n",
       "      <td>auto</td>\n",
       "      <td>300</td>\n",
       "      <td>{'max_depth': None, 'max_features': 'auto', 'n_estimators': 300}</td>\n",
       "      <td>1</td>\n",
       "      <td>0.982063</td>\n",
       "      <td>...</td>\n",
       "      <td>0.970852</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.968575</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.980899</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.335585</td>\n",
       "      <td>0.289018</td>\n",
       "      <td>0.005586</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>28.310359</td>\n",
       "      <td>0.702857</td>\n",
       "      <td>0.976442</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>100</td>\n",
       "      <td>auto</td>\n",
       "      <td>300</td>\n",
       "      <td>{'max_depth': 100, 'max_features': 'auto', 'n_estimators': 300}</td>\n",
       "      <td>1</td>\n",
       "      <td>0.980942</td>\n",
       "      <td>...</td>\n",
       "      <td>0.969731</td>\n",
       "      <td>0.999719</td>\n",
       "      <td>0.970819</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.979775</td>\n",
       "      <td>0.999439</td>\n",
       "      <td>1.839396</td>\n",
       "      <td>0.269796</td>\n",
       "      <td>0.005065</td>\n",
       "      <td>0.000177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>5.898225</td>\n",
       "      <td>0.381109</td>\n",
       "      <td>0.976217</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>100</td>\n",
       "      <td>auto</td>\n",
       "      <td>50</td>\n",
       "      <td>{'max_depth': 100, 'max_features': 'auto', 'n_estimators': 50}</td>\n",
       "      <td>3</td>\n",
       "      <td>0.982063</td>\n",
       "      <td>...</td>\n",
       "      <td>0.965247</td>\n",
       "      <td>0.999719</td>\n",
       "      <td>0.975309</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>0.979775</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.399882</td>\n",
       "      <td>0.035476</td>\n",
       "      <td>0.005904</td>\n",
       "      <td>0.000177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>10.445192</td>\n",
       "      <td>0.538609</td>\n",
       "      <td>0.975768</td>\n",
       "      <td>0.999663</td>\n",
       "      <td>100</td>\n",
       "      <td>auto</td>\n",
       "      <td>100</td>\n",
       "      <td>{'max_depth': 100, 'max_features': 'auto', 'n_estimators': 100}</td>\n",
       "      <td>4</td>\n",
       "      <td>0.982063</td>\n",
       "      <td>...</td>\n",
       "      <td>0.966368</td>\n",
       "      <td>0.999719</td>\n",
       "      <td>0.969697</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>0.980899</td>\n",
       "      <td>0.999439</td>\n",
       "      <td>0.336311</td>\n",
       "      <td>0.137234</td>\n",
       "      <td>0.006444</td>\n",
       "      <td>0.000112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>6.218115</td>\n",
       "      <td>0.366572</td>\n",
       "      <td>0.974422</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>None</td>\n",
       "      <td>auto</td>\n",
       "      <td>50</td>\n",
       "      <td>{'max_depth': None, 'max_features': 'auto', 'n_estimators': 50}</td>\n",
       "      <td>5</td>\n",
       "      <td>0.983184</td>\n",
       "      <td>...</td>\n",
       "      <td>0.963004</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.968575</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.979775</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.582722</td>\n",
       "      <td>0.199845</td>\n",
       "      <td>0.007484</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>4.650476</td>\n",
       "      <td>0.455022</td>\n",
       "      <td>0.974422</td>\n",
       "      <td>0.998373</td>\n",
       "      <td>100</td>\n",
       "      <td>50</td>\n",
       "      <td>50</td>\n",
       "      <td>{'max_depth': 100, 'max_features': 50, 'n_estimators': 50}</td>\n",
       "      <td>5</td>\n",
       "      <td>0.980942</td>\n",
       "      <td>...</td>\n",
       "      <td>0.966368</td>\n",
       "      <td>0.998597</td>\n",
       "      <td>0.968575</td>\n",
       "      <td>0.998598</td>\n",
       "      <td>0.977528</td>\n",
       "      <td>0.998038</td>\n",
       "      <td>0.332224</td>\n",
       "      <td>0.114794</td>\n",
       "      <td>0.005824</td>\n",
       "      <td>0.000275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>10.627047</td>\n",
       "      <td>0.598689</td>\n",
       "      <td>0.974422</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>None</td>\n",
       "      <td>auto</td>\n",
       "      <td>100</td>\n",
       "      <td>{'max_depth': None, 'max_features': 'auto', 'n_estimators': 100}</td>\n",
       "      <td>5</td>\n",
       "      <td>0.982063</td>\n",
       "      <td>...</td>\n",
       "      <td>0.965247</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.968575</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.979775</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.230061</td>\n",
       "      <td>0.204467</td>\n",
       "      <td>0.006475</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>3.743849</td>\n",
       "      <td>0.582610</td>\n",
       "      <td>0.973300</td>\n",
       "      <td>0.999046</td>\n",
       "      <td>100</td>\n",
       "      <td>auto</td>\n",
       "      <td>25</td>\n",
       "      <td>{'max_depth': 100, 'max_features': 'auto', 'n_estimators': 25}</td>\n",
       "      <td>8</td>\n",
       "      <td>0.979821</td>\n",
       "      <td>...</td>\n",
       "      <td>0.960762</td>\n",
       "      <td>0.999719</td>\n",
       "      <td>0.973064</td>\n",
       "      <td>0.998037</td>\n",
       "      <td>0.978652</td>\n",
       "      <td>0.999159</td>\n",
       "      <td>0.259261</td>\n",
       "      <td>0.092646</td>\n",
       "      <td>0.006773</td>\n",
       "      <td>0.000577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>4.008862</td>\n",
       "      <td>0.432861</td>\n",
       "      <td>0.973076</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>None</td>\n",
       "      <td>auto</td>\n",
       "      <td>25</td>\n",
       "      <td>{'max_depth': None, 'max_features': 'auto', 'n_estimators': 25}</td>\n",
       "      <td>9</td>\n",
       "      <td>0.979821</td>\n",
       "      <td>...</td>\n",
       "      <td>0.963004</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.967452</td>\n",
       "      <td>0.999159</td>\n",
       "      <td>0.978652</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>0.300188</td>\n",
       "      <td>0.250203</td>\n",
       "      <td>0.006650</td>\n",
       "      <td>0.000307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>3.608676</td>\n",
       "      <td>0.375041</td>\n",
       "      <td>0.972852</td>\n",
       "      <td>0.997420</td>\n",
       "      <td>100</td>\n",
       "      <td>50</td>\n",
       "      <td>25</td>\n",
       "      <td>{'max_depth': 100, 'max_features': 50, 'n_estimators': 25}</td>\n",
       "      <td>10</td>\n",
       "      <td>0.982063</td>\n",
       "      <td>...</td>\n",
       "      <td>0.965247</td>\n",
       "      <td>0.998036</td>\n",
       "      <td>0.967452</td>\n",
       "      <td>0.996354</td>\n",
       "      <td>0.976404</td>\n",
       "      <td>0.996916</td>\n",
       "      <td>0.279051</td>\n",
       "      <td>0.173165</td>\n",
       "      <td>0.006076</td>\n",
       "      <td>0.000718</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    mean_fit_time  mean_score_time  mean_test_score  mean_train_score  \\\n",
       "74      26.992298         0.621765         0.976442          1.000000   \n",
       "59      28.310359         0.702857         0.976442          0.999720   \n",
       "57       5.898225         0.381109         0.976217          0.999720   \n",
       "58      10.445192         0.538609         0.975768          0.999663   \n",
       "72       6.218115         0.366572         0.974422          1.000000   \n",
       "52       4.650476         0.455022         0.974422          0.998373   \n",
       "73      10.627047         0.598689         0.974422          1.000000   \n",
       "56       3.743849         0.582610         0.973300          0.999046   \n",
       "71       4.008862         0.432861         0.973076          0.999720   \n",
       "51       3.608676         0.375041         0.972852          0.997420   \n",
       "\n",
       "   param_max_depth param_max_features param_n_estimators  \\\n",
       "74            None               auto                300   \n",
       "59             100               auto                300   \n",
       "57             100               auto                 50   \n",
       "58             100               auto                100   \n",
       "72            None               auto                 50   \n",
       "52             100                 50                 50   \n",
       "73            None               auto                100   \n",
       "56             100               auto                 25   \n",
       "71            None               auto                 25   \n",
       "51             100                 50                 25   \n",
       "\n",
       "                                                              params  \\\n",
       "74  {'max_depth': None, 'max_features': 'auto', 'n_estimators': 300}   \n",
       "59   {'max_depth': 100, 'max_features': 'auto', 'n_estimators': 300}   \n",
       "57    {'max_depth': 100, 'max_features': 'auto', 'n_estimators': 50}   \n",
       "58   {'max_depth': 100, 'max_features': 'auto', 'n_estimators': 100}   \n",
       "72   {'max_depth': None, 'max_features': 'auto', 'n_estimators': 50}   \n",
       "52        {'max_depth': 100, 'max_features': 50, 'n_estimators': 50}   \n",
       "73  {'max_depth': None, 'max_features': 'auto', 'n_estimators': 100}   \n",
       "56    {'max_depth': 100, 'max_features': 'auto', 'n_estimators': 25}   \n",
       "71   {'max_depth': None, 'max_features': 'auto', 'n_estimators': 25}   \n",
       "51        {'max_depth': 100, 'max_features': 50, 'n_estimators': 25}   \n",
       "\n",
       "    rank_test_score  split0_test_score       ...         split2_test_score  \\\n",
       "74                1           0.982063       ...                  0.970852   \n",
       "59                1           0.980942       ...                  0.969731   \n",
       "57                3           0.982063       ...                  0.965247   \n",
       "58                4           0.982063       ...                  0.966368   \n",
       "72                5           0.983184       ...                  0.963004   \n",
       "52                5           0.980942       ...                  0.966368   \n",
       "73                5           0.982063       ...                  0.965247   \n",
       "56                8           0.979821       ...                  0.960762   \n",
       "71                9           0.979821       ...                  0.963004   \n",
       "51               10           0.982063       ...                  0.965247   \n",
       "\n",
       "    split2_train_score  split3_test_score  split3_train_score  \\\n",
       "74            1.000000           0.968575            1.000000   \n",
       "59            0.999719           0.970819            1.000000   \n",
       "57            0.999719           0.975309            0.999720   \n",
       "58            0.999719           0.969697            0.999720   \n",
       "72            1.000000           0.968575            1.000000   \n",
       "52            0.998597           0.968575            0.998598   \n",
       "73            1.000000           0.968575            1.000000   \n",
       "56            0.999719           0.973064            0.998037   \n",
       "71            1.000000           0.967452            0.999159   \n",
       "51            0.998036           0.967452            0.996354   \n",
       "\n",
       "    split4_test_score  split4_train_score  std_fit_time  std_score_time  \\\n",
       "74           0.980899            1.000000      4.335585        0.289018   \n",
       "59           0.979775            0.999439      1.839396        0.269796   \n",
       "57           0.979775            1.000000      0.399882        0.035476   \n",
       "58           0.980899            0.999439      0.336311        0.137234   \n",
       "72           0.979775            1.000000      0.582722        0.199845   \n",
       "52           0.977528            0.998038      0.332224        0.114794   \n",
       "73           0.979775            1.000000      0.230061        0.204467   \n",
       "56           0.978652            0.999159      0.259261        0.092646   \n",
       "71           0.978652            0.999720      0.300188        0.250203   \n",
       "51           0.976404            0.996916      0.279051        0.173165   \n",
       "\n",
       "    std_test_score  std_train_score  \n",
       "74        0.005586         0.000000  \n",
       "59        0.005065         0.000177  \n",
       "57        0.005904         0.000177  \n",
       "58        0.006444         0.000112  \n",
       "72        0.007484         0.000000  \n",
       "52        0.005824         0.000275  \n",
       "73        0.006475         0.000000  \n",
       "56        0.006773         0.000577  \n",
       "71        0.006650         0.000307  \n",
       "51        0.006076         0.000718  \n",
       "\n",
       "[10 rows x 23 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid = GridSearchCV(rf,param,cv=5,n_jobs=3)\n",
    "\n",
    "rf_grid_fit_1 = grid.fit(X_train_vect, Y_train)\n",
    "pd.DataFrame(rf_grid_fit_1.cv_results_).sort_values('mean_test_score',ascending=False)[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "RF_results_TFIDF = pd.DataFrame(rf_grid_fit_1.cv_results_).sort_values('mean_test_score',ascending=False)\n",
    "RF_results_TFIDF.to_csv(\"C:\\\\MSA\\\\Spring_Semester\\\\7152\\\\nlp\\\\RF_Results_TFIDF.csv\",header=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RF Tuning for CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_max_features</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>params</th>\n",
       "      <th>rank_test_score</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>...</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split2_train_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split3_train_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>split4_train_score</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>std_train_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>10.187043</td>\n",
       "      <td>0.648410</td>\n",
       "      <td>0.974871</td>\n",
       "      <td>0.999439</td>\n",
       "      <td>100</td>\n",
       "      <td>auto</td>\n",
       "      <td>100</td>\n",
       "      <td>{'max_depth': 100, 'max_features': 'auto', 'n_estimators': 100}</td>\n",
       "      <td>1</td>\n",
       "      <td>0.982063</td>\n",
       "      <td>...</td>\n",
       "      <td>0.965247</td>\n",
       "      <td>0.999719</td>\n",
       "      <td>0.970819</td>\n",
       "      <td>0.999439</td>\n",
       "      <td>0.980899</td>\n",
       "      <td>0.999159</td>\n",
       "      <td>0.262687</td>\n",
       "      <td>0.270129</td>\n",
       "      <td>0.006283</td>\n",
       "      <td>2.507858e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>6.785375</td>\n",
       "      <td>0.390667</td>\n",
       "      <td>0.974871</td>\n",
       "      <td>0.999495</td>\n",
       "      <td>100</td>\n",
       "      <td>auto</td>\n",
       "      <td>50</td>\n",
       "      <td>{'max_depth': 100, 'max_features': 'auto', 'n_estimators': 50}</td>\n",
       "      <td>1</td>\n",
       "      <td>0.983184</td>\n",
       "      <td>...</td>\n",
       "      <td>0.965247</td>\n",
       "      <td>0.999719</td>\n",
       "      <td>0.970819</td>\n",
       "      <td>0.999159</td>\n",
       "      <td>0.978652</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>1.211698</td>\n",
       "      <td>0.282491</td>\n",
       "      <td>0.006246</td>\n",
       "      <td>2.748055e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>10.817990</td>\n",
       "      <td>0.481357</td>\n",
       "      <td>0.974647</td>\n",
       "      <td>0.999944</td>\n",
       "      <td>None</td>\n",
       "      <td>auto</td>\n",
       "      <td>100</td>\n",
       "      <td>{'max_depth': None, 'max_features': 'auto', 'n_estimators': 100}</td>\n",
       "      <td>3</td>\n",
       "      <td>0.982063</td>\n",
       "      <td>...</td>\n",
       "      <td>0.964126</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.970819</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>0.979775</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.309986</td>\n",
       "      <td>0.131527</td>\n",
       "      <td>0.006481</td>\n",
       "      <td>1.121705e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>28.217441</td>\n",
       "      <td>0.599345</td>\n",
       "      <td>0.973974</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>100</td>\n",
       "      <td>auto</td>\n",
       "      <td>300</td>\n",
       "      <td>{'max_depth': 100, 'max_features': 'auto', 'n_estimators': 300}</td>\n",
       "      <td>4</td>\n",
       "      <td>0.982063</td>\n",
       "      <td>...</td>\n",
       "      <td>0.964126</td>\n",
       "      <td>0.999719</td>\n",
       "      <td>0.966330</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>0.979775</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>2.200441</td>\n",
       "      <td>0.163023</td>\n",
       "      <td>0.007315</td>\n",
       "      <td>6.291332e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>4.609863</td>\n",
       "      <td>0.356288</td>\n",
       "      <td>0.973749</td>\n",
       "      <td>0.998822</td>\n",
       "      <td>100</td>\n",
       "      <td>auto</td>\n",
       "      <td>25</td>\n",
       "      <td>{'max_depth': 100, 'max_features': 'auto', 'n_estimators': 25}</td>\n",
       "      <td>5</td>\n",
       "      <td>0.980942</td>\n",
       "      <td>...</td>\n",
       "      <td>0.966368</td>\n",
       "      <td>0.999719</td>\n",
       "      <td>0.969697</td>\n",
       "      <td>0.998878</td>\n",
       "      <td>0.977528</td>\n",
       "      <td>0.998318</td>\n",
       "      <td>1.055001</td>\n",
       "      <td>0.164012</td>\n",
       "      <td>0.005238</td>\n",
       "      <td>4.824089e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>27.118944</td>\n",
       "      <td>0.589048</td>\n",
       "      <td>0.973076</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>None</td>\n",
       "      <td>auto</td>\n",
       "      <td>300</td>\n",
       "      <td>{'max_depth': None, 'max_features': 'auto', 'n_estimators': 300}</td>\n",
       "      <td>6</td>\n",
       "      <td>0.980942</td>\n",
       "      <td>...</td>\n",
       "      <td>0.961883</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.967452</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.978652</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.080222</td>\n",
       "      <td>0.312986</td>\n",
       "      <td>0.007230</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>6.004991</td>\n",
       "      <td>0.357068</td>\n",
       "      <td>0.972403</td>\n",
       "      <td>0.999944</td>\n",
       "      <td>None</td>\n",
       "      <td>auto</td>\n",
       "      <td>50</td>\n",
       "      <td>{'max_depth': None, 'max_features': 'auto', 'n_estimators': 50}</td>\n",
       "      <td>7</td>\n",
       "      <td>0.978700</td>\n",
       "      <td>...</td>\n",
       "      <td>0.964126</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.968575</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>0.976404</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.452314</td>\n",
       "      <td>0.117556</td>\n",
       "      <td>0.005332</td>\n",
       "      <td>1.121705e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>23.628693</td>\n",
       "      <td>0.606407</td>\n",
       "      <td>0.971954</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>None</td>\n",
       "      <td>50</td>\n",
       "      <td>300</td>\n",
       "      <td>{'max_depth': None, 'max_features': 50, 'n_estimators': 300}</td>\n",
       "      <td>8</td>\n",
       "      <td>0.982063</td>\n",
       "      <td>...</td>\n",
       "      <td>0.959641</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.965208</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.977528</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.467595</td>\n",
       "      <td>0.177319</td>\n",
       "      <td>0.008270</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>3.675390</td>\n",
       "      <td>0.550488</td>\n",
       "      <td>0.971954</td>\n",
       "      <td>0.999383</td>\n",
       "      <td>None</td>\n",
       "      <td>50</td>\n",
       "      <td>25</td>\n",
       "      <td>{'max_depth': None, 'max_features': 50, 'n_estimators': 25}</td>\n",
       "      <td>8</td>\n",
       "      <td>0.979821</td>\n",
       "      <td>...</td>\n",
       "      <td>0.964126</td>\n",
       "      <td>0.999719</td>\n",
       "      <td>0.968575</td>\n",
       "      <td>0.999159</td>\n",
       "      <td>0.973034</td>\n",
       "      <td>0.998598</td>\n",
       "      <td>0.152918</td>\n",
       "      <td>0.314125</td>\n",
       "      <td>0.005309</td>\n",
       "      <td>4.823210e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>4.098917</td>\n",
       "      <td>0.419298</td>\n",
       "      <td>0.971730</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>None</td>\n",
       "      <td>auto</td>\n",
       "      <td>25</td>\n",
       "      <td>{'max_depth': None, 'max_features': 'auto', 'n_estimators': 25}</td>\n",
       "      <td>10</td>\n",
       "      <td>0.979821</td>\n",
       "      <td>...</td>\n",
       "      <td>0.960762</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.967452</td>\n",
       "      <td>0.999159</td>\n",
       "      <td>0.975281</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>0.481341</td>\n",
       "      <td>0.174017</td>\n",
       "      <td>0.006779</td>\n",
       "      <td>3.071916e-04</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    mean_fit_time  mean_score_time  mean_test_score  mean_train_score  \\\n",
       "58      10.187043         0.648410         0.974871          0.999439   \n",
       "57       6.785375         0.390667         0.974871          0.999495   \n",
       "73      10.817990         0.481357         0.974647          0.999944   \n",
       "59      28.217441         0.599345         0.973974          0.999720   \n",
       "56       4.609863         0.356288         0.973749          0.998822   \n",
       "74      27.118944         0.589048         0.973076          1.000000   \n",
       "72       6.004991         0.357068         0.972403          0.999944   \n",
       "69      23.628693         0.606407         0.971954          1.000000   \n",
       "66       3.675390         0.550488         0.971954          0.999383   \n",
       "71       4.098917         0.419298         0.971730          0.999720   \n",
       "\n",
       "   param_max_depth param_max_features param_n_estimators  \\\n",
       "58             100               auto                100   \n",
       "57             100               auto                 50   \n",
       "73            None               auto                100   \n",
       "59             100               auto                300   \n",
       "56             100               auto                 25   \n",
       "74            None               auto                300   \n",
       "72            None               auto                 50   \n",
       "69            None                 50                300   \n",
       "66            None                 50                 25   \n",
       "71            None               auto                 25   \n",
       "\n",
       "                                                              params  \\\n",
       "58   {'max_depth': 100, 'max_features': 'auto', 'n_estimators': 100}   \n",
       "57    {'max_depth': 100, 'max_features': 'auto', 'n_estimators': 50}   \n",
       "73  {'max_depth': None, 'max_features': 'auto', 'n_estimators': 100}   \n",
       "59   {'max_depth': 100, 'max_features': 'auto', 'n_estimators': 300}   \n",
       "56    {'max_depth': 100, 'max_features': 'auto', 'n_estimators': 25}   \n",
       "74  {'max_depth': None, 'max_features': 'auto', 'n_estimators': 300}   \n",
       "72   {'max_depth': None, 'max_features': 'auto', 'n_estimators': 50}   \n",
       "69      {'max_depth': None, 'max_features': 50, 'n_estimators': 300}   \n",
       "66       {'max_depth': None, 'max_features': 50, 'n_estimators': 25}   \n",
       "71   {'max_depth': None, 'max_features': 'auto', 'n_estimators': 25}   \n",
       "\n",
       "    rank_test_score  split0_test_score       ...         split2_test_score  \\\n",
       "58                1           0.982063       ...                  0.965247   \n",
       "57                1           0.983184       ...                  0.965247   \n",
       "73                3           0.982063       ...                  0.964126   \n",
       "59                4           0.982063       ...                  0.964126   \n",
       "56                5           0.980942       ...                  0.966368   \n",
       "74                6           0.980942       ...                  0.961883   \n",
       "72                7           0.978700       ...                  0.964126   \n",
       "69                8           0.982063       ...                  0.959641   \n",
       "66                8           0.979821       ...                  0.964126   \n",
       "71               10           0.979821       ...                  0.960762   \n",
       "\n",
       "    split2_train_score  split3_test_score  split3_train_score  \\\n",
       "58            0.999719           0.970819            0.999439   \n",
       "57            0.999719           0.970819            0.999159   \n",
       "73            1.000000           0.970819            0.999720   \n",
       "59            0.999719           0.966330            0.999720   \n",
       "56            0.999719           0.969697            0.998878   \n",
       "74            1.000000           0.967452            1.000000   \n",
       "72            1.000000           0.968575            0.999720   \n",
       "69            1.000000           0.965208            1.000000   \n",
       "66            0.999719           0.968575            0.999159   \n",
       "71            1.000000           0.967452            0.999159   \n",
       "\n",
       "    split4_test_score  split4_train_score  std_fit_time  std_score_time  \\\n",
       "58           0.980899            0.999159      0.262687        0.270129   \n",
       "57           0.978652            0.999720      1.211698        0.282491   \n",
       "73           0.979775            1.000000      0.309986        0.131527   \n",
       "59           0.979775            0.999720      2.200441        0.163023   \n",
       "56           0.977528            0.998318      1.055001        0.164012   \n",
       "74           0.978652            1.000000      4.080222        0.312986   \n",
       "72           0.976404            1.000000      0.452314        0.117556   \n",
       "69           0.977528            1.000000      1.467595        0.177319   \n",
       "66           0.973034            0.998598      0.152918        0.314125   \n",
       "71           0.975281            0.999720      0.481341        0.174017   \n",
       "\n",
       "    std_test_score  std_train_score  \n",
       "58        0.006283     2.507858e-04  \n",
       "57        0.006246     2.748055e-04  \n",
       "73        0.006481     1.121705e-04  \n",
       "59        0.007315     6.291332e-08  \n",
       "56        0.005238     4.824089e-04  \n",
       "74        0.007230     0.000000e+00  \n",
       "72        0.005332     1.121705e-04  \n",
       "69        0.008270     0.000000e+00  \n",
       "66        0.005309     4.823210e-04  \n",
       "71        0.006779     3.071916e-04  \n",
       "\n",
       "[10 rows x 23 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf = RandomForestClassifier(random_state=123,n_jobs=3)\n",
    "param = {'n_estimators' : [10,25,50,100,300], 'max_depth' : [10, 20, 50,100, None],'max_features' : [10,50,'auto']}\n",
    "\n",
    "grid = GridSearchCV(rf,param,cv=5,n_jobs=3)\n",
    "\n",
    "rf_grid_fit_2 = grid.fit(X_train_Count_vect, Y_train)\n",
    "pd.DataFrame(rf_grid_fit_2.cv_results_).sort_values('mean_test_score',ascending=False)[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "RF_results_CountVector = pd.DataFrame(rf_grid_fit_2.cv_results_).sort_values('mean_test_score',ascending=False)\n",
    "RF_results_CountVector.to_csv(\"C:\\\\MSA\\\\Spring_Semester\\\\7152\\\\nlp\\\\RF_Results_CountVector.csv\",header=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient Boosting parameter tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "gb = GradientBoostingClassifier(random_state=123)\n",
    "param = {'n_estimators' : [100,150,250], 'learning_rate' :[0.01,0.1],'max_depth':[3,10],'max_features': [300,'auto',None]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>param_learning_rate</th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_max_features</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>params</th>\n",
       "      <th>rank_test_score</th>\n",
       "      <th>...</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split2_train_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split3_train_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>split4_train_score</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>std_train_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>39.994312</td>\n",
       "      <td>0.237416</td>\n",
       "      <td>0.974647</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.1</td>\n",
       "      <td>10</td>\n",
       "      <td>300</td>\n",
       "      <td>250</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 10, 'max_features': 300, 'n_estimators': 250}</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.963004</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.969697</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.980899</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.678010</td>\n",
       "      <td>0.023927</td>\n",
       "      <td>0.007250</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>26.803623</td>\n",
       "      <td>0.212121</td>\n",
       "      <td>0.974198</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.1</td>\n",
       "      <td>10</td>\n",
       "      <td>300</td>\n",
       "      <td>150</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 10, 'max_features': 300, 'n_estimators': 150}</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0.961883</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.968575</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.979775</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.482787</td>\n",
       "      <td>0.020428</td>\n",
       "      <td>0.007702</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>20.117713</td>\n",
       "      <td>0.186901</td>\n",
       "      <td>0.973525</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.1</td>\n",
       "      <td>10</td>\n",
       "      <td>300</td>\n",
       "      <td>100</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 10, 'max_features': 300, 'n_estimators': 100}</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0.961883</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.966330</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.979775</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.342364</td>\n",
       "      <td>0.008686</td>\n",
       "      <td>0.007851</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>11.104559</td>\n",
       "      <td>0.182823</td>\n",
       "      <td>0.971057</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>0.1</td>\n",
       "      <td>3</td>\n",
       "      <td>300</td>\n",
       "      <td>250</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 3, 'max_features': 300, 'n_estimators': 250}</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0.957399</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.967452</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>0.983146</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>0.126442</td>\n",
       "      <td>0.015359</td>\n",
       "      <td>0.009329</td>\n",
       "      <td>0.000177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>485.688527</td>\n",
       "      <td>0.195584</td>\n",
       "      <td>0.968813</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.1</td>\n",
       "      <td>10</td>\n",
       "      <td>None</td>\n",
       "      <td>250</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 10, 'max_features': None, 'n_estimators': 250}</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.955157</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.961841</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.975281</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>75.116782</td>\n",
       "      <td>0.026989</td>\n",
       "      <td>0.008695</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>542.930090</td>\n",
       "      <td>0.246132</td>\n",
       "      <td>0.968813</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.1</td>\n",
       "      <td>10</td>\n",
       "      <td>auto</td>\n",
       "      <td>250</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 10, 'max_features': 'auto', 'n_estimators': 250}</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.955157</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.961841</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.975281</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.636683</td>\n",
       "      <td>0.037661</td>\n",
       "      <td>0.008695</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>7.969673</td>\n",
       "      <td>0.166501</td>\n",
       "      <td>0.967691</td>\n",
       "      <td>0.984182</td>\n",
       "      <td>0.1</td>\n",
       "      <td>3</td>\n",
       "      <td>300</td>\n",
       "      <td>150</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 3, 'max_features': 300, 'n_estimators': 150}</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0.956278</td>\n",
       "      <td>0.987658</td>\n",
       "      <td>0.962963</td>\n",
       "      <td>0.986259</td>\n",
       "      <td>0.974157</td>\n",
       "      <td>0.980656</td>\n",
       "      <td>0.180202</td>\n",
       "      <td>0.005557</td>\n",
       "      <td>0.007968</td>\n",
       "      <td>0.002491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>336.191472</td>\n",
       "      <td>0.221054</td>\n",
       "      <td>0.965672</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.1</td>\n",
       "      <td>10</td>\n",
       "      <td>None</td>\n",
       "      <td>150</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 10, 'max_features': None, 'n_estimators': 150}</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0.955157</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.958474</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.970787</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.510692</td>\n",
       "      <td>0.025745</td>\n",
       "      <td>0.007458</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>336.762781</td>\n",
       "      <td>0.207033</td>\n",
       "      <td>0.965672</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.1</td>\n",
       "      <td>10</td>\n",
       "      <td>auto</td>\n",
       "      <td>150</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 10, 'max_features': 'auto', 'n_estimators': 150}</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0.955157</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.958474</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.970787</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.161775</td>\n",
       "      <td>0.009401</td>\n",
       "      <td>0.007458</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>165.633316</td>\n",
       "      <td>0.192965</td>\n",
       "      <td>0.965448</td>\n",
       "      <td>0.999832</td>\n",
       "      <td>0.1</td>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>250</td>\n",
       "      <td>{'learning_rate': 0.1, 'max_depth': 3, 'max_features': None, 'n_estimators': 250}</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>0.951794</td>\n",
       "      <td>0.999719</td>\n",
       "      <td>0.965208</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.976404</td>\n",
       "      <td>0.999720</td>\n",
       "      <td>0.869293</td>\n",
       "      <td>0.012998</td>\n",
       "      <td>0.007956</td>\n",
       "      <td>0.000137</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    mean_fit_time  mean_score_time  mean_test_score  mean_train_score  \\\n",
       "29      39.994312         0.237416         0.974647          1.000000   \n",
       "28      26.803623         0.212121         0.974198          1.000000   \n",
       "27      20.117713         0.186901         0.973525          1.000000   \n",
       "20      11.104559         0.182823         0.971057          0.999720   \n",
       "35     485.688527         0.195584         0.968813          1.000000   \n",
       "32     542.930090         0.246132         0.968813          1.000000   \n",
       "19       7.969673         0.166501         0.967691          0.984182   \n",
       "34     336.191472         0.221054         0.965672          1.000000   \n",
       "31     336.762781         0.207033         0.965672          1.000000   \n",
       "26     165.633316         0.192965         0.965448          0.999832   \n",
       "\n",
       "   param_learning_rate param_max_depth param_max_features param_n_estimators  \\\n",
       "29                 0.1              10                300                250   \n",
       "28                 0.1              10                300                150   \n",
       "27                 0.1              10                300                100   \n",
       "20                 0.1               3                300                250   \n",
       "35                 0.1              10               None                250   \n",
       "32                 0.1              10               auto                250   \n",
       "19                 0.1               3                300                150   \n",
       "34                 0.1              10               None                150   \n",
       "31                 0.1              10               auto                150   \n",
       "26                 0.1               3               None                250   \n",
       "\n",
       "                                                                                  params  \\\n",
       "29     {'learning_rate': 0.1, 'max_depth': 10, 'max_features': 300, 'n_estimators': 250}   \n",
       "28     {'learning_rate': 0.1, 'max_depth': 10, 'max_features': 300, 'n_estimators': 150}   \n",
       "27     {'learning_rate': 0.1, 'max_depth': 10, 'max_features': 300, 'n_estimators': 100}   \n",
       "20      {'learning_rate': 0.1, 'max_depth': 3, 'max_features': 300, 'n_estimators': 250}   \n",
       "35    {'learning_rate': 0.1, 'max_depth': 10, 'max_features': None, 'n_estimators': 250}   \n",
       "32  {'learning_rate': 0.1, 'max_depth': 10, 'max_features': 'auto', 'n_estimators': 250}   \n",
       "19      {'learning_rate': 0.1, 'max_depth': 3, 'max_features': 300, 'n_estimators': 150}   \n",
       "34    {'learning_rate': 0.1, 'max_depth': 10, 'max_features': None, 'n_estimators': 150}   \n",
       "31  {'learning_rate': 0.1, 'max_depth': 10, 'max_features': 'auto', 'n_estimators': 150}   \n",
       "26     {'learning_rate': 0.1, 'max_depth': 3, 'max_features': None, 'n_estimators': 250}   \n",
       "\n",
       "    rank_test_score       ...         split2_test_score  split2_train_score  \\\n",
       "29                1       ...                  0.963004            1.000000   \n",
       "28                2       ...                  0.961883            1.000000   \n",
       "27                3       ...                  0.961883            1.000000   \n",
       "20                4       ...                  0.957399            1.000000   \n",
       "35                5       ...                  0.955157            1.000000   \n",
       "32                5       ...                  0.955157            1.000000   \n",
       "19                7       ...                  0.956278            0.987658   \n",
       "34                8       ...                  0.955157            1.000000   \n",
       "31                8       ...                  0.955157            1.000000   \n",
       "26               10       ...                  0.951794            0.999719   \n",
       "\n",
       "    split3_test_score  split3_train_score  split4_test_score  \\\n",
       "29           0.969697            1.000000           0.980899   \n",
       "28           0.968575            1.000000           0.979775   \n",
       "27           0.966330            1.000000           0.979775   \n",
       "20           0.967452            0.999720           0.983146   \n",
       "35           0.961841            1.000000           0.975281   \n",
       "32           0.961841            1.000000           0.975281   \n",
       "19           0.962963            0.986259           0.974157   \n",
       "34           0.958474            1.000000           0.970787   \n",
       "31           0.958474            1.000000           0.970787   \n",
       "26           0.965208            1.000000           0.976404   \n",
       "\n",
       "    split4_train_score  std_fit_time  std_score_time  std_test_score  \\\n",
       "29            1.000000      0.678010        0.023927        0.007250   \n",
       "28            1.000000      0.482787        0.020428        0.007702   \n",
       "27            1.000000      0.342364        0.008686        0.007851   \n",
       "20            0.999720      0.126442        0.015359        0.009329   \n",
       "35            1.000000     75.116782        0.026989        0.008695   \n",
       "32            1.000000      3.636683        0.037661        0.008695   \n",
       "19            0.980656      0.180202        0.005557        0.007968   \n",
       "34            1.000000      3.510692        0.025745        0.007458   \n",
       "31            1.000000      2.161775        0.009401        0.007458   \n",
       "26            0.999720      0.869293        0.012998        0.007956   \n",
       "\n",
       "    std_train_score  \n",
       "29         0.000000  \n",
       "28         0.000000  \n",
       "27         0.000000  \n",
       "20         0.000177  \n",
       "35         0.000000  \n",
       "32         0.000000  \n",
       "19         0.002491  \n",
       "34         0.000000  \n",
       "31         0.000000  \n",
       "26         0.000137  \n",
       "\n",
       "[10 rows x 24 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb_grid = GridSearchCV(gb,param,cv=5,n_jobs=-1)\n",
    "\n",
    "gb_grid_fit = gb_grid.fit(X_train_vect, Y_train)\n",
    "pd.DataFrame(gb_grid_fit.cv_results_).sort_values('mean_test_score',ascending=False)[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "GB_results = pd.DataFrame(gb_grid_fit.cv_results_).sort_values('mean_test_score',ascending=False)\n",
    "GB_results.to_csv(\"C:\\\\MSA\\\\Spring_Semester\\\\7152\\\\nlp\\\\GB_Results.csv\",header=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extreme Gradient Boosting tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
      "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
      "       max_depth=3, min_child_weight=1, missing=None, n_estimators=100,\n",
      "       n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,\n",
      "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
      "       silent=True, subsample=1)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>param_colsample_bytree</th>\n",
       "      <th>param_learning_rate</th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_n_estimators</th>\n",
       "      <th>params</th>\n",
       "      <th>rank_test_score</th>\n",
       "      <th>...</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split2_train_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split3_train_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>split4_train_score</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>std_train_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>177.456918</td>\n",
       "      <td>0.749632</td>\n",
       "      <td>0.974422</td>\n",
       "      <td>0.994223</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.1</td>\n",
       "      <td>15</td>\n",
       "      <td>100</td>\n",
       "      <td>{'colsample_bytree': 0.5, 'learning_rate': 0.1, 'max_depth': 15, 'n_estimators': 100}</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.967489</td>\n",
       "      <td>0.995231</td>\n",
       "      <td>0.969697</td>\n",
       "      <td>0.995233</td>\n",
       "      <td>0.982022</td>\n",
       "      <td>0.994113</td>\n",
       "      <td>19.904182</td>\n",
       "      <td>0.077554</td>\n",
       "      <td>0.005412</td>\n",
       "      <td>0.000880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>348.700755</td>\n",
       "      <td>0.904826</td>\n",
       "      <td>0.973974</td>\n",
       "      <td>0.995569</td>\n",
       "      <td>1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>15</td>\n",
       "      <td>100</td>\n",
       "      <td>{'colsample_bytree': 1, 'learning_rate': 0.1, 'max_depth': 15, 'n_estimators': 100}</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0.963004</td>\n",
       "      <td>0.996073</td>\n",
       "      <td>0.968575</td>\n",
       "      <td>0.996074</td>\n",
       "      <td>0.980899</td>\n",
       "      <td>0.995234</td>\n",
       "      <td>39.874489</td>\n",
       "      <td>0.194613</td>\n",
       "      <td>0.006959</td>\n",
       "      <td>0.000449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>620.893216</td>\n",
       "      <td>0.704965</td>\n",
       "      <td>0.972852</td>\n",
       "      <td>0.998373</td>\n",
       "      <td>1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>15</td>\n",
       "      <td>200</td>\n",
       "      <td>{'colsample_bytree': 1, 'learning_rate': 0.1, 'max_depth': 15, 'n_estimators': 200}</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0.963004</td>\n",
       "      <td>0.998597</td>\n",
       "      <td>0.967452</td>\n",
       "      <td>0.998317</td>\n",
       "      <td>0.977528</td>\n",
       "      <td>0.998598</td>\n",
       "      <td>188.441804</td>\n",
       "      <td>0.166772</td>\n",
       "      <td>0.006475</td>\n",
       "      <td>0.000210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>362.568149</td>\n",
       "      <td>0.751972</td>\n",
       "      <td>0.972627</td>\n",
       "      <td>0.997644</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.1</td>\n",
       "      <td>15</td>\n",
       "      <td>200</td>\n",
       "      <td>{'colsample_bytree': 0.5, 'learning_rate': 0.1, 'max_depth': 15, 'n_estimators': 200}</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0.966368</td>\n",
       "      <td>0.997756</td>\n",
       "      <td>0.967452</td>\n",
       "      <td>0.998317</td>\n",
       "      <td>0.980899</td>\n",
       "      <td>0.997477</td>\n",
       "      <td>30.102964</td>\n",
       "      <td>0.105621</td>\n",
       "      <td>0.005466</td>\n",
       "      <td>0.000456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>141.659736</td>\n",
       "      <td>0.749317</td>\n",
       "      <td>0.971954</td>\n",
       "      <td>0.984126</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.1</td>\n",
       "      <td>3</td>\n",
       "      <td>200</td>\n",
       "      <td>{'colsample_bytree': 0.5, 'learning_rate': 0.1, 'max_depth': 3, 'n_estimators': 200}</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.963004</td>\n",
       "      <td>0.985975</td>\n",
       "      <td>0.967452</td>\n",
       "      <td>0.984857</td>\n",
       "      <td>0.976404</td>\n",
       "      <td>0.984301</td>\n",
       "      <td>36.022152</td>\n",
       "      <td>0.096564</td>\n",
       "      <td>0.005845</td>\n",
       "      <td>0.001275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>234.655857</td>\n",
       "      <td>0.816191</td>\n",
       "      <td>0.971281</td>\n",
       "      <td>0.982556</td>\n",
       "      <td>1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>3</td>\n",
       "      <td>200</td>\n",
       "      <td>{'colsample_bytree': 1, 'learning_rate': 0.1, 'max_depth': 3, 'n_estimators': 200}</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>0.961883</td>\n",
       "      <td>0.984853</td>\n",
       "      <td>0.970819</td>\n",
       "      <td>0.982894</td>\n",
       "      <td>0.970787</td>\n",
       "      <td>0.982058</td>\n",
       "      <td>18.387455</td>\n",
       "      <td>0.111131</td>\n",
       "      <td>0.005385</td>\n",
       "      <td>0.001445</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>374.056465</td>\n",
       "      <td>0.705983</td>\n",
       "      <td>0.967018</td>\n",
       "      <td>0.983173</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.01</td>\n",
       "      <td>15</td>\n",
       "      <td>200</td>\n",
       "      <td>{'colsample_bytree': 0.5, 'learning_rate': 0.01, 'max_depth': 15, 'n_estimators': 200}</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0.952915</td>\n",
       "      <td>0.985975</td>\n",
       "      <td>0.960718</td>\n",
       "      <td>0.983735</td>\n",
       "      <td>0.975281</td>\n",
       "      <td>0.982618</td>\n",
       "      <td>50.774477</td>\n",
       "      <td>0.075085</td>\n",
       "      <td>0.008755</td>\n",
       "      <td>0.001586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>178.342406</td>\n",
       "      <td>0.701015</td>\n",
       "      <td>0.963204</td>\n",
       "      <td>0.978405</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.01</td>\n",
       "      <td>15</td>\n",
       "      <td>100</td>\n",
       "      <td>{'colsample_bytree': 0.5, 'learning_rate': 0.01, 'max_depth': 15, 'n_estimators': 100}</td>\n",
       "      <td>8</td>\n",
       "      <td>...</td>\n",
       "      <td>0.951794</td>\n",
       "      <td>0.980645</td>\n",
       "      <td>0.957351</td>\n",
       "      <td>0.980931</td>\n",
       "      <td>0.969663</td>\n",
       "      <td>0.978694</td>\n",
       "      <td>15.606337</td>\n",
       "      <td>0.027351</td>\n",
       "      <td>0.007310</td>\n",
       "      <td>0.002210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>85.752330</td>\n",
       "      <td>0.664009</td>\n",
       "      <td>0.962531</td>\n",
       "      <td>0.976161</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.1</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>{'colsample_bytree': 0.5, 'learning_rate': 0.1, 'max_depth': 3, 'n_estimators': 100}</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>0.950673</td>\n",
       "      <td>0.980084</td>\n",
       "      <td>0.952862</td>\n",
       "      <td>0.978127</td>\n",
       "      <td>0.967416</td>\n",
       "      <td>0.975049</td>\n",
       "      <td>12.471069</td>\n",
       "      <td>0.072705</td>\n",
       "      <td>0.009156</td>\n",
       "      <td>0.002564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>147.069318</td>\n",
       "      <td>0.829275</td>\n",
       "      <td>0.960287</td>\n",
       "      <td>0.973356</td>\n",
       "      <td>1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>3</td>\n",
       "      <td>100</td>\n",
       "      <td>{'colsample_bytree': 1, 'learning_rate': 0.1, 'max_depth': 3, 'n_estimators': 100}</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>0.947309</td>\n",
       "      <td>0.976718</td>\n",
       "      <td>0.950617</td>\n",
       "      <td>0.972518</td>\n",
       "      <td>0.967416</td>\n",
       "      <td>0.974769</td>\n",
       "      <td>22.306959</td>\n",
       "      <td>0.126858</td>\n",
       "      <td>0.009477</td>\n",
       "      <td>0.002115</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    mean_fit_time  mean_score_time  mean_test_score  mean_train_score  \\\n",
       "6      177.456918         0.749632         0.974422          0.994223   \n",
       "14     348.700755         0.904826         0.973974          0.995569   \n",
       "15     620.893216         0.704965         0.972852          0.998373   \n",
       "7      362.568149         0.751972         0.972627          0.997644   \n",
       "5      141.659736         0.749317         0.971954          0.984126   \n",
       "13     234.655857         0.816191         0.971281          0.982556   \n",
       "3      374.056465         0.705983         0.967018          0.983173   \n",
       "2      178.342406         0.701015         0.963204          0.978405   \n",
       "4       85.752330         0.664009         0.962531          0.976161   \n",
       "12     147.069318         0.829275         0.960287          0.973356   \n",
       "\n",
       "   param_colsample_bytree param_learning_rate param_max_depth  \\\n",
       "6                     0.5                 0.1              15   \n",
       "14                      1                 0.1              15   \n",
       "15                      1                 0.1              15   \n",
       "7                     0.5                 0.1              15   \n",
       "5                     0.5                 0.1               3   \n",
       "13                      1                 0.1               3   \n",
       "3                     0.5                0.01              15   \n",
       "2                     0.5                0.01              15   \n",
       "4                     0.5                 0.1               3   \n",
       "12                      1                 0.1               3   \n",
       "\n",
       "   param_n_estimators  \\\n",
       "6                 100   \n",
       "14                100   \n",
       "15                200   \n",
       "7                 200   \n",
       "5                 200   \n",
       "13                200   \n",
       "3                 200   \n",
       "2                 100   \n",
       "4                 100   \n",
       "12                100   \n",
       "\n",
       "                                                                                    params  \\\n",
       "6    {'colsample_bytree': 0.5, 'learning_rate': 0.1, 'max_depth': 15, 'n_estimators': 100}   \n",
       "14     {'colsample_bytree': 1, 'learning_rate': 0.1, 'max_depth': 15, 'n_estimators': 100}   \n",
       "15     {'colsample_bytree': 1, 'learning_rate': 0.1, 'max_depth': 15, 'n_estimators': 200}   \n",
       "7    {'colsample_bytree': 0.5, 'learning_rate': 0.1, 'max_depth': 15, 'n_estimators': 200}   \n",
       "5     {'colsample_bytree': 0.5, 'learning_rate': 0.1, 'max_depth': 3, 'n_estimators': 200}   \n",
       "13      {'colsample_bytree': 1, 'learning_rate': 0.1, 'max_depth': 3, 'n_estimators': 200}   \n",
       "3   {'colsample_bytree': 0.5, 'learning_rate': 0.01, 'max_depth': 15, 'n_estimators': 200}   \n",
       "2   {'colsample_bytree': 0.5, 'learning_rate': 0.01, 'max_depth': 15, 'n_estimators': 100}   \n",
       "4     {'colsample_bytree': 0.5, 'learning_rate': 0.1, 'max_depth': 3, 'n_estimators': 100}   \n",
       "12      {'colsample_bytree': 1, 'learning_rate': 0.1, 'max_depth': 3, 'n_estimators': 100}   \n",
       "\n",
       "    rank_test_score       ...         split2_test_score  split2_train_score  \\\n",
       "6                 1       ...                  0.967489            0.995231   \n",
       "14                2       ...                  0.963004            0.996073   \n",
       "15                3       ...                  0.963004            0.998597   \n",
       "7                 4       ...                  0.966368            0.997756   \n",
       "5                 5       ...                  0.963004            0.985975   \n",
       "13                6       ...                  0.961883            0.984853   \n",
       "3                 7       ...                  0.952915            0.985975   \n",
       "2                 8       ...                  0.951794            0.980645   \n",
       "4                 9       ...                  0.950673            0.980084   \n",
       "12               10       ...                  0.947309            0.976718   \n",
       "\n",
       "    split3_test_score  split3_train_score  split4_test_score  \\\n",
       "6            0.969697            0.995233           0.982022   \n",
       "14           0.968575            0.996074           0.980899   \n",
       "15           0.967452            0.998317           0.977528   \n",
       "7            0.967452            0.998317           0.980899   \n",
       "5            0.967452            0.984857           0.976404   \n",
       "13           0.970819            0.982894           0.970787   \n",
       "3            0.960718            0.983735           0.975281   \n",
       "2            0.957351            0.980931           0.969663   \n",
       "4            0.952862            0.978127           0.967416   \n",
       "12           0.950617            0.972518           0.967416   \n",
       "\n",
       "    split4_train_score  std_fit_time  std_score_time  std_test_score  \\\n",
       "6             0.994113     19.904182        0.077554        0.005412   \n",
       "14            0.995234     39.874489        0.194613        0.006959   \n",
       "15            0.998598    188.441804        0.166772        0.006475   \n",
       "7             0.997477     30.102964        0.105621        0.005466   \n",
       "5             0.984301     36.022152        0.096564        0.005845   \n",
       "13            0.982058     18.387455        0.111131        0.005385   \n",
       "3             0.982618     50.774477        0.075085        0.008755   \n",
       "2             0.978694     15.606337        0.027351        0.007310   \n",
       "4             0.975049     12.471069        0.072705        0.009156   \n",
       "12            0.974769     22.306959        0.126858        0.009477   \n",
       "\n",
       "    std_train_score  \n",
       "6          0.000880  \n",
       "14         0.000449  \n",
       "15         0.000210  \n",
       "7          0.000456  \n",
       "5          0.001275  \n",
       "13         0.001445  \n",
       "3          0.001586  \n",
       "2          0.002210  \n",
       "4          0.002564  \n",
       "12         0.002115  \n",
       "\n",
       "[10 rows x 24 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "\n",
    "print(XGBClassifier())\n",
    "xgb1 = XGBClassifier(random_state=123,silent=False,n_jobs=4)\n",
    "param = {'learning_rate':[0.01,0.1],'max_depth':[3,15],'n_estimators':[100,200],\n",
    "         'colsample_bytree':[0.5,1]}\n",
    "#'subsample':[0.5,1]\n",
    "xgb_grid = GridSearchCV(xgb1,param,cv=5,n_jobs=4)\n",
    "\n",
    "xgb_grid_fit = xgb_grid.fit(X_train_vect, Y_train)\n",
    "pd.DataFrame(xgb_grid_fit.cv_results_).sort_values('mean_test_score',ascending=False)[0:10]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "XGB_results = pd.DataFrame(xgb_grid_fit.cv_results_).sort_values('mean_test_score',ascending=False)\n",
    "XGB_results.to_csv(\"C:\\\\MSA\\\\Spring_Semester\\\\7152\\\\nlp\\\\XGB_Results.csv\",header=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest with Best Tuning Parameters for TFIDF vectorized data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fit_time : 3.572 / Predict_time : 0.224 / Precision: 1.0 / Recall: 0.863 / Accuracy: 0.981\n",
      "[ 0.06426574  0.01079169  0.00062025 ...,  0.          0.00029655  0.        ]\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "rf_final_1 = RandomForestClassifier(n_estimators = 50, max_depth = 100, max_features='auto',n_jobs=-1,random_state=123)\n",
    "\n",
    "start = time.time()\n",
    "rf_model_1 = rf_final_1.fit(X_train_vect, Y_train)\n",
    "end = time.time()\n",
    "fit_time = end - start\n",
    "\n",
    "start = time.time()\n",
    "Y_pred = rf_model_1.predict(X_test_vect)\n",
    "end = time.time()\n",
    "predict_time = end-start\n",
    "\n",
    "precision, recall, fscore, train_support = score(Y_test, Y_pred, pos_label='spam', average='binary')\n",
    "print('Fit_time : {} / Predict_time : {} / Precision: {} / Recall: {} / Accuracy: {}'.format(round(fit_time,3),round(predict_time,3),\n",
    "    round(precision, 3), round(recall, 3), round((Y_pred==Y_test).sum()/len(Y_pred), 3)))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.064265735980574709, 'text_length'),\n",
       " (0.027785497274975954, 1570),\n",
       " (0.027031553955556422, 6445),\n",
       " (0.022934517772735242, 2735),\n",
       " (0.022415369311974347, 4211)]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(zip(rf_model_1.feature_importances_ ,X_train_vect.columns),reverse=True)[0:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Another RF with similar results and restricted hyperparameters for TFIDF vectorized data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fit_time : 0.922 / Predict_time : 0.188 / Precision: 1.0 / Recall: 0.824 / Accuracy: 0.976\n"
     ]
    }
   ],
   "source": [
    "rf_final_2 = RandomForestClassifier(n_estimators = 10, max_depth = 100, max_features='auto',n_jobs=-1,random_state=123)\n",
    "\n",
    "start = time.time()\n",
    "rf_model_2 = rf_final_2.fit(X_train_vect, Y_train)\n",
    "end = time.time()\n",
    "fit_time = end - start\n",
    "\n",
    "start = time.time()\n",
    "Y_pred = rf_model_2.predict(X_test_vect)\n",
    "end = time.time()\n",
    "predict_time = end-start\n",
    "\n",
    "precision, recall, fscore, train_support = score(Y_test, Y_pred, pos_label='spam', average='binary')\n",
    "print('Fit_time : {} / Predict_time : {} / Precision: {} / Recall: {} / Accuracy: {}'.format(round(fit_time,3),round(predict_time,3),\n",
    "    round(precision, 3), round(recall, 3), round((Y_pred==Y_test).sum()/len(Y_pred), 3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RF with best tuned parameters for Count vectorized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fit_time : 3.86 / Predict_time : 0.321 / Precision: 1.0 / Recall: 0.817 / Accuracy: 0.975\n"
     ]
    }
   ],
   "source": [
    "rf_final_3 = RandomForestClassifier(n_estimators = 50, max_depth = 100, max_features='auto' , n_jobs=-1,random_state=123)\n",
    "\n",
    "start = time.time()\n",
    "rf_model_3 = rf_final_3.fit(X_train_Count_vect, Y_train)\n",
    "end = time.time()\n",
    "fit_time = end - start\n",
    "\n",
    "start = time.time()\n",
    "Y_pred = rf_model_3.predict(X_test_Count_vect)\n",
    "end = time.time()\n",
    "predict_time = end-start\n",
    "\n",
    "precision, recall, fscore, train_support = score(Y_test, Y_pred, pos_label='spam', average='binary')\n",
    "print('Fit_time : {} / Predict_time : {} / Precision: {} / Recall: {} / Accuracy: {}'.format(round(fit_time,3),round(predict_time,3),\n",
    "    round(precision, 3), round(recall, 3), round((Y_pred==Y_test).sum()/len(Y_pred), 3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient Boosting model with Best Tuned hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fit_time : 12.5836923122406 / Predict_time : 0.272244930267334 / Precision: 0.985 / Recall: 0.85 / Accuracy: 0.978\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "gb_final_1 =  GradientBoostingClassifier(n_estimators = 250 , max_depth = 3 , max_features=300, learning_rate = 0.1,random_state=123)\n",
    "\n",
    "start = time.time()\n",
    "gb_model_1 = gb_final_1.fit(X_train_vect, Y_train)\n",
    "end = time.time()\n",
    "fit_time = end - start\n",
    "\n",
    "start = time.time()\n",
    "Y_pred = gb_model_1.predict(X_test_vect)\n",
    "end = time.time()\n",
    "predict_time = end-start\n",
    "\n",
    "precision, recall, fscore, train_support = score(Y_test, Y_pred, pos_label='spam', average='binary')\n",
    "print('Fit_time : {} / Predict_time : {} / Precision: {} / Recall: {} / Accuracy: {}'.format(fit_time,predict_time,\n",
    "    round(precision, 3), round(recall, 3), round((Y_pred==Y_test).sum()/len(Y_pred), 3)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.045961823614495662, 'text_length'),\n",
       " (0.03592707123333648, 1570),\n",
       " (0.028181258450186025, 6445),\n",
       " (0.024202504689814313, 5509),\n",
       " (0.018469949110446127, 591)]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(zip(gb_model_1.feature_importances_ ,X_train_vect.columns),reverse=True)[0:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Xtreme Gradient Boosting with best tuning parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fit_time : 95.34738516807556 / Predict_time : 0.5919156074523926 / Precision: 0.985 / Recall: 0.85 / Accuracy: 0.978\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "xgb_final_1 =  XGBClassifier(max_depth = 15 ,colsample_bytree=0.5, learning_rate = 0.1,n_estimator=100,random_state=123,n_jobs=3)\n",
    "\n",
    "start = time.time()\n",
    "xgb_model_1 = xgb_final_1.fit(X_train_vect, Y_train)\n",
    "end = time.time()\n",
    "fit_time = end - start\n",
    "\n",
    "start = time.time()\n",
    "Y_pred = xgb_model_1.predict(X_test_vect)\n",
    "end = time.time()\n",
    "predict_time = end-start\n",
    "\n",
    "precision, recall, fscore, train_support = score(Y_test, Y_pred, pos_label='spam', average='binary')\n",
    "print('Fit_time : {} / Predict_time : {} / Precision: {} / Recall: {} / Accuracy: {}'.format(fit_time,predict_time,\n",
    "    round(precision, 3), round(recall, 3), round((Y_pred==Y_test).sum()/len(Y_pred), 3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.16893654, 'text_length'),\n",
       " (0.11735729, 'Punct_pc'),\n",
       " (0.063170098, 1570),\n",
       " (0.029266879, 6157),\n",
       " (0.027818024, 6457)]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(zip(xgb_model_1.feature_importances_ ,X_train_vect.columns),reverse=True)[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'text_length'"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_vect.columns[0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}