sentiment analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "from sklearn.model_selection import train_test_split\n",
    "import gc\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation\n",
    "from keras.layers import Bidirectional, GlobalMaxPool1D\n",
    "from keras.models import Model\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers\n",
    "from imblearn.over_sampling import SMOTENC, SMOTE\n",
    "%pylab inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('datasets/text_emotion.csv')\n",
    "df.drop(['tweet_id', 'author'], inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def common_user(s):\n",
    "    return re.sub(r'@\\w+', 'user', s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.content = df.content.apply(common_user)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing && Oversampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>empty</td>\n",
       "      <td>user i know  i was listenin to bad habit earli...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sadness</td>\n",
       "      <td>Layin n bed with a headache  ughhhh...waitin o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sadness</td>\n",
       "      <td>Funeral ceremony...gloomy friday...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>enthusiasm</td>\n",
       "      <td>wants to hang out with friends SOON!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>neutral</td>\n",
       "      <td>user We want to trade with someone who has Hou...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    sentiment                                            content\n",
       "0       empty  user i know  i was listenin to bad habit earli...\n",
       "1     sadness  Layin n bed with a headache  ughhhh...waitin o...\n",
       "2     sadness                Funeral ceremony...gloomy friday...\n",
       "3  enthusiasm               wants to hang out with friends SOON!\n",
       "4     neutral  user We want to trade with someone who has Hou..."
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "df.iloc[:, 0] = le.fit_transform(df.iloc[:, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>user i know  i was listenin to bad habit earli...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>Layin n bed with a headache  ughhhh...waitin o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10</td>\n",
       "      <td>Funeral ceremony...gloomy friday...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>wants to hang out with friends SOON!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>user We want to trade with someone who has Hou...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment                                            content\n",
       "0          2  user i know  i was listenin to bad habit earli...\n",
       "1         10  Layin n bed with a headache  ughhhh...waitin o...\n",
       "2         10                Funeral ceremony...gloomy friday...\n",
       "3          3               wants to hang out with friends SOON!\n",
       "4          8  user We want to trade with someone who has Hou..."
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['anger', 'boredom', 'empty', 'enthusiasm', 'fun', 'happiness',\n",
       "       'hate', 'love', 'neutral', 'relief', 'sadness', 'surprise',\n",
       "       'worry'], dtype=object)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "le.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x20731ac4a90>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAASoElEQVR4nO3df4xd9Xnn8fdn7aYQu8GmtKOsba29qpWW4m1LRkAbqRrHLRgSxfwRJCqamCwrSyua0spSY3ZVscqPFdUmTRPtNisreHFaFJd1s8IKSYnlZBRVKjQ4iTDEzdqbsDDgQio7bpzQptN99o/79e7UjPHM/eF7fef9kqx7z/f8ep65M/OZc+65x6kqJElL2z8bdgGSpOEzDCRJhoEkyTCQJGEYSJKA5cMuoFtXXXVVrV+/vqt1v//977NixYr+FjQk49LLuPQB9jKKxqUP6K2Xw4cP/01V/cR88y7ZMFi/fj1PPvlkV+tOT08zNTXV34KGZFx6GZc+wF5G0bj0Ab31kuR/n2+ep4kkSYaBJMkwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAksQl/AlkSRqm9bseHcp+H9w6mNtqeGQgSTIMJEmGgSQJw0CShGEgScIwkCRhGEiSMAwkSSwgDJLsSfJykqfnjF2Z5GCSY+1xdRtPko8nOZ7kqSTXzllne1v+WJLtc8bfnORIW+fjSdLvJiVJr20hRwYPAlvPGdsFHKqqjcChNg1wM7Cx/dsBfAI64QHcB1wPXAfcdzZA2jI75qx37r4kSQN2wTCoqi8DJ88Z3gbsbc/3ArfOGf9UdTwOrEryRuAm4GBVnayqU8BBYGub94aq+ouqKuBTc7YlSbpIur030URVnQCoqhNJfrKNrwGen7PcTBt7rfGZecbnlWQHnaMIJiYmmJ6e7qr4M2fOdL3uqBmXXsalD7CXUTSIPnZumu3r9hZqUK9Jv29UN9/5/upifF5VtRvYDTA5OVlTU1NdlAjT09N0u+6oGZdexqUPsJdRNIg+7hzijeoG8Zp0ezXRS+0UD+3x5TY+A6ybs9xa4MULjK+dZ1ySdBF1GwYHgLNXBG0HHpkz/u52VdENwOl2Oukx4MYkq9sbxzcCj7V530tyQ7uK6N1ztiVJukgueJooyaeBKeCqJDN0rgq6H3g4yV3Ac8BtbfHPAbcAx4EfAO8BqKqTST4AfKUt9/6qOvum9L+lc8XS5cDn2z9J0kV0wTCoql87z6wt8yxbwN3n2c4eYM88408C11yoDknS4PgJZEmSYSBJMgwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKwfNgFSOqP9bse7fs2d26a5c4FbPfZ+9/W933r4vLIQJJkGEiSDANJEoaBJAnDQJKEYSBJwjCQJGEYSJLoMQyS/HaSZ5I8neTTSS5LsiHJE0mOJfmTJK9ry/5omz7e5q+fs5172/g3k9zUW0uSpMXqOgySrAF+E5isqmuAZcDtwO8BH62qjcAp4K62yl3Aqar6KeCjbTmSXN3W+1lgK/CHSZZ1W5ckafF6PU20HLg8yXLg9cAJ4K3A/jZ/L3Bre76tTdPmb0mSNr6vqv6+qr4NHAeu67EuSdIipKq6Xzm5B/gQ8ArwBeAe4PH21z9J1gGfr6prkjwNbK2qmTbvfwHXA/+hrfPHbfyBts7+efa3A9gBMDEx8eZ9+/Z1VfeZM2dYuXJlV+uOmnHpZVz6gOH1cuSF033f5sTl8NIrF15u05or+r7vfhrEazKIr/dCbLhiWde9bN68+XBVTc43r+sb1SVZTeev+g3Ad4H/Dtw8z6Jn0ybnmXe+8VcPVu0GdgNMTk7W1NTU4opupqen6XbdUTMuvYxLHzC8XhZyQ7nF2rlplo8cufCviWfvmOr7vvtpEK/JIL7eC/Hg1hUD+f7q5TTRrwDfrqrvVNU/AJ8BfglY1U4bAawFXmzPZ4B1AG3+FcDJuePzrCNJugh6CYPngBuSvL6d+98CfAP4EvDOtsx24JH2/ECbps3/YnXOUR0Abm9XG20ANgJ/2UNdkqRF6vo0UVU9kWQ/8FVgFvganVM4jwL7knywjT3QVnkA+KMkx+kcEdzetvNMkofpBMkscHdV/WO3dUmSFq+n/9ymqu4D7jtn+FvMczVQVf0dcNt5tvMhOm9ES5KGwE8gS5IMA0mSYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJ9BgGSVYl2Z/kr5IcTfKLSa5McjDJsfa4ui2bJB9PcjzJU0munbOd7W35Y0m299qUJGlxej0y+BjwZ1X108DPAUeBXcChqtoIHGrTADcDG9u/HcAnAJJcCdwHXA9cB9x3NkAkSRdH12GQ5A3ALwMPAFTVD6vqu8A2YG9bbC9wa3u+DfhUdTwOrEryRuAm4GBVnayqU8BBYGu3dUmSFi9V1d2Kyc8Du4Fv0DkqOAzcA7xQVavmLHeqqlYn+Sxwf1X9eRs/BLwPmAIuq6oPtvHfBV6pqg/Ps88ddI4qmJiYePO+ffu6qv3MmTOsXLmyq3VHzbj0Mi59wPB6OfLC6b5vc+JyeOmVCy+3ac0Vfd93Pw3iNRnE13shNlyxrOteNm/efLiqJuebt7yHmpYD1wLvraonknyM/39KaD6ZZ6xeY/zVg1W76QQQk5OTNTU1taiCz5qenqbbdUfNuPQyLn3A8Hq5c9ejfd/mzk2zfOTIhX9NPHvHVN/33U+DeE0G8fVeiAe3rhjI91cv7xnMADNV9USb3k8nHF5qp39ojy/PWX7dnPXXAi++xrgk6SLpOgyq6q+B55O8qQ1toXPK6ABw9oqg7cAj7fkB4N3tqqIbgNNVdQJ4DLgxyer2xvGNbUySdJH0cpoI4L3AQ0leB3wLeA+dgHk4yV3Ac8BtbdnPAbcAx4EftGWpqpNJPgB8pS33/qo62WNdkqRF6CkMqurrwHxvRmyZZ9kC7j7PdvYAe3qpRZLUPT+BLEkyDCRJhoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIker8dhTSy1g/xrpLSpcYwkPrsyAunh3Z746VmoYG/c9Osr8kFeJpIkmQYSJIMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRJ9CIMky5J8Lcln2/SGJE8kOZbkT5K8ro3/aJs+3uavn7ONe9v4N5Pc1GtNkqTF6ceRwT3A0TnTvwd8tKo2AqeAu9r4XcCpqvop4KNtOZJcDdwO/CywFfjDJMv6UJckaYF6CoMka4G3AZ9s0wHeCuxvi+wFbm3Pt7Vp2vwtbfltwL6q+vuq+jZwHLiul7okSYuzvMf1/wD4HeDH2vSPA9+tqtk2PQOsac/XAM8DVNVsktNt+TXA43O2OXedfyLJDmAHwMTEBNPT010VfebMma7XHTXj0ssg+ti5afbCCw3AxOXD23e/LbSXYX0PLvTrPE6vyaB+5rsOgyRvB16uqsNJps4Oz7NoXWDea63zTwerdgO7ASYnJ2tqamq+xS5oenqabtcdNePSyyD6uHPXo33d3kLt3DTLR470+nfWaFhoL8/eMTX4Yuax0Nd4nF6TB7euGMjPfC9fnbcA70hyC3AZ8AY6RwqrkixvRwdrgRfb8jPAOmAmyXLgCuDknPGz5q4jSboIun7PoKruraq1VbWezhvAX6yqO4AvAe9si20HHmnPD7Rp2vwvVlW18dvb1UYbgI3AX3ZblyRp8QZx3PQ+YF+SDwJfAx5o4w8Af5TkOJ0jgtsBquqZJA8D3wBmgbur6h8HUJck6Tz6EgZVNQ1Mt+ffYp6rgarq74DbzrP+h4AP9aMWSdLi+QlkSZJhIEkyDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSSJHsIgybokX0pyNMkzSe5p41cmOZjkWHtc3caT5ONJjid5Ksm1c7a1vS1/LMn23tuSJC1GL0cGs8DOqvoZ4Abg7iRXA7uAQ1W1ETjUpgFuBja2fzuAT0AnPID7gOuB64D7zgaIJOni6DoMqupEVX21Pf8ecBRYA2wD9rbF9gK3tufbgE9Vx+PAqiRvBG4CDlbVyao6BRwEtnZblyRp8VJVvW8kWQ98GbgGeK6qVs2Zd6qqVif5LHB/Vf15Gz8EvA+YAi6rqg+28d8FXqmqD8+znx10jiqYmJh48759+7qq98yZM6xcubKrdUfNuPQyiD6OvHC6r9tbqInL4aVXhrLrvltoL5vWXDH4Yuax0Nd4nF6TDVcs6/pnZfPmzYeranK+ect7qgpIshL4U+C3qupvk5x30XnG6jXGXz1YtRvYDTA5OVlTU1OLrhdgenqabtcdNePSyyD6uHPXo33d3kLt3DTLR470/KM1Ehbay7N3TA2+mHks9DUep9fkwa0rBvIz39PVREl+hE4QPFRVn2nDL7XTP7THl9v4DLBuzuprgRdfY1ySdJH0cjVRgAeAo1X1+3NmHQDOXhG0HXhkzvi721VFNwCnq+oE8BhwY5LV7Y3jG9uYJOki6eW46S3Au4AjSb7exv4dcD/wcJK7gOeA29q8zwG3AMeBHwDvAaiqk0k+AHylLff+qjrZQ10aIesXcRg/rNM6knoIg/ZG8PneINgyz/IF3H2ebe0B9nRbiySpN34CWZJkGEiSDANJEn34nIEkLfRCAY0ujwwkSYaBJMkwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEl419KLahB3dlzofxf57P1v6/u+JY0Pw2CJ8BbDkl6Lp4kkSYaBJMkwkCRhGEiSMAwkSRgGkiQMA0kShoEkiSX6obMjL5xe0Kd2JWmp8MhAkmQYSJJGKAySbE3yzSTHk+wadj2StJSMRBgkWQb8F+Bm4Grg15JcPdyqJGnpGIkwAK4DjlfVt6rqh8A+YNuQa5KkJSNVNewaSPJOYGtV/Zs2/S7g+qr6jXOW2wHsaJNvAr7Z5S6vAv6my3VHzbj0Mi59gL2MonHpA3rr5V9U1U/MN2NULi3NPGOvSqmq2g3s7nlnyZNVNdnrdkbBuPQyLn2AvYyicekDBtfLqJwmmgHWzZleC7w4pFokackZlTD4CrAxyYYkrwNuBw4MuSZJWjJG4jRRVc0m+Q3gMWAZsKeqnhngLns+1TRCxqWXcekD7GUUjUsfMKBeRuINZEnScI3KaSJJ0hAZBpKkpRUG43LLiyTrknwpydEkzyS5Z9g19SrJsiRfS/LZYdfSiySrkuxP8lft9fnFYdfUjSS/3b63nk7y6SSXDbumhUqyJ8nLSZ6eM3ZlkoNJjrXH1cOscaHO08t/at9fTyX5H0lW9WNfSyYMxuyWF7PAzqr6GeAG4O5LuJez7gGODruIPvgY8GdV9dPAz3EJ9pRkDfCbwGRVXUPnoo7bh1vVojwIbD1nbBdwqKo2Aofa9KXgQV7dy0Hgmqr6V8D/BO7tx46WTBgwRre8qKoTVfXV9vx7dH7hrBluVd1LshZ4G/DJYdfSiyRvAH4ZeACgqn5YVd8dblVdWw5cnmQ58Houoc/9VNWXgZPnDG8D9rbne4FbL2pRXZqvl6r6QlXNtsnH6Xwuq2dLKQzWAM/PmZ7hEv4FelaS9cAvAE8Mt5Ke/AHwO8D/GXYhPfqXwHeA/9ZOeX0yyYphF7VYVfUC8GHgOeAEcLqqvjDcqno2UVUnoPPHFPCTQ66nX/418Pl+bGgphcGCbnlxKUmyEvhT4Leq6m+HXU83krwdeLmqDg+7lj5YDlwLfKKqfgH4PpfO6Yj/p51P3wZsAP45sCLJrw+3Kp0ryb+nc8r4oX5sbymFwVjd8iLJj9AJgoeq6jPDrqcHbwHekeRZOqfu3prkj4dbUtdmgJmqOnuUtp9OOFxqfgX4dlV9p6r+AfgM8EtDrqlXLyV5I0B7fHnI9fQkyXbg7cAd1acPiy2lMBibW14kCZ3z0ker6veHXU8vqureqlpbVevpvCZfrKpL8q/Qqvpr4Pkkb2pDW4BvDLGkbj0H3JDk9e17bQuX4Bvh5zgAbG/PtwOPDLGWniTZCrwPeEdV/aBf210yYdDecDl7y4ujwMMDvuXFIL0FeBedv6K/3v7dMuyiBMB7gYeSPAX8PPAfh1zPorUjm/3AV4EjdH5PXDK3c0jyaeAvgDclmUlyF3A/8KtJjgG/2qZH3nl6+c/AjwEH28/+f+3LvrwdhSRpyRwZSJLOzzCQJBkGkiTDQJKEYSBJwjCQJGEYSJKA/wtlKsL6ptWB7AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.sentiment.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40000, 2)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.content\n",
    "y = df.sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_size = 100 # how big is each word vector\n",
    "max_features = 20000 # how many unique words to use (i.e num rows in embedding vector)\n",
    "maxlen = 100 # max number of words in a comment to use\n",
    "EMBEDDING_FILE = 'datasets/glove.twitter.27B.100d.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words=max_features)\n",
    "tokenizer.fit_on_texts(X)\n",
    "list_tokenized_train = tokenizer.texts_to_sequences(X)\n",
    "X_t = pad_sequences(list_tokenized_train, maxlen=maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sentence_to_sequences(s):\n",
    "    return pad_sequences(tokenizer.texts_to_sequences(s), maxlen=maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(model, s):\n",
    "    return le.inverse_transform(np.argmax(model.predict(sentence_to_sequences(s)), axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_t, y, test_size=0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "sm = SMOTE(random_state=42, n_jobs=-1)\n",
    "X_over, y_over = sm.fit_resample(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({8: 6898,\n",
       "         4: 6898,\n",
       "         12: 6898,\n",
       "         6: 6898,\n",
       "         11: 6898,\n",
       "         10: 6898,\n",
       "         7: 6898,\n",
       "         5: 6898,\n",
       "         9: 6898,\n",
       "         2: 6898,\n",
       "         3: 6898,\n",
       "         1: 6898,\n",
       "         0: 6898})"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "Counter(y_over)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\PC\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:415: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.\n",
      "If you want the future behaviour and silence this warning, you can specify \"categories='auto'\".\n",
      "In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "onehotencoder = OneHotEncoder()\n",
    "y_train_onehot = onehotencoder.fit_transform(y_over.reshape(-1, 1)).toarray()\n",
    "y_test_onehot = onehotencoder.transform(y_test.values.reshape(-1, 1)).toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_coefs(word, *arr): \n",
    "    return word, np.asarray(arr, dtype='float32')\n",
    "embeddings_index = dict(get_coefs(*o.strip().split()) for o in open(EMBEDDING_FILE, encoding='utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\PC\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.02631098, 0.58370703)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_embs = np.stack(embeddings_index.values())\n",
    "emb_mean, emb_std = all_embs.mean(), all_embs.std()\n",
    "emb_mean, emb_std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_index = tokenizer.word_index\n",
    "nb_words = min(max_features, len(word_index))\n",
    "embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n",
    "for word, i in word_index.items():\n",
    "    if i >= max_features: continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None: embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "204"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "inp = Input(shape=(maxlen,))\n",
    "x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n",
    "x = Bidirectional(LSTM(50, return_sequences=True, dropout=0.2, recurrent_dropout=0.1))(x)\n",
    "x = GlobalMaxPool1D()(x)\n",
    "x = Dense(50, activation=\"relu\")(x)\n",
    "x = Dropout(0.2)(x)\n",
    "x = Dense(13, activation=\"softmax\")(x)\n",
    "model = Model(inputs=inp, outputs=x)\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "embedding_2 (Embedding)      (None, 100, 100)          2000000   \n",
      "_________________________________________________________________\n",
      "bidirectional_2 (Bidirection (None, 100, 100)          60400     \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_2 (Glob (None, 100)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 50)                5050      \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 50)                0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 13)                663       \n",
      "=================================================================\n",
      "Total params: 2,066,113\n",
      "Trainable params: 66,113\n",
      "Non-trainable params: 2,000,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\PC\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From C:\\Users\\PC\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "Train on 80706 samples, validate on 8968 samples\n",
      "Epoch 1/1\n",
      "80706/80706 [==============================] - 433s 5ms/step - loss: 0.2564 - acc: 0.9231 - val_loss: 0.4249 - val_acc: 0.9231\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x20730b30e10>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_over, y_train_onehot, batch_size=64, epochs=1, validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8000/8000 [==============================] - 27s 3ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.2324436382651329, 0.9231153883934021]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(X_test, y_test_onehot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(model, open('model.pkl', 'wb'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}