twitter_sentiment_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import nltk\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "import string\n",
    "import re\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>time</th>\n",
       "      <th>date</th>\n",
       "      <th>query</th>\n",
       "      <th>username</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810672</td>\n",
       "      <td>Mon Apr 06 22:19:49 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>scotthamilton</td>\n",
       "      <td>is upset that he can't update his Facebook by ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810917</td>\n",
       "      <td>Mon Apr 06 22:19:53 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>mattycus</td>\n",
       "      <td>@Kenichan I dived many times for the ball. Man...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1467811184</td>\n",
       "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>ElleCTF</td>\n",
       "      <td>my whole body feels itchy and like its on fire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1467811193</td>\n",
       "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>Karoli</td>\n",
       "      <td>@nationwideclass no, it's not behaving at all....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1467811372</td>\n",
       "      <td>Mon Apr 06 22:20:00 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>joy_wolf</td>\n",
       "      <td>@Kwesidei not the whole crew</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label        time                          date     query       username  \\\n",
       "0      0  1467810672  Mon Apr 06 22:19:49 PDT 2009  NO_QUERY  scotthamilton   \n",
       "1      0  1467810917  Mon Apr 06 22:19:53 PDT 2009  NO_QUERY       mattycus   \n",
       "2      0  1467811184  Mon Apr 06 22:19:57 PDT 2009  NO_QUERY        ElleCTF   \n",
       "3      0  1467811193  Mon Apr 06 22:19:57 PDT 2009  NO_QUERY         Karoli   \n",
       "4      0  1467811372  Mon Apr 06 22:20:00 PDT 2009  NO_QUERY       joy_wolf   \n",
       "\n",
       "                                                text  \n",
       "0  is upset that he can't update his Facebook by ...  \n",
       "1  @Kenichan I dived many times for the ball. Man...  \n",
       "2    my whole body feels itchy and like its on fire   \n",
       "3  @nationwideclass no, it's not behaving at all....  \n",
       "4                      @Kwesidei not the whole crew   "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('archive/training.1600000.processed.noemoticon.csv', encoding = \"ISO-8859-1\", engine=\"python\")\n",
    "data.columns = [\"label\", \"time\", \"date\", \"query\", \"username\", \"text\"]\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1599999, 6)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1599999 entries, 0 to 1599998\n",
      "Data columns (total 6 columns):\n",
      " #   Column    Non-Null Count    Dtype \n",
      "---  ------    --------------    ----- \n",
      " 0   label     1599999 non-null  int64 \n",
      " 1   time      1599999 non-null  int64 \n",
      " 2   date      1599999 non-null  object\n",
      " 3   query     1599999 non-null  object\n",
      " 4   username  1599999 non-null  object\n",
      " 5   text      1599999 non-null  object\n",
      "dtypes: int64(2), object(4)\n",
      "memory usage: 73.2+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "label        int64\n",
       "time         int64\n",
       "date        object\n",
       "query       object\n",
       "username    object\n",
       "text        object\n",
       "dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# checking if there is any null value\n",
    "np.sum(data.isnull().any(axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 4])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# checking how many unique labels\n",
    "data['label'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "659775"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# cheking how many unique users\n",
    "len(data['username'].unique())\n",
    "# there are users tweeted multiple times "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Work with only the text and labels\n",
    "data = data[['text', 'label']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert label 4 as probability 1\n",
    "data['label'][data['label'] == 4] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For now work with small amount of data\n",
    "positive = data[data['label'] == 1]\n",
    "negative = data[data['label'] == 0]\n",
    "data = pd.concat([negative[:4000], positive[:4000]])\n",
    "data.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# making text lower case\n",
    "data['text'] = data['text'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove stop words from tweets\n",
    "def remove_stopwords(text):\n",
    "    cleaned_text = []\n",
    "    for word in text.split():\n",
    "        if word not in nltk.corpus.stopwords.words('english'):\n",
    "            cleaned_text.append(word)\n",
    "    return \" \".join(cleaned_text)\n",
    "\n",
    "data['text'] = data['text'].apply(remove_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>upset can't update facebook texting it... migh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>@kenichan dived many times ball. managed save ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>whole body feels itchy like fire</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>@nationwideclass no, behaving all. i'm mad. he...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>@kwesidei whole crew</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7995</th>\n",
       "      <td>803994</td>\n",
       "      <td>lipton tea -so delicious calming (even teine)....</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7996</th>\n",
       "      <td>803995</td>\n",
       "      <td>@bryndrescher lol history belongs last person ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7997</th>\n",
       "      <td>803996</td>\n",
       "      <td>cooked pasta sister. birthday tom</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7998</th>\n",
       "      <td>803997</td>\n",
       "      <td>today's gonna gooooood</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7999</th>\n",
       "      <td>803998</td>\n",
       "      <td>@bigbelbess aha, beat that, minus ska though, ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       index                                               text  label\n",
       "0          0  upset can't update facebook texting it... migh...      0\n",
       "1          1  @kenichan dived many times ball. managed save ...      0\n",
       "2          2                   whole body feels itchy like fire      0\n",
       "3          3  @nationwideclass no, behaving all. i'm mad. he...      0\n",
       "4          4                               @kwesidei whole crew      0\n",
       "...      ...                                                ...    ...\n",
       "7995  803994  lipton tea -so delicious calming (even teine)....      1\n",
       "7996  803995  @bryndrescher lol history belongs last person ...      1\n",
       "7997  803996                  cooked pasta sister. birthday tom      1\n",
       "7998  803997                             today's gonna gooooood      1\n",
       "7999  803998  @bigbelbess aha, beat that, minus ska though, ...      1\n",
       "\n",
       "[8000 rows x 3 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove emails and urls\n",
    "def remove_emails(text):\n",
    "    return re.sub(r'[\\w\\.\\+\\-]+\\@[\\w\\.]+\\.[a-z]{2,3}', '' , text)\n",
    "\n",
    "def remove_urls(text):\n",
    "    return re.sub(r'((http|https)\\:\\/\\/)?[a-zA-Z0-9\\.\\/\\?\\:@\\-_=#]+\\.([a-zA-Z]){2,6}([a-zA-Z0-9\\.\\&\\/\\?\\:@\\-_=#])*', '', text)\n",
    "\n",
    "data['text'] = data['text'].apply(remove_emails)\n",
    "data['text'] = data['text'].apply(remove_urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"broadband plan 'a massive broken promise'  via ~tautao still waiting broadband\""
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.at[49, 'text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# removing punctuations\n",
    "def remove_punctuations(text):\n",
    "    return text.translate(str.maketrans('', '', string.punctuation))\n",
    "data['text'] = data['text'].apply(remove_punctuations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# removing numbers\n",
    "def remove_digits(text):\n",
    "    return text.translate(str.maketrans('', '', string.digits))\n",
    "data['text'] = data['text'].apply(remove_digits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenize tweet text\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "tokenizer = RegexpTokenizer(r'\\w+')\n",
    "data['text'] = data['text'].apply(tokenizer.tokenize)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[upset, cant, update, facebook, texting, it, m...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[kenichan, dived, many, times, ball, managed, ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[whole, body, feels, itchy, like, fire]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[nationwideclass, no, behaving, all, im, mad, ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[kwesidei, whole, crew]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7995</th>\n",
       "      <td>803994</td>\n",
       "      <td>[lipton, tea, so, delicious, calming, even, te...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7996</th>\n",
       "      <td>803995</td>\n",
       "      <td>[bryndrescher, lol, history, belongs, last, pe...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7997</th>\n",
       "      <td>803996</td>\n",
       "      <td>[cooked, pasta, sister, birthday, tom]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7998</th>\n",
       "      <td>803997</td>\n",
       "      <td>[todays, gonna, gooooood]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7999</th>\n",
       "      <td>803998</td>\n",
       "      <td>[bigbelbess, aha, beat, that, minus, ska, thou...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       index                                               text  label\n",
       "0          0  [upset, cant, update, facebook, texting, it, m...      0\n",
       "1          1  [kenichan, dived, many, times, ball, managed, ...      0\n",
       "2          2            [whole, body, feels, itchy, like, fire]      0\n",
       "3          3  [nationwideclass, no, behaving, all, im, mad, ...      0\n",
       "4          4                            [kwesidei, whole, crew]      0\n",
       "...      ...                                                ...    ...\n",
       "7995  803994  [lipton, tea, so, delicious, calming, even, te...      1\n",
       "7996  803995  [bryndrescher, lol, history, belongs, last, pe...      1\n",
       "7997  803996             [cooked, pasta, sister, birthday, tom]      1\n",
       "7998  803997                          [todays, gonna, gooooood]      1\n",
       "7999  803998  [bigbelbess, aha, beat, that, minus, ska, thou...      1\n",
       "\n",
       "[8000 rows x 3 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Applying Stemming\n",
    "# Lemmatization is better than stemming\n",
    "\n",
    "# stemmer = nltk.PorterStemmer()\n",
    "# def replace_by_stem_words(text):\n",
    "#     text = [stemmer.stem(word) for word in text]\n",
    "#     return text\n",
    "\n",
    "# data['text'] = data['text'].apply(replace_by_stem_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Applying Lemmatizer\n",
    "lm = nltk.WordNetLemmatizer()\n",
    "def lemmatizer_on_text(data):\n",
    "    text = [lm.lemmatize(word) for word in data]\n",
    "    return data\n",
    "\n",
    "data['text'] = data['text'].apply(lemmatizer_on_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# adding <bos> and <eos> tokens\n",
    "def add_start_and_end_tokens(tokens):\n",
    "    tokens.insert(0, '<bos>')\n",
    "    tokens.append('<eos>')\n",
    "    return tokens\n",
    "data['text'] = data['text'].apply(add_start_and_end_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total number of lemmatize words in all tweets 78446\n",
      "total number of unique lemmatize words in all tweets 15182\n"
     ]
    }
   ],
   "source": [
    "# finding out number of unique words\n",
    "all_words = []\n",
    "for tweet in data['text']:\n",
    "    for word in tweet:\n",
    "        all_words.append(word)\n",
    "print(\"total number of lemmatize words in all tweets\", len(all_words))\n",
    "unique_words = list(set(all_words))\n",
    "print(\"total number of unique lemmatize words in all tweets\", len(unique_words))\n",
    "\n",
    "word_counts = [all_words.count(word) for word in unique_words]\n",
    "unique_words = [word for _, word in sorted(zip(word_counts, unique_words), key=lambda pair: pair[0], reverse=True)]\n",
    "unique_words = unique_words[:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# indexing words\n",
    "word_to_ix = {word:i for i,word in enumerate(unique_words)}\n",
    "ix_to_word = {i:word for i,word in enumerate(unique_words)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating train and test set\n",
    "X = data.text\n",
    "y = data.label\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "X_train = list(X_train)\n",
    "X_test = list(X_test)\n",
    "y_train = list(y_train)\n",
    "y_test = list(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# word to vec\n",
    "class FormattedDataset(Dataset):\n",
    "    def __init__(self, X, y, unique_words, max_len):\n",
    "        super().__init__()\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.unique_words = unique_words\n",
    "        self.word_to_ix = {word:i for i,word in enumerate(unique_words)}\n",
    "        self.max_len = max_len\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        selected_tweet = self.X[index]\n",
    "        tweet_length = len(selected_tweet)\n",
    "        tweet_vec = torch.zeros((len(self.unique_words), self.max_len))\n",
    "        for i in range(tweet_length):\n",
    "            if self.word_to_ix.get(selected_tweet[i], 0):\n",
    "                tweet_vec[self.word_to_ix[selected_tweet[i]]][self.max_len - tweet_length + i] = 1\n",
    "        return {'text':tweet_vec.T, 'label':self.y[index]}\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.y)\n",
    "\n",
    "\n",
    "train_data = FormattedDataset(X_train, y_train, unique_words, max([len(x) for x in X]))\n",
    "test_data = FormattedDataset(X_test, y_test, unique_words, max([len(x) for x in X]))\n",
    "\n",
    "train_loader = DataLoader(dataset=train_data, batch_size=32, shuffle=True)\n",
    "test_loader = DataLoader(dataset=test_data, batch_size=32, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "#device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(torch.cuda.is_available())\n",
    "device = \"cpu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNN(nn.Module):\n",
    "    def __init__(self, input_size, hidden_sizes, num_layers, num_classes):\n",
    "        super().__init__()\n",
    "        self.num_layers = num_layers\n",
    "        self.hidden_sizes = hidden_sizes\n",
    "        self.rnn = nn.GRU(input_size, hidden_sizes[0], num_layers, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_sizes[0], hidden_sizes[1])\n",
    "        self.fc2 = nn.Linear(hidden_sizes[1], num_classes)\n",
    "\n",
    "    def forward(self, x):\n",
    "        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_sizes[0]).to(device)\n",
    " \n",
    "        out, _ = self.rnn(x, h0)\n",
    "        # out: batch_size, seq_length, hidden_size \n",
    "        out = out[:, -1, :]\n",
    "        out = F.relu(self.fc(out))\n",
    "        out = F.dropout(out, p=0.2)\n",
    "        out = self.fc2(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_size = len(unique_words)\n",
    "hidden_size = [512, 128]\n",
    "num_layers = 2\n",
    "num_classes = 2\n",
    "learning_rate = 0.001\n",
    "\n",
    "model = RNN(input_size, hidden_size, num_layers, num_classes).to(device)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1 / 2, step 20/200, loss = 0.6936\n",
      "epoch 1 / 2, step 40/200, loss = 0.6930\n",
      "epoch 1 / 2, step 60/200, loss = 0.6847\n",
      "epoch 1 / 2, step 80/200, loss = 0.6890\n",
      "epoch 1 / 2, step 100/200, loss = 0.6848\n",
      "epoch 1 / 2, step 120/200, loss = 0.6036\n",
      "epoch 1 / 2, step 140/200, loss = 0.6023\n",
      "epoch 1 / 2, step 160/200, loss = 0.6412\n",
      "epoch 1 / 2, step 180/200, loss = 0.5669\n",
      "epoch 1 / 2, step 200/200, loss = 0.6102\n",
      "accuracy = 65.5\n",
      "epoch 2 / 2, step 20/200, loss = 0.5022\n",
      "epoch 2 / 2, step 40/200, loss = 0.4131\n",
      "epoch 2 / 2, step 60/200, loss = 0.5875\n",
      "epoch 2 / 2, step 80/200, loss = 0.5911\n",
      "epoch 2 / 2, step 100/200, loss = 0.4226\n",
      "epoch 2 / 2, step 120/200, loss = 0.5808\n",
      "epoch 2 / 2, step 140/200, loss = 0.4618\n",
      "epoch 2 / 2, step 160/200, loss = 0.5281\n",
      "epoch 2 / 2, step 180/200, loss = 0.6574\n",
      "epoch 2 / 2, step 200/200, loss = 0.5082\n",
      "accuracy = 69.625\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 2\n",
    "n_total_steps = len(train_loader)\n",
    "\n",
    "# training\n",
    "for epoch in range(num_epochs):\n",
    "    for i, example in enumerate(train_loader):\n",
    "        tweets = example['text'].to(device)\n",
    "        labels = example['label'].to(device)\n",
    "        \n",
    "        # forward pass\n",
    "        outputs = model(tweets)\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        # backward pass\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if (i+1)%20 == 0:\n",
    "            print(f'epoch {epoch+1} / {num_epochs}, step {i+1}/{n_total_steps}, loss = {loss.item():.4f}')\n",
    "            \n",
    "    # testing\n",
    "    with torch.no_grad():\n",
    "        n_correct = 0\n",
    "        n_samples = 0\n",
    "        for example in test_loader:\n",
    "            tweets = example['text'].to(device)\n",
    "            labels = example['label'].to(device)\n",
    "            outputs = model(tweets)\n",
    "\n",
    "            # value, index\n",
    "            _, predictions = torch.max(outputs, 1)\n",
    "            n_samples += labels.shape[0]\n",
    "            n_correct += (predictions == labels).sum().item()\n",
    "\n",
    "        accuracy = 100.0 * (n_correct/ n_samples)\n",
    "        print(f'accuracy = {accuracy}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}