deeplearning_intro.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "deeplearning_intro.ipynb",
      "provenance": [],
      "private_outputs": true,
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ajfisch/deeplearning_bootcamp_2020/blob/master/deeplearning_intro.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GA8LLAO9dchw",
        "colab_type": "text"
      },
      "source": [
        "# NLP Task: Beer Review Sentiment Analysis (Deep Learning)\n",
        "\n",
        "In this tutorial, we'll extend on the tutorial from lab1 to implement neural networks to learn to analyze beer reviews. \n",
        "\n",
        "Let's get started! First run the following cells to install PyTorch and get the data again."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mcLJrU16dch0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We use the CountVectorizer again to create the vocab.\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "\n",
        "# Torch modules.\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from torchvision import datasets, transforms\n",
        "\n",
        "# Gives a progress bar\n",
        "from tqdm import tqdm\n",
        "\n",
        "# Utilities for plotting.\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import pickle\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DMJfz2f-JC9E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!apt-get install wget\n",
        "!wget https://raw.githubusercontent.com/yala/MLCodeLab/master/lab1/data/beer/overall_train.p\n",
        "!wget https://raw.githubusercontent.com/yala/MLCodeLab/master/lab1/data/beer/overall_dev.p\n",
        "!wget https://raw.githubusercontent.com/yala/MLCodeLab/master/lab1/data/beer/overall_test.p\n",
        "\n",
        "train_set =  pickle.load(open(\"overall_train.p\", \"rb\"))\n",
        "dev_set =  pickle.load(open(\"overall_dev.p\", \"rb\"))\n",
        "test_set =  pickle.load(open(\"overall_test.p\", \"rb\"))\n",
        "\n",
        "# Extract tweets and labels into 2 lists\n",
        "def preprocess_data(data):\n",
        "    for indx, sample in enumerate(data):\n",
        "        text, label = sample['text'], sample['y']\n",
        "        text = text.lower().strip()\n",
        "        data[indx] = text, label\n",
        "    return data\n",
        "\n",
        "# Preprocess all the data splits.\n",
        "train_set = preprocess_data(train_set)\n",
        "dev_set = preprocess_data(dev_set)\n",
        "test_set =  preprocess_data(test_set)\n",
        "\n",
        "# Separate components into X and Y lists.\n",
        "trainText = [t[0] for t in train_set]\n",
        "trainY = [t[1] for t in train_set]\n",
        "\n",
        "devText = [t[0] for t in dev_set]\n",
        "devY = [t[1] for t in dev_set]\n",
        "\n",
        "testText = [t[0] for t in test_set]\n",
        "testY = [t[1] for t in test_set]\n",
        "\n",
        "# Set that word has to appear at least 5 times to be in vocab\n",
        "min_df = 5\n",
        "max_features = 1000\n",
        "countVec = CountVectorizer(min_df=min_df, max_features=max_features )\n",
        "\n",
        "# Learn vocabulary from train set\n",
        "countVec.fit(trainText)\n",
        "\n",
        "# Transform list of review to matrix of bag-of-word vectors\n",
        "trainX = countVec.transform(trainText)\n",
        "devX = countVec.transform(devText)\n",
        "testX = countVec.transform(testText)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DcaqZk9ouova",
        "colab_type": "text"
      },
      "source": [
        "# Step 1: Pytorch Dataset\n",
        "\n",
        "Datasets are abstractions that hold data for you. As long as you define a __len__ and __getitem__, they can be used to pipe data into your training routine."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4smEQXOddciA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Define a Beer review dataset\n",
        "class BeerReviewDataset(torch.utils.data.Dataset):\n",
        "    \n",
        "    def __init__(self, X, Y):\n",
        "      self.dataset = (X, Y)\n",
        "      assert X.shape[0] == len(Y)\n",
        "    \n",
        "    def __len__(self):\n",
        "       # Returns the number of points in the dataset.\n",
        "       return self.dataset[0].shape[0]\n",
        "\n",
        "    def __getitem__(self, i):\n",
        "      # Returns count vector as x and the label as y.\n",
        "      return np.array(self.dataset[0][i].todense()[0]), self.dataset[1][i]\n",
        "\n",
        "train = BeerReviewDataset(trainX, trainY)\n",
        "dev =   BeerReviewDataset(devX, devY)\n",
        "test =   BeerReviewDataset(testX, testY)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pFWyga62Muf9",
        "colab_type": "text"
      },
      "source": [
        "# Step 2: Define the Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c0NnnFWgdci-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Model(nn.Module):\n",
        "   \n",
        "    def __init__(self):\n",
        "        super(Model, self).__init__()\n",
        "        self.fully_connected = nn.Linear(1000, 3)\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.fully_connected(x)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l4HkAz4jvmKk",
        "colab_type": "text"
      },
      "source": [
        "# Exercise 1:\n",
        "\n",
        "This is just a linear model!\n",
        "\n",
        "Add a non-linearity (F.relu) and an extra layer."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M9hvStGzdcjC",
        "colab_type": "text"
      },
      "source": [
        "# Step 3: Training\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n3Ht1mhhdcjE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Training settings\n",
        "batch_size = 64\n",
        "epochs = 10\n",
        "lr = .01\n",
        "momentum = 0.5\n",
        "\n",
        "model = Model()\n",
        "optimizer = optim.SGD(model.parameters(), lr=lr, momentum=momentum)\n",
        "\n",
        "train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n",
        "dev_loader = torch.utils.data.DataLoader(dev, batch_size=batch_size, shuffle=True)\n",
        "test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=True)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GA2CzvprxUsk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "for batch in train_loader:\n",
        "  print(batch[0].shape)\n",
        "  print(batch[1].shape)\n",
        "  \n",
        "  break\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jAl05-rTdcjH",
        "colab_type": "text"
      },
      "source": [
        "To train our model:\n",
        "\n",
        "1) we'll randomly sample batches from our train loader\n",
        "\n",
        "2) compute our loss (using standard `cross_entropy`)\n",
        "\n",
        "3) compute our gradients (by calling `backward()` on our loss)\n",
        "\n",
        "4) update our neural network with an `optimizer.step()`, and go back to 1)\n",
        "\n",
        "I've added some extra stuff here to log our accuracy and average loss for the epoch.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wxIj0eWsdcjJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train_epoch( model, train_loader, optimizer, epoch):\n",
        "    model.train() # Set the nn.Module to train mode. \n",
        "    total_loss = 0\n",
        "    correct = 0\n",
        "    num_samples = len(train_loader.dataset)\n",
        "    \n",
        "    # Iterate over batches of data.\n",
        "    for batch_idx, (x, target) in enumerate(train_loader):\n",
        "        x = x.float().squeeze(1)\n",
        "        \n",
        "        # Reset gradient data to 0\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        # 1) Get the prediction for batch\n",
        "        output = model(x)\n",
        "        \n",
        "        # 2) Compute loss\n",
        "        loss = F.cross_entropy(output, target)\n",
        "        \n",
        "        # 3) Do backprop\n",
        "        loss.backward()\n",
        "        \n",
        "        # 4) Update model\n",
        "        optimizer.step()\n",
        "        \n",
        "        # Do book-keeping to track accuracy and avg loss\n",
        "        pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability\n",
        "        correct += pred.eq(target.view_as(pred)).sum().item()\n",
        "        total_loss += loss.detach() # Don't keep computation graph \n",
        "\n",
        "    print('Train Epoch: {} \\tLoss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'.format(\n",
        "            epoch, total_loss / num_samples, \n",
        "            correct, \n",
        "            num_samples,\n",
        "            100. * correct / num_samples))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xGZnlI58dcjN",
        "colab_type": "text"
      },
      "source": [
        "# Step 4: Evaluation\n",
        "Similar to above, we'll also loop through our dev or test set, and compute our loss and accuracy. \n",
        "This lets us see how well our model is generalizing. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GPzqSuY3dcjO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def eval_epoch(model, test_loader, name):\n",
        "    model.eval()\n",
        "    test_loss = 0\n",
        "    correct = 0\n",
        "    for data, target in test_loader:\n",
        "        data = data.float().squeeze(1)\n",
        "        target = target.long()\n",
        "        output = model(data)\n",
        "        test_loss += F.cross_entropy(output, target).item() # sum up batch loss\n",
        "        pred = output.max(1, keepdim=True)[1] # get the index of the max log-probability\n",
        "        correct += pred.eq(target.view_as(pred)).sum().item()\n",
        "\n",
        "    test_loss /= len(test_loader.dataset)\n",
        "    print('\\n{} set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
        "        name,\n",
        "        test_loss, \n",
        "        correct, \n",
        "        len(test_loader.dataset),\n",
        "        100. * correct / len(test_loader.dataset)))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pp9H5tWEdcjR",
        "colab_type": "text"
      },
      "source": [
        "# Step 5: Everything Together"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vVpU_N0idcjS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "for epoch in range(1, epochs + 1):\n",
        "    train_epoch(model, train_loader, optimizer, epoch)\n",
        "    eval_epoch(model,  dev_loader, \"Dev\")\n",
        "    print(\"---\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dNWqKY72dcjs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "eval_epoch(model,  test_loader, \"Test\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jr2JLiJiwkHr",
        "colab_type": "text"
      },
      "source": [
        "# Exercise 2:\n",
        "\n",
        "1. What is the training accuracy?\n",
        "2. Try changing the learning rate or the batch size!"
      ]
    }
  ]
}