P2_KLUE.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0RmcmqTvs1_T"
   },
   "source": [
    "# Setting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "y5ZMzORj6Xxn"
   },
   "source": [
    "라이브러리 다운로드"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "oeI3L25s6XZP"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting mxnet\n",
      "  Downloading mxnet-1.8.0.post0-py2.py3-none-manylinux2014_x86_64.whl (46.9 MB)\n",
      "\u001b[K     |████████████████████████████████| 46.9 MB 239 kB/s  eta 0:00:01    |████████████████▊               | 24.5 MB 8.5 MB/s eta 0:00:03     |███████████████████▎            | 28.3 MB 8.5 MB/s eta 0:00:03\n",
      "\u001b[?25hCollecting graphviz<0.9.0,>=0.8.1\n",
      "  Downloading graphviz-0.8.4-py2.py3-none-any.whl (16 kB)\n",
      "Requirement already satisfied: requests<3,>=2.20.0 in /opt/conda/lib/python3.7/site-packages (from mxnet) (2.23.0)\n",
      "Requirement already satisfied: numpy<2.0.0,>1.16.0 in /opt/conda/lib/python3.7/site-packages (from mxnet) (1.18.5)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.20.0->mxnet) (3.0.4)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.20.0->mxnet) (2.9)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.20.0->mxnet) (1.25.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.20.0->mxnet) (2020.6.20)\n",
      "Installing collected packages: graphviz, mxnet\n",
      "Successfully installed graphviz-0.8.4 mxnet-1.8.0.post0\n",
      "Collecting gluonnlp\n",
      "  Downloading gluonnlp-0.10.0.tar.gz (344 kB)\n",
      "\u001b[K     |████████████████████████████████| 344 kB 1.9 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: pandas in /opt/conda/lib/python3.7/site-packages (1.1.5)\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.7/site-packages (4.46.0)\n",
      "Requirement already satisfied: numpy>=1.16.0 in /opt/conda/lib/python3.7/site-packages (from gluonnlp) (1.18.5)\n",
      "Collecting cython\n",
      "  Using cached Cython-0.29.23-cp37-cp37m-manylinux1_x86_64.whl (2.0 MB)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from gluonnlp) (20.9)\n",
      "Requirement already satisfied: pytz>=2017.2 in /opt/conda/lib/python3.7/site-packages (from pandas) (2020.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /opt/conda/lib/python3.7/site-packages (from pandas) (2.8.1)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging->gluonnlp) (2.4.7)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.7/site-packages (from python-dateutil>=2.7.3->pandas) (1.14.0)\n",
      "Building wheels for collected packages: gluonnlp\n",
      "  Building wheel for gluonnlp (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for gluonnlp: filename=gluonnlp-0.10.0-cp37-cp37m-linux_x86_64.whl size=595774 sha256=f6b2c7a85c0ab4faa5efee0bfd0ca2d1033ad7f06859e9b9c007cf80ac83b806\n",
      "  Stored in directory: /opt/ml/.cache/pip/wheels/be/b4/06/7f3fdfaf707e6b5e98b79c041e023acffbe395d78a527eae00\n",
      "Successfully built gluonnlp\n",
      "Installing collected packages: cython, gluonnlp\n",
      "Successfully installed cython-0.29.23 gluonnlp-0.10.0\n",
      "Requirement already satisfied: sentencepiece in /opt/conda/lib/python3.7/site-packages (0.1.95)\n",
      "Collecting transformers==3\n",
      "  Downloading transformers-3.0.0-py3-none-any.whl (754 kB)\n",
      "\u001b[K     |████████████████████████████████| 754 kB 537 kB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from transformers==3) (20.9)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from transformers==3) (3.0.12)\n",
      "Requirement already satisfied: sentencepiece in /opt/conda/lib/python3.7/site-packages (from transformers==3) (0.1.95)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from transformers==3) (2.23.0)\n",
      "Requirement already satisfied: sacremoses in /opt/conda/lib/python3.7/site-packages (from transformers==3) (0.0.44)\n",
      "Collecting tokenizers==0.8.0-rc4\n",
      "  Downloading tokenizers-0.8.0rc4-cp37-cp37m-manylinux1_x86_64.whl (3.0 MB)\n",
      "\u001b[K     |████████████████████████████████| 3.0 MB 4.0 MB/s eta 0:00:01     |███████████████████▎            | 1.8 MB 4.0 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from transformers==3) (1.18.5)\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.7/site-packages (from transformers==3) (4.46.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers==3) (2021.4.4)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging->transformers==3) (2.4.7)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /opt/conda/lib/python3.7/site-packages (from requests->transformers==3) (3.0.4)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->transformers==3) (1.25.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->transformers==3) (2020.6.20)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->transformers==3) (2.9)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers==3) (1.0.1)\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers==3) (7.1.2)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers==3) (1.14.0)\n",
      "Installing collected packages: tokenizers, transformers\n",
      "  Attempting uninstall: tokenizers\n",
      "    Found existing installation: tokenizers 0.9.4\n",
      "    Uninstalling tokenizers-0.9.4:\n",
      "      Successfully uninstalled tokenizers-0.9.4\n",
      "  Attempting uninstall: transformers\n",
      "    Found existing installation: transformers 4.2.0\n",
      "    Uninstalling transformers-4.2.0:\n",
      "      Successfully uninstalled transformers-4.2.0\n",
      "Successfully installed tokenizers-0.8.0rc4 transformers-3.0.0\n",
      "Requirement already satisfied: torch in /opt/conda/lib/python3.7/site-packages (1.6.0)\n",
      "Requirement already satisfied: future in /opt/conda/lib/python3.7/site-packages (from torch) (0.18.2)\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from torch) (1.18.5)\n",
      "Collecting git+https://****@github.com/SKTBrain/KoBERT.git@master\n",
      "  Cloning https://****@github.com/SKTBrain/KoBERT.git (to revision master) to /tmp/pip-req-build-qrsgi56u\n",
      "  Running command git clone -q 'https://****@github.com/SKTBrain/KoBERT.git' /tmp/pip-req-build-qrsgi56u\n",
      "Building wheels for collected packages: kobert\n",
      "  Building wheel for kobert (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for kobert: filename=kobert-0.1.2-py3-none-any.whl size=12705 sha256=7e6be853aa8d7789901848a64c89fa7fc12079f6b5540cd8a50d1afcbc4517a5\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-knwbqaex/wheels/d3/68/ca/334747dfb038313b49cf71f84832a33372f3470d9ddfd051c0\n",
      "Successfully built kobert\n",
      "Installing collected packages: kobert\n",
      "Successfully installed kobert-0.1.2\n"
     ]
    }
   ],
   "source": [
    "!pip install mxnet\n",
    "!pip install gluonnlp pandas tqdm\n",
    "!pip install sentencepiece\n",
    "!pip install transformers==3\n",
    "!pip install torch\n",
    "!pip install git+https://git@github.com/SKTBrain/KoBERT.git@master"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rcI3nARqs9qg"
   },
   "source": [
    "라이브러리 불러오기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "ETROhbNxsuXQ"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import gluonnlp as nlp\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import tarfile\n",
    "import pickle as pickle\n",
    "from tqdm import tqdm\n",
    "from kobert.utils import get_tokenizer\n",
    "from kobert.pytorch_kobert import get_pytorch_kobert_model\n",
    "from transformers import AdamW\n",
    "from transformers.optimization import get_cosine_schedule_with_warmup\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KcobKDe9tAuQ"
   },
   "source": [
    "GPU 설정"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "i8v0khrlswNx"
   },
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0hZw_ITPtCgp"
   },
   "source": [
    "kobert 불러오기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "nhsub2pBsx1q"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[██████████████████████████████████████████████████]\n",
      "[██████████████████████████████████████████████████]\n"
     ]
    }
   ],
   "source": [
    "bertmodel, vocab = get_pytorch_kobert_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "p9_lv7GMtE1_"
   },
   "source": [
    "# Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "5mr-nvcjOzLF"
   },
   "outputs": [],
   "source": [
    "def load_data(dataset_dir):\n",
    "    with open('/opt/ml/input/data/label_type.pkl', 'rb') as f:\n",
    "        label_type = pickle.load(f)\n",
    "    dataset = pd.read_csv(dataset_dir, delimiter='\\t', header=None)\n",
    "    dataset = preprocessing_dataset(dataset, label_type)\n",
    "    return dataset\n",
    "\n",
    "def preprocessing_dataset(dataset, label_type):\n",
    "    label = []\n",
    "    for i in dataset[8]:\n",
    "        if i == 'blind':\n",
    "            label.append(100)\n",
    "        else:\n",
    "            label.append(label_type[i])\n",
    "    out_dataset = pd.DataFrame({'sentence':dataset[1],'entity_01':dataset[2],'entity_02':dataset[5],'label':label,})\n",
    "    return out_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "xkI-E7PauxGq"
   },
   "outputs": [],
   "source": [
    "dataset_path = r\"/opt/ml/input/data/train/train.tsv\"\n",
    "\n",
    "dataset = load_data(dataset_path)\n",
    "\n",
    "dataset['sentence'] = dataset['entity_01'] + ' [SEP] ' + dataset['entity_02'] + ' [SEP] ' + dataset['sentence']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "YIE_tnYq6AYL"
   },
   "outputs": [],
   "source": [
    "train, vali = train_test_split(dataset, test_size=0.2, random_state=42)\n",
    "train[['sentence','label']].to_csv(\"/opt/ml/input/data/train/train_train.txt\", sep='\\t', index=False)\n",
    "vali[['sentence','label']].to_csv(\"/opt/ml/input/data/train/train_vali.txt\", sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "2tI-jupiCwpE"
   },
   "outputs": [],
   "source": [
    "dataset_train = nlp.data.TSVDataset(\"/opt/ml/input/data/train/train_train.txt\", field_indices=[0,1], num_discard_samples=1)\n",
    "dataset_vali = nlp.data.TSVDataset(\"/opt/ml/input/data/train/train_vali.txt\", field_indices=[0,1], num_discard_samples=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "ca54j41sN-0L"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using cached model\n"
     ]
    }
   ],
   "source": [
    "tokenizer = get_tokenizer()\n",
    "tok = nlp.data.BERTSPTokenizer(tokenizer, vocab, lower=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "eRRaHwF_C28c"
   },
   "outputs": [],
   "source": [
    "class BERTDataset(Dataset):\n",
    "    def __init__(self, dataset, sent_idx, label_idx, bert_tokenizer, max_len, pad, pair):\n",
    "        transform = nlp.data.BERTSentenceTransform(bert_tokenizer, max_seq_length=max_len, pad=pad, pair=pair)\n",
    "        self.sentences = [transform([i[sent_idx]]) for i in dataset]\n",
    "        self.labels = [np.int32(i[label_idx]) for i in dataset]\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return (self.sentences[i] + (self.labels[i], ))\n",
    "\n",
    "    def __len__(self):\n",
    "        return (len(self.labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "4BKznxZotPrl"
   },
   "outputs": [],
   "source": [
    "max_len = 128\n",
    "batch_size = 32\n",
    "warmup_ratio = 0.01\n",
    "num_epochs = 20\n",
    "max_grad_norm = 1\n",
    "log_interval = 50\n",
    "learning_rate = 5e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "WtW5knVCC6ZC"
   },
   "outputs": [],
   "source": [
    "data_train = BERTDataset(dataset_train, 0, 1, tok, max_len, True, False)\n",
    "data_vali = BERTDataset(dataset_vali, 0, 1, tok, max_len, True, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "spDs0h8tC7fX"
   },
   "outputs": [],
   "source": [
    "train_dataloader = torch.utils.data.DataLoader(data_train, batch_size=batch_size, num_workers=5)\n",
    "vali_dataloader = torch.utils.data.DataLoader(data_vali, batch_size=batch_size, num_workers=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "U0I1L7EVtShS"
   },
   "source": [
    "# Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "eR9IqXuStUbL"
   },
   "outputs": [],
   "source": [
    "class BERTClassifier(nn.Module):\n",
    "    def __init__(self,\n",
    "                 bert,\n",
    "                 hidden_size = 768,\n",
    "                 num_classes = 42,\n",
    "                 dr_rate=None,\n",
    "                 params=None):\n",
    "        super(BERTClassifier, self).__init__()\n",
    "        self.bert = bert\n",
    "        self.dr_rate = dr_rate \n",
    "        self.classifier = nn.Linear(hidden_size , num_classes)\n",
    "        if dr_rate:\n",
    "            self.dropout = nn.Dropout(p=dr_rate)\n",
    "    \n",
    "    def gen_attention_mask(self, token_ids, valid_length):\n",
    "        attention_mask = torch.zeros_like(token_ids)\n",
    "        for i, v in enumerate(valid_length):\n",
    "            attention_mask[i][:v] = 1\n",
    "        return attention_mask.float()\n",
    "\n",
    "    def forward(self, token_ids, valid_length, segment_ids):\n",
    "        attention_mask = self.gen_attention_mask(token_ids, valid_length)\n",
    "        _, pooler = self.bert(input_ids = token_ids, token_type_ids = segment_ids.long(), attention_mask = attention_mask.float().to(token_ids.device))\n",
    "        if self.dr_rate:\n",
    "            out = self.dropout(pooler)\n",
    "        return self.classifier(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "piJyyUoutWWt"
   },
   "outputs": [],
   "source": [
    "model = BERTClassifier(bertmodel, dr_rate=0.5).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "TqaRnWqwtXii"
   },
   "outputs": [],
   "source": [
    "no_decay = ['bias', 'LayerNorm.weight']\n",
    "optimizer_grouped_parameters = [\n",
    "    {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n",
    "    {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "YYExV_Uwqdpi"
   },
   "outputs": [],
   "source": [
    "class LabelSmoothingLoss(nn.Module):\n",
    "    def __init__(self, classes=42, smoothing=0.0, dim=-1):\n",
    "        super(LabelSmoothingLoss, self).__init__()\n",
    "        self.confidence = 1.0 - smoothing\n",
    "        self.smoothing = smoothing\n",
    "        self.cls = classes\n",
    "        self.dim = dim\n",
    "\n",
    "    def forward(self, pred, target):\n",
    "        pred = pred.log_softmax(dim=self.dim)\n",
    "        with torch.no_grad():\n",
    "            true_dist = torch.zeros_like(pred)\n",
    "            true_dist.fill_(self.smoothing / (self.cls - 1))\n",
    "            true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)\n",
    "        return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "SvLPsHAMtYp4"
   },
   "outputs": [],
   "source": [
    "optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate)\n",
    "loss_fn = LabelSmoothingLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "wJrYbrK5taVC"
   },
   "outputs": [],
   "source": [
    "t_total = len(train_dataloader) * num_epochs\n",
    "warmup_step = int(t_total * warmup_ratio)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "4PDk3f8ctasE"
   },
   "outputs": [],
   "source": [
    "scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=warmup_step, num_training_steps=t_total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "7uxhVAqWtcbJ"
   },
   "outputs": [],
   "source": [
    "def calc_accuracy(X,Y):\n",
    "    max_vals, max_indices = torch.max(X, 1)\n",
    "    train_acc = (max_indices == Y).sum().data.cpu().numpy()/max_indices.size()[0]\n",
    "    return train_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "ASK6KHOTtd2H"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1 batch id 1 loss 2.607132911682129 train acc 0.4375\n",
      "epoch 1 batch id 51 loss 1.6226816177368164 train acc 0.5275735294117647\n",
      "epoch 1 batch id 101 loss 1.5394706726074219 train acc 0.5529084158415841\n",
      "epoch 1 batch id 151 loss 1.4411141872406006 train acc 0.5666390728476821\n",
      "epoch 1 batch id 201 loss 1.116940975189209 train acc 0.584110696517413\n",
      "epoch 1 train acc 0.5915277777777778\n",
      "epoch 1 test acc 0.6458333333333334\n",
      "epoch 2 batch id 1 loss 1.673290491104126 train acc 0.625\n",
      "epoch 2 batch id 51 loss 1.6937164068222046 train acc 0.6427696078431373\n",
      "epoch 2 batch id 101 loss 0.9597698450088501 train acc 0.6652227722772277\n",
      "epoch 2 batch id 151 loss 0.8569899797439575 train acc 0.6709437086092715\n",
      "epoch 2 batch id 201 loss 0.5642976760864258 train acc 0.6814365671641791\n",
      "epoch 2 train acc 0.6894444444444444\n",
      "epoch 2 test acc 0.6606359649122807\n",
      "epoch 3 batch id 1 loss 1.1957015991210938 train acc 0.6875\n",
      "epoch 3 batch id 51 loss 1.0850778818130493 train acc 0.7322303921568627\n",
      "epoch 3 batch id 101 loss 0.6523208022117615 train acc 0.7534034653465347\n",
      "epoch 3 batch id 151 loss 0.6339395046234131 train acc 0.7595198675496688\n",
      "epoch 3 batch id 201 loss 0.3107388913631439 train acc 0.7683457711442786\n",
      "epoch 3 train acc 0.7738888888888888\n",
      "epoch 3 test acc 0.6759868421052632\n",
      "epoch 4 batch id 1 loss 0.9990034103393555 train acc 0.75\n",
      "epoch 4 batch id 51 loss 0.7654902935028076 train acc 0.8020833333333334\n",
      "epoch 4 batch id 101 loss 0.5522868633270264 train acc 0.817759900990099\n",
      "epoch 4 batch id 151 loss 0.454212486743927 train acc 0.8230546357615894\n",
      "epoch 4 batch id 201 loss 0.28601932525634766 train acc 0.830068407960199\n",
      "epoch 4 train acc 0.8340277777777778\n",
      "epoch 4 test acc 0.6765350877192983\n",
      "epoch 5 batch id 1 loss 0.807449221611023 train acc 0.75\n",
      "epoch 5 batch id 51 loss 0.515419065952301 train acc 0.8596813725490197\n",
      "epoch 5 batch id 101 loss 0.4243314564228058 train acc 0.8688118811881188\n",
      "epoch 5 batch id 151 loss 0.3449571132659912 train acc 0.8704470198675497\n",
      "epoch 5 batch id 201 loss 0.16029833257198334 train acc 0.8756218905472637\n",
      "epoch 5 train acc 0.8761111111111111\n",
      "epoch 5 test acc 0.7088815789473685\n",
      "epoch 6 batch id 1 loss 0.6422731280326843 train acc 0.84375\n",
      "epoch 6 batch id 51 loss 0.5930699706077576 train acc 0.8854166666666666\n",
      "epoch 6 batch id 101 loss 0.35308247804641724 train acc 0.8898514851485149\n",
      "epoch 6 batch id 151 loss 0.3153834939002991 train acc 0.8923841059602649\n",
      "epoch 6 batch id 201 loss 0.29020363092422485 train acc 0.8962997512437811\n",
      "epoch 6 train acc 0.8959722222222222\n",
      "epoch 6 test acc 0.6984649122807017\n",
      "epoch 7 batch id 1 loss 0.5975848436355591 train acc 0.8125\n",
      "epoch 7 batch id 51 loss 0.41929733753204346 train acc 0.8964460784313726\n",
      "epoch 7 batch id 101 loss 0.2970639169216156 train acc 0.9102722772277227\n",
      "epoch 7 batch id 151 loss 0.18571817874908447 train acc 0.9099751655629139\n",
      "epoch 7 batch id 201 loss 0.17240563035011292 train acc 0.9116915422885572\n",
      "epoch 7 train acc 0.9151388888888888\n",
      "epoch 7 test acc 0.7088815789473685\n",
      "epoch 8 batch id 1 loss 0.38309553265571594 train acc 0.875\n",
      "epoch 8 batch id 51 loss 0.3673267960548401 train acc 0.9252450980392157\n",
      "epoch 8 batch id 101 loss 0.27876394987106323 train acc 0.9303836633663366\n",
      "epoch 8 batch id 151 loss 0.3607330918312073 train acc 0.9273592715231788\n",
      "epoch 8 batch id 201 loss 0.06138664111495018 train acc 0.9311256218905473\n",
      "epoch 8 train acc 0.9329166666666666\n",
      "epoch 8 test acc 0.7050438596491229\n",
      "epoch 9 batch id 1 loss 0.38850629329681396 train acc 0.875\n",
      "epoch 9 batch id 51 loss 0.30791956186294556 train acc 0.9301470588235294\n",
      "epoch 9 batch id 101 loss 0.19681043922901154 train acc 0.9384282178217822\n",
      "epoch 9 batch id 151 loss 0.2988731265068054 train acc 0.9395695364238411\n",
      "epoch 9 batch id 201 loss 0.05578959733247757 train acc 0.9416977611940298\n",
      "epoch 9 train acc 0.9433333333333334\n",
      "epoch 9 test acc 0.7039473684210527\n",
      "epoch 10 batch id 1 loss 0.3451826572418213 train acc 0.875\n",
      "epoch 10 batch id 51 loss 0.15554288029670715 train acc 0.9491421568627451\n",
      "epoch 10 batch id 101 loss 0.2023722380399704 train acc 0.9535891089108911\n",
      "epoch 10 batch id 151 loss 0.10997848957777023 train acc 0.9540562913907285\n",
      "epoch 10 batch id 201 loss 0.02117171138525009 train acc 0.9553793532338308\n",
      "epoch 10 train acc 0.9572222222222222\n",
      "epoch 10 test acc 0.7033991228070176\n",
      "epoch 11 batch id 1 loss 0.3255789279937744 train acc 0.90625\n",
      "epoch 11 batch id 51 loss 0.1649569422006607 train acc 0.9595588235294118\n",
      "epoch 11 batch id 101 loss 0.1800413429737091 train acc 0.9628712871287128\n",
      "epoch 11 batch id 151 loss 0.23035238683223724 train acc 0.9631622516556292\n",
      "epoch 11 batch id 201 loss 0.013898182660341263 train acc 0.9643967661691543\n",
      "epoch 11 train acc 0.9659722222222222\n",
      "epoch 11 test acc 0.7033991228070176\n",
      "epoch 12 batch id 1 loss 0.254984974861145 train acc 0.9375\n",
      "epoch 12 batch id 51 loss 0.12183711677789688 train acc 0.9669117647058824\n",
      "epoch 12 batch id 101 loss 0.1299206018447876 train acc 0.9709158415841584\n",
      "epoch 12 batch id 151 loss 0.06411467492580414 train acc 0.9722682119205298\n",
      "epoch 12 batch id 201 loss 0.01588389277458191 train acc 0.9735696517412935\n",
      "epoch 12 train acc 0.9747222222222223\n",
      "epoch 12 test acc 0.7209429824561403\n",
      "epoch 13 batch id 1 loss 0.1800939291715622 train acc 0.9375\n",
      "epoch 13 batch id 51 loss 0.14019674062728882 train acc 0.977328431372549\n",
      "epoch 13 batch id 101 loss 0.11297675222158432 train acc 0.9795792079207921\n",
      "epoch 13 batch id 151 loss 0.11392006278038025 train acc 0.9811672185430463\n",
      "epoch 13 batch id 201 loss 0.007623476907610893 train acc 0.9810323383084577\n",
      "epoch 13 train acc 0.9819444444444444\n",
      "epoch 13 test acc 0.7094298245614035\n",
      "epoch 14 batch id 1 loss 0.12819932401180267 train acc 0.9375\n",
      "epoch 14 batch id 51 loss 0.021332092583179474 train acc 0.9846813725490197\n",
      "epoch 14 batch id 101 loss 0.10595191270112991 train acc 0.9882425742574258\n",
      "epoch 14 batch id 151 loss 0.07489223778247833 train acc 0.9875827814569537\n",
      "epoch 14 batch id 201 loss 0.006914549507200718 train acc 0.9877176616915423\n",
      "epoch 14 train acc 0.9881944444444445\n",
      "epoch 14 test acc 0.7023026315789473\n",
      "epoch 15 batch id 1 loss 0.11336752027273178 train acc 0.96875\n",
      "epoch 15 batch id 51 loss 0.013257134705781937 train acc 0.9889705882352942\n",
      "epoch 15 batch id 101 loss 0.09660407900810242 train acc 0.9922648514851485\n",
      "epoch 15 batch id 151 loss 0.10099480301141739 train acc 0.9917218543046358\n",
      "epoch 15 batch id 201 loss 0.006759616080671549 train acc 0.9914490049751243\n",
      "epoch 15 train acc 0.9915277777777778\n",
      "epoch 15 test acc 0.7171052631578947\n",
      "epoch 16 batch id 1 loss 0.051641806960105896 train acc 1.0\n",
      "epoch 16 batch id 51 loss 0.018738441169261932 train acc 0.9908088235294118\n",
      "epoch 16 batch id 101 loss 0.09985600411891937 train acc 0.9931930693069307\n",
      "epoch 16 batch id 151 loss 0.16732867062091827 train acc 0.9923427152317881\n",
      "epoch 16 batch id 201 loss 0.005542983300983906 train acc 0.9928482587064676\n",
      "epoch 16 train acc 0.9933333333333333\n",
      "epoch 16 test acc 0.7127192982456141\n",
      "epoch 17 batch id 1 loss 0.059301089495420456 train acc 1.0\n",
      "epoch 17 batch id 51 loss 0.019563529640436172 train acc 0.9932598039215687\n",
      "epoch 17 batch id 101 loss 0.09088771790266037 train acc 0.9941212871287128\n",
      "epoch 17 batch id 151 loss 0.08927685767412186 train acc 0.9937913907284768\n",
      "epoch 17 batch id 201 loss 0.005064973141998053 train acc 0.9945584577114428\n",
      "epoch 17 train acc 0.9948611111111111\n",
      "epoch 17 test acc 0.7154605263157895\n",
      "epoch 18 batch id 1 loss 0.043095678091049194 train acc 1.0\n",
      "epoch 18 batch id 51 loss 0.011966804042458534 train acc 0.9944852941176471\n",
      "epoch 18 batch id 101 loss 0.10693957656621933 train acc 0.9956683168316832\n",
      "epoch 18 batch id 151 loss 0.04466027766466141 train acc 0.9954470198675497\n",
      "epoch 18 batch id 201 loss 0.006011196877807379 train acc 0.9951803482587065\n",
      "epoch 18 train acc 0.9955555555555555\n",
      "epoch 18 test acc 0.7160087719298246\n",
      "epoch 19 batch id 1 loss 0.04708477482199669 train acc 1.0\n",
      "epoch 19 batch id 51 loss 0.011093584820628166 train acc 0.9944852941176471\n",
      "epoch 19 batch id 101 loss 0.0880388393998146 train acc 0.995049504950495\n",
      "epoch 19 batch id 151 loss 0.0322967991232872 train acc 0.9950331125827815\n",
      "epoch 19 batch id 201 loss 0.004775755573064089 train acc 0.9950248756218906\n",
      "epoch 19 train acc 0.9954166666666666\n",
      "epoch 19 test acc 0.7160087719298246\n",
      "epoch 20 batch id 1 loss 0.05008593574166298 train acc 1.0\n",
      "epoch 20 batch id 51 loss 0.011030284687876701 train acc 0.9944852941176471\n",
      "epoch 20 batch id 101 loss 0.08221601694822311 train acc 0.9956683168316832\n",
      "epoch 20 batch id 151 loss 0.05611453950405121 train acc 0.9946192052980133\n",
      "epoch 20 batch id 201 loss 0.006238740868866444 train acc 0.9947139303482587\n",
      "epoch 20 train acc 0.995\n",
      "epoch 20 test acc 0.7160087719298246\n"
     ]
    }
   ],
   "source": [
    "for e in range(num_epochs):\n",
    "    train_acc = 0.0\n",
    "    test_acc = 0.0\n",
    "    best_acc = 0.0\n",
    "    model.train()\n",
    "    for batch_id, (token_ids, valid_length, segment_ids, label) in enumerate(train_dataloader):\n",
    "        optimizer.zero_grad()\n",
    "        token_ids = token_ids.long().to(device)\n",
    "        segment_ids = segment_ids.long().to(device)\n",
    "        valid_length= valid_length\n",
    "        label = label.long().to(device)\n",
    "        out = model(token_ids, valid_length, segment_ids)\n",
    "        loss = loss_fn(out, label)\n",
    "        loss.backward()\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)\n",
    "        optimizer.step()\n",
    "        scheduler.step()\n",
    "        train_acc += calc_accuracy(out, label)\n",
    "        if batch_id % log_interval == 0:\n",
    "            print(\"epoch {} batch id {} loss {} train acc {}\".format(e+1, batch_id+1, loss.data.cpu().numpy(), train_acc / (batch_id+1)))\n",
    "    print(\"epoch {} train acc {}\".format(e+1, train_acc / (batch_id+1)))\n",
    "    model.eval()\n",
    "    for batch_id, (token_ids, valid_length, segment_ids, label) in enumerate(vali_dataloader):\n",
    "        token_ids = token_ids.long().to(device)\n",
    "        segment_ids = segment_ids.long().to(device)\n",
    "        valid_length = valid_length\n",
    "        label = label.long().to(device)\n",
    "        out = model(token_ids, valid_length, segment_ids)\n",
    "        test_acc += calc_accuracy(out, label)\n",
    "    print(\"epoch {} test acc {}\".format(e+1, test_acc / (batch_id+1)))\n",
    "    if test_acc >= best_acc:\n",
    "        best_acc = test_acc\n",
    "        torch.save(model.state_dict(), \"/opt/ml/model/model_state_dict.pt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "h7ASgrTpfdZh"
   },
   "source": [
    "# Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "Siuxwi6SdiTW"
   },
   "outputs": [],
   "source": [
    "dataset_path = r\"/opt/ml/input/data/test/test.tsv\"\n",
    "\n",
    "dataset = load_data(dataset_path)\n",
    "\n",
    "dataset['sentence'] = dataset['entity_01'] + ' [SEP] ' + dataset['entity_02'] + ' [SEP] ' + dataset['sentence']\n",
    "\n",
    "dataset[['sentence','label']].to_csv(\"/opt/ml/input/data/test/test.txt\", sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "yPfoO4ym6AYU"
   },
   "outputs": [],
   "source": [
    "dataset_test = nlp.data.TSVDataset(\"/opt/ml/input/data/test/test.txt\", field_indices=[0,1], num_discard_samples=1)\n",
    "\n",
    "data_test = BERTDataset(dataset_test, 0, 1, tok, max_len, True, False)\n",
    "\n",
    "test_dataloader = torch.utils.data.DataLoader(data_test, batch_size=batch_size, num_workers=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "id": "g3TFf_YgtjDG"
   },
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load(\"/opt/ml/model/model_state_dict.pt\"))\n",
    "\n",
    "model.eval()\n",
    "\n",
    "Predict = []\n",
    "\n",
    "for batch_id, (token_ids, valid_length, segment_ids, label) in enumerate(test_dataloader):\n",
    "    token_ids = token_ids.long().to(device)\n",
    "    segment_ids = segment_ids.long().to(device)\n",
    "    valid_length = valid_length\n",
    "    label = label.long().to(device)\n",
    "    out = model(token_ids, valid_length, segment_ids)\n",
    "    _, predict = torch.max(out,1)\n",
    "    Predict.extend(predict.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "id": "_aV-Fgpffp4s"
   },
   "outputs": [],
   "source": [
    "output = pd.DataFrame(Predict, columns=['pred'])\n",
    "output.to_csv('/opt/ml/result/submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "P2-KLUE.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}