preprocessing_choice_correct.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from multiprocessing import Pool, cpu_count\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "SNIPPET_DIR_PATH = '/home/unnc/Documents/_data/_snippet/'\n",
    "ORIGIN_DIR_PATH = '/home/unnc/Documents/_data/_original_data/'\n",
    "\n",
    "\n",
    "TRAIN_NAME = f'train'\n",
    "\n",
    "TEST_17_NAME = f'test_17_choice_correct'\n",
    "TEST_18_NAME = f'test_18_choice_correct'\n",
    "TRAIN_3000_NAME = f'train_3000'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "OUTPUT_DIR_PATH = '/home/unnc/Documents/_data/choice_correct/' # QC1 QC2 QC3 QC4 QC5\n",
    "# SWAG_OUTPUT_DIR_PATH = '/home/unnc/Documents/_data/swag/' # Q C1 C2 C3 C4 C5\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# original data\n",
    "# train_original_data = pd.read_excel(f'{ORIGIN_DIR_PATH}train.xlsx', header=None)\n",
    "test_original_data_17 = pd.read_excel(f'{ORIGIN_DIR_PATH}test_17-18.xlsx', sheet_name='2017', header=None)\n",
    "test_original_data_18 = pd.read_excel(f'{ORIGIN_DIR_PATH}test_17-18.xlsx', sheet_name='2018', header=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Level 5_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 540,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_postfix = '4_longTable_tf'\n",
    "snippet_type = '4_long_tf'\n",
    "output_dir = f'lvl_{snippet_type}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 541,
   "metadata": {},
   "outputs": [],
   "source": [
    "# snippet \n",
    "# snippet_train = pd.read_excel(f'{SNIPPET_DIR_PATH}/训练集查询结果/带答案搜索/level{file_postfix}.xlsx', header=None)\n",
    "snippet_test_17 = pd.read_excel(f'{SNIPPET_DIR_PATH}/2017真题查询结果/带答案搜索/indexLevels{file_postfix}.xlsx', header=None)\n",
    "snippet_test_18 = pd.read_excel(f'{SNIPPET_DIR_PATH}/2018真题查询结果/带答案搜索/indexLevels{file_postfix}.xlsx', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 542,
   "metadata": {},
   "outputs": [],
   "source": [
    "# concatenate orginal data with snippet\n",
    "# train_with_snippet = pd.concat([train_original_data, snippet_train], axis=1)\n",
    "test_17_all = pd.concat([test_original_data_17, snippet_test_17], axis=1)\n",
    "test_18_all = pd.concat([test_original_data_18, snippet_test_18], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### drop corruped data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 543,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. handle corrupted row at 10040\n",
    "# if train_with_snippet[train_with_snippet.iloc[:,8].isnull() != True].iloc[:,0].tolist() != []:\n",
    "#     print(train_with_snippet[train_with_snippet.iloc[:,8].isnull() != True].iloc[:,0].tolist())\n",
    "#     train_with_snippet.drop(train_with_snippet.index[10040], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 544,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. handle wrong choice (longer than 400) at 21511\n",
    "# if len(train_with_snippet.iloc[21509,2]) > 400:\n",
    "#     print(train_with_snippet.iloc[21509,2])\n",
    "#     train_with_snippet.drop(train_with_snippet.index[21509], inplace=True)\n",
    "#     print('dropped')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### drop nan entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 545,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 0: question\n",
    "# 1: question type\n",
    "# 2-6: choices\n",
    "# 7: answer\n",
    "# 10: textbook snippet\n",
    "# 11-14: title level 1-4\n",
    "# train_data = train_with_snippet.iloc[:,[0,1,2,3,4,5,6,7,10,11,12,13,14]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 546,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_with_snippet.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 547,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 0: question\n",
    "# 1: question type\n",
    "# 2-6: choices\n",
    "# 7: answer\n",
    "# 8: textbook snippet: best\n",
    "# 9-14: title level 1-4\n",
    "test_17_data = test_17_all.iloc[:,[0,1,2,3,4,5,6,7, 8, 9,10,11,12,13]]\n",
    "test_18_data = test_18_all.iloc[:,[0,1,2,3,4,5,6,7, 8, 9,10,11,12,13]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 548,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>最有助于诊断室性心动过速的心电图特点是</td>\n",
       "      <td>T波和主波方向相反</td>\n",
       "      <td>心室夺获</td>\n",
       "      <td>QRS波群宽大畸形</td>\n",
       "      <td>QRS波群呈束支传导阻滞图形</td>\n",
       "      <td>PR间期延长</td>\n",
       "      <td>A1</td>\n",
       "      <td>2017</td>\n",
       "      <td>2</td>\n",
       "      <td>(1)加速性心室自主节律也称缓慢型室速。心电图表现为：①连续发生3～lO个起源于心室的QRS...</td>\n",
       "      <td>内科学</td>\n",
       "      <td>心律失常</td>\n",
       "      <td>心动过速</td>\n",
       "      <td>特殊类型的室性心动过速</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     0          1     2          3               4       5  \\\n",
       "0  最有助于诊断室性心动过速的心电图特点是  T波和主波方向相反  心室夺获  QRS波群宽大畸形  QRS波群呈束支传导阻滞图形  PR间期延长   \n",
       "\n",
       "    6     7  8                                                  0    1     2  \\\n",
       "0  A1  2017  2  (1)加速性心室自主节律也称缓慢型室速。心电图表现为：①连续发生3～lO个起源于心室的QRS...  内科学  心律失常   \n",
       "\n",
       "      3            4  \n",
       "0  心动过速  特殊类型的室性心动过速  "
      ]
     },
     "execution_count": 548,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_17_data.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 549,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop entries that has any nan and, \n",
    "# print num of nan in each col\n",
    "def any_nan_values(df):\n",
    "    return df.isnull().values.any()\n",
    "\n",
    "def drop_nan(df):\n",
    "    if any_nan_values(df):\n",
    "        print(\"nan in each col:\\n\", df.isnull().sum(), sep='')\n",
    "        \n",
    "        return df.dropna()\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 550,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_data = drop_nan(train_data)\n",
    "# # reset index after dropping rows\n",
    "# train_data.reset_index(inplace=True)\n",
    "# train_data = train_data.drop(columns='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 551,
   "metadata": {},
   "outputs": [],
   "source": [
    "# assert every cell is not nan\n",
    "# assert not any_nan_values(train_data) \n",
    "assert not any_nan_values(test_17_data) and not any_nan_values(test_18_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 552,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### add column headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 553,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_data.columns = ['q', 'q_type', 'c1', 'c2', 'c3', 'c4', 'c5', 'a', 's1', 's1t1', 's1t2', 's1t3', 's1t4']\n",
    "\n",
    "test_17_data.columns = ['q','c1','c2','c3','c4','c5', 'q_type', 'year', 'a', 's1','s1t1','s1t2','s1t3','s1t4']\n",
    "test_18_data.columns = ['q','c1','c2','c3','c4','c5', 'q_type', 'year', 'a', 's1','s1t1','s1t2','s1t3','s1t4']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 554,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_data.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 555,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rearange df columns to match the ones in training data\n",
    "# test_17_data = test_17_data[train_data.columns]\n",
    "# test_18_data = test_18_data[train_data.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 556,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Draw question len && snippet len graph \n",
    "def draw_q_s_len(df):\n",
    "    q_len = [len(str(i).replace(' ', '')) for i in df['q']]\n",
    "    s1_len = [len(str(i).replace(' ', '')) for i in df['s1']]\n",
    "\n",
    "    len_512 = round(sum([i > 512 for i in s1_len]) / len(s1_len), 2)\n",
    "    len_756 = round(sum([i > 756 for i in s1_len]) / len(s1_len), 2)\n",
    "    len_1024 = round(sum([i > 1024 for i in s1_len]) / len(s1_len), 2)\n",
    "\n",
    "    label = f'snippet:   >512: {len_512}       >756: {len_756}      >1024: {len_1024}'\n",
    "\n",
    "    a = sns.distplot(s1_len, kde=False, axlabel=label)\n",
    "    a = sns.distplot(q_len, kde=False)\n",
    "    a = a.get_figure()\n",
    "    a.savefig(f'/{TRUNC_OUTPUT_DIR_PATH}/{output_dir}/qs.png', dpi=120, bbox_inches='tight')\n",
    "    print('saved to:', TRUNC_OUTPUT_DIR_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 557,
   "metadata": {},
   "outputs": [],
   "source": [
    "# draw_q_s_len(train_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add textbook snippit - only the best match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 558,
   "metadata": {},
   "outputs": [],
   "source": [
    "def combine_titles(df_titles):\n",
    "    \"\"\"\n",
    "    df_titles: any cleaned df that contains title information \n",
    "    \n",
    "    return: list\n",
    "    \"\"\"\n",
    "    return [' '.join(titles) for titles in df_titles[['s1t1', 's1t2', 's1t3', 's1t4']].values]\n",
    "\n",
    "\n",
    "def combine_qst(df_data, add_q=True, add_s=True, add_t=True):\n",
    "    \"\"\"\n",
    "    return: list\n",
    "    \n",
    "    output sequence is [title + question + snippet]\n",
    "    \"\"\"\n",
    "    result = []\n",
    "    if add_t:\n",
    "        t = combine_titles(df_data)\n",
    "    q = df_data['q'].values\n",
    "    s1 = df_data['s1'].values\n",
    "    \n",
    "    for i in range(df_data.shape[0]):\n",
    "        assert 'æ' not in t[i] and 'æ' not in q[i] and 'æ' not in s1[i], 'Input data contains æ sign'\n",
    "        all_tqs = [t[i], q[i], s1[i]] # change output sequence here\n",
    "        input_tqs = [] \n",
    "        for i, add in enumerate([add_t, add_q, add_s]): # should be the same sequence as all_tqs\n",
    "            if add:\n",
    "                input_tqs.append(all_tqs[i])\n",
    "        input_row = '  æ  '.join(input_tqs)\n",
    "        result.append(input_row)\n",
    "    print(f'First output is \\n{result[0]}')\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 559,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_trunc_df(df_data):\n",
    "    qst_list = combine_qst(df_data, True, True, True)\n",
    "    df_trunc = df_data[['q','q_type','c1','c2','c3','c4','c5','a']].copy()\n",
    "    df_trunc['q'] = qst_list\n",
    "    return df_trunc\n",
    "def add_question_index(df_data):\n",
    "    question_len = df_data.shape[0]\n",
    "    df_data['q_index'] = range(question_len)\n",
    "    return df_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 560,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_trunc = get_trunc_df(train_data)\n",
    "# train_trunc = add_question_index(train_trunc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 561,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First output is \n",
      "内科学 心律失常 心动过速 特殊类型的室性心动过速  æ  最有助于诊断室性心动过速的心电图特点是  æ  (1)加速性心室自主节律也称缓慢型室速。心电图表现为：①连续发生3～lO个起源于心室的QRS波群，心率常为60～110次／分。②心动过速的开始和终止呈渐进性。③由于心室与窦房结两个起搏点轮流控制心室节律，融合波常出现于心律失常的开始与终止时，心室夺获亦很常见。(2)尖端扭转型室速尖端扭转是多形性室性心动过速的一个特殊类型，因发作时QRS波群的振幅与波峰呈周期性改变，宛如围绕等电位线连续扭转而得名。频率200～250次／分。QT间期>0．5S U波显著。治疗：①静脉注射硫酸镁可终止和预防尖端扭转型室速的发作；②I A类、Ⅲ类药物均可使QT间期延长，不宜使用；I B类药物(利多卡因、苯妥英钠)对本病无效。\n"
     ]
    }
   ],
   "source": [
    "test_17_trunc = get_trunc_df(test_17_data)\n",
    "test_17_trunc = add_question_index(test_17_trunc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 562,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First output is \n",
      "外科学 前列腺炎、附睾炎与肾结核 肾结核 诊断  æ  要了解肾结核患者分侧肾功能、病变程度与范围。首选的方法是  æ  (1)尿液抗酸杆菌尿沉渣涂片抗酸染色约50％～70％的病例可找到抗酸杆菌。但找到抗酸杆菌不能作为诊断的唯一依据，因枯草杆菌、包皮垢杆菌也是抗酸杆菌。(2)尿液结核杆菌培养阳性率可达90％，对诊断有决定性意义，但需时较长(4～8周)。(3)尿路平片(KUB)  可见到病肾局灶或斑点状钙化影或全肾广泛钙化。(4)静脉尿路造影(IVU)  可了解分肾功能、病变程度和范围，是确诊肾结核的检查方法，对肾结核治疗方案的选择必不可少。早期表现为肾盏边缘不光滑如虫蛀状，随着病变进展，肾盏失去杯形，不规则扩大或模糊变形。若肾盏颈纤维化狭窄，可见空洞充盈不全或完全不显影。肾脏广泛被破坏、肾功能丧失时，不能显示出典型的结核破坏性病变。根据临床表现，如尿中找到结核杆菌，静脉尿路造影一侧肾正常，另一侧“无功能”未显影，可确诊肾结核。(5)逆行肾孟造影可显示病肾空洞性破坏，输尿管僵硬，管腔节段性狭窄且边缘不整。(6)B超简单易行，对中晚期病例可初步确定病变部位，较易发现对侧肾积水及膀胱挛缩。(7)CT和MRI CT对中晚期肾结核能清楚显示扩大的肾盏肾盂、皮质空洞及钙化灶，三维成像还可显示输尿管全长病变。MRI水成像对诊断肾结核。肾积水有独到之处。(8)膀胱镜检查病变以膀胱三角区和患侧输尿管口周围最明显。当膀胱挛缩容量<50ml、有急性膀胱炎时禁忌检查。注意：①对肾结核最有价值的确诊方法是_一尿液结核杆菌培养，但少用；其次为静脉尿路造影(IVU)。\n"
     ]
    }
   ],
   "source": [
    "test_18_trunc = get_trunc_df(test_18_data)\n",
    "test_18_trunc = add_question_index(test_18_trunc)\n",
    "assert test_18_trunc.shape[0] == 599 and test_17_trunc.shape[0] == 600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 563,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_input_len(df):\n",
    "    input_len = [len(i.replace(' ', '')) for i in df['q']]\n",
    "    len_512 = round(sum([i > 512 for i in input_len]) / len(input_len), 2)\n",
    "    len_756 = round(sum([i > 756 for i in input_len]) / len(input_len), 2)\n",
    "    len_1024 = round(sum([i > 1024 for i in input_len]) / len(input_len), 2)\n",
    "\n",
    "    label = f'test_18_only_correct: >512: {len_512}       >756: {len_756}      >1024: {len_1024}'\n",
    "    a = sns.distplot(input_len, kde=False, axlabel=label)\n",
    "    a = a.get_figure()\n",
    "    a.savefig(f'/{OUTPUT_DIR_PATH}/{output_dir}/only_correct_18.png', dpi=120, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 564,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAELCAYAAADX3k30AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGcpJREFUeJzt3Xu4XFWZ5/Hvz4SIoBACBycSMGCnlbRjAx6B9gaCg4AKOsLTCM1EpZ+Md2hbBcee1rZHBXTwMjrSEZA4LQoiCg/jDWOAnlaCJ1xCQqQTIIY0aXJUoEVmaJF3/lirSOWk7rvqVGXx+zzPec6+1V5vrV311qq1916liMDMzMr1tGEHYGZmg+VEb2ZWOCd6M7PCOdGbmRXOid7MrHBO9GZmhXOiNzMrnBO9mVnhnOjNzAo3c9gBAOy1114xf/78YYdhZrZDWbly5S8jYqzddiOR6OfPn8/ExMSwwzAz26FI+kUn27nrxsyscE70ZmaFc6I3MyucE72ZWeGc6M3MCudEb2ZWOCd6M7PCOdGbmRXOid7MrHAjcWfssF22YmPbbU49bL9piMTMrP/cojczK1zbRC/pEklbJK2uWzZH0nWS1uX/e+TlkvR5SeslrZJ0yCCDNzOz9jpp0V8KHDtl2TnAsohYACzL8wDHAQvy32LgS/0J08zMetU20UfEjcCvpyw+EViap5cCb6hb/tVIbgJmS5rbr2DNzKx7vfbRPzsiNgPk/3vn5fsA99VttykvMzOzIen3yVg1WBYNN5QWS5qQNDE5OdnnMMzMrKbXRP9ArUsm/9+Sl28C9q3bbh5wf6MdRMSSiBiPiPGxsbY/kGJmZj3q9Tr6a4BFwLn5/9V1y98t6RvAYcDDtS6eHV27a+19nb2Zjaq2iV7S14Ejgb0kbQI+QkrwV0g6A9gInJw3/y5wPLAeeBR46wBiNjOzLrRN9BHx5iarjm6wbQDvqhqUmZn1j++MNTMrnBO9mVnhnOjNzArnRG9mVjgnejOzwjnRm5kVzonezKxwTvRmZoVzojczK5wTvZlZ4ZzozcwK50RvZlY4J3ozs8I50ZuZFc6J3syscE70ZmaFc6I3MyucE72ZWeGc6M3MCudEb2ZWOCd6M7PCOdGbmRXOid7MrHBO9GZmhXOiNzMrnBO9mVnhnOjNzArnRG9mVjgnejOzwjnRm5kVzonezKxwlRK9pL+QtEbSaklfl7SzpP0lrZC0TtLlkmb1K1gzM+tez4le0j7Ae4HxiHghMAM4BTgP+ExELAAeBM7oR6BmZtabmX14/DMk/Q7YBdgMHAWcmtcvBT4KfKliOZVctmLjMIs3Mxuqnlv0EfHPwKeBjaQE/zCwEngoIh7Pm20C9qkapJmZ9a5K180ewInA/sBzgF2B4xpsGk0ev1jShKSJycnJXsMwM7M2qpyMfTVwb0RMRsTvgKuAlwKzJdW6hOYB9zd6cEQsiYjxiBgfGxurEIaZmbVSJdFvBA6XtIskAUcDdwLLgZPyNouAq6uFaGZmVVTpo18BXAncAtyR97UEOBt4n6T1wJ7AxX2I08zMelTpqpuI+AjwkSmL7wEOrbJfMzPrH98Za2ZWOCd6M7PCOdGbmRXOid7MrHBO9GZmhXOiNzMrnBO9mVnhnOjNzArnRG9mVjgnejOzwjnRm5kVzonezKxwTvRmZoVzojczK5wTvZlZ4SqNR29bXbZiY9ttTj1sv2mIxMxsW27Rm5kVzonezKxwTvRmZoVzojczK5wTvZlZ4ZzozcwK50RvZlY4J3ozs8I50ZuZFc6J3syscE70ZmaFc6I3MyucE72ZWeGc6M3MCudEb2ZWOCd6M7PCVUr0kmZLulLSzyWtlfQnkuZIuk7Suvx/j34Fa2Zm3avaov8c8P2IeAHwx8Ba4BxgWUQsAJbleTMzG5KeE72k3YBXAhcDRMS/RcRDwInA0rzZUuANVYM0M7PeVWnRHwBMAl+RdKukiyTtCjw7IjYD5P97N3qwpMWSJiRNTE5OVgjDzMxaqZLoZwKHAF+KiIOB39JFN01ELImI8YgYHxsbqxCGmZm1UiXRbwI2RcSKPH8lKfE/IGkuQP6/pVqIZmZWRc+JPiL+BbhP0vPzoqOBO4FrgEV52SLg6koRmplZJTMrPv49wNckzQLuAd5K+vC4QtIZwEbg5IplmJlZBZUSfUTcBow3WHV0lf2amVn/+M5YM7PCOdGbmRXOid7MrHBO9GZmhXOiNzMrnBO9mVnhnOjNzArnRG9mVjgnejOzwjnRm5kVzonezKxwTvRmZoVzojczK5wTvZlZ4aqOR29duGzFxpbrTz1sv2mKxMyeStyiNzMrnBO9mVnhnOjNzArnRG9mVjgnejOzwjnRm5kVzonezKxwTvRmZoVzojczK5wTvZlZ4ZzozcwK50RvZlY4J3ozs8I50ZuZFc6J3syscE70ZmaFq5zoJc2QdKuka/P8/pJWSFon6XJJs6qHaWZmvepHi/5MYG3d/HnAZyJiAfAgcEYfyjAzsx5VSvSS5gGvBS7K8wKOAq7MmywF3lClDDMzq6Zqi/6zwAeBJ/L8nsBDEfF4nt8E7NPogZIWS5qQNDE5OVkxDDMza6bnRC/pdcCWiFhZv7jBptHo8RGxJCLGI2J8bGys1zDMzKyNmRUe+zLgBEnHAzsDu5Fa+LMlzcyt+nnA/dXDNDOzXvXcoo+ID0XEvIiYD5wC/DgiTgOWAyflzRYBV1eO0szMejaI6+jPBt4naT2pz/7iAZRhZmYdqtJ186SIuB64Pk/fAxzaj/2amVl1vjPWzKxwTvRmZoVzojczK5wTvZlZ4ZzozcwK50RvZlY4J3ozs8I50ZuZFc6J3syscE70ZmaFc6I3MyucE72ZWeGc6M3MCudEb2ZWOCd6M7PCOdGbmRXOid7MrHBO9GZmhXOiNzMrXF9+M9b647IVGyvv49TD9utDJGZWErfozcwK50RvZlY4J3ozs8I50ZuZFc6J3syscE70ZmaFc6I3MyvcDn8dfT+uPTczK5lb9GZmhXOiNzMrnBO9mVnhek70kvaVtFzSWklrJJ2Zl8+RdJ2kdfn/Hv0L18zMulWlRf848JcRcSBwOPAuSQuBc4BlEbEAWJbnzcxsSHq+6iYiNgOb8/RvJK0F9gFOBI7Mmy0FrgfOrhSldazdVUge3dLsqacvffSS5gMHAyuAZ+cPgdqHwd79KMPMzHpTOdFLeibwLeCsiPjXLh63WNKEpInJycmqYZiZWROVEr2knUhJ/msRcVVe/ICkuXn9XGBLo8dGxJKIGI+I8bGxsSphmJlZC1WuuhFwMbA2Ii6oW3UNsChPLwKu7j08MzOrqsoQCC8DTgfukHRbXvZfgHOBKySdAWwETq4WopmZVVHlqpv/A6jJ6qN73a8NVidjA/nKHLOy+M5YM7PCOdGbmRXOid7MrHA7/Hj01n++u9asLG7Rm5kVzonezKxwTvRmZoVzojczK5wTvZlZ4ZzozcwK50RvZlY4X0dvA+Fr8c1Gh1v0ZmaFc6I3MyucE72ZWeGc6M3MCudEb2ZWOCd6M7PCOdGbmRXOid7MrHBO9GZmhfOdsWYV+S5gG3Vu0ZuZFc4teutauxZsSZ5Kz9XK5Ra9mVnh3KK3HVYnrW33j5u5RW9mVjy36G0o3Bq3Zvza6D+36M3MCucWvY2sflzx4mvcbZB2lNeXW/RmZoUbSIte0rHA54AZwEURce4gyjGrytfJd8f9590Zlfrqe4te0gzgi8BxwELgzZIW9rscMzPrzCBa9IcC6yPiHgBJ3wBOBO4cQFlmI29UWnXWnZK+7Q2ij34f4L66+U15mZmZDcEgWvRqsCy220haDCzOs49IuqvNfvcCflkxtn4bxZhgNOMaxZhgROI6bdvZkYhpio5jOq39JpXVlbFD1xVUrq/ndrLRIBL9JmDfuvl5wP1TN4qIJcCSTncqaSIixquH1z+jGBOMZlyjGBOMZlyOqXOjGNcoxjSIrpufAQsk7S9pFnAKcM0AyjEzsw70vUUfEY9LejfwA9LllZdExJp+l2NmZp0ZyHX0EfFd4Lt93m3H3TzTaBRjgtGMaxRjgtGMyzF1bhTjGrmYFLHdeVIzMyuIh0AwMyvcDpHoJR0r6S5J6yWdM81lb5B0h6TbJE3kZXMkXSdpXf6/R14uSZ/Pca6SdEifYrhE0hZJq+uWdR2DpEV5+3WSFg0oro9K+udcX7dJOr5u3YdyXHdJek3d8r4dX0n7Slouaa2kNZLOzMuHVl8tYhp2Xe0s6WZJt+e4/iYv31/Sivy8L88XVSDp6Xl+fV4/v128fYzpUkn31tXVQXn5dL7eZ0i6VdK1eX5o9dS1iBjpP9IJ3buBA4BZwO3AwmksfwOw15Rl5wPn5OlzgPPy9PHA90j3EhwOrOhTDK8EDgFW9xoDMAe4J//fI0/vMYC4Pgq8v8G2C/Oxezqwfz6mM/p9fIG5wCF5+lnAP+Wyh1ZfLWIadl0JeGae3glYkevgCuCUvPxC4B15+p3AhXn6FODyVvH2OaZLgZMabD+dr/f3AZcB1+b5odVTt387Qov+ySEVIuLfgNqQCsN0IrA0Ty8F3lC3/KuR3ATMljS3amERcSPw64oxvAa4LiJ+HREPAtcBxw4grmZOBL4REY9FxL3AetKx7evxjYjNEXFLnv4NsJZ0Z/bQ6qtFTM1MV11FRDySZ3fKfwEcBVyZl0+tq1odXgkcLUkt4u1nTM1My+td0jzgtcBFeV4MsZ66tSMk+mEPqRDADyWtVLqbF+DZEbEZ0psY2Dsvn85Yu41hOmN7d/4afUmti2QYceWvzAeTWoUjUV9TYoIh11XujrgN2EJKhncDD0XE4w3KeLL8vP5hYM9+xzU1poio1dXHc119RtLTp8Y0pex+19VngQ8CT+T5PRlyPXVjR0j0HQ2pMEAvi4hDSKNxvkvSK1tsO+xYW8UwXbF9CXgecBCwGfjvw4hL0jOBbwFnRcS/ttp0uuJqENPQ6yoifh8RB5HuYD8UOLBFGdMS19SYJL0Q+BDwAuAlpO6Ys6crJkmvA7ZExMr6xS32P+z34HZ2hETf0ZAKgxIR9+f/W4Bvk94MD9S6ZPL/LUOItdsYpiW2iHggv1GfAL7M1q+m0xaXpJ1ICfVrEXFVXjzU+moU0yjUVU1EPARcT+rnni2pdo9NfRlPlp/X707quhtIXHUxHZu7vyIiHgO+wvTW1cuAEyRtIHWXHUVq4Y9EPXVkOk4EVPkj3dR1D+nkRe0E1B9NU9m7As+qm/4JqZ/vU2x7Yu/8PP1atj0xdHMfY5nPtic9u4qB1Aq6l3Riao88PWcAcc2tm/4LUp8kwB+x7Ymoe0gnF/t6fPPz/irw2SnLh1ZfLWIadl2NAbPz9DOAfwBeB3yTbU8yvjNPv4ttTzJe0SrePsc0t64uPwucO6TX+5FsPRk7tHrqOu7pKKQPlXs86UqFu4EPT2O5B+QDczuwplY2qb9tGbAu/59T9yL8Yo7zDmC8T3F8nfTV/nekVsEZvcQAvI10Amg98NYBxfW/crmrSGMc1SezD+e47gKOG8TxBV5O+jq8Crgt/x0/zPpqEdOw6+pFwK25/NXAX9e97m/Oz/ubwNPz8p3z/Pq8/oB28fYxph/nuloN/D1br8yZttd73ueRbE30Q6unbv98Z6yZWeF2hD56MzOrwInezKxwTvRmZoVzojczK5wTvZlZ4ZzozcwKN7KJXtJsSe+s8PizJO3SZpuPS7pP0iNTlu+nNKzsrXlsjeOb7aPH2DZI2quf+xy0Tuqzw/3Ml/R/64abvbBuXbPj8T5Jd+ZjsUzSczsop+EQslVjGRZJH6iLc7Wk30uak9dtN5R23ePek4fEXSPp/A7KeXHe13ql4X+3u21f0mn5WKyS9BNJf1y37swc3xpJZ/XjuXdL0sm5/CckjU9Zt90wwWoyjPSUx71fUnTyvlUHQ0lLeqWkWyQ9LumkKevOz3GsbXYMujZdF+z3cGPCfOruuOzh8RuYMrxwg20OJw0h+8iU5UvYOuToQmBDn59b29j6XN7MVvP9jJk2Q8G2Oq4tjsergF3y9DvIw762KafhELJVYxngMep4CF3g9cCP2x2bXG8/YuuNPHt3sO+bgT8h3Yj0PRrc1AO8tBYvaQyo2tDALyTd0LQL6S7eHwELpruuSOP1PJ80fEL9DVTNhn9uOIx03eP2Jf0G9i86yCkdDSWdX3svIt0xfVLd8pcC/8jWYal/ChxZtc5GtkUPnAs8L7dSPgVPtmp+llsStR8k2FXS/1b6oYLVkv5U0nuB5wDLJS1vVkBE3BR5RMOpq4Dd8vTutBiPQumHEr6SW0G3SnpVXv4WSVdJ+n5uVW7XmpL0t/Wth9yKfG+Lsj6Yy7ld0rl52UGSbsp18m1t/UGN6yV9QtINwJlKP9xwQa6P83K9XZLr81ZJJ+bHzZD06VzOqtwi7Kg+sz/Nx+H9ksbabLuNZscjIpZHxKN59ibSGCFN5RZQsyFkK8UyQB9Q+sGN/yxptzbbvpl0V3I77yANFfAYPDleU1NKYwDtFhE/jZR1vkqDeouIn0Qa+he2PR4HAjdFxKORRm28AXhjB3F26zuSrpF0graONVMf39qIuKvB4xoOExzth5H+DGnkyk7uLu1oKOmI2BARq9g6GuaTq0h31s4ifSDtBDzQQbmt9fvTto+f2vPZdgyVY0gtbZG6nK4l/fDFm4Av1223e7Ro5TQpa2oLci7pdupNwIPAi1s89i+Br+TpFwAb84F6C2ksi93z/C+Afetjy8/xlrzsaaSWwJ5NyjmONNZOrWVbu4V/FXBEnv4YeTwVUmvmf9Y9/tJcZzPy/CeAP8vTs0mtmF1JyeFb5FZ/XTnb1CdpXO6GQzyQWkD/lfSGuZI0PtDT6o7rb0m3ud8AvKLd8Ziy7gvAX9XN39Zgm71Ib7b6eLZruVeNZQCv+ecD55ES0KXAyxtsswtpgKw5dcvuBW4BVgKL6+sG+BvSkMg3AC/Jy58DfLfBvseBH9XNv4J8u3+LmN8PXJSnD8yvoz1znD8F/scA6knAEaQP8PXAJ4E/aLDd9Wzbov9C7TWf5y9myo+Z5NfERtIHHsAJwOemvgdyXV3UoMyT6pcDpwNfaPFcLm0Qw6eBh0jDG3+8H3U2yi36qY7Jf7eSXtQvABaQEvKrJZ0n6RUR8XAfynozcGlEzCOPSSKpWV29nDRmCRHxc1JC/8O8bllEPBwR/w+4E9imbzkiNgC/knRw7blFxK+alPNq0gfKo/mxv5a0O2kAqBvyNktJH341l0/Zxzcj4vd5+hjgHKVxv68nfRjtl8u5MPI42xHR8IdFIuLPI2Kiybr7IuJvSV+VL85/38mrNwP7RcTB5F/s6aAFC4CkPyO9wT5VV9ZBjTZtFFaDZT3HMggRcVdEnE1K+NcB10r6/JTNXg/845Tj0mwo7ZmkAb0OBz4AXCFJEXF/RDQ679TVMLr52+sZ5CGDI2It6YPqOuD7pG6Lx5s9vleR3BARi0i/cPYE8HNJb2rz0JbPT1OGkVY6J/Vh4K8bxDAREX/ebRntSPoD0gfmPNK3iqPUemj0jmz3tWeECfhkRPzddiukF5MS8icl/TAiPlaxrDPIv0YTET+VtDOpldjoq2+rEyWP1U3/nsb1fRGp9f/vgEta7Et0P3b1b1vMC3hTTPmKm7s9Kg+AJOlQ4K3AfyAN8PRlgEjdCLWuhJWS7iZ9MDb80Kjb36tJb7oj8j5a+SV5CNn8gdVwONheYxmUXPevItXbYaQW6EVTNjuFKd02UTeUtqTaUNo3kr6RXhWpmXizpCdIr+PJJiFsYttusabD6Ep6UY7tuPrGSUTUPtiR9Im8z76T9AxSt9DbSN9IzyR9wLTSdJhgNR7a+nmkvvzb8/nQecAtkg6NiH/ptowOvZHU/fVIjut7pA/qG7vYx3ZGuUX/G9KJkZofAG/Ln7pI2kfS3pKeAzwaEX9P+spzSJPHd2MjcHQu50BSa7fZm+NG4LS87R+SWsWN+geb+TbpQ+UlpOfYzA9Jz3+XXNac/O3lQUmvyNucTvqK3okfAO+pndHP3ypq5by91vepfGUHHdanpGMkrQL+G+mbwsKIOCsi1uT1Y5Jm5OkDSN/K7mmzz4OBvwNOiDb9zJBafMBy0tdogEXA1Q3223UsgyLpNODnpCFuvw4cGBF/lb/11bbZndRlcXXdsl0lPas2TfqmVvux9u+QzlXUXpuzSB+CDUU6J/EbSYfn18V/onG97QdcBZweEf80Zd3eddv8Rzo7l9CVfL7rTtI48R+IiPGI+GK0/oEZSCOEnqL04937k473zfm5XgysjYgLahtHxB0RsXdEzI+I+aQkfkiLJA/wM2CB0lVfs0gfzNd08fQ2AkdImpk/fI4gdYFW04/+n0H9kX6IdzXwqTx/Jqmr5g5S/9/zSL8NWRv+9WfkPjngPaQ3zvIW+z+fdPCeyP8/mpcvJJ35vj3v95gW+9iZ1M92B6lb6VV5+Vuo65sj9Y8fGVP6+vL8heTxtdvUxzmkF/htwCfysoNIJ8RWkd7Ytashrmfb/slL2fbs/jNIybM29Gtt6NWZwAW5nNuBdzeqT5r00QMvBp7b4jm8iTTk8+2kLrjXd3A8fkQ6IVUb4veausds10eflzcbQvYE4GO9xjLA1/rLaXNVTH5NfaPB89xuKO28bhZpSN/V+fkdlZc37KPP68bz9neTvlHURrh9O/D2umP/YN3xmKh7/D/UvXaOHlBdHQ/s3GL9G/Mxeyy/bn5Qt267YYJpMox0g/1uoE0ffV182w0lTTqHdkKefkmO8bfAr4A1efkM0vtyba7HC/pRZx6meMhy3/8twMkRsW7Y8ZhZeUa566Z4khaSWpzLnOTNbFCeEi16SStI16TWOz0i7uhiH68hXVFQ796I6Ot1wpL+PfkqnjqPRcRh/SzHzJ46nhKJ3szsqcxdN2ZmhXOiNzMrnBO9mVnhnOjNzArnRG9mVrj/D8woJfNpOf+qAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_input_len(test_18_trunc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 565,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shuffle_df(df, frac=1, random_state=42):\n",
    "    return df.sample(frac=frac, random_state=random_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 566,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_trunc_shuffled = shuffle_df(train_trunc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 567,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_trunc_shuffled.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. TRUNC method"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For a single question\n",
    "\n",
    "$QC_1...C_5 \\longrightarrow \\sum_{c=1}^{5} QC_c$\n",
    "\n",
    "question + 5 choice -> 5 * (question + 1 choice)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 568,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expand_choices(df):\n",
    "    \"\"\"\n",
    "    return:\n",
    "    df with expanded choice\n",
    "    \"\"\"\n",
    "    \n",
    "    result = []\n",
    "    question_index = df.columns.get_loc('q')\n",
    "    first_choice_index = df.columns.get_loc('c1')\n",
    "    answer_index = df.columns.get_loc('a')\n",
    "    # iterate through all entries in df\n",
    "    for i in tqdm(range(df.shape[0])):\n",
    "        one_entry = df.iloc[i,:] \n",
    "        # for each entry, take its 5 choices in sequence into 5 [question, one_choice, label] outputs\n",
    "        for choice_index in range(5):\n",
    "            label = 1 if (choice_index + 1)  == one_entry[answer_index] else 0\n",
    "            result.append({'input': one_entry[question_index], \n",
    "                       'choice': one_entry[first_choice_index+choice_index],\n",
    "                       'q_type': one_entry['q_type'],\n",
    "                       'c_index': choice_index, \n",
    "                       'q_index': one_entry['q_index'],\n",
    "                       'label': label})\n",
    "    return pd.DataFrame(result)[['input','q_index','choice','c_index','label','q_type']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 569,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_trunc_df = expand_choices(train_trunc_shuffled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 570,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f315f30faac64db7b4665c61294dd921",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=600), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test_17_trunc_df = expand_choices(test_17_trunc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 571,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b319ffbd174f45d9a28a36e981c050e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=599), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test_18_trunc_df = expand_choices(test_18_trunc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save files to help calculating accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 572,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_debug_file(df, file_name):\n",
    "    df.to_excel(f'{TRUNC_OUTPUT_DIR_PATH}/{output_dir}/{file_name}_debug.xlsx', index=None)\n",
    "    print(f'{file_name} saved!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 573,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save_debug_file(test_17_trunc_df, TEST_17_NAME)\n",
    "# save_debug_file(test_18_trunc_df, TEST_18_NAME)\n",
    "# save_debug_file(train_trunc_df.head(15000), TRAIN_3000_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing for bert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 574,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_for_bert(df):\n",
    "    return pd.DataFrame({\n",
    "        'id':range(df.shape[0]),\n",
    "        'label':df['label'],\n",
    "        'alpha':['a']*df.shape[0],\n",
    "        'text_a': df['input'].replace(r'\\n', ' ', regex=True),\n",
    "        'text_b': df['choice'].replace(r'\\n', ' ', regex=True)\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 575,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_trunc_bert = prepare_for_bert(train_trunc_df)\n",
    "\n",
    "test_17_trunc_bert = prepare_for_bert(test_17_trunc_df)\n",
    "test_18_trunc_bert = prepare_for_bert(test_18_trunc_df)\n",
    "# train_3000_trunc_bert = prepare_for_bert(train_trunc_df.head(15000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 576,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_input_file(df_bert, file_name):\n",
    "    df_bert.to_csv(f'{OUTPUT_DIR_PATH}/{output_dir}/{file_name}.tsv', \n",
    "                sep='\\t', index=False, header=False)\n",
    "    print(f'{file_name} saved!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 577,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save_input_file(train_trunc_bert, TRAIN_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 578,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_17_choice_correct saved!\n",
      "test_18_choice_correct saved!\n"
     ]
    }
   ],
   "source": [
    "save_input_file(test_17_trunc_bert, TEST_17_NAME)\n",
    "save_input_file(test_18_trunc_bert, TEST_18_NAME)\n",
    "# save_input_file(train_3000_trunc_bert, TRAIN_3000_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}