training_model_on_WikiQA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "74acdbcd",
   "metadata": {},
   "source": [
    "<div align=\"center\"> <h1 align=\"center\"> Chatbot Assignment: QA Chatbot with WikiQA </h1> </div>\n",
    "\n",
    "<div align=\"center\"> <h3 align=\"center\"> December 12, 2021 </h3> </div>\n",
    "<div align=\"center\"> <h3 align=\"center\"> Maria Aroca </h3> </div>\n",
    "<br />"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74e2305b",
   "metadata": {},
   "source": [
    "#### Code Adapted from\n",
    "https://towardsdatascience.com/how-to-fine-tune-a-q-a-transformer-86f91ec92997\n",
    "https://colab.research.google.com/github/fastforwardlabs/ff14_blog/blob/master/_notebooks/2020-05-19-Getting_Started_with_QA.ipynb?pli=1&authuser=1#scrollTo=bgYVkF2RmHPK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b3fbb3d0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default\n",
      "Reusing dataset wiki_qa (C:\\Users\\User\\.cache\\huggingface\\datasets\\wiki_qa\\default\\0.1.0\\d2d236b5cbdc6fbdab45d168b4d678a002e06ddea3525733a24558150585951c)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0e3c3d5727a4489824301714f881436",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Load WikiQA Dataset from huggingface library\n",
    "\n",
    "from datasets import load_dataset\n",
    "\n",
    "dataset = load_dataset(\"wiki_qa\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af5791bf",
   "metadata": {},
   "source": [
    "## Preparing the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fcccd320",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Extract correct answers from training and validation WikiQA dataset (only answers with label ==1)\n",
    "\n",
    "train_questions = []\n",
    "train_answers = []\n",
    "train_q_ids = []\n",
    "\n",
    "val_questions = []\n",
    "val_answers = []\n",
    "val_q_ids = []\n",
    "\n",
    "for i in range(0, len(dataset['validation'])):\n",
    "    label = dataset['validation']['label'][i]\n",
    "    question = dataset['validation']['question'][i]\n",
    "    answer = dataset['validation']['answer'][i]\n",
    "    q_id = dataset['validation']['question_id'][i]\n",
    "    if label==1:\n",
    "        val_questions.append(question)\n",
    "        val_answers.append(answer)\n",
    "        val_q_ids.append(q_id)\n",
    "        \n",
    "for i in range(0, len(dataset['train'])):\n",
    "    label = dataset['train']['label'][i]\n",
    "    question = dataset['train']['question'][i]\n",
    "    answer = dataset['train']['answer'][i]\n",
    "    q_id = dataset['train']['question_id'][i]\n",
    "    if label==1:\n",
    "        train_questions.append(question)\n",
    "        train_answers.append(answer)\n",
    "        train_q_ids.append(q_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "144d8804",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create 'context' by concatenating strings in possible answers from WikiQA\n",
    "\n",
    "train_contexts = []\n",
    "val_contexts = []\n",
    "\n",
    "for q_id in val_q_ids:\n",
    "    list_question_id = [i for i,_ in enumerate(dataset['validation']) if _['question_id'] == q_id]\n",
    "    val_contexts.append(' '.join(dataset['validation']['answer'][list_question_id[0]:list_question_id[-1]+1]))\n",
    "    \n",
    "for q_id in train_q_ids:\n",
    "    list_question_id = [i for i,_ in enumerate(dataset['train']) if _['question_id'] == q_id]\n",
    "    train_contexts.append(' '.join(dataset['train']['answer'][list_question_id[0]:list_question_id[-1]+1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "49ed3b17",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Remove one problematic line (encoding problem)\n",
    "del val_answers[32]\n",
    "del val_questions[32]\n",
    "del val_contexts[32]\n",
    "del train_answers[314]\n",
    "del train_questions[314]\n",
    "del train_contexts[314]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3cfe7f7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import pandas as pd\n",
    "# Get start and end positions from answers in context\n",
    "train_answer_start=[]\n",
    "train_answer_end=[]\n",
    "for i in range(0, len(train_answers)):\n",
    "    for match in re.finditer(train_answers[i], train_contexts[i]):\n",
    "        answer_start= match.start()\n",
    "        answer_end = match.end()\n",
    "    train_answer_start.append(answer_start)\n",
    "    train_answer_end.append(answer_end)\n",
    "        \n",
    "val_answer_start=[]\n",
    "val_answer_end=[]\n",
    "for i in range(0, len(val_answers)):\n",
    "    for match in re.finditer(val_answers[i], val_contexts[i]):\n",
    "        answer_start= match.start()\n",
    "        answer_end = match.end()\n",
    "    val_answer_start.append(answer_start)\n",
    "    val_answer_end.append(answer_end)\n",
    "    \n",
    "train_answers2 = pd.DataFrame(list(zip(train_answers, train_answer_start, train_answer_end)),\n",
    "               columns =['text', 'answer_start', 'answer_end']).to_dict(orient='records')\n",
    "val_answers2 = pd.DataFrame(list(zip(val_answers, val_answer_start, val_answer_end)),\n",
    "               columns =['text', 'answer_start', 'answer_end']).to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68ce07f5",
   "metadata": {},
   "source": [
    "## Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "54147d0b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "750f4fd9e1914c44b3347f946874012e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/79.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bac9cd5abd9d4137adc19e0ee8aa9861",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/571 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b728de8659954b4a955f7d36015cb8cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/878k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "663187bc0c014768a3c2d05690ac03eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/446k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bdcb9aa1b1724b1498201b6869badd65",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/772 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a2efa8a05ed4c9883ea3707d0c9015d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/473M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Fine-tuning on https://huggingface.co/deepset/roberta-base-squad2\n",
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForQuestionAnswering\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"deepset/roberta-base-squad2\") \n",
    "model = AutoModelForQuestionAnswering.from_pretrained(\"deepset/roberta-base-squad2\")\n",
    "train_encodings = tokenizer(train_contexts, train_questions, truncation=True, padding=True)\n",
    "val_encodings = tokenizer(val_contexts, val_questions, truncation=True, padding=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "6c890b90",
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_token_positions(encodings, answers):\n",
    "    # initialize lists to contain the token indices of answer start/end\n",
    "    start_positions = []\n",
    "    end_positions = []\n",
    "    for i in range(len(answers)):\n",
    "        # append start/end token position using char_to_token method\n",
    "        start_positions.append(encodings.char_to_token(i, answers[i]['answer_start']))\n",
    "        end_positions.append(encodings.char_to_token(i, answers[i]['answer_end']))\n",
    "\n",
    "        # if start position is None, the answer passage has been truncated\n",
    "        if start_positions[-1] is None:\n",
    "            start_positions[-1] = tokenizer.model_max_length\n",
    "        # end position cannot be found, char_to_token found space, so shift position until found\n",
    "        shift = 1\n",
    "        while end_positions[-1] is None:\n",
    "            end_positions[-1] = encodings.char_to_token(i, answers[i]['answer_end'] - shift)\n",
    "            shift += 1\n",
    "    # update our encodings object with the new token-based start/end positions\n",
    "    encodings.update({'start_positions': start_positions, 'end_positions': end_positions})\n",
    "\n",
    "# apply function to our data\n",
    "add_token_positions(train_encodings, train_answers2)\n",
    "add_token_positions(val_encodings, val_answers2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "38156bf7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['input_ids', 'attention_mask', 'start_positions', 'end_positions'])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_encodings.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "fe6ee13e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "class WikiQADataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, encodings):\n",
    "        self.encodings = encodings\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        return {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.encodings.input_ids)\n",
    "\n",
    "# build datasets for both our training and validation sets\n",
    "train_dataset = WikiQADataset(train_encodings)\n",
    "val_dataset = WikiQADataset(val_encodings)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0aadb01",
   "metadata": {},
   "source": [
    "## Fine-Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "ab53abba",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0: 100%|██████████| 65/65 [21:44<00:00, 20.08s/it, loss=3.01]\n",
      "Epoch 1: 100%|██████████| 65/65 [21:36<00:00, 19.95s/it, loss=3.21]\n",
      "Epoch 2: 100%|██████████| 65/65 [21:35<00:00, 19.93s/it, loss=1.96] \n"
     ]
    }
   ],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "from transformers import AdamW\n",
    "from tqdm import tqdm\n",
    "\n",
    "# setup GPU/CPU\n",
    "device =  torch.device('cpu')\n",
    "# move model over to detected device\n",
    "model.to(device)\n",
    "# activate training mode of model\n",
    "model.train()\n",
    "# initialize adam optimizer with weight decay (reduces chance of overfitting)\n",
    "optim = AdamW(model.parameters(), lr=5e-5)\n",
    "\n",
    "# initialize data loader for training data\n",
    "train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)\n",
    "\n",
    "for epoch in range(3):\n",
    "    # set model to train mode\n",
    "    model.train()\n",
    "    # setup loop (we use tqdm for the progress bar)\n",
    "    loop = tqdm(train_loader, leave=True)\n",
    "    for batch in loop:\n",
    "        # initialize calculated gradients (from prev step)\n",
    "        optim.zero_grad()\n",
    "        # pull all the tensor batches required for training\n",
    "        input_ids = batch['input_ids'].to(device)\n",
    "        attention_mask = batch['attention_mask'].to(device)\n",
    "        start_positions = batch['start_positions'].to(device)\n",
    "        end_positions = batch['end_positions'].to(device)\n",
    "        # train model on batch and return outputs (incl. loss)\n",
    "        outputs = model(input_ids, attention_mask=attention_mask,\n",
    "                        start_positions=start_positions,\n",
    "                        end_positions=end_positions)\n",
    "        # extract loss\n",
    "        loss = outputs[0]\n",
    "        # calculate loss for every parameter that needs grad update\n",
    "        loss.backward()\n",
    "        # update parameters\n",
    "        optim.step()\n",
    "        # print relevant info to progress bar\n",
    "        loop.set_description(f'Epoch {epoch}')\n",
    "        loop.set_postfix(loss=loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "5ed4bbe6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('models/roberta-custom\\\\tokenizer_config.json',\n",
       " 'models/roberta-custom\\\\special_tokens_map.json',\n",
       " 'models/roberta-custom\\\\vocab.json',\n",
       " 'models/roberta-custom\\\\merges.txt',\n",
       " 'models/roberta-custom\\\\added_tokens.json',\n",
       " 'models/roberta-custom\\\\tokenizer.json')"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save Model\n",
    "model_path = 'models/roberta-custom'\n",
    "model.save_pretrained(model_path)\n",
    "tokenizer.save_pretrained(model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6ba16f0",
   "metadata": {},
   "source": [
    "## Model Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f558f757",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Model\n",
    "from transformers import AutoTokenizer, AutoModelForQuestionAnswering\n",
    "model = AutoModelForQuestionAnswering.from_pretrained(model_path)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "e42cf056",
   "metadata": {},
   "outputs": [],
   "source": [
    "start_pred = torch.argmax(outputs['start_logits'], dim=1)\n",
    "end_pred = torch.argmax(outputs['end_logits'], dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "870a4028",
   "metadata": {},
   "outputs": [],
   "source": [
    "# switch model out of training mode\n",
    "model.eval()\n",
    "# initialize validation set data loader\n",
    "val_loader = DataLoader(val_dataset, batch_size=16)\n",
    "# initialize list to store accuracies\n",
    "acc = []\n",
    "# loop through batches\n",
    "for batch in val_loader:\n",
    "    # we don't need to calculate gradients as we're not training\n",
    "    with torch.no_grad():\n",
    "        # pull batched items from loader\n",
    "        input_ids = batch['input_ids']\n",
    "        attention_mask = batch['attention_mask']\n",
    "        # we will use true positions for accuracy calc\n",
    "        start_true = batch['start_positions']\n",
    "        end_true = batch['end_positions']\n",
    "        # make predictions\n",
    "        outputs = model(input_ids, attention_mask=attention_mask)\n",
    "        # pull prediction tensors out and argmax to get predicted tokens\n",
    "        start_pred = torch.argmax(outputs['start_logits'], dim=1)\n",
    "        end_pred = torch.argmax(outputs['end_logits'], dim=1)\n",
    "        # calculate accuracy for both and append to accuracy list\n",
    "        acc.append(((start_pred == start_true).sum()/len(start_pred)).item())\n",
    "        acc.append(((end_pred == end_true).sum()/len(end_pred)).item())\n",
    "# calculate average accuracy in total\n",
    "acc = sum(acc)/len(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "b47e497f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.46496212151315475"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "760b084c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T/F\tstart\tend\n",
      "\n",
      "true\t1\t32\n",
      "pred\t1\t32\n",
      "\n",
      "true\t131\t165\n",
      "pred\t151\t165\n",
      "\n",
      "true\t1\t24\n",
      "pred\t1\t24\n",
      "\n",
      "true\t1\t38\n",
      "pred\t45\t155\n",
      "\n",
      "true\t36\t57\n",
      "pred\t36\t57\n",
      "\n",
      "true\t31\t56\n",
      "pred\t1\t11\n",
      "\n",
      "true\t1\t58\n",
      "pred\t1\t58\n",
      "\n",
      "true\t1\t30\n",
      "pred\t1\t115\n",
      "\n",
      "true\t70\t121\n",
      "pred\t70\t121\n",
      "\n",
      "true\t8\t39\n",
      "pred\t1\t39\n",
      "\n",
      "true\t5\t36\n",
      "pred\t46\t89\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"T/F\\tstart\\tend\\n\")\n",
    "for i in range (len(start_true)):\n",
    "    print(f\"true\\t{start_true[i]}\\t{end_true[i]}\\n\"\n",
    "         f\"pred\\t{start_pred[i]}\\t{end_pred[i]}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f67275ef",
   "metadata": {},
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "c7332405",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = AutoModelForQuestionAnswering.from_pretrained(model_path, return_dict=False)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
    "outputs = model(**inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "2d402530",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"how are glacier caves formed?\"\n",
    "\n",
    "context = \"\"\"A partly submerged glacier cave on Perito Moreno Glacier . \n",
    "The ice facade is approximately 60 m high Ice formations in the Titlis glacier cave. A glacier cave is a \n",
    "cave formed within the ice of a glacier . Glacier caves are often called ice caves , \n",
    "but this term is properly used to describe bedrock caves that contain year-round ice.\"\"\"\n",
    "\n",
    "\n",
    "# 1. TOKENIZE THE INPUT\n",
    "# note: if you don't include return_tensors='pt' you'll get a list of lists which is easier for \n",
    "# exploration but you cannot feed that into a model. \n",
    "inputs = tokenizer.encode_plus(question, context, return_tensors=\"pt\") \n",
    "\n",
    "# 2. OBTAIN MODEL SCORES\n",
    "# the AutoModelForQuestionAnswering class includes a span predictor on top of the model. \n",
    "# the model returns answer start and end scores for each word in the text\n",
    "answer_start_scores, answer_end_scores = model(**inputs)\n",
    "answer_start = torch.argmax(answer_start_scores)  # get the most likely beginning of answer with the argmax of the score\n",
    "answer_end = torch.argmax(answer_end_scores) + 1  # get the most likely end of answer with the argmax of the score\n",
    "\n",
    "# 3. GET THE ANSWER SPAN\n",
    "# once we have the most likely start and end tokens, we grab all the tokens between them\n",
    "# and convert tokens back to words!\n",
    "answer = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"][0][answer_start:answer_end]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "1b5d655d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "how are glacier caves formed?\n",
      " A glacier cave is a \n",
      "cave formed within the ice of a glacier .\n"
     ]
    }
   ],
   "source": [
    "print(question)\n",
    "print(answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "10100b4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"how much is 1 tablespoon of water?\"\n",
    "\n",
    "context = \"\"\"This tablespoon has a capacity of about 15 mL. \n",
    "Measuring Spoons In the US and parts of Canada, a tablespoon is the largest type of spoon used for eating from a bowl. \n",
    "In the UK, Europe and most Commonwealth countries, a tablespoon is a type of large spoon usually used for serving. \n",
    "In countries where a tablespoon is a serving spoon, the nearest equivalent to the US tablespoon is either the dessert spoon or the soup spoon . \n",
    "A tablespoonful, nominally the capacity of one tablespoon, is commonly used as a measure of volume in cooking . It is abbreviated as T, tb, tbs, tbsp, tblsp, or tblspn. The capacity of ordinary tablespoons is not regulated by law and is subject to considerable variation. In the USA one tablespoon (measurement unit) is approximately 15 mL; the capacity of an actual tablespoon (dining utensil) ranges from 7 mL to 14 mL. \n",
    "In Australia one tablespoon (measurement unit) is 20 mL.\"\"\"\n",
    "\n",
    "\n",
    "# 1. TOKENIZE THE INPUT\n",
    "# note: if you don't include return_tensors='pt' you'll get a list of lists which is easier for \n",
    "# exploration but you cannot feed that into a model. \n",
    "inputs = tokenizer.encode_plus(question, context, return_tensors=\"pt\") \n",
    "\n",
    "# 2. OBTAIN MODEL SCORES\n",
    "# the AutoModelForQuestionAnswering class includes a span predictor on top of the model. \n",
    "# the model returns answer start and end scores for each word in the text\n",
    "answer_start_scores, answer_end_scores = model(**inputs)\n",
    "answer_start = torch.argmax(answer_start_scores)  # get the most likely beginning of answer with the argmax of the score\n",
    "answer_end = torch.argmax(answer_end_scores) + 1  # get the most likely end of answer with the argmax of the score\n",
    "\n",
    "# 3. GET THE ANSWER SPAN\n",
    "# once we have the most likely start and end tokens, we grab all the tokens between them\n",
    "# and convert tokens back to words!\n",
    "answer = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"][0][answer_start:answer_end]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "a73b5353",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "question: how much is 1 tablespoon of water?\n",
      "answer: This tablespoon has a capacity of about 15 mL.\n"
     ]
    }
   ],
   "source": [
    "print('question:',question)\n",
    "print('answer:',answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "9d41d8d6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'how much is 1 tablespoon of water'"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_questions[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "61884b5c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'This tablespoon has a capacity of about 15 mL. Measuring Spoons In the US and parts of Canada, a tablespoon is the largest type of spoon used for eating from a bowl. In the UK, Europe and most Commonwealth countries, a tablespoon is a type of large spoon usually used for serving. In countries where a tablespoon is a serving spoon, the nearest equivalent to the US tablespoon is either the dessert spoon or the soup spoon . A tablespoonful, nominally the capacity of one tablespoon, is commonly used as a measure of volume in cooking . It is abbreviated as T, tb, tbs, tbsp, tblsp, or tblspn. The capacity of ordinary tablespoons is not regulated by law and is subject to considerable variation. In the USA one tablespoon (measurement unit) is approximately 15 mL; the capacity of an actual tablespoon (dining utensil) ranges from 7 mL to 14 mL. In Australia one tablespoon (measurement unit) is 20 mL.'"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_contexts[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "46c5d146",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"how are cholera and typhus transmitted and prevented?\"\n",
    "\n",
    "context = \"\"\"Cholera is an infection in the small intestine caused by the bacterium Vibrio cholerae . \n",
    "The main symptoms are watery diarrhea and vomiting . Transmission occurs primarily by drinking water or eating food \n",
    "that has been contaminated by the feces (waste product) of an infected person, including one with no apparent symptoms. \n",
    "The severity of the diarrhea and vomiting can lead to rapid dehydration and electrolyte imbalance, and death in some cases. \n",
    "The primary treatment is oral rehydration therapy , typically with oral rehydration solution , to replace water and electrolytes. If this is not tolerated or does not provide improvement fast enough, intravenous fluids can also be used. Antibacterial drugs are beneficial in those with severe disease to shorten its duration and severity. Worldwide, it affects 3–5 million people and causes 100,000–130,000 deaths a year . \n",
    "Cholera was one of the earliest infections to be studied by epidemiological methods.\"\"\"\n",
    "\n",
    "\n",
    "# 1. TOKENIZE THE INPUT\n",
    "# note: if you don't include return_tensors='pt' you'll get a list of lists which is easier for \n",
    "# exploration but you cannot feed that into a model. \n",
    "inputs = tokenizer.encode_plus(question, context, return_tensors=\"pt\") \n",
    "\n",
    "# 2. OBTAIN MODEL SCORES\n",
    "# the AutoModelForQuestionAnswering class includes a span predictor on top of the model. \n",
    "# the model returns answer start and end scores for each word in the text\n",
    "answer_start_scores, answer_end_scores = model(**inputs)\n",
    "answer_start = torch.argmax(answer_start_scores)  # get the most likely beginning of answer with the argmax of the score\n",
    "answer_end = torch.argmax(answer_end_scores) + 1  # get the most likely end of answer with the argmax of the score\n",
    "\n",
    "# 3. GET THE ANSWER SPAN\n",
    "# once we have the most likely start and end tokens, we grab all the tokens between them\n",
    "# and convert tokens back to words!\n",
    "answer = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"][0][answer_start:answer_end]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "da871d0d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "question: how are cholera and typhus transmitted and prevented?\n",
      "answer:  Transmission occurs primarily by drinking water or eating food \n"
     ]
    }
   ],
   "source": [
    "print('question:',question)\n",
    "print('answer:',answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "718ab903",
   "metadata": {},
   "outputs": [],
   "source": [
    "import wikipedia as wiki\n",
    "import pprint as pp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "b714bb92",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"how old was monica lewinsky during the affair?\"\n",
    "\n",
    "context = \"\"\"Monica Samille Lewinsky (born July 23, 1973) is an American woman with whom United States President Bill Clinton \n",
    "admitted to having had an \"improper relationship\" while she worked at the White House in 1995 and 1996. \n",
    "The affair and its repercussions (which included Clinton\\'s impeachment ) became known as the Lewinsky scandal .\"\"\"\n",
    "\n",
    "\n",
    "# 1. TOKENIZE THE INPUT\n",
    "# note: if you don't include return_tensors='pt' you'll get a list of lists which is easier for \n",
    "# exploration but you cannot feed that into a model. \n",
    "inputs = tokenizer.encode_plus(question, context, return_tensors=\"pt\") \n",
    "\n",
    "# 2. OBTAIN MODEL SCORES\n",
    "# the AutoModelForQuestionAnswering class includes a span predictor on top of the model. \n",
    "# the model returns answer start and end scores for each word in the text\n",
    "answer_start_scores, answer_end_scores = model(**inputs)\n",
    "answer_start = torch.argmax(answer_start_scores)  # get the most likely beginning of answer with the argmax of the score\n",
    "answer_end = torch.argmax(answer_end_scores) + 1  # get the most likely end of answer with the argmax of the score\n",
    "\n",
    "# 3. GET THE ANSWER SPAN\n",
    "# once we have the most likely start and end tokens, we grab all the tokens between them\n",
    "# and convert tokens back to words!\n",
    "answer = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"][0][answer_start:answer_end]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "3212d06b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "question: how old was monica lewinsky during the affair?\n",
      "answer: Monica Samille Lewinsky (born July 23, 1973) is an American woman with whom United States President Bill Clinton \n",
      "admitted to having had an \"improper relationship\" while she worked at the White House in 1995 and 1996.\n"
     ]
    }
   ],
   "source": [
    "print('question:',question)\n",
    "print('answer:',answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "id": "00deddbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import wikipedia as wiki\n",
    "import pprint as pp\n",
    "\n",
    "question = 'Why is the sky blue?'\n",
    "results = wiki.search(question)\n",
    "page = wiki.page(results[0])\n",
    "text = page.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "7d7804b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "context = text[:model.config.max_position_embeddings]\n",
    "\n",
    "\n",
    "# 1. TOKENIZE THE INPUT\n",
    "# note: if you don't include return_tensors='pt' you'll get a list of lists which is easier for \n",
    "# exploration but you cannot feed that into a model. \n",
    "inputs = tokenizer.encode_plus(question, context, return_tensors=\"pt\") \n",
    "\n",
    "# 2. OBTAIN MODEL SCORES\n",
    "# the AutoModelForQuestionAnswering class includes a span predictor on top of the model. \n",
    "# the model returns answer start and end scores for each word in the text\n",
    "answer_start_scores, answer_end_scores = model(**inputs)\n",
    "answer_start = torch.argmax(answer_start_scores)  # get the most likely beginning of answer with the argmax of the score\n",
    "answer_end = torch.argmax(answer_end_scores) + 1  # get the most likely end of answer with the argmax of the score\n",
    "\n",
    "# 3. GET THE ANSWER SPAN\n",
    "# once we have the most likely start and end tokens, we grab all the tokens between them\n",
    "# and convert tokens back to words!\n",
    "answer = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"][0][answer_start:answer_end]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "e7b35d99",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "question: Why is the sky blue?\n",
      "answer: Diffuse sky radiation is solar radiation reaching the Earth's surface after having been scattered from the direct solar beam by molecules or particulates in the atmosphere.\n"
     ]
    }
   ],
   "source": [
    "print('question:',question)\n",
    "print('answer:',answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "id": "fc19af63",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = 'How did Edgar Allan Poe die?'\n",
    "results = wiki.search(question)\n",
    "page = wiki.page(results[0])\n",
    "text = page.content\n",
    "context = text[:model.config.max_position_embeddings]\n",
    "inputs = tokenizer.encode_plus(question, context, return_tensors=\"pt\") \n",
    "answer_start_scores, answer_end_scores = model(**inputs)\n",
    "answer_start = torch.argmax(answer_start_scores)  # get the most likely beginning of answer with the argmax of the score\n",
    "answer_end = torch.argmax(answer_end_scores) + 1  # get the most likely end of answer with the argmax of the score\n",
    "answer = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"][0][answer_start:answer_end]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "4f0beeb4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "question: How did Edgar Allan Poe die?\n",
      "answer:  He was taken to the Washington College Hospital, where he died at 5 a.m. on Sunday, October 7.\n"
     ]
    }
   ],
   "source": [
    "print('question:',question)\n",
    "print('answer:',answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "0296edfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = 'what does the president of the usa do?'\n",
    "results = wiki.search(question)\n",
    "page = wiki.page(results[0])\n",
    "text = page.content\n",
    "context = text[:model.config.max_position_embeddings]\n",
    "inputs = tokenizer.encode_plus(question, context, return_tensors=\"pt\") \n",
    "answer_start_scores, answer_end_scores = model(**inputs)\n",
    "answer_start = torch.argmax(answer_start_scores)  # get the most likely beginning of answer with the argmax of the score\n",
    "answer_end = torch.argmax(answer_end_scores) + 1  # get the most likely end of answer with the argmax of the score\n",
    "answer = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"][0][answer_start:answer_end]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "31504391",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "question: what does the president of the usa do?\n",
      "answer:  The president directs the executive branch of the federal government and is the commander-in-chief of the United States Armed Forces.\n"
     ]
    }
   ],
   "source": [
    "print('question:',question)\n",
    "print('answer:',answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "cabb1aa2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_answer(question):\n",
    "    results = wiki.search(question)\n",
    "    page = wiki.page(results[0])\n",
    "    text = page.content\n",
    "    context = text[:model.config.max_position_embeddings]\n",
    "    inputs = tokenizer.encode_plus(question, context, return_tensors=\"pt\") \n",
    "    answer_start_scores, answer_end_scores = model(**inputs)\n",
    "    answer_start = torch.argmax(answer_start_scores)  # get the most likely beginning of answer with the argmax of the score\n",
    "    answer_end = torch.argmax(answer_end_scores) + 1  # get the most likely end of answer with the argmax of the score\n",
    "    answer = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"][0][answer_start:answer_end]))\n",
    "    return answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "3c3aab56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' The president directs the executive branch of the federal government and is the commander-in-chief of the United States Armed Forces.'"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_answer(\"what does the president of the usa do?\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15a050c1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}