eval_MLM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation of RoBERTa Masked Language Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/flickr_test_masked.json\") as f:\n",
    "    results = json.load(f)\n",
    "\n",
    "def build_row(img_inst):\n",
    "    new_inst = copy.deepcopy(img_inst)\n",
    "    new_inst[\"tokens_positive_eval\"] = None\n",
    "    new_inst[\"orig_tokens_positive_eval\"] = None\n",
    "    new_inst[\"tokens_negative\"] = None\n",
    "    return new_inst\n",
    "\n",
    "new_imgs = [build_row(x) for x in results[\"images\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>height</th>\n",
       "      <th>width</th>\n",
       "      <th>id</th>\n",
       "      <th>caption</th>\n",
       "      <th>dataset_name</th>\n",
       "      <th>tokens_negative</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>original_img_id</th>\n",
       "      <th>tokens_positive_eval</th>\n",
       "      <th>...</th>\n",
       "      <th>orig_id</th>\n",
       "      <th>orig_sentence_id</th>\n",
       "      <th>orig_tokens_positive_eval</th>\n",
       "      <th>pred_word</th>\n",
       "      <th>masked_caption</th>\n",
       "      <th>masked_word</th>\n",
       "      <th>score_raw</th>\n",
       "      <th>score_scaled</th>\n",
       "      <th>score_k</th>\n",
       "      <th>tokens_positive_eval_idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1016887272.jpg</td>\n",
       "      <td>500</td>\n",
       "      <td>333</td>\n",
       "      <td>0</td>\n",
       "      <td>Several climbers in a row are climbing the roc...</td>\n",
       "      <td>flickr</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>1016887272</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>153901</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>ropes</td>\n",
       "      <td>Several climbers in a row are climbing the roc...</td>\n",
       "      <td>line</td>\n",
       "      <td>0.060851</td>\n",
       "      <td>0.122919</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1016887272.jpg</td>\n",
       "      <td>500</td>\n",
       "      <td>333</td>\n",
       "      <td>1</td>\n",
       "      <td>Several climbers in a row are climbing the roc...</td>\n",
       "      <td>flickr</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>1016887272</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>153901</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>reins</td>\n",
       "      <td>Several climbers in a row are climbing the roc...</td>\n",
       "      <td>line</td>\n",
       "      <td>0.009116</td>\n",
       "      <td>0.018414</td>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7162685234.jpg</td>\n",
       "      <td>333</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>Two children playing on the swings in the sand...</td>\n",
       "      <td>flickr</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>7162685234</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>153906</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>swings</td>\n",
       "      <td>Two children playing on the &lt;mask&gt; in the sand...</td>\n",
       "      <td>beach</td>\n",
       "      <td>0.078458</td>\n",
       "      <td>0.256047</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7162685234.jpg</td>\n",
       "      <td>333</td>\n",
       "      <td>500</td>\n",
       "      <td>3</td>\n",
       "      <td>Two children playing on the bikes in the sand ...</td>\n",
       "      <td>flickr</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>7162685234</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>153906</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>bikes</td>\n",
       "      <td>Two children playing on the &lt;mask&gt; in the sand...</td>\n",
       "      <td>beach</td>\n",
       "      <td>0.038087</td>\n",
       "      <td>0.124297</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7162685234.jpg</td>\n",
       "      <td>333</td>\n",
       "      <td>500</td>\n",
       "      <td>4</td>\n",
       "      <td>Two children playing on the rocks in the sand ...</td>\n",
       "      <td>flickr</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "      <td>7162685234</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>153906</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>rocks</td>\n",
       "      <td>Two children playing on the &lt;mask&gt; in the sand...</td>\n",
       "      <td>beach</td>\n",
       "      <td>0.014063</td>\n",
       "      <td>0.045894</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7888</th>\n",
       "      <td>1015118661.jpg</td>\n",
       "      <td>375</td>\n",
       "      <td>500</td>\n",
       "      <td>7888</td>\n",
       "      <td>A little boy is standing on the sidelines whil...</td>\n",
       "      <td>flickr</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>1015118661</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>158866</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>sidelines</td>\n",
       "      <td>A little boy is standing on the &lt;mask&gt; while a...</td>\n",
       "      <td>street</td>\n",
       "      <td>0.015828</td>\n",
       "      <td>0.054147</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7889</th>\n",
       "      <td>1015118661.jpg</td>\n",
       "      <td>375</td>\n",
       "      <td>500</td>\n",
       "      <td>7889</td>\n",
       "      <td>A little boy is standing on the street while a...</td>\n",
       "      <td>flickr</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "      <td>1015118661</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>158866</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>Brooklyn</td>\n",
       "      <td>A little boy is standing on the street while a...</td>\n",
       "      <td>overalls</td>\n",
       "      <td>0.035121</td>\n",
       "      <td>0.502833</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7890</th>\n",
       "      <td>1015118661.jpg</td>\n",
       "      <td>375</td>\n",
       "      <td>500</td>\n",
       "      <td>7890</td>\n",
       "      <td>A little boy is standing on the street while a...</td>\n",
       "      <td>flickr</td>\n",
       "      <td>None</td>\n",
       "      <td>3</td>\n",
       "      <td>1015118661</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>158866</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>France</td>\n",
       "      <td>A little boy is standing on the street while a...</td>\n",
       "      <td>overalls</td>\n",
       "      <td>0.018215</td>\n",
       "      <td>0.260784</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7891</th>\n",
       "      <td>1015118661.jpg</td>\n",
       "      <td>375</td>\n",
       "      <td>500</td>\n",
       "      <td>7891</td>\n",
       "      <td>A little boy is standing on the street while a...</td>\n",
       "      <td>flickr</td>\n",
       "      <td>None</td>\n",
       "      <td>4</td>\n",
       "      <td>1015118661</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>158866</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>Bangladesh</td>\n",
       "      <td>A little boy is standing on the street while a...</td>\n",
       "      <td>overalls</td>\n",
       "      <td>0.015795</td>\n",
       "      <td>0.226137</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7892</th>\n",
       "      <td>1015118661.jpg</td>\n",
       "      <td>375</td>\n",
       "      <td>500</td>\n",
       "      <td>7892</td>\n",
       "      <td>A young boy runs aross the tracks.</td>\n",
       "      <td>flickr</td>\n",
       "      <td>None</td>\n",
       "      <td>5</td>\n",
       "      <td>1015118661</td>\n",
       "      <td>None</td>\n",
       "      <td>...</td>\n",
       "      <td>158869</td>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>tracks</td>\n",
       "      <td>A young boy runs aross the &lt;mask&gt; .</td>\n",
       "      <td>street</td>\n",
       "      <td>0.018413</td>\n",
       "      <td>0.061118</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7893 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           file_name height width    id  \\\n",
       "0     1016887272.jpg    500   333     0   \n",
       "1     1016887272.jpg    500   333     1   \n",
       "2     7162685234.jpg    333   500     2   \n",
       "3     7162685234.jpg    333   500     3   \n",
       "4     7162685234.jpg    333   500     4   \n",
       "...              ...    ...   ...   ...   \n",
       "7888  1015118661.jpg    375   500  7888   \n",
       "7889  1015118661.jpg    375   500  7889   \n",
       "7890  1015118661.jpg    375   500  7890   \n",
       "7891  1015118661.jpg    375   500  7891   \n",
       "7892  1015118661.jpg    375   500  7892   \n",
       "\n",
       "                                                caption dataset_name  \\\n",
       "0     Several climbers in a row are climbing the roc...       flickr   \n",
       "1     Several climbers in a row are climbing the roc...       flickr   \n",
       "2     Two children playing on the swings in the sand...       flickr   \n",
       "3     Two children playing on the bikes in the sand ...       flickr   \n",
       "4     Two children playing on the rocks in the sand ...       flickr   \n",
       "...                                                 ...          ...   \n",
       "7888  A little boy is standing on the sidelines whil...       flickr   \n",
       "7889  A little boy is standing on the street while a...       flickr   \n",
       "7890  A little boy is standing on the street while a...       flickr   \n",
       "7891  A little boy is standing on the street while a...       flickr   \n",
       "7892                 A young boy runs aross the tracks.       flickr   \n",
       "\n",
       "     tokens_negative  sentence_id  original_img_id tokens_positive_eval  ...  \\\n",
       "0               None            0       1016887272                 None  ...   \n",
       "1               None            1       1016887272                 None  ...   \n",
       "2               None            0       7162685234                 None  ...   \n",
       "3               None            1       7162685234                 None  ...   \n",
       "4               None            2       7162685234                 None  ...   \n",
       "...              ...          ...              ...                  ...  ...   \n",
       "7888            None            1       1015118661                 None  ...   \n",
       "7889            None            2       1015118661                 None  ...   \n",
       "7890            None            3       1015118661                 None  ...   \n",
       "7891            None            4       1015118661                 None  ...   \n",
       "7892            None            5       1015118661                 None  ...   \n",
       "\n",
       "     orig_id  orig_sentence_id  orig_tokens_positive_eval   pred_word  \\\n",
       "0     153901                 0                       None       ropes   \n",
       "1     153901                 0                       None       reins   \n",
       "2     153906                 0                       None      swings   \n",
       "3     153906                 0                       None       bikes   \n",
       "4     153906                 0                       None       rocks   \n",
       "...      ...               ...                        ...         ...   \n",
       "7888  158866                 1                       None   sidelines   \n",
       "7889  158866                 1                       None    Brooklyn   \n",
       "7890  158866                 1                       None      France   \n",
       "7891  158866                 1                       None  Bangladesh   \n",
       "7892  158869                 4                       None      tracks   \n",
       "\n",
       "                                         masked_caption masked_word score_raw  \\\n",
       "0     Several climbers in a row are climbing the roc...        line  0.060851   \n",
       "1     Several climbers in a row are climbing the roc...        line  0.009116   \n",
       "2     Two children playing on the <mask> in the sand...       beach  0.078458   \n",
       "3     Two children playing on the <mask> in the sand...       beach  0.038087   \n",
       "4     Two children playing on the <mask> in the sand...       beach  0.014063   \n",
       "...                                                 ...         ...       ...   \n",
       "7888  A little boy is standing on the <mask> while a...      street  0.015828   \n",
       "7889  A little boy is standing on the street while a...    overalls  0.035121   \n",
       "7890  A little boy is standing on the street while a...    overalls  0.018215   \n",
       "7891  A little boy is standing on the street while a...    overalls  0.015795   \n",
       "7892                A young boy runs aross the <mask> .      street  0.018413   \n",
       "\n",
       "      score_scaled  score_k  tokens_positive_eval_idx  \n",
       "0         0.122919        3                         4  \n",
       "1         0.018414        7                         4  \n",
       "2         0.256047        2                         1  \n",
       "3         0.124297        3                         1  \n",
       "4         0.045894        9                         1  \n",
       "...            ...      ...                       ...  \n",
       "7888      0.054147        8                         1  \n",
       "7889      0.502833        1                         3  \n",
       "7890      0.260784        8                         3  \n",
       "7891      0.226137        9                         3  \n",
       "7892      0.061118        7                         1  \n",
       "\n",
       "[7893 rows x 21 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df = pd.DataFrame.from_dict(new_imgs)\n",
    "results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEVCAYAAAAM3jVmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZEUlEQVR4nO3de5xcZX3H8c/XcIsgNwPbkASCGq1cBCXS1Eu7FVvCRcG+RFMRIqWNWmptTatBrYIaTS9QCoq+olICImlUbCIXFaPjpVxiUCEERCIJSUxKJAhkU4ps+PWP8ywchpmds5vZ2cvzfb9e89rZ55zznOc3M/nOc86ZnSgiMDOzPDxnuAdgZmad49A3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ3+UkPQ5Sf/Ypr4OltQjaVz6vSbpL9rRd+rvBkmz29XfAPb7CUkPSvqfTu97LJP0Jkkb0mvm5ZJWS+pusm63pI2dHaENhEN/BJC0TtJjkrZJeljSTZLeJemp5yci3hURH6/Y1+v7Wyci1kfEXhGxow1jP0/Sl+r6PyEiFu1s3wMcxxRgLnBYRPxOg+Xdkp5MwbVN0j2SzqrYd3nbHkm/knR+3TohaXtpnR5J70/LzpP0RGrre35/X9JrS+tuT32Utz94APVPlPRFSZtTfT+XdL6kPav20Y9/Bf46vWZ+GhGHR0StDf3aMHDojxxviIjnAYcAC4APAF9s904k7dLuPkeIQ4CtEbGln3U2RcRewN7A3wGfl/SSiv1vSqG3F/Aa4GxJp9atc1TfOun2z6Vl/5m2nQB8D/hKRPyw1Ofhab19S9uvrzIwSfsDNwPjgd9Pr6M/BvYFXlixvv4cAqxuQz82Ajj0R5iIeCQilgFvBWZLOgJA0uWSPpHuT5B0bZo1PiTph5KeI+lK4GDgG30zTUlT0wzybEnrge+W2spvAC+UtELSI5KWpiBpeLjedzQhaSbwQeCtaX+3p+VPnS5K4/qwpPslbZF0haR90rK+ccyWtD6dmvlQs8dG0j5p+1+n/j6c+n89cCNwUBrH5S0e44iI64GHgJelvneXdJGkTel2kaTdm2y/FrgJOKy//TTZthe4Cpgk6YB+aj1L0t1p1n6fpHf20+37gG3A2yNiXdrPhoh4b0Tckfp7laQfp+f3x5JeVdpXTdLHJf132t+302tsd0k9wDjgdkm/TOs/dTQpaXx6bf5G0l3AK+vqOEjS19JztlbS35SWnSdpSXpOt6k4bTS9tHyKpGvStlslfbq07M/T4/MbSd+SdEjLB98Ah/6IFRErgI3AaxssnpuWHQB0UQRvRMQZwHqKo4b6meYfAi8Fjm+yyzOBPwcOAnqBiyuM8ZvAJ0mz2Ig4qsFq70i3PwJeAOwFfLpundcALwGOAz4i6aVNdnkJsE/q5w/TmM+KiO8AJ/D0bPwd/Y07vVG8kWLWvSY1fwiYARwNHAUcC3y4yfbTgFcDt/S3nybb7pbGvRX4TT+rbgFOpjgqOQv4N0mvaLLu64FrIuLJJvvcH7iO4jl9PnAhcJ2k55dWe1vaz4HAbsDfR8Tj6SgEiqOYRkcNH6U4mnghxWvrqWs5Kk5PfgO4HZhE8fz+raTya/CNwGKKo5JlpNeGiutN1wL3A1PT9ovTslMpXvN/SvFv4IfA1U0eG6vj0B/ZNgH7N2h/ApgIHBIRT6TTBK2+ROm8iNgeEY81WX5lRNwZEduBfwTekv7h7azTgQsj4r6I6AHOBWbVHWWcHxGPRcTtFAHxrDePNJa3AudGxLY0o70AOGMAYzlI0sPAY8DXgfdFxE9L4/xYRGyJiF8D59f1fVA6snoU+AVwK/Cjuv5/ktbpu5XD7S2lff8l8OY0628oIq6LiF+mo5LvA9+m8QQAiiDf3E/dJwH3RsSVEdEbEVcDPwfeUFrnPyLiF+n1sYTiza+KtwDzI+KhiNjAMycLrwQOiIiPRcRvI+I+4PPArNI6P4qI69P1pSt5+rk/lmIC8g/pdft/EdH3eL8T+FRE3J0ew08CR3u2X41Df2SbRHEKot6/UMxQv50O/edV6GvDAJbfD+xKMRPeWQel/sp970JxhNKn/Gmb/6U4Gqg3gWIGWt/XpAGMZVNE7Esxe74YeF2LcR5Uv21E7E0xK30MqL9Y/Yq0Tt/tW6VlS9K+u4A7gWP6G6ikEyTdouL03cPAiTR/PrZSTAKaqa8Nnv3YVXkOmvVd/9rpcwhPv1k+nOr4IP0/93ukCcEU4P4mb4yHAP9e6vMhQAzstZAth/4IJemVFC/i+tkkaaY7NyJeQDFbe5+k4/oWN+my1ZHAlNL9gymOJh4EtgPPLY1rHMUhddV+N1H8Iy333Qs80GK7eg+mMdX39asB9kNEPE5xofxIPX0xttE4NzXZ/hHgyzxzplx13w9SzFTPk9QwqNO1hK9RfGqmK71ZXE8RbI18B3iTSp/2qlNfGwzysWtgM89+7fTZAKyteyN8XkScWKHfDcDBavzBgw3AO+v6HR8RNw2+jHw49EcYSXtLOpni/OWXImJVg3VOlvQiSQIeBXakGxRh+oJB7Prtkg6T9FzgY8BX0yH3LyhmXydJ2pXiPHf5AucDwNR+Audq4O8kHSppL56+BtD01EYjaSxLgPmSnpcO5d8HfKn/LZv291uK00MfKY3zw5IOkDQhtTfsO9Uxi0F+oiUifg58C3h/k1V2o3iMfw30SjoB+JN+uryQ4uhlUd8pDkmTJF0o6WUUbxgvlvQ2SbtIeivFRehrBzP+OkuAcyXtJ2ky8J7SshXAo5I+kC74jpN0RJrQtLKC4g1lgaQ9Je0h6dVp2efSPg9Pte4j6bQ21JIFh/7I8Q1J2yhmMR+i+Ifc7HPk0yhmdz0UH9W7tPS56U9RhNfDkv5+APu/Eric4nB7D+Bv4KlZ7V8BX6CYGW6nuIjc5yvp51ZJP2nQ72Wp7x8Aa4H/45nBMBDvSfu/j+II6Mup/8G6jGI2+QbgE8BK4A5gFfCT1Nan75NBPRSnMPanuA5Qdrue+Tn7i/rZ978AcyQdWL8gIrZRPP5LKC72vo3iImdDEfEQ8CqKI6Fb0+toOfAIsCYitlJcFJ5LcSro/cDJ6ahjZ51P8XispbjucGVpXDsojoaOTssfpHgd7dOq09K2L6L4cMJGims6RMTXgX8CFqdrLHdSXMi3CtT6+p+ZmY0VnumbmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWWk0X86PKJMmDAhpk6dOqhtt2/fzp577tneAY1wrjkPudWcW72w8zXfdtttD0bEAfXtIz70p06dysqVKwe1ba1Wo7u7u70DGuFccx5yqzm3emHna5Z0f6N2n94xM8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8vIiP+L3J2x6leP8I551zVdvm7BSR0cjZnZ8PNM38wsIw59M7OMOPTNzDLi0Dczy4hD38wsIw59M7OMOPTNzDLi0Dczy4hD38wsIw59M7OMOPTNzDLi0Dczy4hD38wsIw59M7OMOPTNzDJSOfQljZP0U0nXpt/3l3SjpHvTz/1K654raY2keyQdX2o/RtKqtOxiSWpvOWZm1p+BzPTfC9xd+n0esDwipgHL0+9IOgyYBRwOzAQulTQubfNZYA4wLd1m7tTozcxsQCqFvqTJwEnAF0rNpwCL0v1FwKml9sUR8XhErAXWAMdKmgjsHRE3R0QAV5S2MTOzDqg6078IeD/wZKmtKyI2A6SfB6b2ScCG0nobU9ukdL++3czMOqTl/5Er6WRgS0TcJqm7Qp+NztNHP+2N9jmH4jQQXV1d1Gq1Crt9tq7xMPfI3qbLB9vvSNbT0zMm6+qPax77cqsXhq7mKv8x+quBN0o6EdgD2FvSl4AHJE2MiM3p1M2WtP5GYEpp+8nAptQ+uUH7s0TEQmAhwPTp06O7u7t6RSWXXLWUC1Y1L3Hd6YPrdySr1WoM9vEarVzz2JdbvTB0Nbc8vRMR50bE5IiYSnGB9rsR8XZgGTA7rTYbWJruLwNmSdpd0qEUF2xXpFNA2yTNSJ/aObO0jZmZdUCVmX4zC4Alks4G1gOnAUTEaklLgLuAXuCciNiRtnk3cDkwHrgh3czMrEMGFPoRUQNq6f5W4Lgm680H5jdoXwkcMdBBmplZe/gvcs3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLSMvQl7SHpBWSbpe0WtL5qX1/STdKujf93K+0zbmS1ki6R9LxpfZjJK1Kyy6WpKEpy8zMGqky038ceF1EHAUcDcyUNAOYByyPiGnA8vQ7kg4DZgGHAzOBSyWNS319FpgDTEu3me0rxczMWmkZ+lHoSb/umm4BnAIsSu2LgFPT/VOAxRHxeESsBdYAx0qaCOwdETdHRABXlLYxM7MO2KXKSmmmfhvwIuAzEXGrpK6I2AwQEZslHZhWnwTcUtp8Y2p7It2vb2+0vzkURwR0dXVRq9UqF1TWNR7mHtnbdPlg+x3Jenp6xmRd/XHNY19u9cLQ1Vwp9CNiB3C0pH2Br0s6op/VG52nj37aG+1vIbAQYPr06dHd3V1lmM9yyVVLuWBV8xLXnT64fkeyWq3GYB+v0co1j3251QtDV/OAPr0TEQ8DNYpz8Q+kUzakn1vSahuBKaXNJgObUvvkBu1mZtYhVT69c0Ca4SNpPPB64OfAMmB2Wm02sDTdXwbMkrS7pEMpLtiuSKeCtkmakT61c2ZpGzMz64Aqp3cmAovSef3nAEsi4lpJNwNLJJ0NrAdOA4iI1ZKWAHcBvcA56fQQwLuBy4HxwA3pZmZmHdIy9CPiDuDlDdq3Asc12WY+ML9B+0qgv+sBZmY2hPwXuWZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llpGXoS5oi6XuS7pa0WtJ7U/v+km6UdG/6uV9pm3MlrZF0j6TjS+3HSFqVll0sSUNTlpmZNVJlpt8LzI2IlwIzgHMkHQbMA5ZHxDRgefqdtGwWcDgwE7hU0rjU12eBOcC0dJvZxlrMzKyFlqEfEZsj4ifp/jbgbmAScAqwKK22CDg13T8FWBwRj0fEWmANcKykicDeEXFzRARwRWkbMzPrgF0GsrKkqcDLgVuBrojYDMUbg6QD02qTgFtKm21MbU+k+/XtjfYzh+KIgK6uLmq12kCG+ZSu8TD3yN6mywfb70jW09MzJuvqj2se+3KrF4au5sqhL2kv4GvA30bEo/2cjm+0IPppf3ZjxEJgIcD06dOju7u76jCf4ZKrlnLBquYlrjt9cP2OZLVajcE+XqOVax77cqsXhq7mSp/ekbQrReBfFRHXpOYH0ikb0s8tqX0jMKW0+WRgU2qf3KDdzMw6pMqndwR8Ebg7Ii4sLVoGzE73ZwNLS+2zJO0u6VCKC7Yr0qmgbZJmpD7PLG1jZmYdUOX0zquBM4BVkn6W2j4ILACWSDobWA+cBhARqyUtAe6i+OTPORGxI233buByYDxwQ7qZmVmHtAz9iPgRjc/HAxzXZJv5wPwG7SuBIwYyQDMzax//Ra6ZWUYc+mZmGXHom5llxKFvZpaRAf1F7lgzdd51/S5ft+CkDo3EzKwzPNM3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8tIy9CXdJmkLZLuLLXtL+lGSfemn/uVlp0raY2keyQdX2o/RtKqtOxiSWp/OWZm1p8qM/3LgZl1bfOA5RExDViefkfSYcAs4PC0zaWSxqVtPgvMAaalW32fZmY2xFqGfkT8AHiorvkUYFG6vwg4tdS+OCIej4i1wBrgWEkTgb0j4uaICOCK0jZmZtYhuwxyu66I2AwQEZslHZjaJwG3lNbbmNqeSPfr2xuSNIfiqICuri5qtdrgBjke5h7ZO6htgUHvdzj19PSMynHvDNc89uVWLwxdzYMN/WYanaePftobioiFwEKA6dOnR3d396AGc8lVS7lg1eBLXHf64PY7nGq1GoN9vEYr1zz25VYvDF3Ng/30zgPplA3p55bUvhGYUlpvMrAptU9u0G5mZh002NBfBsxO92cDS0vtsyTtLulQigu2K9KpoG2SZqRP7ZxZ2sbMzDqk5bkPSVcD3cAESRuBjwILgCWSzgbWA6cBRMRqSUuAu4Be4JyI2JG6ejfFJ4HGAzekm5mZdVDL0I+IP2uy6Lgm688H5jdoXwkcMaDRmZlZW/kvcs3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMtLu/zlrTJk677qmy9YtOKmDIzEzaw/P9M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuJv2Ryk/r6BE/wtnGY2Mnmmb2aWEYe+mVlGHPpmZhlx6JuZZcQXcoeIL/Sa2Ujkmb6ZWUYc+mZmGfHpnWHi0z9mNhw80zczy0jHZ/qSZgL/DowDvhARCzo9htHARwJmNhQ6GvqSxgGfAf4Y2Aj8WNKyiLirk+MYC5q9Kcw9spd3tHjDaMVvKGZjV6dn+scCayLiPgBJi4FTAIf+CNLqKKMVv2mYjVydDv1JwIbS7xuB3+vwGGyI7eybxkC14+imilZvZjt7Sm4gj9tAax7KN2KfihxdFBGd25l0GnB8RPxF+v0M4NiIeE/denOAOenXlwD3DHKXE4AHB7ntaOWa85BbzbnVCztf8yERcUB9Y6dn+huBKaXfJwOb6leKiIXAwp3dmaSVETF9Z/sZTVxzHnKrObd6Yehq7vRHNn8MTJN0qKTdgFnAsg6PwcwsWx2d6UdEr6S/Br5F8ZHNyyJidSfHYGaWs45/Tj8irgeu79DudvoU0SjkmvOQW8251QtDVHNHL+Samdnw8tcwmJllZEyEvqSZku6RtEbSvAbLJenitPwOSa8YjnG2S4V6T0913iHpJklHDcc426lVzaX1Xilph6Q3d3J8Q6FKzZK6Jf1M0mpJ3+/0GNutwmt7H0nfkHR7qvms4Rhnu0i6TNIWSXc2Wd7+7IqIUX2juCD8S+AFwG7A7cBhdeucCNwACJgB3Drc4x7iel8F7JfunzCa661ac2m971JcM3rzcI+7A8/zvhR/zX5w+v3A4R53B2r+IPBP6f4BwEPAbsM99p2o+Q+AVwB3Nlne9uwaCzP9p77aISJ+C/R9tUPZKcAVUbgF2FfSxE4PtE1a1hsRN0XEb9Kvt1D8PcRoVuU5BngP8DVgSycHN0Sq1Pw24JqIWA8QEaO97io1B/A8SQL2ogj93s4Os30i4gcUNTTT9uwaC6Hf6KsdJg1indFioLWcTTFTGM1a1ixpEvAm4HMdHNdQqvI8vxjYT1JN0m2SzuzY6IZGlZo/DbyU4o86VwHvjYgnOzO8YdH27BoL/4mKGrTVfySpyjqjReVaJP0RRei/ZkhHNPSq1HwR8IGI2FFMAke9KjXvAhwDHAeMB26WdEtE/GKoBzdEqtR8PPAz4HXAC4EbJf0wIh4d4rENl7Zn11gI/Spf7VDp6x9GiUq1SHoZ8AXghIjY2qGxDZUqNU8HFqfAnwCcKKk3Iv6rIyNsv6qv6wcjYjuwXdIPgKOA0Rr6VWo+C1gQxQnvNZLWAr8LrOjMEDuu7dk1Fk7vVPlqh2XAmelK+AzgkYjY3OmBtknLeiUdDFwDnDGKZ31lLWuOiEMjYmpETAW+CvzVKA58qPa6Xgq8VtIukp5L8Y21d3d4nO1Upeb1FEc2SOqi+ELG+zo6ys5qe3aN+pl+NPlqB0nvSss/R/FpjhOBNcD/UswWRqWK9X4EeD5waZr59sYo/rKqijWPKVVqjoi7JX0TuAN4kuJ/omv40b/RoOLz/HHgckmrKE59fCAiRu23b0q6GugGJkjaCHwU2BWGLrv8F7lmZhkZC6d3zMysIoe+mVlGHPpmZhlx6JuZZcShb2aWEYe+mVlGHPpmZhlx6JuZZeT/AVnGWmAtwjT1AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# score_raw\tscore_scaled\tscore_k\n",
    "import pylab as pl\n",
    "axes = results_df[\"score_raw\"].hist(bins=np.linspace(0, 1, 40))\n",
    "_ = pl.suptitle(\"Distribution of RoBERTa Confidence\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEVCAYAAAAM3jVmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAfRklEQVR4nO3de5hcVZnv8e/PRBAIhEuwT0iABAVHApJn0iIe0ekIHgLqoI5okMPdCeBlxmNmBJQjjIjihXEEBCYKA0ggIohBBRFxWrwQMXEi4aoBAjSJiZAQaEA04Z0/9mpmU1R1V1dVV6d7/T7PU09XrbX32uvdtfuttdfeXa2IwMzM8vCy4e6AmZm1j5O+mVlGnPTNzDLipG9mlhEnfTOzjDjpm5llxEm/xSRdJOn/t6itXST1ShqTXndL+mAr2k7t3Sjp6Fa1N4jtflbSY5L+0MZtNrzvWr3frVA+DiqP9SrLniHpinb3cTRy0h8ESSskPSvpKUlPSPqlpBMlvbAfI+LEiDizzrYO7G+ZiHg4IsZFxMYW9P0lvzQRcXBEXNZs24Psx87AXGDPiPhfNZb5pKQHUxLokfStdvZxMNJ+/Uvqa98x8cYG1u2VdI+kvyvVd0l6vlTf93hjqu+W9KdU9pik70iamPZf37J/krSx9PquQca3r6QbUmxrJd0u6djB7aWq7b7oOGjlsW79c9IfvHdGxNbArsDZwMnAxa3eiKSxrW5zE7Er8HhErKlWmc48jgQOjIhxQCdwSxv714hvpb5OAP4T+PZg103rfwy4QlJHqX5lX33pcVup/iNp3VcD44AvR8TnSm2eCNxWWndavR1LHy4/AX6a2t8BOAk4eBDx1dLvcWBDx0m/QRGxPiKuB94PHC1pLwBJl0r6bHo+QdL3S6Okn0l6maRvArsA30ujr09ImiIpJB0v6WHgJ6Wy8gfAq9Joa72khZK2T9vqktRT7mPf2YSkWcAngfen7f021b8wbZH6dZqkhyStkXS5pPGprq8fR0t6OI0qP1Vr30gan9b/Y2rvtNT+gcDNwE6pH5dWWf31wE0RcX/az3+IiHmltreX9B+SVkpaJ+m7qXy7tK//mMq/L2lyP308Lo2s10m6SdKupbq3Sbo37ePzAdVqpywiNgDzgUmSdkxt7STp+vT+L5f09/2sfxPwFPCqerZXse4TwHeB6f0tJ+mrkh6R9KSkJZLe3M/iXwIui4gvRMRjUVgSEe8rtff3Ka61Kc6dSnWh4kz492k/f02FlxwHlce6pKmSfqrirPpmig/Uchz7qTirekLSbyV1leq6JZ0p6Rdp/R9JmlCq37+07iOSjknlm0v6cjrGV6uYqt2i3x0/AjnpNykibgd6gGq/PHNT3Y5AB0XijYg4EniY4qxhXER8sbTO3wCvBQ6qscmjgOOAnYANwLl19PGHwOf4n1HlPlUWOyY9ZgK7UYwaz69YZn/gNcABwKclvbbGJs8Dxqd2/ib1+diI+DHFKLFv9HpMlXUXAUdJ+mdJnXrpHO83gS2BacArga+k8pcB/0ExgtwFeLZK/wGQ9C6K9+I9FO/Nz4CrUt0E4FrgNIpEcz/wphpxVra7WYr1cWBdKr6K4hjYCXgv8DlJB1RZV5LeDmwG3F3P9irW3yHFs3yARX9N8cGwPXAl8G1Jr6jS3pbAG4Fr+tnmW4HPA+8DJgIPAQsqFnsHxQf5Pmm5g+o8Dq4EllC8B2cCL1x7kjQJ+AHw2RTHPwHX9n3QJh8AjqU4RjZLyyBpF+BGimN0x7QvlqZ1vgDskcpeDUwCPl0r/hErIvyo8wGsoJh2qCxfBHwqPb8U+Gx6/hlgIfDqgdoCpgAB7FalbGx63Q2cXarfE/gzMAboAnpqbQM4A7iior4b+GB6fgvwoVLda4C/AGNL/Zhcqr8dmF0lrjHAcxRztX1lJwDd6flL+lmljSOAHwNPUyTQU1L5ROB5YLs63qvpwLoasd4IHF+qexnwDMUHxlHAolKdKJL2B2ts54z0HjwBbEz97Up1O6eyrUvLfx64tMq6z6RlP1FativF+0TFY6tSTM8A69P7sxTYpaJ/xwA/72c/rQP2qVI+KbX5V/2sezHwxdLrcemYmZJeB7B/qf7q0nv5ouOgdIyNpfjQ3tAXZ6q/knT8UkypfrOiLzcBR5f2y2mlug8BP0zPTwWuqxKLKI63V5XK3gg8WE9uGEkPj/RbYxKwtkr5lyhGXj+S9ICkU+po65FB1D8EvJyKU98G7ZTaK7c9luIMpU/5bptnKH7JK02gGFlVtjWp3o5ExPyIOBDYlmJO+jOSDqJIomsjYl3lOpK2lPTvaTrpSeBWYNsqZwpQJPevptP7JyjeO6U+7kRpH0fx2z/Qe3J1RGxLsa/uBGak8p1Sf58qLVu5L66OiG0jYkuKaZ2jJJ1Qql+Z6suPp0v1/xAR44HXAdsBNae0ACTNTdNa61Ps46l+/Kyj+MCZ2E9zLzpmIqKX4kOvHF89x0y1dtdVxFk+nnYFDut7/1Ic+1f0tdZ2d6Y4e6u0I8UZ5JJSmz9M5aOKk36TJL2e4iD/eWVdRDwVEXMjYjfgncDHS6f2tb7edKCvPd259HwXipHVYxSjlC1L/RrDiw/YgdpdSfHLVG57A7B6gPUqPZb6VNnWo4Nsh4j4S0R8G7gD2Isi+W4vadsqi8+lODt5Q0RsA7wllVebj38EOKEikW4REb8EVlHax5LEi/d5f/19jOKs5gxJEyn26faSti4tVnNfRMQKirOQd9azvYp1l1FMd3wt9fkl0vz9yRTTLNulD6r1VNlHEfEMcBvwd5V1JS86ZiRtRXGxd9DvdYVVwHapvT67lJ4/QjHSL79/W0XE2XW0/QjVr5k8RjElOK3U5vgoLoaPKk76DZK0jaR3UMxhXpF+6SqXeYekV6dfwicpTt/7bklbTTHnPVj/V9Keac71M8A1Udzm9jvgFZLeLunlFHPSm5fWWw1MUen20gpXAf8vXUAbx/9cA9gwmM6lvlwNnCVpaxUXSD8O1HWPtaRjUgxbq7j4ezDF/P2vImIVRVK8QMWF25dL6kvuW1P80j6h4uL26f1s5iLgVEnT0jbHSzos1f0AmCbpPemi4j8AVW8trRH/vRRTDZ+IiEeAXwKfl/QKSa8Djqe42Fst9snALGBQt1WWXEYxh/23Neq3pvgg/yMwVtKngW36ae8TwDHp+soOqY/7SOqbt78SOFbSdEmbUxwzv0ofXg2LiIeAxcC/SNpM0v68+IPwCuCdkg6SNCbt2y71c+G+ZD5woKT3SRoraQdJ0yPieeDrwFckvTLFOimdYY4qTvqD9z1JT1GMGD4F/CvFBaNqdqeYm+6lGDVdEBHdqe7zwGnpVPKfBrH9b1JcN/gD8AqKpERErKeYu/wGxUjraYq56D59txE+Luk3Vdq9JLV9K/Ag8Cfgo4PoV9lH0/YfoDgDujK1X48nKS6yPkwxf/1F4KSI6DuTOpLiTOJeYA3FbY4A/wZsQTFiW0Rxal5VRFxHcdFuQZoKupN0G2IarR9GcTvu4xTv4S/q7HufLwFzUvI4nGK+eiVwHXB6RNxcWrbvjqpeiousvwD+pVTfd4dL+VF19B0Rf6a4sF/rjwNvovjQ/B3FdMmf6GfqKp35vDU9HpC0FpgH3JDqb0nbupZidP4qYHat9gbpA8AbKKbeTgcuL/XrEeBQiuPkjymGf6aOfBYRDwOHUJwZrqW4DtJ3Y8PJFNOxi9Jx8WOKs8dRRemChZmZZcAjfTOzjDjpm5llxEnfzCwjTvpmZhlx0jczy4iTvplZRpz0zcwy4qRvZpYRJ30zs4w46ZuZZcRJ38wsI076ZmYZcdI3M8uIk76ZWUac9M3MMuKkb2aWESd9M7OMjB3uDgxkwoQJMWXKlIbWffrpp9lqq60GXnAUccx5yC3m3OKF5mNesmTJYxGxY2X5Jp/0p0yZwuLFixtat7u7m66urtZ2aBPnmPOQW8y5xQvNxyzpoWrlnt4xM8uIk76ZWUac9M3MMuKkb2aWESd9M7OMOOmbmWXESd/MLCNO+mZmGXHSNzPLyCb/F7nNWPboeo455Qc161ec/fY29sbMbPh5pG9mlhEnfTOzjDjpm5llxEnfzCwjAyZ9SZdIWiPpzlLZtyQtTY8Vkpam8imSni3VXVRaZ4akZZKWSzpXkoYkIjMzq6meu3cuBc4HLu8riIj39z2XdA6wvrT8/RExvUo7FwJzgEXADcAs4MZB99jMzBo24Eg/Im4F1larS6P19wFX9deGpInANhFxW0QExQfIuwbdWzMza0qz9+m/GVgdEb8vlU2V9F/Ak8BpEfEzYBLQU1qmJ5VVJWkOxVkBHR0ddHd3N9S5ji1g7t4batY32u6mrLe3d1TG1R/HPPrlFi8MXczNJv3DefEofxWwS0Q8LmkG8F1J04Bq8/dRq9GImAfMA+js7IxG/2XYefMXcs6y2iGuOKKxdjdl/rdyecgt5tzihaGLueGkL2ks8B5gRl9ZRDwHPJeeL5F0P7AHxch+cmn1ycDKRrdtZmaNaeaWzQOBeyPihWkbSTtKGpOe7wbsDjwQEauApyTtl64DHAUsbGLbZmbWgHpu2bwKuA14jaQeScenqtm89ALuW4A7JP0WuAY4MSL6LgKfBHwDWA7cj+/cMTNruwGndyLi8Brlx1Qpuxa4tsbyi4G9Btk/MzNrIf9FrplZRpz0zcwy4qRvZpYRJ30zs4w46ZuZZcRJ38wsI076ZmYZcdI3M8uIk76ZWUac9M3MMuKkb2aWESd9M7OMOOmbmWXESd/MLCNO+mZmGXHSNzPLiJO+mVlGnPTNzDLipG9mlpF6/jH6JZLWSLqzVHaGpEclLU2PQ0p1p0paLuk+SQeVymdIWpbqzpWk1odjZmb9qWekfykwq0r5VyJienrcACBpT2A2MC2tc4GkMWn5C4E5wO7pUa1NMzMbQgMm/Yi4FVhbZ3uHAgsi4rmIeBBYDuwraSKwTUTcFhEBXA68q8E+m5lZg8Y2se5HJB0FLAbmRsQ6YBKwqLRMTyr7S3peWV6VpDkUZwV0dHTQ3d3dUAc7toC5e2+oWd9ou5uy3t7eURlXfxzz6JdbvDB0MTea9C8EzgQi/TwHOA6oNk8f/ZRXFRHzgHkAnZ2d0dXV1VAnz5u/kHOW1Q5xxRGNtbsp6+7uptH9NVI55tEvt3hh6GJu6O6diFgdERsj4nng68C+qaoH2Lm06GRgZSqfXKXczMzaqKGkn+bo+7wb6Luz53pgtqTNJU2luGB7e0SsAp6StF+6a+coYGET/TYzswYMOL0j6SqgC5ggqQc4HeiSNJ1iimYFcAJARNwl6WrgbmAD8OGI2JiaOoniTqAtgBvTw8zM2mjApB8Rh1cpvrif5c8CzqpSvhjYa1C9MzOzlvJf5JqZZcRJ38wsI076ZmYZcdI3M8uIk76ZWUac9M3MMuKkb2aWESd9M7OMOOmbmWXESd/MLCNO+mZmGXHSNzPLiJO+mVlGnPTNzDLipG9mlhEnfTOzjDjpm5llxEnfzCwjTvpmZhkZMOlLukTSGkl3lsq+JOleSXdIuk7Stql8iqRnJS1Nj4tK68yQtEzScknnStKQRGRmZjXVM9K/FJhVUXYzsFdEvA74HXBqqe7+iJieHieWyi8E5gC7p0dlm2ZmNsQGTPoRcSuwtqLsRxGxIb1cBEzurw1JE4FtIuK2iAjgcuBdDfXYzMwa1oo5/eOAG0uvp0r6L0k/lfTmVDYJ6Ckt05PKzMysjcY2s7KkTwEbgPmpaBWwS0Q8LmkG8F1J04Bq8/fRT7tzKKaC6OjooLu7u6H+dWwBc/feULO+0XY3Zb29vaMyrv445tEvt3hh6GJuOOlLOhp4B3BAmrIhIp4DnkvPl0i6H9iDYmRfngKaDKys1XZEzAPmAXR2dkZXV1dDfTxv/kLOWVY7xBVHNNbupqy7u5tG99dI5ZhHv9zihaGLuaHpHUmzgJOBv42IZ0rlO0oak57vRnHB9oGIWAU8JWm/dNfOUcDCpntvZmaDMuBIX9JVQBcwQVIPcDrF3TqbAzenOy8XpTt13gJ8RtIGYCNwYkT0XQQ+ieJOoC0orgGUrwOYmVkbDJj0I+LwKsUX11j2WuDaGnWLgb0G1TszM2sp/0WumVlGnPTNzDLipG9mlhEnfTOzjDjpm5llxEnfzCwjTvpmZhlx0jczy4iTvplZRpz0zcwy4qRvZpYRJ30zs4w46ZuZZcRJ38wsI076ZmYZcdI3M8uIk76ZWUac9M3MMuKkb2aWESd9M7OMDJj0JV0iaY2kO0tl20u6WdLv08/tSnWnSlou6T5JB5XKZ0halurOlaTWh2NmZv2pZ6R/KTCrouwU4JaI2B24Jb1G0p7AbGBaWucCSWPSOhcCc4Dd06OyTTMzG2IDJv2IuBVYW1F8KHBZen4Z8K5S+YKIeC4iHgSWA/tKmghsExG3RUQAl5fWMTOzNhnb4HodEbEKICJWSXplKp8ELCot15PK/pKeV5ZXJWkOxVkBHR0ddHd3N9bJLWDu3htq1jfa7qast7d3VMbVH8c8+uUWLwxdzI0m/VqqzdNHP+VVRcQ8YB5AZ2dndHV1NdSZ8+Yv5JxltUNccURj7W7Kuru7aXR/jVSOefTLLV4YupgbvXtndZqyIf1ck8p7gJ1Ly00GVqbyyVXKzcysjRpN+tcDR6fnRwMLS+WzJW0uaSrFBdvb01TQU5L2S3ftHFVax8zM2mTA6R1JVwFdwARJPcDpwNnA1ZKOBx4GDgOIiLskXQ3cDWwAPhwRG1NTJ1HcCbQFcGN6mJlZGw2Y9CPi8BpVB9RY/izgrCrli4G9BtU7MzNrKf9FrplZRpz0zcwy4qRvZpaRVt+nP6JMOeUH/davOPvtbeqJmVl7eKRvZpYRJ30zs4w46ZuZZcRJ38wsI076ZmYZcdI3M8uIk76ZWUac9M3MMuKkb2aWESd9M7OMOOmbmWXESd/MLCNO+mZmGXHSNzPLiJO+mVlGGk76kl4jaWnp8aSkj0k6Q9KjpfJDSuucKmm5pPskHdSaEMzMrF4N/xOViLgPmA4gaQzwKHAdcCzwlYj4cnl5SXsCs4FpwE7AjyXtEREbG+2DmZkNTqumdw4A7o+Ih/pZ5lBgQUQ8FxEPAsuBfVu0fTMzq0Orkv5s4KrS649IukPSJZK2S2WTgEdKy/SkMjMzaxNFRHMNSJsBK4FpEbFaUgfwGBDAmcDEiDhO0teA2yLiirTexcANEXFtlTbnAHMAOjo6ZixYsKChvq1Zu57Vzza0KgB7Txrf+MrDpLe3l3Hjxg13N9rKMY9+ucULzcc8c+bMJRHRWVnein+MfjDwm4hYDdD3E0DS14Hvp5c9wM6l9SZTfFi8RETMA+YBdHZ2RldXV0MdO2/+Qs5Z1niIK45obLvDqbu7m0b310jlmEe/3OKFoYu5FdM7h1Oa2pE0sVT3buDO9Px6YLakzSVNBXYHbm/B9s3MrE5NjfQlbQm8DTihVPxFSdMppndW9NVFxF2SrgbuBjYAH/adO2Zm7dVU0o+IZ4AdKsqO7Gf5s4CzmtmmmZk1zn+Ra2aWESd9M7OMtOLunVFryik/qFm34uy3t7EnZmat4ZG+mVlGnPTNzDLipG9mlhEnfTOzjDjpm5llxEnfzCwjTvpmZhlx0jczy4iTvplZRpz0zcwy4qRvZpYRJ30zs4w46ZuZZcRJ38wsI076ZmYZ8ffpN6i/79oHf9++mW2aPNI3M8tIU0lf0gpJyyQtlbQ4lW0v6WZJv08/tystf6qk5ZLuk3RQs503M7PBacVIf2ZETI+IzvT6FOCWiNgduCW9RtKewGxgGjALuEDSmBZs38zM6jQUc/qHAl3p+WVAN3ByKl8QEc8BD0paDuwL3DYEfTAz26QNdF3w0llbDcl2FRGNryw9CKwDAvj3iJgn6YmI2La0zLqI2E7S+cCiiLgilV8M3BgR11Rpdw4wB6Cjo2PGggULGurfmrXrWf1sQ6s2be9J44dlu729vYwbN25Ytj1cHPPoNxrjXfbo+n7rp44f01TMM2fOXFKagXlBsyP9N0XESkmvBG6WdG8/y6pKWdVPnIiYB8wD6OzsjK6uroY6d978hZyzbHhuUFpxRNewbLe7u5tG99dI5ZhHv9EY7zF1jPSHIuam5vQjYmX6uQa4jmK6ZrWkiQDp55q0eA+wc2n1ycDKZrZvZmaD03DSl7SVpK37ngP/B7gTuB44Oi12NLAwPb8emC1pc0lTgd2B2xvdvpmZDV4zcx8dwHWS+tq5MiJ+KOnXwNWSjgceBg4DiIi7JF0N3A1sAD4cERub6r2ZmQ1Kw0k/Ih4A9qlS/jhwQI11zgLOanSbZmbWHP9FrplZRpz0zcwy4qRvZpYRf8vmEPG3cJrZpsgjfTOzjDjpm5llxEnfzCwjTvpmZhlx0jczy4jv3hkmvrvHzIaDR/pmZhlx0jczy4iTvplZRpz0zcwy4qRvZpYRJ30zs4z4ls1NlG/pNLOh4JG+mVlGnPTNzDLScNKXtLOk/5R0j6S7JP1jKj9D0qOSlqbHIaV1TpW0XNJ9kg5qRQBmZla/Zub0NwBzI+I3krYGlki6OdV9JSK+XF5Y0p7AbGAasBPwY0l7RMTGJvpgZmaD0PBIPyJWRcRv0vOngHuASf2sciiwICKei4gHgeXAvo1u38zMBk8R0Xwj0hTgVmAv4OPAMcCTwGKKs4F1ks4HFkXEFWmdi4EbI+KaKu3NAeYAdHR0zFiwYEFD/Vqzdj2rn21o1U3e3pPGVy3v7e1l3Lhxbe7N8HLMo99ojHfZo+v7rZ86fkxTMc+cOXNJRHRWljd9y6akccC1wMci4klJFwJnApF+ngMcB6jK6lU/cSJiHjAPoLOzM7q6uhrq23nzF3LOstF5V+qKI7qqlnd3d9Po/hqpHPPoNxrjPWaA27IvnbXVkMTcVEaU9HKKhD8/Ir4DEBGrS/VfB76fXvYAO5dWnwysbGb7Oat1H//cvTfQ1d6umNkI0szdOwIuBu6JiH8tlU8sLfZu4M70/HpgtqTNJU0Fdgdub3T7ZmY2eM2M9N8EHAksk7Q0lX0SOFzSdIqpmxXACQARcZekq4G7Ke78+bDv3DEza6+Gk35E/Jzq8/Q39LPOWcBZjW7T6uOvcDCzWkbnVU7rlz8UzPLlr2EwM8uIk76ZWUY8vWMv4ekfs9HLI30zs4w46ZuZZcTTO9Zy/U0PeWrIbHg56dugDTTnb2abLk/vmJllxCN9ayvfGWQ2vDzSNzPLiEf6tklp5HrB3L03vPDd5D5TMOufk76NKp4+Muufp3fMzDLikb5lpZnbTX2WYKOBk75Zi3hqyUYCJ32zOjX7R2n+ULBNgef0zcwy4pG+2QhQeZZQvk21FZo5y/AZzMjS9qQvaRbwVWAM8I2IOLvdfTDbFA3ndxoN5bb9obBpaWvSlzQG+BrwNqAH+LWk6yPi7nb2w8w2HfV84PR3ZtPfh0azH2aj8QOp3SP9fYHlEfEAgKQFwKGAk76ZNWQ4z1JGonZfyJ0EPFJ63ZPKzMysDRQR7duYdBhwUER8ML0+Etg3Ij5asdwcYE56+RrgvgY3OQF4rMF1RyrHnIfcYs4tXmg+5l0jYsfKwnZP7/QAO5deTwZWVi4UEfOAec1uTNLiiOhstp2RxDHnIbeYc4sXhi7mdk/v/BrYXdJUSZsBs4Hr29wHM7NstXWkHxEbJH0EuInils1LIuKudvbBzCxnbb9PPyJuAG5o0+aaniIagRxzHnKLObd4YYhibuuFXDMzG17+7h0zs4yMiqQvaZak+yQtl3RKlXpJOjfV3yHpr4ejn61SR7xHpDjvkPRLSfsMRz9baaCYS8u9XtJGSe9tZ/+GQj0xS+qStFTSXZJ+2u4+tlodx/Z4Sd+T9NsU87HD0c9WkXSJpDWS7qxR3/rcFREj+kFxQfh+YDdgM+C3wJ4VyxwC3AgI2A/41XD3e4jj/d/Adun5wSM53npjLi33E4prRu8d7n634X3eluKv2XdJr1853P1uQ8yfBL6Qnu8IrAU2G+6+NxHzW4C/Bu6sUd/y3DUaRvovfLVDRPwZ6Ptqh7JDgcujsAjYVtLEdne0RQaMNyJ+GRHr0stFFH8PMZLV8x4DfBS4FljTzs4NkXpi/gDwnYh4GCAiRnrc9cQcwNaSBIyjSPob2tvN1omIWyliqKXluWs0JP16vtphNH39w2BjOZ5ipDCSDRizpEnAu4GL2tivoVTP+7wHsJ2kbklLJB3Vtt4NjXpiPh94LcUfdS4D/jEinm9P94ZFy3PXaPg+fVUpq7wlqZ5lRoq6Y5E0kyLp7z+kPRp69cT8b8DJEbGxGASOePXEPBaYARwAbAHcJmlRRPxuqDs3ROqJ+SBgKfBW4FXAzZJ+FhFPDnHfhkvLc9doSPr1fLVDXV//MELUFYuk1wHfAA6OiMfb1LehUk/MncCClPAnAIdI2hAR321LD1uv3uP6sYh4Gnha0q3APsBITfr1xHwscHYUE97LJT0I/BVwe3u62HYtz12jYXqnnq92uB44Kl0J3w9YHxGr2t3RFhkwXkm7AN8BjhzBo76yAWOOiKkRMSUipgDXAB8awQkf6juuFwJvljRW0pbAG4B72tzPVqon5ocpzmyQ1EHxhYwPtLWX7dXy3DXiR/pR46sdJJ2Y6i+iuJvjEGA58AzFaGFEqjPeTwM7ABekke+GGMFfVlVnzKNKPTFHxD2SfgjcATxP8Z/oqt76NxLU+T6fCVwqaRnF1MfJETFiv31T0lVAFzBBUg9wOvByGLrc5b/INTPLyGiY3jEzszo56ZuZZcRJ38wsI076ZmYZcdI3M8uIk76ZWUac9M3MMuKkb2aWkf8GnjJkQiv1mL0AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# score_raw\tscore_scaled\tscore_k\n",
    "import pylab as pl\n",
    "axes = results_df[\"score_scaled\"].hist(bins=np.linspace(0, 1, 40))\n",
    "_ = pl.suptitle(\"Distribution of Scaled RoBERTa Confidence\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEVCAYAAAAb/KWvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAZHUlEQVR4nO3df7hdVX3n8ffHRPkVIGGAKyQpAUmBQEbFiFin9tLQB1Q0dFra9AmaUJwMTxFQY20Af6AllTqiMFLaiUJNBY1pZB4yIlYN3na0AxkClBgiYyAxhED4UQIEEQh854+1rtn35Jx7zr05h3Oz7uf1PHly9t5rr732Ovt8zjp773OuIgIzMyvLa7rdADMzaz+Hu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgUZ9uEv6O0mfbFNdvyFpu6QxebpP0gfbUXeu71ZJc9tV3xC2e7mkJyQ92oVth6Sj8+NhP1f5eTmqva3rrKEeP5I+J+nDw9xWV46tkUBSr6TNQyi/StLxnWxTOxQd7pI2Snpe0rOStkn6V0nnSfr1fkfEeRHxly3WdepgZSJiU0SMi4iX29D2yyTdUFP/uyJiye7WPcR2TAYWANMi4vV1lvdKeiWH57OS7pd0TifaMoTnapdQzM/Lg+1uU+UY2y7pUUlfkzSu3dtpoR2HAB8A/kdl3kxJP5P0S0k/knREo/W7cWztwb4AfLbbjWim6HDP3hsR+wNHAFcAfwFc1+6NSBrb7jpHiCOAJyPisUHKbImIccABpP79iqRptYUK7qP35v1/E/Bm4OIutGEe8N2IeB5A0sHATcAngYOAO4FvdaFdXdeB424FcIqkw9pcb3tFRLH/gI3AqTXzTgJeAU7I018DLs+PDwa+A2wD/h3436Q3wK/ndZ4HtgMfB6YAAZwLbAL+pTJvbK6vD/gcsAp4GrgZOCgv6wU212svcDrwIvBS3t6/Ver7YH78GuATwC+Ax4B/AA7My/rbMTe37Qng0kH66cC8/uO5vk/k+k/N+/xKbsfX6qxbbz8eB/6QFDg/Ab6U+/NyYC/SyGcTsBX4O2Cfyrp/DjwCbAH+NO/H0bXPVZ6eBdwDPAM8kPttEfAy8Kvc5mty2Wo9dfc3L5sH/Di38SlgA/CuVo8x4PPALZXphbltzwL3Ab9fWTbotmqe78OAe4GPNWjHbcDZlen5wL9WpvfLz+WxDdavbuto4J9Jx+wTwLcarDMl9+s5wEN5H84D3prbuq2//3P5N+R2PpnrvREYX1n278CJefrwXKZ3kH6/OPfpU8DfA3tXj0nSQONR0ut3L+Aq0nG1JT/eq6b8JXmbG4E5TbLlB8DcbmfcYP9Gw8h9gIhYRXoif7vO4gV52SFAD+nJjoh4PymM3hvp4/3nK+v8DnAccFqDTX6AFFKHAzuA/95CG78H/BXpRTUuIt5Yp9i8/O8U4ChgHHBNTZn/BBwDzAQ+Jem4Bpv8Minwjsr78wHgnIj4IfAu8sg8IuYN1m5Jr5H0+8B4YE2e/TbgQeBQUvD+NfCbpFHu0cBE4FN5/dOBjwG/B0wlvbk02tZJpID+87y9dwIbI+JS0pvyh3KbP9Tq/laWvw24n/Rm/3ngOkkabN9zmyaR+mt9ZfYDpGPtQOAzwA01I76m25I0hRS210TEFxpsfnqup9/xwL/1T0TEc7ktrZwr/kvg+8AEYBKpvwbzNtLz9cek0LyU9NwdD/yRpN/p3xXSYOdw0mtmMnBZbt8DpDC+UdK+pLD+WkT0DbLdOaTX3RtIx9QnKsteT/rEcgTpje5S4GTScfdG0iCvtvzBpONxLrBY0jGDbHtdrmfk6va7Syf/UWfknuffTh7JMnDk/lnS6ProZnWxc9RyVJ151ZH7FZXl00gj8jEMMnLPjy8DbqhZ3sfO0dVK4M8qy44hjfTHVtoxqbJ8FTC7zn6NAV4gnVPvn/dfgb78eJd21qzfSxrZbyONvO7p3w7pzWdTpayA54A3VOa9HdiQH19f01+/SYORO+nc8pcatOnX/VSZF6Q3k2b7Ow9YX1m2b1739YMcY9tJI/PIz8v4QfrrHmBWK9vK+/HFvI0/aXKsv0RlVE469XhFTZmfAPOa9RnpTXNx9fhpsE7/cTaxMu9J4I8r098GPtxg/TOBu2vmrSANDO4lj6wH6ffzKtPvBh6oHJMvkkfyed4DwLsr06eRBgP95XcA+1WWLwM+Ocj2FwHXD9Y/3f436kbu2URSENX6b6RR1/clPShpYQt1PTSE5b8AXksaIeyuw3N91brHkj5x9Kve3fJL0ui+1sHA6+rUNXEIbdkSEeMj4qCIeFNELK0sq+7/IaQAW50vcG8DvpfnQ9qn2v5qZDLpBTtUrezvr/stIn6ZHw52kfTMSNd1eoFjqTy/kj4g6Z7K/p7AwOe/2bbmAA8Dywfdq3RqYv/K9HbSNZCqA0hvQs18nPRGvErSWkl/2qT81srj5+tMjwOQdKikpZIelvQMcAO7vha+QuqjL0fEC022W3usHF6ZfjwiflWZrvd6qZZ/KtKnm0bLa+1PGtCMWKMu3CW9lfRC/nHtsoh4NiIWRMRRwHuBj0qa2b+4QZWN5vebXHn8G6QR1hOkEey+lXaNYWfItVLvFtJHzmrdOxj4wmrFE7lNtXU9PMR6GqnuxxOkF/vx+c1gfEQcGOliJKRz7bX91chDpI/jzbZZq2P7GxH/TPp08QWAfHfKV4APAf8hIsYDPyUFZ6suy23+Rv8ttg3cS/qk028tldMGkvYj9dfaZhuMiEcj4r9ExOGkTzXX9t+Oups+R3pu/mNEHACcTaUv8l1GV5E+dVwm6aAm9dUeK1sq07XHQL3XS7X8hNxHjZbXOo7Kaa+RaNSEu6QDJJ0BLCWd7lhTp8wZko7O5zyfIV2Y67+tcSvpHO1QnS1pWj6P+FlgeaRbJf8fsLek90h6Len8316V9bYCU6q3bdb4JvARSUfmF0X/OfodQ2lcbssyYJGk/XMgfZQ0qmqriHiFFHZfknQogKSJkvqvVywD5lX669ODVHcdcE6+3e81uZ5j87KGz9WrsL9XAb8n6U2ki5hBunBLvkX0hCHW9xJwVq7r64McD98lXT/o9z+BEyT9gaS9Sdc17o2InzXboKSz8vUDSJ8Igp2vg92xP+kTxTZJE0nXS6quBlZHxAeBW0gX2wdzvqRJ+U3gEga/G+ibwCckHZLvJPoUuz7nn5H0Okm/DZwB/GO9iiTtBbyFdFF1xBoN4f6/JD1LGuldSjqH2eg+7KnAD0kH4P8Bro2dF3Q+Rzo4tkn62BC2/3XSaO5RYG/gQoCIeBr4M+CrpFHjc6SLuf36D6wnJd1Vp97rc93/QrrL4lfABUNoV9UFefsPkj7RfCPX3wl/QTr1dXv+aP5D0vUCIuJWUjjelsvc1qiSSBfGzyHdifM06YJj/8jsauAPJT0lqd4F7I7tb0Q8Tjpn/cmIuA+4knQsbSVd9PzJMOp8EfjPpIvS1zcI+H8A3i1pn0o7/oB0bvgp0kXP2S1u8q3AHZK2k86BXxQRG4ba7jo+A5xIer5uId2qCYCkWaS7nc7Lsz4KnChpziD1fYN04ffB/O/yQcpeTrod9F7SOf27aso/SuqnLaS7eM7rfyOUdImkWytl30e6RjPYyL7rlC8OmNkeTtJfAY9FxFXdbkunSdpIugD8wy5s+w7g3Ij46au97aEo9UslZqNORFzS7TaMBhHxtm63oRWj4bSMmdmo49MyZmYF8sjdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswKNiD/WcfDBB8eUKVOGvf5zzz3Hfvvt17zgKOC+GMj9sZP7YqAS+mP16tVPRMQh9ZaNiHCfMmUKd95557DX7+vro7e3t30N2oO5LwZyf+zkvhiohP6Q9ItGy3xaxsysQA53M7MCOdzNzArkcDczK5DD3cysQA53M7MCOdzNzArkcDczK5DD3cysQCPiG6pmZt00ZeEtXdv2xive05F6PXI3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEC+FdJshGr37XkLpu9gXot1dur2PHv1ONzNbMR4Ne83H8qb3Z7Ip2XMzArkcDczK5DD3cysQA53M7MCOdzNzArku2XMmujmLwZ2y2jc59J45G5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgRzuZmYFcribmRXI4W5mViCHu5lZgVr6+QFJHwE+CASwBjgH2Bf4FjAF2Aj8UUQ8lctfDJwLvAxcGBH/1O6G2+gy3K/Dl/4HGcwaaTpylzQRuBCYEREnAGOA2cBCYGVETAVW5mkkTcvLjwdOB66VNKYzzTczs3paPS0zFthH0ljSiH0LMAtYkpcvAc7Mj2cBSyPihYjYAKwHTmpbi83MrClFRPNC0kXAIuB54PsRMUfStogYXynzVERMkHQNcHtE3JDnXwfcGhHLa+qcD8wH6OnpecvSpUuHvRPbt29n3Lhxw16/JKX2xZqHnx7Wej37wNbn29yYPZT7YqCR0h/TJx447HVPOeWU1RExo96ypufcJU0gjcaPBLYB/yjp7MFWqTNvl3eQiFgMLAaYMWNG9Pb2NmtKQ319fezO+iUptS+Ge958wfQdXLnGv2wN7otaI6U/Ns7p7Ui9rZyWORXYEBGPR8RLwE3AbwFbJR0GkP9/LJffDEyurD+JdBrHzMxeJa28bW0CTpa0L+m0zEzgTuA5YC5wRf7/5lx+BfANSV8EDgemAqva3G7rEv8RB7M9Q9Nwj4g7JC0H7gJ2AHeTTqeMA5ZJOpf0BnBWLr9W0jLgvlz+/Ih4uUPtNzOzOlo64RQRnwY+XTP7BdIovl75RaQLsGZm1gX+hqqZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlag7v9SvQ3ZYD+76z8IbWbgkbuZWZEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyOFuZlYgh7uZWYEc7mZmBXK4m5kVyH9DdTcM9rdMzcy6ySN3M7MCOdzNzArkcDczK1BL4S5pvKTlkn4maZ2kt0s6SNIPJP08/z+hUv5iSesl3S/ptM4138zM6ml15H418L2IOBZ4I7AOWAisjIipwMo8jaRpwGzgeOB04FpJY9rdcDMza6xpuEs6AHgncB1ARLwYEduAWcCSXGwJcGZ+PAtYGhEvRMQGYD1wUnubbWZmg1FEDF5AehOwGLiPNGpfDVwEPBwR4yvlnoqICZKuAW6PiBvy/OuAWyNieU2984H5AD09PW9ZunTpsHdi+/btjBs3btjrD9eah59+1bfZTM8+sPX5brdi5HB/7OS+GGik9Mf0iQcOe91TTjlldUTMqLeslfvcxwInAhdExB2SriafgmlAdebt8g4SEYtJbxrMmDEjent7W2hKfX19fezO+sM1bwTe575g+g6uXOOvL/Rzf+zkvhhopPTHxjm9Ham3lXPum4HNEXFHnl5OCvutkg4DyP8/Vik/ubL+JGBLe5prZmataBruEfEo8JCkY/KsmaRTNCuAuXneXODm/HgFMFvSXpKOBKYCq9raajMzG1Srn0kuAG6U9DrgQeAc0hvDMknnApuAswAiYq2kZaQ3gB3A+RHxcttbbmZmDbUU7hFxD1DvpP3MBuUXAYuG3ywzM9sd/oaqmVmBHO5mZgVyuJuZFcjhbmZWIIe7mVmBHO5mZgVyuJuZFcjhbmZWIIe7mVmBHO5mZgVyuJuZFcjhbmZWIIe7mVmBHO5mZgVyuJuZFcjhbmZWIIe7mVmBHO5mZgVq9W+ojmhrHn6aeQtv6XYzzMxGDI/czcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzArUc7pLGSLpb0nfy9EGSfiDp5/n/CZWyF0taL+l+Sad1ouFmZtbYUEbuFwHrKtMLgZURMRVYmaeRNA2YDRwPnA5cK2lMe5prZmataCncJU0C3gN8tTJ7FrAkP14CnFmZvzQiXoiIDcB64KS2tNbMzFrS6sj9KuDjwCuVeT0R8QhA/v/QPH8i8FCl3OY8z8zMXiVN/0C2pDOAxyJitaTeFupUnXlRp975wHyAnp4e+vr6Wqi6vp59YMH0HcNevyTui4HcHzu5LwYaKf2xO9k3mKbhDrwDeJ+kdwN7AwdIugHYKumwiHhE0mHAY7n8ZmByZf1JwJbaSiNiMbAYYMaMGdHb2zvsnfjyjTdz5ZpWdqV8C6bvcF9UuD92cl8MNFL6Y+Oc3o7U2/S0TERcHBGTImIK6ULpbRFxNrACmJuLzQVuzo9XALMl7SXpSGAqsKrtLTczs4Z2523rCmCZpHOBTcBZABGxVtIy4D5gB3B+RLy82y01M7OWDSncI6IP6MuPnwRmNii3CFi0m20zM7Nh8jdUzcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjUNd0mTJf1I0jpJayVdlOcfJOkHkn6e/59QWediSesl3S/ptE7ugJmZ7aqVkfsOYEFEHAecDJwvaRqwEFgZEVOBlXmavGw2cDxwOnCtpDGdaLyZmdXXNNwj4pGIuCs/fhZYB0wEZgFLcrElwJn58SxgaUS8EBEbgPXASW1ut5mZDWLsUApLmgK8GbgD6ImIRyC9AUg6NBebCNxeWW1znldb13xgPkBPTw99fX1Dbfuv9ewDC6bvGPb6JXFfDOT+2Ml9MdBI6Y/dyb7BtBzuksYB3wY+HBHPSGpYtM682GVGxGJgMcCMGTOit7e31abs4ss33syVa4b0PlWsBdN3uC8q3B87uS8GGin9sXFOb0fqbeluGUmvJQX7jRFxU569VdJheflhwGN5/mZgcmX1ScCW9jTXzMxa0crdMgKuA9ZFxBcri1YAc/PjucDNlfmzJe0l6UhgKrCqfU02M7NmWvlM8g7g/cAaSffkeZcAVwDLJJ0LbALOAoiItZKWAfeR7rQ5PyJebnfDzcyssabhHhE/pv55dICZDdZZBCzajXaZmdlu8DdUzcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAOdzOzAjnczcwK5HA3MyuQw93MrEAdC3dJp0u6X9J6SQs7tR0zM9tVR8Jd0hjgb4B3AdOAP5E0rRPbMjOzXXVq5H4SsD4iHoyIF4GlwKwObcvMzGp0KtwnAg9VpjfneWZm9ioY26F6VWdeDCggzQfm58ntku7fje0dDDyxG+sX40L3xQDuj53cFwONlP7QX+/W6kc0WtCpcN8MTK5MTwK2VAtExGJgcTs2JunOiJjRjrr2dO6LgdwfO7kvBiq9Pzp1Wub/AlMlHSnpdcBsYEWHtmVmZjU6MnKPiB2SPgT8EzAGuD4i1nZiW2ZmtqtOnZYhIr4LfLdT9ddoy+mdQrgvBnJ/7OS+GKjo/lBENC9lZmZ7FP/8gJlZgfbocPdPHOwkabKkH0laJ2mtpIu63aZukzRG0t2SvtPttnSbpPGSlkv6WT5G3t7tNnWTpI/k18lPJX1T0t7dblO77bHh7p842MUOYEFEHAecDJw/yvsD4CJgXbcbMUJcDXwvIo4F3sgo7hdJE4ELgRkRcQLppo/Z3W1V++2x4Y5/4mCAiHgkIu7Kj58lvXhH7beCJU0C3gN8tdtt6TZJBwDvBK4DiIgXI2JbVxvVfWOBfSSNBfal5ns4JdiTw90/cdCApCnAm4E7utyUbroK+DjwSpfbMRIcBTwO/H0+TfVVSft1u1HdEhEPA18ANgGPAE9HxPe726r225PDvelPHIxGksYB3wY+HBHPdLs93SDpDOCxiFjd7baMEGOBE4G/jYg3A88Bo/YalaQJpE/5RwKHA/tJOru7rWq/PTncm/7EwWgj6bWkYL8xIm7qdnu66B3A+yRtJJ2u+11JN3S3SV21GdgcEf2f5JaTwn60OhXYEBGPR8RLwE3Ab3W5TW23J4e7f+KgQpJI51TXRcQXu92eboqIiyNiUkRMIR0Xt0VEcSOzVkXEo8BDko7Js2YC93WxSd22CThZ0r75dTOTAi8wd+wbqp3mnzjYxTuA9wNrJN2T512SvylsdgFwYx4IPQic0+X2dE1E3CFpOXAX6S6zuynw26r+hqqZWYH25NMyZmbWgMPdzKxADnczswI53M3MCuRwNzMrkMPdzKxADnczswI53M3MCvT/Ae1kwRi8MBh5AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pylab as pl\n",
    "axes = results_df[\"score_k\"].hist()\n",
    "_ = pl.suptitle(\"Distribution of Prediction Rank (0 is max prob.)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rtx8000_mdetr_efficientnetB3_10_eval_time.csv\n",
      "v100_mdetr_resnet101_25_eval_time.csv\n",
      "rtx8000_mdetr_efficientnetB5_25_eval_time.csv\n",
      "rtx8000_mdetr_efficientnetB5_5_eval_time.csv\n",
      "v100_mdetr_efficientnetB5_5_eval_time.csv\n",
      "v100_mdetr_efficientnetB5_1_eval_time.csv\n",
      "rtx8000_mdetr_efficientnetB5_1_eval_time.csv\n",
      "rtx8000_mdetr_resnet101_50_eval_time.csv\n",
      "v100_mdetr_resnet101_1_eval_time.csv\n",
      "v100_mdetr_efficientnetB5_10_eval_time.csv\n",
      "v100_mdetr_resnet101_5_eval_time.csv\n",
      "rtx8000_mdetr_resnet101_10_eval_time.csv\n",
      "v100_mdetr_efficientnetB3_25_eval_time.csv\n",
      "rtx8000_mdetr_efficientnetB3_1_eval_time.csv\n",
      "v100_mdetr_efficientnetB3_1_eval_time.csv\n",
      "v100_mdetr_efficientnetB3_5_eval_time.csv\n",
      "rtx8000_mdetr_efficientnetB3_5_eval_time.csv\n",
      "rtx8000_mdetr_resnet101_5_eval_time.csv\n",
      "rtx8000_mdetr_resnet101_1_eval_time.csv\n",
      "v100_mdetr_resnet101_10_eval_time.csv\n",
      "rtx8000_mdetr_efficientnetB5_10_eval_time.csv\n",
      "rtx8000_mdetr_efficientnetB3_25_eval_time.csv\n",
      "v100_mdetr_efficientnetB3_10_eval_time.csv\n",
      "rtx8000_mdetr_resnet101_25_eval_time.csv\n",
      "v100_mdetr_efficientnetB5_25_eval_time.csv\n",
      "v100_mdetr_resnet101_50_eval_time.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>total_cumulative_time</th>\n",
       "      <th>batch_time</th>\n",
       "      <th>model_scoring_time</th>\n",
       "      <th>post_processing_time</th>\n",
       "      <th>gpu_type</th>\n",
       "      <th>model_type</th>\n",
       "      <th>bs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7.652452</td>\n",
       "      <td>5.411121</td>\n",
       "      <td>3.733840</td>\n",
       "      <td>1.552557</td>\n",
       "      <td>rtx8000</td>\n",
       "      <td>efficientnetB3</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9.725491</td>\n",
       "      <td>1.765453</td>\n",
       "      <td>0.281142</td>\n",
       "      <td>1.451199</td>\n",
       "      <td>rtx8000</td>\n",
       "      <td>efficientnetB3</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11.503588</td>\n",
       "      <td>1.756125</td>\n",
       "      <td>0.280393</td>\n",
       "      <td>1.442852</td>\n",
       "      <td>rtx8000</td>\n",
       "      <td>efficientnetB3</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13.331980</td>\n",
       "      <td>1.804251</td>\n",
       "      <td>0.306628</td>\n",
       "      <td>1.460782</td>\n",
       "      <td>rtx8000</td>\n",
       "      <td>efficientnetB3</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15.153791</td>\n",
       "      <td>1.795350</td>\n",
       "      <td>0.306558</td>\n",
       "      <td>1.452703</td>\n",
       "      <td>rtx8000</td>\n",
       "      <td>efficientnetB3</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>313</th>\n",
       "      <td>1769.536860</td>\n",
       "      <td>5.236986</td>\n",
       "      <td>1.217451</td>\n",
       "      <td>3.896962</td>\n",
       "      <td>v100</td>\n",
       "      <td>efficientnetB5</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>314</th>\n",
       "      <td>1775.943139</td>\n",
       "      <td>5.286016</td>\n",
       "      <td>1.254733</td>\n",
       "      <td>3.908134</td>\n",
       "      <td>v100</td>\n",
       "      <td>efficientnetB5</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>315</th>\n",
       "      <td>1780.676564</td>\n",
       "      <td>3.623870</td>\n",
       "      <td>0.793819</td>\n",
       "      <td>2.755013</td>\n",
       "      <td>v100</td>\n",
       "      <td>efficientnetB5</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>25.623204</td>\n",
       "      <td>18.446871</td>\n",
       "      <td>5.274828</td>\n",
       "      <td>12.918848</td>\n",
       "      <td>v100</td>\n",
       "      <td>resnet101</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>40.763370</td>\n",
       "      <td>14.350885</td>\n",
       "      <td>1.952987</td>\n",
       "      <td>12.266667</td>\n",
       "      <td>v100</td>\n",
       "      <td>resnet101</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>63628 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     total_cumulative_time  batch_time  model_scoring_time  \\\n",
       "0                 7.652452    5.411121            3.733840   \n",
       "1                 9.725491    1.765453            0.281142   \n",
       "2                11.503588    1.756125            0.280393   \n",
       "3                13.331980    1.804251            0.306628   \n",
       "4                15.153791    1.795350            0.306558   \n",
       "..                     ...         ...                 ...   \n",
       "313            1769.536860    5.236986            1.217451   \n",
       "314            1775.943139    5.286016            1.254733   \n",
       "315            1780.676564    3.623870            0.793819   \n",
       "0                25.623204   18.446871            5.274828   \n",
       "1                40.763370   14.350885            1.952987   \n",
       "\n",
       "     post_processing_time gpu_type      model_type  bs  \n",
       "0                1.552557  rtx8000  efficientnetB3  10  \n",
       "1                1.451199  rtx8000  efficientnetB3  10  \n",
       "2                1.442852  rtx8000  efficientnetB3  10  \n",
       "3                1.460782  rtx8000  efficientnetB3  10  \n",
       "4                1.452703  rtx8000  efficientnetB3  10  \n",
       "..                    ...      ...             ...  ..  \n",
       "313              3.896962     v100  efficientnetB5  25  \n",
       "314              3.908134     v100  efficientnetB5  25  \n",
       "315              2.755013     v100  efficientnetB5  25  \n",
       "0               12.918848     v100       resnet101  50  \n",
       "1               12.266667     v100       resnet101  50  \n",
       "\n",
       "[63628 rows x 7 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import glob\n",
    "from pathlib import Path\n",
    "\n",
    "dfs = []\n",
    "\n",
    "for f in glob.glob(\"output/*eval_time.csv\"):\n",
    "    fname = Path(f).name\n",
    "    print(fname)\n",
    "    split = fname.split(\"_\")\n",
    "\n",
    "    gpu_type = split[0]\n",
    "    model_type = split[2]\n",
    "    bs = split[3]\n",
    "\n",
    "    df = pd.read_csv(f)\n",
    "\n",
    "    df[\"gpu_type\"] = gpu_type\n",
    "    df[\"model_type\"] = model_type\n",
    "    df[\"bs\"] = int(bs)\n",
    "    \n",
    "    dfs.append(df)\n",
    "\n",
    "combined = pd.concat(dfs).drop(\"Unnamed: 0\", axis=1)\n",
    "combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "gpu_type  model_type      bs\n",
       "rtx8000   efficientnetB3  1     5985.057900\n",
       "                          5     2315.462034\n",
       "                          10    1879.812081\n",
       "                          25    1952.657409\n",
       "          efficientnetB5  1     6097.589397\n",
       "                          5     2333.660515\n",
       "                          10    2024.211313\n",
       "                          25    2155.363988\n",
       "          resnet101       1     5984.471876\n",
       "                          5     2398.830361\n",
       "                          10    2002.745179\n",
       "                          25    2126.245727\n",
       "                          50    1906.240563\n",
       "v100      efficientnetB3  1     8985.096552\n",
       "                          5     2254.231734\n",
       "                          10    2611.136887\n",
       "                          25    2455.410847\n",
       "          efficientnetB5  1     6328.430410\n",
       "                          5     2349.030134\n",
       "                          10    2093.149725\n",
       "                          25    1780.676564\n",
       "          resnet101       1     5880.704913\n",
       "                          5     2401.964148\n",
       "                          10    3110.835262\n",
       "                          25    2598.700232\n",
       "                          50      40.763370\n",
       "Name: total_cumulative_time, dtype: float64"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "runtimes = combined.groupby([\"gpu_type\", \"model_type\", \"bs\"])[\"total_cumulative_time\"].max()\n",
    "runtimes.to_csv(\"data/runtimes.csv\")\n",
    "runtimes"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "478565d84f56a7791dc0111d293f8eab97e93db4509df281991a2b24d01be195"
  },
  "kernelspec": {
   "display_name": "Python 3.8.13 ('dls')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}