analysis_of_errors.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_colwidth', 150)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_opener_en_test = pd.read_json(\"data/opener_en/test.json\")\n",
    "df_en_BERT_review = pd.read_json(\"data/preds/en_BERT_review_pred.json\")\n",
    "df_en_bert_large = pd.read_json(\"data/preds/en_bert-large-uncased_pred.json\")\n",
    "\n",
    "\n",
    "\n",
    "df_opener_es_test = pd.read_json(\"data/opener_es/test.json\")\n",
    "df_es_roberta_bne = pd.read_json(\"data/preds/es_roberta-large-bne_pred.json\")\n",
    "df_es_xlm_roberta = pd.read_json(\"data/preds/es_xlm-roberta-large_pred.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_jsons(filepath):\n",
    "\n",
    "    with open(filepath, 'r') as json_file:\n",
    "        json_data = json.load(json_file)\n",
    "        df = pd.json_normalize(\n",
    "            json_data,\n",
    "            record_path=[\"opinions\"],\n",
    "            sep=\"_\",\n",
    "            meta=[\n",
    "                ['sent_id'],\n",
    "                ['text']\n",
    "            ],\n",
    "            errors='ignore'\n",
    "        )\n",
    "\n",
    "    df[['Source_string','Source_index']] = pd.DataFrame(df['Source'].tolist(), index = df.index)\n",
    "    df[['Target_string','Target_index']] = pd.DataFrame(df['Target'].tolist(), index = df.index)\n",
    "    df[['Polar_expression_string','Polar_expression_index']] = pd.DataFrame(df['Polar_expression'].tolist(), index = df.index)\n",
    "    df['Implicit_holder'] = df['Source_string'].apply(lambda x: 1 if len(x) == 0 else 0)\n",
    "    df['Implicit_target'] = df['Target_string'].apply(lambda x: 1 if len(x) == 0 else 0)\n",
    "    df['Implicit_holder_target'] = (df['Implicit_holder'] == 1) & (df['Implicit_target'] == 1)\n",
    "    df['Implicit_expression'] = df['Polar_expression_string'].apply(lambda x: 1 if len(x) == 0 else 0)\n",
    "    df['Implicit_target_exclusive'] = (df['Implicit_target']==1) & (df['Implicit_holder'] == 0) & (df['Implicit_expression']==0 ) \n",
    "    df['Implicit_holder_exclusive'] = (df['Implicit_target']==0) & (df['Implicit_holder'] == 1) & (df['Implicit_expression']==0 )\n",
    "    \n",
    "    df['H-A-O'] = df.apply(lambda row: 1 if (row['Implicit_target'] == 0 and row['Implicit_holder'] ==0) and row['Implicit_expression'] ==0 else 0, axis=1)\n",
    "    \n",
    "#    df['H-A-O'] =  (df['Implicit_target']==0) & (df['Implicit_holder'] == 0) & (df['Implicit_expression']==0 )\n",
    "    df['H-IA-IO'] =(df['Implicit_target']==1) & (df['Implicit_holder'] == 0) & (df['Implicit_expression']==1 )\n",
    "    df['H-A-IO'] = (df['Implicit_target']==0) & (df['Implicit_holder'] == 0) & (df['Implicit_expression']==1 )\n",
    "    df['H-IA-O']=  (df['Implicit_target']==1) & (df['Implicit_holder'] == 0) & (df['Implicit_expression']==0 )\n",
    "    df['IH-A-O']=  (df['Implicit_target']==0) & (df['Implicit_holder'] == 1) & (df['Implicit_expression']==0 )\n",
    "    df['IH-A-IO']= (df['Implicit_target']==0) & (df['Implicit_holder'] == 1) & (df['Implicit_expression']==1 )\n",
    "    df['IH-IA-O']= (df['Implicit_target']==1) & (df['Implicit_holder'] == 1) & (df['Implicit_expression']==0 )\n",
    "    df['IH-IA-IO']= (df['Implicit_target']==1) & (df['Implicit_holder'] == 1) & (df['Implicit_expression']==1 )\n",
    "\n",
    "\n",
    "    df = df.reindex(\n",
    "        columns= [\n",
    "        'text',\n",
    "        'sent_id',\n",
    "        'Source',\n",
    "        'Source_string',\n",
    "        'Source_index',\n",
    "        'Target',\n",
    "        'Target_string',\n",
    "        'Target_index',\n",
    "        'Polar_expression',\n",
    "        'Polar_expression_string',\n",
    "        'Polar_expression_index',\n",
    "        'Polarity',\n",
    "        'Intensity',\n",
    "        'Implicit_holder',\n",
    "        'Implicit_target',\n",
    "        'Implicit_holder_target',\n",
    "        'Implicit_expression',\n",
    "        'Implicit_target_exclusive',\n",
    "        'Implicit_holder_exclusive',\n",
    "        'H-A-O', 'H-IA-IO','H-A-IO','H-IA-O','IH-A-O', 'IH-A-IO', 'IH-IA-O', 'IH-IA-IO'\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_opener_en_test = normalize_jsons(\"data/opener_en/test.json\")\n",
    "df_en_BERT_review = normalize_jsons(\"data/preds/en_BERT_review_pred.json\")\n",
    "df_en_bert_large = normalize_jsons(\"data/preds/en_bert-large-uncased_pred.json\")\n",
    "\n",
    "df_opener_es_test = normalize_jsons(\"data/opener_es/test.json\")\n",
    "df_es_roberta_bne = normalize_jsons(\"data/preds/es_roberta-large-bne_pred.json\")\n",
    "df_es_xlm_roberta = normalize_jsons(\"data/preds/es_xlm-roberta-large_pred.json\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['text', 'sent_id', 'Source', 'Source_string', 'Source_index', 'Target',\n",
       "       'Target_string', 'Target_index', 'Polar_expression',\n",
       "       'Polar_expression_string', 'Polar_expression_index', 'Polarity',\n",
       "       'Intensity', 'Implicit_holder', 'Implicit_target',\n",
       "       'Implicit_holder_target', 'Implicit_expression',\n",
       "       'Implicit_target_exclusive', 'Implicit_holder_exclusive', 'H-A-O',\n",
       "       'H-IA-IO', 'H-A-IO', 'H-IA-O', 'IH-A-O', 'IH-A-IO', 'IH-IA-O',\n",
       "       'IH-IA-IO'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_opener_en_test.columns\n",
    "\n",
    "# df_en.dtypes\n",
    "# df_en.columns\n",
    "# df_en.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>Source</th>\n",
       "      <th>Source_string</th>\n",
       "      <th>Source_index</th>\n",
       "      <th>Target</th>\n",
       "      <th>Target_string</th>\n",
       "      <th>Target_index</th>\n",
       "      <th>Polar_expression</th>\n",
       "      <th>Polar_expression_string</th>\n",
       "      <th>...</th>\n",
       "      <th>Implicit_target_exclusive</th>\n",
       "      <th>Implicit_holder_exclusive</th>\n",
       "      <th>H-A-O</th>\n",
       "      <th>H-IA-IO</th>\n",
       "      <th>H-A-IO</th>\n",
       "      <th>H-IA-O</th>\n",
       "      <th>IH-A-O</th>\n",
       "      <th>IH-A-IO</th>\n",
       "      <th>IH-IA-O</th>\n",
       "      <th>IH-IA-IO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>They were there just to keep us happy .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-6</td>\n",
       "      <td>[[us], [29:31]]</td>\n",
       "      <td>[us]</td>\n",
       "      <td>[29:31]</td>\n",
       "      <td>[[They], [0:4]]</td>\n",
       "      <td>[They]</td>\n",
       "      <td>[0:4]</td>\n",
       "      <td>[[to keep, happy], [21:28, 32:37]]</td>\n",
       "      <td>[to keep, happy]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A wonderful place to go and we are planning to return as soon as we can .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-7</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[place], [12:17]]</td>\n",
       "      <td>[place]</td>\n",
       "      <td>[12:17]</td>\n",
       "      <td>[[wonderful], [2:11]]</td>\n",
       "      <td>[wonderful]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A wonderful place to go and we are planning to return as soon as we can .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-7</td>\n",
       "      <td>[[we], [28:30]]</td>\n",
       "      <td>[we]</td>\n",
       "      <td>[28:30]</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[planning to return], [35:53]]</td>\n",
       "      <td>[planning to return]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Thank you for such a wonderful experience .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-8</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[experience], [31:41]]</td>\n",
       "      <td>[experience]</td>\n",
       "      <td>[31:41]</td>\n",
       "      <td>[[wonderful], [21:30]]</td>\n",
       "      <td>[wonderful]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Hotel was under construction .</td>\n",
       "      <td>opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-2</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[Hotel], [0:5]]</td>\n",
       "      <td>[Hotel]</td>\n",
       "      <td>[0:5]</td>\n",
       "      <td>[[under construction], [10:28]]</td>\n",
       "      <td>[under construction]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                        text  \\\n",
       "0                                    They were there just to keep us happy .   \n",
       "1  A wonderful place to go and we are planning to return as soon as we can .   \n",
       "2  A wonderful place to go and we are planning to return as soon as we can .   \n",
       "3                                Thank you for such a wonderful experience .   \n",
       "4                                             Hotel was under construction .   \n",
       "\n",
       "                                                               sent_id  \\\n",
       "0  opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-6   \n",
       "1  opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-7   \n",
       "2  opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-7   \n",
       "3  opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-8   \n",
       "4  opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-2   \n",
       "\n",
       "            Source Source_string Source_index                   Target  \\\n",
       "0  [[us], [29:31]]          [us]      [29:31]          [[They], [0:4]]   \n",
       "1         [[], []]            []           []       [[place], [12:17]]   \n",
       "2  [[we], [28:30]]          [we]      [28:30]                 [[], []]   \n",
       "3         [[], []]            []           []  [[experience], [31:41]]   \n",
       "4         [[], []]            []           []         [[Hotel], [0:5]]   \n",
       "\n",
       "  Target_string Target_index                    Polar_expression  \\\n",
       "0        [They]        [0:4]  [[to keep, happy], [21:28, 32:37]]   \n",
       "1       [place]      [12:17]               [[wonderful], [2:11]]   \n",
       "2            []           []     [[planning to return], [35:53]]   \n",
       "3  [experience]      [31:41]              [[wonderful], [21:30]]   \n",
       "4       [Hotel]        [0:5]     [[under construction], [10:28]]   \n",
       "\n",
       "  Polar_expression_string  ... Implicit_target_exclusive  \\\n",
       "0        [to keep, happy]  ...                     False   \n",
       "1             [wonderful]  ...                     False   \n",
       "2    [planning to return]  ...                      True   \n",
       "3             [wonderful]  ...                     False   \n",
       "4    [under construction]  ...                     False   \n",
       "\n",
       "  Implicit_holder_exclusive H-A-O  H-IA-IO  H-A-IO  H-IA-O  IH-A-O  IH-A-IO  \\\n",
       "0                     False     1    False   False   False   False    False   \n",
       "1                      True     0    False   False   False    True    False   \n",
       "2                     False     0    False   False    True   False    False   \n",
       "3                      True     0    False   False   False    True    False   \n",
       "4                      True     0    False   False   False    True    False   \n",
       "\n",
       "   IH-IA-O  IH-IA-IO  \n",
       "0    False     False  \n",
       "1    False     False  \n",
       "2    False     False  \n",
       "3    False     False  \n",
       "4    False     False  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_opener_en_test.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>Source</th>\n",
       "      <th>Source_string</th>\n",
       "      <th>Source_index</th>\n",
       "      <th>Target</th>\n",
       "      <th>Target_string</th>\n",
       "      <th>Target_index</th>\n",
       "      <th>Polar_expression</th>\n",
       "      <th>Polar_expression_string</th>\n",
       "      <th>...</th>\n",
       "      <th>Implicit_target_exclusive</th>\n",
       "      <th>Implicit_holder_exclusive</th>\n",
       "      <th>H-A-O</th>\n",
       "      <th>H-IA-IO</th>\n",
       "      <th>H-A-IO</th>\n",
       "      <th>H-IA-O</th>\n",
       "      <th>IH-A-O</th>\n",
       "      <th>IH-A-IO</th>\n",
       "      <th>IH-IA-O</th>\n",
       "      <th>IH-IA-IO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>They were there just to keep us happy .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-6</td>\n",
       "      <td>[[They], [0:4]]</td>\n",
       "      <td>[They]</td>\n",
       "      <td>[0:4]</td>\n",
       "      <td>[[They], [0:4]]</td>\n",
       "      <td>[They]</td>\n",
       "      <td>[0:4]</td>\n",
       "      <td>[[keep], [24:28]]</td>\n",
       "      <td>[keep]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>They were there just to keep us happy .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-6</td>\n",
       "      <td>[[They], [0:4]]</td>\n",
       "      <td>[They]</td>\n",
       "      <td>[0:4]</td>\n",
       "      <td>[[They], [0:4]]</td>\n",
       "      <td>[They]</td>\n",
       "      <td>[0:4]</td>\n",
       "      <td>[[happy], [32:37]]</td>\n",
       "      <td>[happy]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A wonderful place to go and we are planning to return as soon as we can .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-7</td>\n",
       "      <td>[[we], [28:30]]</td>\n",
       "      <td>[we]</td>\n",
       "      <td>[28:30]</td>\n",
       "      <td>[[place], [12:17]]</td>\n",
       "      <td>[place]</td>\n",
       "      <td>[12:17]</td>\n",
       "      <td>[[wonderful], [2:11]]</td>\n",
       "      <td>[wonderful]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Thank you for such a wonderful experience .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-8</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[experience], [31:41]]</td>\n",
       "      <td>[experience]</td>\n",
       "      <td>[31:41]</td>\n",
       "      <td>[[such a wonderful], [14:30]]</td>\n",
       "      <td>[such a wonderful]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Hotel was under construction .</td>\n",
       "      <td>opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-2</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[Hotel], [0:5]]</td>\n",
       "      <td>[Hotel]</td>\n",
       "      <td>[0:5]</td>\n",
       "      <td>[[under construction], [10:28]]</td>\n",
       "      <td>[under construction]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                        text  \\\n",
       "0                                    They were there just to keep us happy .   \n",
       "1                                    They were there just to keep us happy .   \n",
       "2  A wonderful place to go and we are planning to return as soon as we can .   \n",
       "3                                Thank you for such a wonderful experience .   \n",
       "4                                             Hotel was under construction .   \n",
       "\n",
       "                                                               sent_id  \\\n",
       "0  opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-6   \n",
       "1  opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-6   \n",
       "2  opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-7   \n",
       "3  opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-8   \n",
       "4  opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-2   \n",
       "\n",
       "            Source Source_string Source_index                   Target  \\\n",
       "0  [[They], [0:4]]        [They]        [0:4]          [[They], [0:4]]   \n",
       "1  [[They], [0:4]]        [They]        [0:4]          [[They], [0:4]]   \n",
       "2  [[we], [28:30]]          [we]      [28:30]       [[place], [12:17]]   \n",
       "3         [[], []]            []           []  [[experience], [31:41]]   \n",
       "4         [[], []]            []           []         [[Hotel], [0:5]]   \n",
       "\n",
       "  Target_string Target_index                 Polar_expression  \\\n",
       "0        [They]        [0:4]                [[keep], [24:28]]   \n",
       "1        [They]        [0:4]               [[happy], [32:37]]   \n",
       "2       [place]      [12:17]            [[wonderful], [2:11]]   \n",
       "3  [experience]      [31:41]    [[such a wonderful], [14:30]]   \n",
       "4       [Hotel]        [0:5]  [[under construction], [10:28]]   \n",
       "\n",
       "  Polar_expression_string  ... Implicit_target_exclusive  \\\n",
       "0                  [keep]  ...                     False   \n",
       "1                 [happy]  ...                     False   \n",
       "2             [wonderful]  ...                     False   \n",
       "3      [such a wonderful]  ...                     False   \n",
       "4    [under construction]  ...                     False   \n",
       "\n",
       "  Implicit_holder_exclusive  H-A-O  H-IA-IO  H-A-IO  H-IA-O  IH-A-O  IH-A-IO  \\\n",
       "0                     False      1    False   False   False   False    False   \n",
       "1                     False      1    False   False   False   False    False   \n",
       "2                     False      1    False   False   False   False    False   \n",
       "3                      True      0    False   False   False    True    False   \n",
       "4                      True      0    False   False   False    True    False   \n",
       "\n",
       "   IH-IA-O  IH-IA-IO  \n",
       "0    False     False  \n",
       "1    False     False  \n",
       "2    False     False  \n",
       "3    False     False  \n",
       "4    False     False  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_en_BERT_review.head(5)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Highest Scoring F1 Score Predictions and Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_opener_en_test = normalize_jsons(\"data/opener_en/test.json\")\n",
    "df_opener_es_test = normalize_jsons(\"data/opener_es/test.json\")\n",
    "\n",
    "df_es_xlm_roberta = normalize_jsons(\"es_xlm-roberta-large_pred.json\")\n",
    "df_en_roberta_largev1 = normalize_jsons(\"en_all-roberta-large-v1_pred.json\")\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set of Aspects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Target\n",
    "\n",
    "en_target_tokens_test = set(sum(df_opener_en_test['Target_string'].tolist(), []))\n",
    "en_target_tokens_pred = set(sum(df_en_roberta_largev1['Target_string'].tolist(), []))\n",
    "\n",
    "es_target_tokens_test = set(sum(df_opener_es_test['Target_string'].tolist(), []))\n",
    "es_target_tokens_pred = set(sum(df_es_xlm_roberta['Target_string'].tolist(), []))\n",
    "\n",
    "# Holder\n",
    "\n",
    "en_holder_tokens_test = set(sum(df_opener_en_test['Source_string'].tolist(), []))\n",
    "en_holder_tokens_pred = set(sum(df_en_roberta_largev1['Source_string'].tolist(), []))\n",
    "\n",
    "es_holder_tokens_test = set(sum(df_opener_es_test['Source_string'].tolist(), []))\n",
    "es_holder_tokens_pred = set(sum(df_es_xlm_roberta['Source_string'].tolist(), []))\n",
    "\n",
    "# Expression\n",
    "\n",
    "en_expression_tokens_test = set(sum(df_opener_en_test['Polar_expression_string'].tolist(), []))\n",
    "en_expression_tokens_pred = set(sum(df_en_roberta_largev1['Polar_expression_string'].tolist(), []))\n",
    "\n",
    "es_expression_tokens_test = set(sum(df_opener_es_test['Polar_expression_string'].tolist(), []))\n",
    "es_expression_tokens_pred = set(sum(df_es_xlm_roberta['Polar_expression_string'].tolist(), []))\n",
    "\n",
    "\n",
    "# Polarity\n",
    "en_pos_test = sum(df_opener_en_test['Polarity'] == \"Positive\") \n",
    "en_pos_pred = sum(df_en_roberta_largev1['Polarity'] == \"Positive\") \n",
    "en_neg_test = sum(df_opener_en_test['Polarity'] == \"Negative\") \n",
    "en_neg_pred = sum(df_en_roberta_largev1['Polarity'] == \"Negative\") \n",
    "\n",
    "es_pos_test = sum(df_opener_es_test['Polarity'] == \"Positive\") \n",
    "es_pos_pred = sum(df_es_xlm_roberta['Polarity'] == \"Positive\") \n",
    "es_neg_test = sum(df_opener_es_test['Polarity'] == \"Negative\") \n",
    "es_neg_pred = sum(df_es_xlm_roberta['Polarity'] == \"Negative\") \n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Polarity Counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "596 589 269 257\n"
     ]
    }
   ],
   "source": [
    "print(en_pos_test, en_pos_pred, en_neg_test, en_neg_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "768 603 189 142\n"
     ]
    }
   ],
   "source": [
    "print(es_pos_test, es_pos_pred, es_neg_test, es_neg_pred)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set Target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "136 396 380\n"
     ]
    }
   ],
   "source": [
    "en_target_diff_test = en_target_tokens_test.difference(en_target_tokens_pred)\n",
    "en_target_diff_pred = en_target_tokens_test.difference(en_target_tokens_pred)\n",
    "\n",
    "print(len(en_target_diff_test),len(en_target_tokens_test), len(en_target_tokens_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'24 hr bar',\n",
       " 'A la carte restaurant steak House',\n",
       " 'A stay',\n",
       " 'All inclusive',\n",
       " 'Busdriver Peter Langhout',\n",
       " 'Coffee maker',\n",
       " 'French cuisine',\n",
       " 'He',\n",
       " 'Hotel and Villa Dievole',\n",
       " 'Kettle',\n",
       " 'Melia Grand Hermitage',\n",
       " 'Metro',\n",
       " 'Numerous shopping',\n",
       " 'One bed',\n",
       " 'Our room',\n",
       " 'Pool',\n",
       " 'RS375 off',\n",
       " 'The TV',\n",
       " 'The all inclusive',\n",
       " 'The house · keeping',\n",
       " 'The technology provided',\n",
       " 'This',\n",
       " 'This B & amp ; B',\n",
       " 'Tuscan location',\n",
       " 'Wellness offer and quality',\n",
       " 'Wi · Fi in the room',\n",
       " 'a choice',\n",
       " 'a hotel',\n",
       " 'a hotel with such staff',\n",
       " 'a rubber duck',\n",
       " 'a waiter on hand',\n",
       " 'access to the city and Messe',\n",
       " 'all Steigenburger hotels',\n",
       " 'all rounder',\n",
       " 'anything to complain about',\n",
       " 'back massage',\n",
       " 'bars',\n",
       " 'bath',\n",
       " 'bathroom door',\n",
       " 'caffees',\n",
       " 'chairs / tables',\n",
       " 'check in',\n",
       " 'chill out',\n",
       " 'choice of drinks',\n",
       " 'cocktails',\n",
       " 'cocktails.',\n",
       " 'complex',\n",
       " 'curtains',\n",
       " 'customer',\n",
       " 'distance from firenze',\n",
       " 'dvd player',\n",
       " 'electrical outlet',\n",
       " 'enough seats',\n",
       " 'ever.staff',\n",
       " 'everything else',\n",
       " 'fitnes',\n",
       " 'foot · paths',\n",
       " 'forks',\n",
       " 'fridge in 1 of the rooms',\n",
       " 'garden',\n",
       " 'glasses',\n",
       " 'guys',\n",
       " 'hot water',\n",
       " 'internet',\n",
       " 'italian',\n",
       " 'key station',\n",
       " 'live show',\n",
       " 'my 3 · night stay',\n",
       " 'my next stay',\n",
       " 'nothing to complain',\n",
       " 'offer',\n",
       " 'people.',\n",
       " 'playground',\n",
       " 'programme',\n",
       " 'public transportation',\n",
       " 'radio',\n",
       " 'range',\n",
       " 'reason to eat in',\n",
       " 'relax time',\n",
       " 'repair',\n",
       " 'room lock',\n",
       " 'room.staff',\n",
       " 'sea',\n",
       " 'sea.very',\n",
       " 'selection for the buffet',\n",
       " 'shower',\n",
       " 'size',\n",
       " 'size of the room',\n",
       " 'skype',\n",
       " 'some aircraft noise',\n",
       " 'some of them',\n",
       " 'some rennovation',\n",
       " 'spa',\n",
       " 'speed',\n",
       " 'staff.The',\n",
       " 'sunbeds',\n",
       " 'swimming pool water',\n",
       " 'swimmingpool',\n",
       " 'tea / coffee',\n",
       " 'terrace',\n",
       " 'that place.',\n",
       " 'the 4 stars',\n",
       " 'the Metro',\n",
       " 'the balcony',\n",
       " 'the bed',\n",
       " 'the centre of Amsterdam',\n",
       " 'the centre of Koln Altstadt',\n",
       " 'the city center',\n",
       " 'the co · ordination between the working personnel',\n",
       " 'the elevator',\n",
       " 'the garden',\n",
       " 'the little gift when leaving',\n",
       " 'the manager',\n",
       " 'the only one visitor',\n",
       " 'the people who are working',\n",
       " 'the shopping center in the hotel',\n",
       " 'the smilling faces of the emploeys',\n",
       " 'the spa',\n",
       " 'the waiters in pirates',\n",
       " 'the water in the lagune and pond',\n",
       " 'the way they solved the food situation',\n",
       " 'this room',\n",
       " 'tiling round toilet area',\n",
       " 'time',\n",
       " 'to get there',\n",
       " 'toiletries',\n",
       " 'towels in the room',\n",
       " 'two main buffet restaurants',\n",
       " 'up to date flat screen',\n",
       " 'view of the mountains from my balcony',\n",
       " 'views',\n",
       " 'vouchers for meals and drinks',\n",
       " 'wash basin',\n",
       " 'water in the fridge',\n",
       " 'with regard to food',\n",
       " 'zone 1'}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_target_diff_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>Source</th>\n",
       "      <th>Source_string</th>\n",
       "      <th>Source_index</th>\n",
       "      <th>Target</th>\n",
       "      <th>Target_string</th>\n",
       "      <th>Target_index</th>\n",
       "      <th>Polar_expression</th>\n",
       "      <th>Polar_expression_string</th>\n",
       "      <th>...</th>\n",
       "      <th>Implicit_holder_exclusive</th>\n",
       "      <th>H-A-O</th>\n",
       "      <th>H-IA-IO</th>\n",
       "      <th>H-A-IO</th>\n",
       "      <th>H-IA-O</th>\n",
       "      <th>IH-A-O</th>\n",
       "      <th>IH-A-IO</th>\n",
       "      <th>IH-IA-O</th>\n",
       "      <th>IH-IA-IO</th>\n",
       "      <th>Token Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>They were there just to keep us happy .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-6</td>\n",
       "      <td>[[us], [29:31]]</td>\n",
       "      <td>[us]</td>\n",
       "      <td>[29:31]</td>\n",
       "      <td>[[They], [0:4]]</td>\n",
       "      <td>[They]</td>\n",
       "      <td>[0:4]</td>\n",
       "      <td>[[to keep, happy], [21:28, 32:37]]</td>\n",
       "      <td>[to keep, happy]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A wonderful place to go and we are planning to return as soon as we can .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-7</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[place], [12:17]]</td>\n",
       "      <td>[place]</td>\n",
       "      <td>[12:17]</td>\n",
       "      <td>[[wonderful], [2:11]]</td>\n",
       "      <td>[wonderful]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A wonderful place to go and we are planning to return as soon as we can .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-7</td>\n",
       "      <td>[[we], [28:30]]</td>\n",
       "      <td>[we]</td>\n",
       "      <td>[28:30]</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[planning to return], [35:53]]</td>\n",
       "      <td>[planning to return]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Thank you for such a wonderful experience .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-8</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[experience], [31:41]]</td>\n",
       "      <td>[experience]</td>\n",
       "      <td>[31:41]</td>\n",
       "      <td>[[wonderful], [21:30]]</td>\n",
       "      <td>[wonderful]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Hotel was under construction .</td>\n",
       "      <td>opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-2</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[Hotel], [0:5]]</td>\n",
       "      <td>[Hotel]</td>\n",
       "      <td>[0:5]</td>\n",
       "      <td>[[under construction], [10:28]]</td>\n",
       "      <td>[under construction]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>860</th>\n",
       "      <td>Front desk wasn 't the best , quite slow for passport registration and clerk insisted that we pay up for a better room .</td>\n",
       "      <td>opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-8</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[Front desk], [0:10]]</td>\n",
       "      <td>[Front desk]</td>\n",
       "      <td>[0:10]</td>\n",
       "      <td>[[quite slow], [30:40]]</td>\n",
       "      <td>[quite slow]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>861</th>\n",
       "      <td>Front desk wasn 't the best , quite slow for passport registration and clerk insisted that we pay up for a better room .</td>\n",
       "      <td>opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-8</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[Front desk], [0:10]]</td>\n",
       "      <td>[Front desk]</td>\n",
       "      <td>[0:10]</td>\n",
       "      <td>[[wasn 't the best], [11:27]]</td>\n",
       "      <td>[wasn 't the best]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>862</th>\n",
       "      <td>In · room breakfast was tasty and quite large , awesome .</td>\n",
       "      <td>opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[breakfast], [10:19]]</td>\n",
       "      <td>[breakfast]</td>\n",
       "      <td>[10:19]</td>\n",
       "      <td>[[quite large], [34:45]]</td>\n",
       "      <td>[quite large]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>863</th>\n",
       "      <td>In · room breakfast was tasty and quite large , awesome .</td>\n",
       "      <td>opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[breakfast], [10:19]]</td>\n",
       "      <td>[breakfast]</td>\n",
       "      <td>[10:19]</td>\n",
       "      <td>[[tasty], [24:29]]</td>\n",
       "      <td>[tasty]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>864</th>\n",
       "      <td>In · room breakfast was tasty and quite large , awesome .</td>\n",
       "      <td>opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[breakfast], [10:19]]</td>\n",
       "      <td>[breakfast]</td>\n",
       "      <td>[10:19]</td>\n",
       "      <td>[[awesome], [48:55]]</td>\n",
       "      <td>[awesome]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>865 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                         text  \\\n",
       "0                                                                                     They were there just to keep us happy .   \n",
       "1                                                   A wonderful place to go and we are planning to return as soon as we can .   \n",
       "2                                                   A wonderful place to go and we are planning to return as soon as we can .   \n",
       "3                                                                                 Thank you for such a wonderful experience .   \n",
       "4                                                                                              Hotel was under construction .   \n",
       "..                                                                                                                        ...   \n",
       "860  Front desk wasn 't the best , quite slow for passport registration and clerk insisted that we pay up for a better room .   \n",
       "861  Front desk wasn 't the best , quite slow for passport registration and clerk insisted that we pay up for a better room .   \n",
       "862                                                                 In · room breakfast was tasty and quite large , awesome .   \n",
       "863                                                                 In · room breakfast was tasty and quite large , awesome .   \n",
       "864                                                                 In · room breakfast was tasty and quite large , awesome .   \n",
       "\n",
       "                                                                 sent_id  \\\n",
       "0    opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-6   \n",
       "1    opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-7   \n",
       "2    opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-7   \n",
       "3    opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-8   \n",
       "4    opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-2   \n",
       "..                                                                   ...   \n",
       "860  opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-8   \n",
       "861  opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-8   \n",
       "862  opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9   \n",
       "863  opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9   \n",
       "864  opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9   \n",
       "\n",
       "              Source Source_string Source_index                   Target  \\\n",
       "0    [[us], [29:31]]          [us]      [29:31]          [[They], [0:4]]   \n",
       "1           [[], []]            []           []       [[place], [12:17]]   \n",
       "2    [[we], [28:30]]          [we]      [28:30]                 [[], []]   \n",
       "3           [[], []]            []           []  [[experience], [31:41]]   \n",
       "4           [[], []]            []           []         [[Hotel], [0:5]]   \n",
       "..               ...           ...          ...                      ...   \n",
       "860         [[], []]            []           []   [[Front desk], [0:10]]   \n",
       "861         [[], []]            []           []   [[Front desk], [0:10]]   \n",
       "862         [[], []]            []           []   [[breakfast], [10:19]]   \n",
       "863         [[], []]            []           []   [[breakfast], [10:19]]   \n",
       "864         [[], []]            []           []   [[breakfast], [10:19]]   \n",
       "\n",
       "    Target_string Target_index                    Polar_expression  \\\n",
       "0          [They]        [0:4]  [[to keep, happy], [21:28, 32:37]]   \n",
       "1         [place]      [12:17]               [[wonderful], [2:11]]   \n",
       "2              []           []     [[planning to return], [35:53]]   \n",
       "3    [experience]      [31:41]              [[wonderful], [21:30]]   \n",
       "4         [Hotel]        [0:5]     [[under construction], [10:28]]   \n",
       "..            ...          ...                                 ...   \n",
       "860  [Front desk]       [0:10]             [[quite slow], [30:40]]   \n",
       "861  [Front desk]       [0:10]       [[wasn 't the best], [11:27]]   \n",
       "862   [breakfast]      [10:19]            [[quite large], [34:45]]   \n",
       "863   [breakfast]      [10:19]                  [[tasty], [24:29]]   \n",
       "864   [breakfast]      [10:19]                [[awesome], [48:55]]   \n",
       "\n",
       "    Polar_expression_string  ... Implicit_holder_exclusive H-A-O H-IA-IO  \\\n",
       "0          [to keep, happy]  ...                     False     1   False   \n",
       "1               [wonderful]  ...                      True     0   False   \n",
       "2      [planning to return]  ...                     False     0   False   \n",
       "3               [wonderful]  ...                      True     0   False   \n",
       "4      [under construction]  ...                      True     0   False   \n",
       "..                      ...  ...                       ...   ...     ...   \n",
       "860            [quite slow]  ...                      True     0   False   \n",
       "861      [wasn 't the best]  ...                      True     0   False   \n",
       "862           [quite large]  ...                      True     0   False   \n",
       "863                 [tasty]  ...                      True     0   False   \n",
       "864               [awesome]  ...                      True     0   False   \n",
       "\n",
       "     H-A-IO  H-IA-O  IH-A-O  IH-A-IO  IH-IA-O  IH-IA-IO  Token Count  \n",
       "0     False   False   False    False    False     False           11  \n",
       "1     False   False    True    False    False     False           19  \n",
       "2     False    True   False    False    False     False           19  \n",
       "3     False   False    True    False    False     False           10  \n",
       "4     False   False    True    False    False     False            8  \n",
       "..      ...     ...     ...      ...      ...       ...          ...  \n",
       "860   False   False    True    False    False     False           27  \n",
       "861   False   False    True    False    False     False           27  \n",
       "862   False   False    True    False    False     False           14  \n",
       "863   False   False    True    False    False     False           14  \n",
       "864   False   False    True    False    False     False           14  \n",
       "\n",
       "[865 rows x 28 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_opener_en_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['They']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_opener_en_test.loc[0]['Target_string']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>Source</th>\n",
       "      <th>Source_string</th>\n",
       "      <th>Source_index</th>\n",
       "      <th>Target</th>\n",
       "      <th>Target_string</th>\n",
       "      <th>Target_index</th>\n",
       "      <th>Polar_expression</th>\n",
       "      <th>Polar_expression_string</th>\n",
       "      <th>...</th>\n",
       "      <th>Implicit_holder_exclusive</th>\n",
       "      <th>H-A-O</th>\n",
       "      <th>H-IA-IO</th>\n",
       "      <th>H-A-IO</th>\n",
       "      <th>H-IA-O</th>\n",
       "      <th>IH-A-O</th>\n",
       "      <th>IH-A-IO</th>\n",
       "      <th>IH-IA-O</th>\n",
       "      <th>IH-IA-IO</th>\n",
       "      <th>Token Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>565</th>\n",
       "      <td>The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .</td>\n",
       "      <td>opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[a waiter on hand], [87:103]]</td>\n",
       "      <td>[a waiter on hand]</td>\n",
       "      <td>[87:103]</td>\n",
       "      <td>[[there was always], [70:86]]</td>\n",
       "      <td>[there was always]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>566</th>\n",
       "      <td>The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .</td>\n",
       "      <td>opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[24 hr bar], [16:25]]</td>\n",
       "      <td>[24 hr bar]</td>\n",
       "      <td>[16:25]</td>\n",
       "      <td>[[well run], [30:38]]</td>\n",
       "      <td>[well run]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                   text  \\\n",
       "565  The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .   \n",
       "566  The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .   \n",
       "\n",
       "                                                                 sent_id  \\\n",
       "565  opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6   \n",
       "566  opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6   \n",
       "\n",
       "       Source Source_string Source_index                          Target  \\\n",
       "565  [[], []]            []           []  [[a waiter on hand], [87:103]]   \n",
       "566  [[], []]            []           []          [[24 hr bar], [16:25]]   \n",
       "\n",
       "          Target_string Target_index               Polar_expression  \\\n",
       "565  [a waiter on hand]     [87:103]  [[there was always], [70:86]]   \n",
       "566         [24 hr bar]      [16:25]          [[well run], [30:38]]   \n",
       "\n",
       "    Polar_expression_string  ... Implicit_holder_exclusive H-A-O H-IA-IO  \\\n",
       "565      [there was always]  ...                      True     0   False   \n",
       "566              [well run]  ...                      True     0   False   \n",
       "\n",
       "     H-A-IO  H-IA-O  IH-A-O  IH-A-IO  IH-IA-O  IH-IA-IO  Token Count  \n",
       "565   False   False    True    False    False     False           28  \n",
       "566   False   False    True    False    False     False           28  \n",
       "\n",
       "[2 rows x 28 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_opener_en_test[df_opener_en_test['sent_id'] == \"opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>Source</th>\n",
       "      <th>Source_string</th>\n",
       "      <th>Source_index</th>\n",
       "      <th>Target</th>\n",
       "      <th>Target_string</th>\n",
       "      <th>Target_index</th>\n",
       "      <th>Polar_expression</th>\n",
       "      <th>Polar_expression_string</th>\n",
       "      <th>...</th>\n",
       "      <th>Implicit_target_exclusive</th>\n",
       "      <th>Implicit_holder_exclusive</th>\n",
       "      <th>H-A-O</th>\n",
       "      <th>H-IA-IO</th>\n",
       "      <th>H-A-IO</th>\n",
       "      <th>H-IA-O</th>\n",
       "      <th>IH-A-O</th>\n",
       "      <th>IH-A-IO</th>\n",
       "      <th>IH-IA-O</th>\n",
       "      <th>IH-IA-IO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>523</th>\n",
       "      <td>The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .</td>\n",
       "      <td>opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[The first floor], [0:15]]</td>\n",
       "      <td>[The first floor]</td>\n",
       "      <td>[0:15]</td>\n",
       "      <td>[[24 hr], [16:21]]</td>\n",
       "      <td>[24 hr]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524</th>\n",
       "      <td>The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .</td>\n",
       "      <td>opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[The first floor], [0:15]]</td>\n",
       "      <td>[The first floor]</td>\n",
       "      <td>[0:15]</td>\n",
       "      <td>[[well run], [30:38]]</td>\n",
       "      <td>[well run]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>525</th>\n",
       "      <td>The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .</td>\n",
       "      <td>opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[bar], [22:25]]</td>\n",
       "      <td>[bar]</td>\n",
       "      <td>[22:25]</td>\n",
       "      <td>[[well run], [30:38]]</td>\n",
       "      <td>[well run]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>526</th>\n",
       "      <td>The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .</td>\n",
       "      <td>opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[a waiter], [87:95]]</td>\n",
       "      <td>[a waiter]</td>\n",
       "      <td>[87:95]</td>\n",
       "      <td>[[there was always], [70:86]]</td>\n",
       "      <td>[there was always]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>527</th>\n",
       "      <td>The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .</td>\n",
       "      <td>opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[a waiter], [87:95]]</td>\n",
       "      <td>[a waiter]</td>\n",
       "      <td>[87:95]</td>\n",
       "      <td>[[on hand], [96:103]]</td>\n",
       "      <td>[on hand]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                   text  \\\n",
       "523  The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .   \n",
       "524  The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .   \n",
       "525  The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .   \n",
       "526  The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .   \n",
       "527  The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .   \n",
       "\n",
       "                                                                 sent_id  \\\n",
       "523  opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6   \n",
       "524  opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6   \n",
       "525  opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6   \n",
       "526  opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6   \n",
       "527  opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6   \n",
       "\n",
       "       Source Source_string Source_index                       Target  \\\n",
       "523  [[], []]            []           []  [[The first floor], [0:15]]   \n",
       "524  [[], []]            []           []  [[The first floor], [0:15]]   \n",
       "525  [[], []]            []           []             [[bar], [22:25]]   \n",
       "526  [[], []]            []           []        [[a waiter], [87:95]]   \n",
       "527  [[], []]            []           []        [[a waiter], [87:95]]   \n",
       "\n",
       "         Target_string Target_index               Polar_expression  \\\n",
       "523  [The first floor]       [0:15]             [[24 hr], [16:21]]   \n",
       "524  [The first floor]       [0:15]          [[well run], [30:38]]   \n",
       "525              [bar]      [22:25]          [[well run], [30:38]]   \n",
       "526         [a waiter]      [87:95]  [[there was always], [70:86]]   \n",
       "527         [a waiter]      [87:95]          [[on hand], [96:103]]   \n",
       "\n",
       "    Polar_expression_string  ... Implicit_target_exclusive  \\\n",
       "523                 [24 hr]  ...                     False   \n",
       "524              [well run]  ...                     False   \n",
       "525              [well run]  ...                     False   \n",
       "526      [there was always]  ...                     False   \n",
       "527               [on hand]  ...                     False   \n",
       "\n",
       "    Implicit_holder_exclusive  H-A-O  H-IA-IO  H-A-IO  H-IA-O  IH-A-O  \\\n",
       "523                      True      0    False   False   False    True   \n",
       "524                      True      0    False   False   False    True   \n",
       "525                      True      0    False   False   False    True   \n",
       "526                      True      0    False   False   False    True   \n",
       "527                      True      0    False   False   False    True   \n",
       "\n",
       "     IH-A-IO  IH-IA-O  IH-IA-IO  \n",
       "523    False    False     False  \n",
       "524    False    False     False  \n",
       "525    False    False     False  \n",
       "526    False    False     False  \n",
       "527    False    False     False  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_en_roberta_largev1[df_en_roberta_largev1['sent_id'] == \"opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_opener_en_test[df_opener_en_test['sent_id'] == \"opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>Source</th>\n",
       "      <th>Source_string</th>\n",
       "      <th>Source_index</th>\n",
       "      <th>Target</th>\n",
       "      <th>Target_string</th>\n",
       "      <th>Target_index</th>\n",
       "      <th>Polar_expression</th>\n",
       "      <th>Polar_expression_string</th>\n",
       "      <th>...</th>\n",
       "      <th>Implicit_target_exclusive</th>\n",
       "      <th>Implicit_holder_exclusive</th>\n",
       "      <th>H-A-O</th>\n",
       "      <th>H-IA-IO</th>\n",
       "      <th>H-A-IO</th>\n",
       "      <th>H-IA-O</th>\n",
       "      <th>IH-A-O</th>\n",
       "      <th>IH-A-IO</th>\n",
       "      <th>IH-IA-O</th>\n",
       "      <th>IH-IA-IO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>523</th>\n",
       "      <td>The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .</td>\n",
       "      <td>opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[The first floor], [0:15]]</td>\n",
       "      <td>[The first floor]</td>\n",
       "      <td>[0:15]</td>\n",
       "      <td>[[24 hr], [16:21]]</td>\n",
       "      <td>[24 hr]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524</th>\n",
       "      <td>The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .</td>\n",
       "      <td>opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[The first floor], [0:15]]</td>\n",
       "      <td>[The first floor]</td>\n",
       "      <td>[0:15]</td>\n",
       "      <td>[[well run], [30:38]]</td>\n",
       "      <td>[well run]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>525</th>\n",
       "      <td>The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .</td>\n",
       "      <td>opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[bar], [22:25]]</td>\n",
       "      <td>[bar]</td>\n",
       "      <td>[22:25]</td>\n",
       "      <td>[[well run], [30:38]]</td>\n",
       "      <td>[well run]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>526</th>\n",
       "      <td>The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .</td>\n",
       "      <td>opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[a waiter], [87:95]]</td>\n",
       "      <td>[a waiter]</td>\n",
       "      <td>[87:95]</td>\n",
       "      <td>[[there was always], [70:86]]</td>\n",
       "      <td>[there was always]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>527</th>\n",
       "      <td>The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .</td>\n",
       "      <td>opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[a waiter], [87:95]]</td>\n",
       "      <td>[a waiter]</td>\n",
       "      <td>[87:95]</td>\n",
       "      <td>[[on hand], [96:103]]</td>\n",
       "      <td>[on hand]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                   text  \\\n",
       "523  The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .   \n",
       "524  The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .   \n",
       "525  The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .   \n",
       "526  The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .   \n",
       "527  The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .   \n",
       "\n",
       "                                                                 sent_id  \\\n",
       "523  opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6   \n",
       "524  opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6   \n",
       "525  opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6   \n",
       "526  opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6   \n",
       "527  opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6   \n",
       "\n",
       "       Source Source_string Source_index                       Target  \\\n",
       "523  [[], []]            []           []  [[The first floor], [0:15]]   \n",
       "524  [[], []]            []           []  [[The first floor], [0:15]]   \n",
       "525  [[], []]            []           []             [[bar], [22:25]]   \n",
       "526  [[], []]            []           []        [[a waiter], [87:95]]   \n",
       "527  [[], []]            []           []        [[a waiter], [87:95]]   \n",
       "\n",
       "         Target_string Target_index               Polar_expression  \\\n",
       "523  [The first floor]       [0:15]             [[24 hr], [16:21]]   \n",
       "524  [The first floor]       [0:15]          [[well run], [30:38]]   \n",
       "525              [bar]      [22:25]          [[well run], [30:38]]   \n",
       "526         [a waiter]      [87:95]  [[there was always], [70:86]]   \n",
       "527         [a waiter]      [87:95]          [[on hand], [96:103]]   \n",
       "\n",
       "    Polar_expression_string  ... Implicit_target_exclusive  \\\n",
       "523                 [24 hr]  ...                     False   \n",
       "524              [well run]  ...                     False   \n",
       "525              [well run]  ...                     False   \n",
       "526      [there was always]  ...                     False   \n",
       "527               [on hand]  ...                     False   \n",
       "\n",
       "    Implicit_holder_exclusive  H-A-O  H-IA-IO  H-A-IO  H-IA-O  IH-A-O  \\\n",
       "523                      True      0    False   False   False    True   \n",
       "524                      True      0    False   False   False    True   \n",
       "525                      True      0    False   False   False    True   \n",
       "526                      True      0    False   False   False    True   \n",
       "527                      True      0    False   False   False    True   \n",
       "\n",
       "     IH-A-IO  IH-IA-O  IH-IA-IO  \n",
       "523    False    False     False  \n",
       "524    False    False     False  \n",
       "525    False    False     False  \n",
       "526    False    False     False  \n",
       "527    False    False     False  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_en_roberta_largev1[df_en_roberta_largev1['text'].str.contains(\"24 hr bar\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "215 472 370\n"
     ]
    }
   ],
   "source": [
    "es_target_diff_test = es_target_tokens_test.difference(es_target_tokens_pred)\n",
    "es_target_diff_pred = es_target_tokens_test.difference(es_target_tokens_pred)\n",
    "\n",
    "print(len(es_target_diff_test),len(es_target_tokens_test), len(es_target_tokens_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'7 dias',\n",
       " 'APARTAMENTO',\n",
       " 'Algo',\n",
       " 'Alojamiento',\n",
       " 'Camas de 1,05',\n",
       " 'Camp Nou',\n",
       " 'El Gran Bahía Principe Ambar',\n",
       " 'El complejo',\n",
       " 'El tamaño de la habitación y el baño',\n",
       " 'Fin de semana',\n",
       " 'La comida y cena',\n",
       " 'Los de recepción',\n",
       " 'Museos',\n",
       " 'Neptuno',\n",
       " 'Puerta del Sol',\n",
       " 'Sol',\n",
       " 'Tiendas',\n",
       " 'WIFI',\n",
       " 'a lounge',\n",
       " 'a todos lados',\n",
       " 'acogimiento por parte de todos los empleados',\n",
       " 'actividades',\n",
       " 'acudir a un musical',\n",
       " 'aire acondicionado',\n",
       " 'al estadio de futbol del barca',\n",
       " 'algo',\n",
       " 'amplitud de menu',\n",
       " 'autobus',\n",
       " 'bandeja de infusiones y café',\n",
       " 'bar',\n",
       " 'barandillas',\n",
       " 'baños',\n",
       " 'bollería',\n",
       " 'brares',\n",
       " 'bus urbanos.se',\n",
       " 'cadena Playa Senator',\n",
       " 'cafes',\n",
       " 'cafetera eléctrica',\n",
       " 'calidad',\n",
       " 'camas queen',\n",
       " 'centro de Peñiscola',\n",
       " 'cereales',\n",
       " 'cesta de frutas',\n",
       " 'cines',\n",
       " 'con Moet Chandon',\n",
       " 'congreso de los diputados',\n",
       " 'conocer la ciudad',\n",
       " 'cualquier punto monumental de la ciudad',\n",
       " 'cuarto de baño',\n",
       " 'cuartos',\n",
       " 'cuidado',\n",
       " 'cultura',\n",
       " 'de Sevilla capital',\n",
       " 'de cocoteros',\n",
       " 'de nuestra estancia',\n",
       " 'de todo',\n",
       " 'decoracion',\n",
       " 'del Nou Camp',\n",
       " 'del camp nou',\n",
       " 'del servicio',\n",
       " 'del transporte público',\n",
       " 'descansa',\n",
       " 'descansar',\n",
       " 'detallitos',\n",
       " 'diseño',\n",
       " 'dos hamacas',\n",
       " 'el aire acondicionado',\n",
       " 'el apartamento',\n",
       " 'el bullicio de la gente',\n",
       " 'el clima',\n",
       " 'el de recepcion',\n",
       " 'el del hotel',\n",
       " 'el desayuno bufet',\n",
       " 'el detalle de una botella de cava',\n",
       " 'el fiambre',\n",
       " 'el gimnasio',\n",
       " 'el hote',\n",
       " 'el lobby del Hotel',\n",
       " 'el mantel',\n",
       " 'el olor a orine viejo',\n",
       " 'el personal de limpieza',\n",
       " 'el personal de recepción',\n",
       " 'el queso',\n",
       " 'el resto',\n",
       " 'el servicio de habitaciones',\n",
       " 'el telefono',\n",
       " 'el trato recibido',\n",
       " 'en alojarme otra vez alli',\n",
       " 'en las paredes',\n",
       " 'escritorio',\n",
       " 'ese aspecto',\n",
       " 'ese tipo de instalaciones',\n",
       " 'eso',\n",
       " 'esta cadena',\n",
       " 'este',\n",
       " 'etc.las habitaciones',\n",
       " 'excesivo ruido dentro de la habitación',\n",
       " 'fachada',\n",
       " 'fin de semana',\n",
       " 'friobar',\n",
       " 'gastronoma',\n",
       " 'hotel de Madrid',\n",
       " 'hotel palladium punta cana',\n",
       " 'iluminosidad',\n",
       " 'indicacion concreta ni acompanamiento de personal',\n",
       " 'infusiones',\n",
       " 'internet',\n",
       " 'inversión en formación en idiomas',\n",
       " 'ir de compras',\n",
       " 'la animacion',\n",
       " 'la azote',\n",
       " 'la cama supletoria',\n",
       " 'la estación de tren',\n",
       " 'la estación del AVE',\n",
       " 'la fachada',\n",
       " 'la impieza',\n",
       " 'la localizacion',\n",
       " 'la oferta',\n",
       " 'la operadora',\n",
       " 'la plaza del duomo',\n",
       " 'la reposicion del minibar',\n",
       " 'la restauración',\n",
       " 'la situación',\n",
       " 'la ubicación del hotel',\n",
       " 'la zona de los múseos',\n",
       " 'la zona donde está',\n",
       " 'las sabanas',\n",
       " 'lo cual',\n",
       " 'lo demás',\n",
       " 'localización',\n",
       " 'los 2 jacuzzy',\n",
       " 'los jardines',\n",
       " 'los nórdicos',\n",
       " 'los trabajadores',\n",
       " 'lugar para descansar en familia',\n",
       " 'medidas de seguridad apropiadas',\n",
       " 'miinibar',\n",
       " 'mis expectativas de comodidad y tranquilida',\n",
       " 'mogollon de reclamaciones',\n",
       " 'muchos desperfectos',\n",
       " 'nada de nada',\n",
       " 'ninguno de los dos',\n",
       " 'no oir ni un solo ruido',\n",
       " 'nuestra decision',\n",
       " 'otras bebidas',\n",
       " 'para llegar a la playa',\n",
       " 'para ver el musical \" El Rey León',\n",
       " 'parking',\n",
       " 'patio interior',\n",
       " 'pequeña cafetera',\n",
       " 'pequeñas frutas tropicales',\n",
       " 'piscina',\n",
       " 'piscinas',\n",
       " 'plaza Mayor',\n",
       " 'por dejar unas horas mas el coche en el garaje',\n",
       " 'prensa',\n",
       " 'puerta del sol',\n",
       " 'recepción 24 horas',\n",
       " 'reclamar',\n",
       " 'relación calidad precio',\n",
       " 'ruidos',\n",
       " 'sala',\n",
       " 'semejante complejo',\n",
       " 'servicio de fax',\n",
       " 'shushi',\n",
       " 'sitios de ocio',\n",
       " 'sitios para tomar algo',\n",
       " 'sol',\n",
       " 'spa',\n",
       " 'su albornoz',\n",
       " 'su arteria principal ( Gran Via',\n",
       " 'su buffet libre',\n",
       " 'su ubicacon',\n",
       " 'tapear por los numerosos bares de la zona',\n",
       " 'tema aparcamiento',\n",
       " 'terraza',\n",
       " 'tiendas',\n",
       " 'todas las facilidades',\n",
       " 'todo el personal',\n",
       " 'todo eso',\n",
       " 'todo incluído',\n",
       " 'todo lo que deseas',\n",
       " 'todo lo que se necesita',\n",
       " 'todo lo que se necesite.',\n",
       " 'todos los aspectos',\n",
       " 'trato que recibimos del personal',\n",
       " 'ubicacion',\n",
       " 'ubicación para visitar los lugares de interés turístico',\n",
       " 'un calentador de agua',\n",
       " 'un desayuno',\n",
       " 'un lavado de cara',\n",
       " 'un personal',\n",
       " 'un poco de mantenimiento',\n",
       " 'un sitio',\n",
       " 'un vehiculo',\n",
       " 'una entrada de metro',\n",
       " 'una entrada gratuita al spa',\n",
       " 'una experiencia',\n",
       " 'una habitación',\n",
       " 'una intimidad para acceder',\n",
       " 'una parada',\n",
       " 'una recepción más amplia y luminosa',\n",
       " 'una zona de Madrid',\n",
       " 'unas mantas para abrigarnos',\n",
       " 'unas vistas',\n",
       " 'unos bares',\n",
       " 'unos dias',\n",
       " 'unos jacuzzy',\n",
       " 'varios detalles',\n",
       " 'verlo todo eso',\n",
       " 'visitarlo',\n",
       " 'wifi con acceso abierto a internet',\n",
       " 'zona',\n",
       " 'zona de playa',\n",
       " 'zonas de aparcamiento'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es_target_diff_test"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Holder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 12\n",
      "5 7\n"
     ]
    }
   ],
   "source": [
    "en_holder_diff_test = en_holder_tokens_test.difference(en_holder_tokens_pred)\n",
    "en_holder_diff_pred = en_holder_tokens_pred.difference(en_holder_tokens_test)\n",
    "\n",
    "print(len(en_holder_tokens_test), len(en_holder_tokens_pred))\n",
    "\n",
    "print(len(en_holder_diff_test),len(en_holder_diff_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'all eight couples', 'eyes.I', 'me', 'my experience', 'you'}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_holder_diff_test\n",
    "\n",
    "# find you, me holder sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'building',\n",
       " 'everyone',\n",
       " 'he',\n",
       " 'many different activities',\n",
       " 'reason',\n",
       " 'the linen',\n",
       " 'the room'}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_holder_diff_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8 3\n",
      "5 0\n"
     ]
    }
   ],
   "source": [
    "es_holder_diff_test = es_holder_tokens_test.difference(es_holder_tokens_pred)\n",
    "es_holder_diff_pred = es_holder_tokens_pred.difference(es_holder_tokens_test)\n",
    "\n",
    "print(len(es_holder_tokens_test), len(es_holder_tokens_pred))\n",
    "\n",
    "print(len(es_holder_diff_test),len(es_holder_diff_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'me', 'hemos', 'Yo', 'nosotros', 'yo'} set()\n"
     ]
    }
   ],
   "source": [
    "print(es_holder_diff_test , es_holder_diff_pred) # check yo, nosotros"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Set Expression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "594 547\n",
      "220 173\n"
     ]
    }
   ],
   "source": [
    "en_express_diff_test = en_expression_tokens_test.difference(en_expression_tokens_pred)\n",
    "en_express_diff_pred = en_expression_tokens_pred.difference(en_expression_tokens_test)\n",
    "\n",
    "print(len(en_expression_tokens_test), len(en_expression_tokens_pred))\n",
    "\n",
    "print(len(en_express_diff_test),len(en_express_diff_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\"'s not worth\",\n",
       " '12 times',\n",
       " 'BEST',\n",
       " 'Central',\n",
       " 'Clean',\n",
       " 'Dirty.our',\n",
       " 'Friendly',\n",
       " 'Greek dishes with a general European twist',\n",
       " 'Many people',\n",
       " 'NOT to eat in the buffet restaurant',\n",
       " 'Not',\n",
       " 'One of the best holidays',\n",
       " 'Only',\n",
       " 'Relax and enjoy',\n",
       " 'Robbed in elevator of hotel !',\n",
       " 'Similar to',\n",
       " 'There are not',\n",
       " 'There is no',\n",
       " 'To be avoided at all costs',\n",
       " 'To expensive',\n",
       " 'Took',\n",
       " 'Totally overpriced',\n",
       " 'Within 7 minutes',\n",
       " 'a good option in this category',\n",
       " 'a happy',\n",
       " 'a real 4star hotel',\n",
       " 'a top hotel !',\n",
       " 'abit repetitive at times',\n",
       " 'accompanied by a live singer',\n",
       " 'advise everyone',\n",
       " \"all in length , which isn 't the most convenient\",\n",
       " 'all tastes',\n",
       " 'already booked',\n",
       " 'already went 14 times',\n",
       " 'always will remember with joy and happiness',\n",
       " 'annoying to wait for',\n",
       " 'are satisfied',\n",
       " 'at affordable prices',\n",
       " 'attrackts back packing , party focused youngsters',\n",
       " 'bad quality',\n",
       " 'be definitely back soon !',\n",
       " 'bit dated',\n",
       " 'black mould on tiling',\n",
       " 'boasts outstanding architecture',\n",
       " 'booked again',\n",
       " 'brocken',\n",
       " 'budget',\n",
       " 'buffet',\n",
       " 'can be considered if you can cope with the noise',\n",
       " 'candy',\n",
       " 'cannot wait until I can go back again',\n",
       " 'centrally located',\n",
       " 'changed',\n",
       " 'chic !',\n",
       " 'chipped',\n",
       " 'cleaned',\n",
       " 'cleaned every day',\n",
       " 'close to',\n",
       " 'close to airport',\n",
       " 'could be a little improved',\n",
       " 'could do with less jelly fish',\n",
       " 'could not have been better',\n",
       " 'dark',\n",
       " 'decent',\n",
       " 'did not understand much english',\n",
       " \"didn 't care at all for the complaints\",\n",
       " \"didn 't know the hotel restaurant / bar menu\",\n",
       " 'disaster',\n",
       " 'does not include any other restaurant',\n",
       " \"don 't go\",\n",
       " \"don 't refund\",\n",
       " 'easy to get',\n",
       " 'efficient',\n",
       " 'empty',\n",
       " 'encountered',\n",
       " 'enjoyed',\n",
       " 'everything you need',\n",
       " 'extra',\n",
       " 'fantastically clean',\n",
       " 'fist hole in bedroom door',\n",
       " 'for a week',\n",
       " 'for dinner',\n",
       " 'for dinner you have to wait in a queue',\n",
       " 'friendly.',\n",
       " 'fully visible from the outside',\n",
       " 'get',\n",
       " 'good quality',\n",
       " 'got',\n",
       " 'great fun',\n",
       " 'great value',\n",
       " 'had a brilliant time',\n",
       " 'had a table all to ourselves',\n",
       " 'had to pay 20 Euro',\n",
       " 'had to pay for',\n",
       " 'had to sit on towels',\n",
       " 'had to squeeze in order to fit between the shower and wall to get to the toilet',\n",
       " 'happy',\n",
       " 'hard to leave it !',\n",
       " 'have',\n",
       " 'have spoken very highly of this hotel to my friends and colleagues',\n",
       " 'have tea coffee making facilties',\n",
       " 'have to queue for',\n",
       " 'have to wait',\n",
       " 'helpful in providing directions and general info',\n",
       " 'helpfull',\n",
       " 'highly recommend',\n",
       " 'hours',\n",
       " 'in a very nice location on the golf field',\n",
       " 'in other part',\n",
       " 'including',\n",
       " 'just cross the boulevar du Periferique',\n",
       " 'just functional',\n",
       " 'let alone',\n",
       " 'like',\n",
       " 'loads of',\n",
       " 'located among the vineyards and olive trees',\n",
       " 'looking to come back next year',\n",
       " 'looks a bit dated',\n",
       " 'lots of space',\n",
       " 'lousy',\n",
       " 'loved staying',\n",
       " 'maintained',\n",
       " 'maximum',\n",
       " 'may look great',\n",
       " 'meals in the main self sevice restaurant',\n",
       " 'might',\n",
       " 'mini fridge.',\n",
       " 'more than 700',\n",
       " 'moved out',\n",
       " 'near',\n",
       " 'nearby',\n",
       " 'negative',\n",
       " 'never have',\n",
       " 'new',\n",
       " 'nice alternation',\n",
       " 'no diversity',\n",
       " 'no place to hang something more than two shirts',\n",
       " 'no place to put luggage',\n",
       " 'noise',\n",
       " 'nor too small',\n",
       " 'not have been able to cook a meal',\n",
       " 'not international',\n",
       " 'not kind',\n",
       " 'not like',\n",
       " 'not pleasant',\n",
       " 'not run down or anything',\n",
       " 'odd stains on sofa',\n",
       " 'offered',\n",
       " 'ok.',\n",
       " 'open daily',\n",
       " 'over look the pool.',\n",
       " 'painting',\n",
       " 'perfekt',\n",
       " 'planning to return',\n",
       " 'plenty of very good',\n",
       " 'private',\n",
       " 'profession',\n",
       " 'provides a prefect \" base camp \"',\n",
       " 'quite easily by using a nearby bus',\n",
       " 'quite slow',\n",
       " 'reach',\n",
       " 'reachable after 166 steps downstairs ( or upstairs )',\n",
       " 'really kind',\n",
       " 'really small',\n",
       " 'reasonable in terms of a place to sleep',\n",
       " 'relaxing',\n",
       " 'rotten',\n",
       " 'seemed to be taken',\n",
       " 'service minded',\n",
       " 'shopping center \" Diagonal Mar \"',\n",
       " 'should be included',\n",
       " 'slendid',\n",
       " 'smoked everywhere',\n",
       " 'so helpful',\n",
       " 'so kind',\n",
       " 'spend the extra money',\n",
       " 'spoilt',\n",
       " 'stay in a better one',\n",
       " 'stopped foor several moments',\n",
       " 'supported our needs throughout the visit',\n",
       " 'takes no responsibility',\n",
       " 'the most picturesque of Tuscany',\n",
       " 'there are many',\n",
       " 'to consider when you just want to spend a night and get fast to the airport first thing in the morning',\n",
       " 'to keep',\n",
       " 'to make your holiday perfect',\n",
       " 'to stay when you want to visit Paris',\n",
       " 'too huge',\n",
       " 'treated to a full breakfast daily',\n",
       " 'trying very , very , very friendly',\n",
       " 'uncomfortable',\n",
       " 'understanding',\n",
       " 'very cozy',\n",
       " 'very easy to travel to anywhere',\n",
       " 'very funny',\n",
       " 'very spacious',\n",
       " 'very sunny',\n",
       " 'very warm',\n",
       " 'walking distance to the metro',\n",
       " 'warmly welcomed by all the staff',\n",
       " 'was',\n",
       " 'was not cleaned properly',\n",
       " 'was taking place',\n",
       " \"wasn 't sufficient\",\n",
       " 'well laid out',\n",
       " 'were kept tidy',\n",
       " 'were made to feel very welcome',\n",
       " 'were not sure',\n",
       " 'will like',\n",
       " 'will not forget',\n",
       " 'willing to assist',\n",
       " 'with a lake',\n",
       " 'with a view on Nevskii',\n",
       " 'with all necessary infrastructure',\n",
       " 'with swipe card access',\n",
       " 'with twin mattresses',\n",
       " 'work very hard',\n",
       " 'worked fine',\n",
       " 'working',\n",
       " 'would take an hour or so'}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "en_express_diff_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "688 530\n",
      "331 173\n"
     ]
    }
   ],
   "source": [
    "es_express_diff_test = es_expression_tokens_test.difference(es_expression_tokens_pred)\n",
    "es_express_diff_pred = es_expression_tokens_pred.difference(es_expression_tokens_test)\n",
    "\n",
    "print(len(es_expression_tokens_test), len(es_expression_tokens_pred))\n",
    "\n",
    "print(len(es_express_diff_test),len(es_express_diff_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'%',\n",
       " '0,50 €',\n",
       " '10 · 15 min.',\n",
       " '2,5 €',\n",
       " 'Bien',\n",
       " 'Comodidad',\n",
       " 'Descanso',\n",
       " 'ESTUPENDO',\n",
       " 'El gran problema',\n",
       " 'En este sentido deja mucho que desear',\n",
       " 'Enhorabuena',\n",
       " 'Ideal para pasar un fin de semana de turismo por la capital',\n",
       " 'Imposible',\n",
       " 'Inútil',\n",
       " 'La cercanía',\n",
       " 'Lamentable',\n",
       " 'Llegando a la madrugada no hubo',\n",
       " 'Lo unico q vale',\n",
       " 'Muy amable',\n",
       " 'Muy amables',\n",
       " 'Muy bien',\n",
       " 'Muy bien ubicado',\n",
       " 'Muy buenas',\n",
       " 'NI OS ACERQUÉIS',\n",
       " 'No',\n",
       " 'No es un',\n",
       " 'Nos encantaría volver',\n",
       " 'Problemas con el estado de la cama',\n",
       " 'Remoñones',\n",
       " 'Repetiremos',\n",
       " 'Repetiremos sin duda',\n",
       " 'Tranquilo',\n",
       " 'Tuvieron',\n",
       " 'Una verdader tortura',\n",
       " 'a 15 minutos andando',\n",
       " 'a 5 minutos',\n",
       " 'a cualquier hora',\n",
       " 'a la altura de un resort !',\n",
       " 'a la entrada',\n",
       " 'a pesar de tener ya algunos años',\n",
       " 'a una parada de metro',\n",
       " 'abusando',\n",
       " 'acogedor',\n",
       " 'aconsejan',\n",
       " 'aconsejaria',\n",
       " 'acorde a la categoria de hotel',\n",
       " 'acorde con las características que ofrecen',\n",
       " 'al igual que',\n",
       " 'al lado',\n",
       " 'alejado',\n",
       " 'algo apartado del centro',\n",
       " 'algo caro',\n",
       " 'algo falto de calidad para un cuatro estrellas.',\n",
       " 'alguna grieta',\n",
       " 'amplio',\n",
       " 'amplios',\n",
       " 'arrepiento',\n",
       " 'atentos',\n",
       " 'aunque en Bormujos',\n",
       " 'aunque parezca lejana del centro de la ciudad',\n",
       " 'bastante usadas',\n",
       " 'bien agradable',\n",
       " 'bien cuidado / reformado',\n",
       " 'bien dispuesta',\n",
       " 'bordes',\n",
       " 'buen estado',\n",
       " 'bueno ver el Barsa',\n",
       " 'buffet',\n",
       " 'caro ( 15 €',\n",
       " 'casi casi',\n",
       " 'cercana',\n",
       " 'cerraba a las 23:00 h',\n",
       " 'cobrando 9 € diarios',\n",
       " 'como en todos los hoteles playa',\n",
       " 'con',\n",
       " 'con apenas',\n",
       " 'con balcón / mirador a G.Vía.',\n",
       " 'con encanto familiar',\n",
       " 'con gran calidad',\n",
       " 'con terraza',\n",
       " 'contaba',\n",
       " 'continuarán con',\n",
       " 'correcto para un cuatro estrellas',\n",
       " 'cortado muy , muy fino',\n",
       " 'crei',\n",
       " 'cuenta',\n",
       " 'cumple con su \" obligación \"',\n",
       " 'cumple con todo lo que uno puede esperar',\n",
       " 'da',\n",
       " 'de 4 estrellas',\n",
       " 'de ocio en madrid',\n",
       " 'del tipo casero',\n",
       " 'desastroso',\n",
       " 'desconocido para los huespedes',\n",
       " 'discuten',\n",
       " 'disfrutar',\n",
       " 'distincto',\n",
       " 'divinos',\n",
       " 'dormido fenomenal',\n",
       " 'educado',\n",
       " 'el alardeo',\n",
       " 'el relax',\n",
       " 'en el centro Madrid',\n",
       " 'en el corazón de Madrid',\n",
       " 'en la almohada',\n",
       " 'en la animada',\n",
       " 'en la azotea tenian',\n",
       " 'en pleno centro de Madrid',\n",
       " 'en pleno centro de la ciudad',\n",
       " 'encantada',\n",
       " 'encantador',\n",
       " 'entrada directa a la playa',\n",
       " 'era amplia',\n",
       " 'era escasìsimo',\n",
       " 'es aceptable',\n",
       " 'es genial',\n",
       " 'es ideal si vas a la zona de los teatros',\n",
       " 'es inmejorable',\n",
       " 'es muy atento',\n",
       " 'es muy grande',\n",
       " 'es poco amplio',\n",
       " 'es un poco elevado para la zona en la que esta situada en la ciudad',\n",
       " 'espacio natural',\n",
       " 'espectaculares',\n",
       " 'esta a un paso',\n",
       " 'esta en plena',\n",
       " 'esta enfrente del hotel',\n",
       " 'esta muy bien ubicado',\n",
       " 'esta situado en lo mejor de',\n",
       " 'estaban',\n",
       " 'estafaron',\n",
       " 'estamos muy contentos',\n",
       " 'estava cerrado',\n",
       " 'estava un poco turvia por no decir sucia',\n",
       " 'estupendos',\n",
       " 'está a tan solo 15 minutos de esta',\n",
       " 'está bien',\n",
       " 'está situado en todo el centro de Madrid',\n",
       " 'están rehabilitando',\n",
       " 'evitar',\n",
       " 'excelente !',\n",
       " 'falta de pintura en',\n",
       " 'fatal',\n",
       " 'feliz con',\n",
       " 'fue acertada.',\n",
       " 'fuimos',\n",
       " 'funcinando',\n",
       " 'funcionaban los chorros',\n",
       " 'gestionaba una empresa externa',\n",
       " 'grades',\n",
       " 'gran cosa',\n",
       " 'gratuíto',\n",
       " 'gustado mucho',\n",
       " 'gustó',\n",
       " 'gustó mucho',\n",
       " 'ha decepcionado',\n",
       " 'ha sido una gozada',\n",
       " 'había toda la variedad de otros hoteles playa',\n",
       " 'hay que pedir reserva',\n",
       " 'hemos sentido bien atendidos',\n",
       " 'hicimos',\n",
       " 'ideal para conocer Mequinez',\n",
       " 'ideal para escapadas urbanas',\n",
       " 'ideal para poder visitar cualquier punto de Madrid',\n",
       " 'ideal para viajes relacionados con el fútbol',\n",
       " 'incluido',\n",
       " 'inexistente en la zona',\n",
       " 'inmensa',\n",
       " 'inolvidable',\n",
       " 'joven',\n",
       " 'la calidad',\n",
       " 'la puerta del hotel',\n",
       " 'lejos',\n",
       " 'limpia',\n",
       " 'lo he pasado muy bien',\n",
       " 'lo suben al momento.',\n",
       " 'lo unico bueno',\n",
       " 'lujoso',\n",
       " 'magnifica',\n",
       " 'magnifico',\n",
       " 'mala',\n",
       " 'malo malisimo',\n",
       " 'maravilloso',\n",
       " 'maravilloso !',\n",
       " 'mejor de lo que esperaba',\n",
       " 'mis expectativas eran mayores',\n",
       " 'muchas',\n",
       " 'muchos arboles',\n",
       " 'muy aceptable',\n",
       " 'muy bien aisladas',\n",
       " 'muy bien comunicado con',\n",
       " 'muy bien cuidados',\n",
       " 'muy bien situado',\n",
       " 'muy bonita',\n",
       " 'muy buen aspecto',\n",
       " 'muy comodo',\n",
       " 'muy concurrida',\n",
       " 'muy cuidadas',\n",
       " 'muy céntrico',\n",
       " 'muy inferiores a un hotel de esta categoría',\n",
       " 'muy lento',\n",
       " 'muy muy flojo',\n",
       " 'muy recomendable',\n",
       " 'muy silenciosa',\n",
       " 'muy tranquilo',\n",
       " 'nada',\n",
       " 'nada de lo prometido',\n",
       " 'nada mas salir de la puerta del hotel',\n",
       " 'nada que ver con lo que',\n",
       " 'necesita una reforma',\n",
       " 'ni',\n",
       " 'ni caso',\n",
       " 'no aguantamos más',\n",
       " 'no cobrar nada',\n",
       " 'no creo que regrese',\n",
       " 'no desprecio',\n",
       " 'no dudare',\n",
       " 'no en la misma puerta del sol',\n",
       " 'no encontramos',\n",
       " 'no era nada espectacular',\n",
       " 'no es necesario',\n",
       " 'no escojais',\n",
       " 'no estaría mal',\n",
       " 'no está céntrico',\n",
       " 'no había',\n",
       " 'no hay',\n",
       " 'no hay ruidos',\n",
       " 'no he tenido problema para aparcar en la zona',\n",
       " 'no se escucha nada de ruido',\n",
       " 'no se oye absolutamente nada',\n",
       " 'no son muy grandes',\n",
       " 'no tiene',\n",
       " 'no todos los dias cambiaban',\n",
       " 'no vendria nada mal',\n",
       " 'noche',\n",
       " 'numerosos',\n",
       " 'ofertaron',\n",
       " 'ofrecen',\n",
       " 'pagamos 280 euros mas',\n",
       " 'para ir al Spa cada persona tiene',\n",
       " 'para nada',\n",
       " 'para solventar el problema',\n",
       " 'para sus huéspedes',\n",
       " 'para todos',\n",
       " 'pasamos de total relax',\n",
       " 'pelin pequeñas',\n",
       " 'perfecta para acceder de forma rápida al aeropuerto',\n",
       " 'pesimas',\n",
       " 'pesimo',\n",
       " 'poco donde elegir',\n",
       " 'poco servicial y orientado al cliente extrajero',\n",
       " 'poder aparcar',\n",
       " 'podiamos subir',\n",
       " 'ponen',\n",
       " 'por unos 35 euros por cabeza dormimos estupendamente',\n",
       " 'preciosa',\n",
       " 'preciosas',\n",
       " 'proxima',\n",
       " 'puedes encontrar',\n",
       " 'puedes ir andando',\n",
       " 'pusieron',\n",
       " 'párrafo aparte',\n",
       " 'que funciona de maravilla.No',\n",
       " 'que no reponen cuando se acaba',\n",
       " 'que pocos hoteles lo ofrecen',\n",
       " 'que te absorba la tranquilidad',\n",
       " 'realmente atento',\n",
       " 'realmente se puede',\n",
       " 'recién reformado',\n",
       " 'recomiendo 100',\n",
       " 'recomiendo para familias',\n",
       " 'relajarse',\n",
       " 'respetaron',\n",
       " 'resultó cómodo',\n",
       " 'resultó de gran utilidad para los niños',\n",
       " 'retirado',\n",
       " 'retirado del centro',\n",
       " 'satisfactorio',\n",
       " 'se merece',\n",
       " 'se puede esperar más',\n",
       " 'se puede ir paseando',\n",
       " 'se quedan cortos.',\n",
       " 'se sale',\n",
       " 'se tarda en urbano mas de media hora',\n",
       " 'sentir como reyes',\n",
       " 'serviciales.',\n",
       " 'si estas solo',\n",
       " 'si para parejas',\n",
       " 'siempre dispuesto',\n",
       " 'siempre nos reciben con',\n",
       " 'simpatico',\n",
       " 'simpaticos',\n",
       " 'sin dudas elegiria',\n",
       " 'sin faltarle',\n",
       " 'sin ningún interes en atender al cliente',\n",
       " 'sin variedad',\n",
       " 'situado al lado',\n",
       " 'situado en el centro de madrid',\n",
       " 'situado en plena',\n",
       " 'situado en primera linea de playa',\n",
       " 'situado practicamente en el centro de Milan',\n",
       " 'son amplias',\n",
       " 'son de calidad',\n",
       " 'sorprendio muchisimo',\n",
       " 'su cercanía',\n",
       " 'suficiente cantidad',\n",
       " 'suficiente para pasar un rato agradable',\n",
       " 'super recomiendo',\n",
       " 'super · relajante',\n",
       " 'tambièn tienes que estar mendigando',\n",
       " 'tan poca',\n",
       " 'te atendienden con mucho respeto , saber estar y con educación',\n",
       " 'te cogen',\n",
       " 'te incluia',\n",
       " 'te sientes muy bien',\n",
       " 'tenido que dejar',\n",
       " 'tiene todo para satisfacer a los clientes',\n",
       " 'tiene unas vistas espectaculares',\n",
       " 'tienen',\n",
       " 'tienes',\n",
       " 'todo un lujo',\n",
       " 'tuvieron',\n",
       " 'un paraíso',\n",
       " 'un poco escaso',\n",
       " 'una desilusion',\n",
       " 'vendría magníficamente',\n",
       " 'verse mal casi todos los canales',\n",
       " 'visitase',\n",
       " 'vistas a la rotonda',\n",
       " 'volveremos',\n",
       " 'vovere'}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es_express_diff_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transformer, token length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fpath = !pwd\n",
    "\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "plm_model_name = 'roberta-large'\n",
    "plm_model_path= f'{fpath[0]}/pretrained_models/roberta-large'\n",
    "tokenizer = AutoTokenizer.from_pretrained(plm_model_path)\n",
    "\n",
    "length = len(tokenizer.encode('text'))\n",
    "length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a new column with the number of words from the 'text' column\n",
    "df_opener_en_test['Token Count'] = df_opener_en_test['text'].apply(lambda x: len(tokenizer.encode(x)))\n",
    "df_opener_es_test['Token Count'] = df_opener_es_test['text'].apply(lambda x: len(tokenizer.encode(x)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>Source</th>\n",
       "      <th>Source_string</th>\n",
       "      <th>Source_index</th>\n",
       "      <th>Target</th>\n",
       "      <th>Target_string</th>\n",
       "      <th>Target_index</th>\n",
       "      <th>Polar_expression</th>\n",
       "      <th>Polar_expression_string</th>\n",
       "      <th>...</th>\n",
       "      <th>Implicit_holder_exclusive</th>\n",
       "      <th>H-A-O</th>\n",
       "      <th>H-IA-IO</th>\n",
       "      <th>H-A-IO</th>\n",
       "      <th>H-IA-O</th>\n",
       "      <th>IH-A-O</th>\n",
       "      <th>IH-A-IO</th>\n",
       "      <th>IH-IA-O</th>\n",
       "      <th>IH-IA-IO</th>\n",
       "      <th>Token Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>565</th>\n",
       "      <td>The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .</td>\n",
       "      <td>opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[a waiter on hand], [87:103]]</td>\n",
       "      <td>[a waiter on hand]</td>\n",
       "      <td>[87:103]</td>\n",
       "      <td>[[there was always], [70:86]]</td>\n",
       "      <td>[there was always]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>566</th>\n",
       "      <td>The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .</td>\n",
       "      <td>opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[24 hr bar], [16:25]]</td>\n",
       "      <td>[24 hr bar]</td>\n",
       "      <td>[16:25]</td>\n",
       "      <td>[[well run], [30:38]]</td>\n",
       "      <td>[well run]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                   text  \\\n",
       "565  The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .   \n",
       "566  The first floor 24 hr bar was well run and no matter what time of day there was always a waiter on hand to serve .   \n",
       "\n",
       "                                                                 sent_id  \\\n",
       "565  opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6   \n",
       "566  opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6   \n",
       "\n",
       "       Source Source_string Source_index                          Target  \\\n",
       "565  [[], []]            []           []  [[a waiter on hand], [87:103]]   \n",
       "566  [[], []]            []           []          [[24 hr bar], [16:25]]   \n",
       "\n",
       "          Target_string Target_index               Polar_expression  \\\n",
       "565  [a waiter on hand]     [87:103]  [[there was always], [70:86]]   \n",
       "566         [24 hr bar]      [16:25]          [[well run], [30:38]]   \n",
       "\n",
       "    Polar_expression_string  ... Implicit_holder_exclusive H-A-O H-IA-IO  \\\n",
       "565      [there was always]  ...                      True     0   False   \n",
       "566              [well run]  ...                      True     0   False   \n",
       "\n",
       "     H-A-IO  H-IA-O  IH-A-O  IH-A-IO  IH-IA-O  IH-IA-IO  Token Count  \n",
       "565   False   False    True    False    False     False           28  \n",
       "566   False   False    True    False    False     False           28  \n",
       "\n",
       "[2 rows x 28 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_en_roberta_largev1[df_en_roberta_largev1['sent_id'] == \"opener_en/kaf/hotel/english00134_94658e39af0d72b00ead2ac314bf7074-2\"]\n",
    "df_opener_en_test[df_opener_en_test['sent_id'] == \"opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6\"] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "56    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "57    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "58    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "59    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "60    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "61    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "62    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "63    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "64    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "65    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "66    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "67    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "68    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "69    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "70    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "71    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "72    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "73    really nice hotel , well laid out all rooms over look the pool. rooms are nice and large have tea coffee making facilties , mini fridge. loads of ...\n",
       "Name: text, dtype: object"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df_opener_en_test.sort_values(by=\"Token Count\", ascending=False)\n",
    "\n",
    "df_opener_en_test[df_opener_en_test['sent_id'] =='opener_en/kaf/hotel/english00134_94658e39af0d72b00ead2ac314bf7074-2']['text']\n",
    "\n",
    "# df_opener_en_test[df_opener_en_test['sent_id'] =='opener_en/kaf/hotel/english00134_94658e39af0d72b00ead2ac314bf7074-2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>Source</th>\n",
       "      <th>Source_string</th>\n",
       "      <th>Source_index</th>\n",
       "      <th>Target</th>\n",
       "      <th>Target_string</th>\n",
       "      <th>Target_index</th>\n",
       "      <th>Polar_expression</th>\n",
       "      <th>Polar_expression_string</th>\n",
       "      <th>...</th>\n",
       "      <th>Implicit_holder_exclusive</th>\n",
       "      <th>H-A-O</th>\n",
       "      <th>H-IA-IO</th>\n",
       "      <th>H-A-IO</th>\n",
       "      <th>H-IA-O</th>\n",
       "      <th>IH-A-O</th>\n",
       "      <th>IH-A-IO</th>\n",
       "      <th>IH-IA-O</th>\n",
       "      <th>IH-IA-IO</th>\n",
       "      <th>Token Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Es un hotel muy casero en todo lo que respecta al trato y a la comida , personal muy atento y simpatico , esta situado en primera linea de playa y...</td>\n",
       "      <td>opener_es/kaf/hotel/spanish00172_bf69ca6910c0aa687b0eab91d0353fd1-2</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[un lavado de cara], [535:552]]</td>\n",
       "      <td>[un lavado de cara]</td>\n",
       "      <td>[535:552]</td>\n",
       "      <td>[[necesita], [526:534]]</td>\n",
       "      <td>[necesita]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Es un hotel muy casero en todo lo que respecta al trato y a la comida , personal muy atento y simpatico , esta situado en primera linea de playa y...</td>\n",
       "      <td>opener_es/kaf/hotel/spanish00172_bf69ca6910c0aa687b0eab91d0353fd1-2</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[trato], [50:55]]</td>\n",
       "      <td>[trato]</td>\n",
       "      <td>[50:55]</td>\n",
       "      <td>[[muy casero], [12:22]]</td>\n",
       "      <td>[muy casero]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Es un hotel muy casero en todo lo que respecta al trato y a la comida , personal muy atento y simpatico , esta situado en primera linea de playa y...</td>\n",
       "      <td>opener_es/kaf/hotel/spanish00172_bf69ca6910c0aa687b0eab91d0353fd1-2</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[mobiliario], [647:657]]</td>\n",
       "      <td>[mobiliario]</td>\n",
       "      <td>[647:657]</td>\n",
       "      <td>[[bastante antiguo], [658:674]]</td>\n",
       "      <td>[bastante antiguo]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>220</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                     text  \\\n",
       "15  Es un hotel muy casero en todo lo que respecta al trato y a la comida , personal muy atento y simpatico , esta situado en primera linea de playa y...   \n",
       "9   Es un hotel muy casero en todo lo que respecta al trato y a la comida , personal muy atento y simpatico , esta situado en primera linea de playa y...   \n",
       "18  Es un hotel muy casero en todo lo que respecta al trato y a la comida , personal muy atento y simpatico , esta situado en primera linea de playa y...   \n",
       "\n",
       "                                                                sent_id  \\\n",
       "15  opener_es/kaf/hotel/spanish00172_bf69ca6910c0aa687b0eab91d0353fd1-2   \n",
       "9   opener_es/kaf/hotel/spanish00172_bf69ca6910c0aa687b0eab91d0353fd1-2   \n",
       "18  opener_es/kaf/hotel/spanish00172_bf69ca6910c0aa687b0eab91d0353fd1-2   \n",
       "\n",
       "      Source Source_string Source_index                            Target  \\\n",
       "15  [[], []]            []           []  [[un lavado de cara], [535:552]]   \n",
       "9   [[], []]            []           []                [[trato], [50:55]]   \n",
       "18  [[], []]            []           []         [[mobiliario], [647:657]]   \n",
       "\n",
       "          Target_string Target_index                 Polar_expression  \\\n",
       "15  [un lavado de cara]    [535:552]          [[necesita], [526:534]]   \n",
       "9               [trato]      [50:55]          [[muy casero], [12:22]]   \n",
       "18         [mobiliario]    [647:657]  [[bastante antiguo], [658:674]]   \n",
       "\n",
       "   Polar_expression_string  ... Implicit_holder_exclusive H-A-O H-IA-IO  \\\n",
       "15              [necesita]  ...                      True     0   False   \n",
       "9             [muy casero]  ...                      True     0   False   \n",
       "18      [bastante antiguo]  ...                      True     0   False   \n",
       "\n",
       "    H-A-IO  H-IA-O  IH-A-O  IH-A-IO  IH-IA-O  IH-IA-IO  Token Count  \n",
       "15   False   False    True    False    False     False          220  \n",
       "9    False   False    True    False    False     False          220  \n",
       "18   False   False    True    False    False     False          220  \n",
       "\n",
       "[3 rows x 28 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_opener_es_test.sort_values(by=\"Token Count\", ascending=False)[:3]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Implicit Percentages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def json_eda(df):\n",
    "# some basic counts and percentages\n",
    "    \n",
    "    sentances = df['sent_id'].nunique()\n",
    "    holders = len(df[df['Source_string'].apply(len) > 0])\n",
    "    targets = len(df[df['Target_string'].apply(len) > 0])\n",
    "    polarity = len(df[df['Polarity'].apply(len) > 0])\n",
    "    expression = len(df[df['Polar_expression_string'].apply(len) > 0])\n",
    "\n",
    "    print(f'n sentances: {sentances} \\nn holders: {holders} \\nn targets: {targets} \\nn polarity: {polarity} \\nn expression: {expression}')\n",
    "\n",
    "    perIH = df['Implicit_holder'].sum()/len(df)\n",
    "    perIT = df['Implicit_target'].sum()/len(df)\n",
    "    perIE = df['Implicit_expression'].sum()/len(df)\n",
    "    perIHIT = df['Implicit_holder_target'].sum()/len(df)\n",
    "    exclIT = df['Implicit_target_exclusive'].sum()/len(df)\n",
    "    exclIH = df['Implicit_holder_exclusive'].sum()/len(df)\n",
    "\n",
    "    print(f'\\nImplicit percentages in the dataset: \\npercent implicit holder: {perIH},\\npercent implicit target: {perIT},\\npercent implicit holder implicit target: {perIHIT}, \\npercent implicit expression: {perIE}')\n",
    "\n",
    "    print(f'\\nImplicit target exclusive: {exclIT}')\n",
    "    print(f'\\nImplicit holder exclusive: {exclIH}')\n",
    "\n",
    "    HAO=df['H-A-O'].sum()/len(df)\n",
    "    HIAIO=df['H-IA-IO'].sum()/len(df)\n",
    "    HAIO=df['H-A-IO'].sum()/len(df)\n",
    "    HIAO=df['H-IA-O'].sum()/len(df)\n",
    "    IHAO=df['IH-A-O'].sum()/len(df)\n",
    "    IHAIO=df['IH-A-IO'].sum()/len(df)\n",
    "    IHIAO=df['IH-IA-O'].sum()/len(df)\n",
    "\n",
    "    print(f'H-A-O: {HAO}')\n",
    "    print(f'H-IA-IO: {HIAIO}')\n",
    "    print(f'H-A-IO: {HAIO}')\n",
    "    print(f'H-IA-O: {HIAIO}')\n",
    "    print(f'IH-A-O: {IHAO}')\n",
    "    print(f'IH-A-IO: {IHAIO}')\n",
    "    print(f'IH-IA-O: {IHIAO}')\n",
    "        \n",
    "\n",
    "    cross_polarity_intensity = pd.crosstab(df['Polarity'], df['Intensity'], margins = True, normalize='all') * 100\n",
    "    display(cross_polarity_intensity)\n",
    "\n",
    "    df_count_per_sent = pd.DataFrame(df.groupby(['sent_id'])['Polar_expression'].count())\n",
    "\n",
    "    count_mult_opinion =  (df_count_per_sent['Polar_expression'] > 1).sum()\n",
    "    perc_mult_opinion = (df_count_per_sent['Polar_expression'] > 1).sum() / df.shape[0]\n",
    "\n",
    "    print(f'The total count of sentiments with multiple opinions is: {count_mult_opinion}')\n",
    "    print(f'The percent of sentiments with multiple opinions is: {perc_mult_opinion}')\n",
    "    \n",
    "    \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'df_en_train' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[31], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m json_eda(df_en_train)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'df_en_train' is not defined"
     ]
    }
   ],
   "source": [
    "json_eda(df_en_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_en_train = normalize_jsons(\"data/opener_en/train.json\")\n",
    "# df_es_train = normalize_jsons(\"data/opener_es/train.json\")\n",
    "# df_en_dev = normalize_jsons(\"data/opener_en/dev.json\")\n",
    "# df_es_dev = normalize_jsons(\"data/opener_es/dev.json\")\n",
    "# df_en_test = normalize_jsons(\"data/opener_en/test.json\")\n",
    "# df_es_test = normalize_jsons(\"data/opener_es/test.json\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n sentances: 1252 \n",
      "n holders: 176 \n",
      "n targets: 2748 \n",
      "n polarity: 3042 \n",
      "n expression: 3042\n",
      "\n",
      "Implicit percentages in the dataset: \n",
      "percent implicit holder: 0.9421433267587114,\n",
      "percent implicit target: 0.09664694280078895,\n",
      "percent implicit holder implicit target: 0.08481262327416174, \n",
      "percent implicit expression: 0.0\n",
      "\n",
      "Implicit target exclusive: 0.011834319526627219\n",
      "\n",
      "Implicit holder exclusive: 0.8573307034845496\n",
      "H-A-O: 0.04602235371466141\n",
      "H-IA-IO: 0.0\n",
      "H-A-IO: 0.0\n",
      "H-IA-O: 0.0\n",
      "IH-A-O: 0.8573307034845496\n",
      "IH-A-IO: 0.0\n",
      "IH-IA-O: 0.08481262327416174\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Intensity</th>\n",
       "      <th>Standard</th>\n",
       "      <th>Strong</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Polarity</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Negative</th>\n",
       "      <td>16.074951</td>\n",
       "      <td>2.662722</td>\n",
       "      <td>18.737673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Positive</th>\n",
       "      <td>51.314924</td>\n",
       "      <td>29.947403</td>\n",
       "      <td>81.262327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>67.389875</td>\n",
       "      <td>32.610125</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Intensity   Standard     Strong         All\n",
       "Polarity                                   \n",
       "Negative   16.074951   2.662722   18.737673\n",
       "Positive   51.314924  29.947403   81.262327\n",
       "All        67.389875  32.610125  100.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The total count of sentiments with multiple opinions is: 752\n",
      "The percent of sentiments with multiple opinions is: 0.24720578566732412\n"
     ]
    }
   ],
   "source": [
    "json_eda(df_es_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n sentances: 198 \n",
      "n holders: 49 \n",
      "n targets: 371 \n",
      "n polarity: 400 \n",
      "n expression: 400\n",
      "\n",
      "Implicit percentages in the dataset: \n",
      "percent implicit holder: 0.8775,\n",
      "percent implicit target: 0.0725,\n",
      "percent implicit holder implicit target: 0.0525, \n",
      "percent implicit expression: 0.0\n",
      "\n",
      "Implicit target exclusive: 0.02\n",
      "\n",
      "Implicit holder exclusive: 0.825\n",
      "H-A-O: 0.1025\n",
      "H-IA-IO: 0.0\n",
      "H-A-IO: 0.0\n",
      "H-IA-O: 0.0\n",
      "IH-A-O: 0.825\n",
      "IH-A-IO: 0.0\n",
      "IH-IA-O: 0.0525\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Intensity</th>\n",
       "      <th>Standard</th>\n",
       "      <th>Strong</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Polarity</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Negative</th>\n",
       "      <td>24.75</td>\n",
       "      <td>4.25</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Positive</th>\n",
       "      <td>42.50</td>\n",
       "      <td>28.50</td>\n",
       "      <td>71.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>67.25</td>\n",
       "      <td>32.75</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Intensity  Standard  Strong    All\n",
       "Polarity                          \n",
       "Negative      24.75    4.25   29.0\n",
       "Positive      42.50   28.50   71.0\n",
       "All           67.25   32.75  100.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The total count of sentiments with multiple opinions is: 115\n",
      "The percent of sentiments with multiple opinions is: 0.2875\n"
     ]
    }
   ],
   "source": [
    "json_eda(df_en_dev)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n sentances: 174 \n",
      "n holders: 23 \n",
      "n targets: 363 \n",
      "n polarity: 387 \n",
      "n expression: 387\n",
      "\n",
      "Implicit percentages in the dataset: \n",
      "percent implicit holder: 0.9405684754521964,\n",
      "percent implicit target: 0.06201550387596899,\n",
      "percent implicit holder implicit target: 0.06201550387596899, \n",
      "percent implicit expression: 0.0\n",
      "\n",
      "Implicit target exclusive: 0.0\n",
      "\n",
      "Implicit holder exclusive: 0.8785529715762274\n",
      "H-A-O: 0.059431524547803614\n",
      "H-IA-IO: 0.0\n",
      "H-A-IO: 0.0\n",
      "H-IA-O: 0.0\n",
      "IH-A-O: 0.8785529715762274\n",
      "IH-A-IO: 0.0\n",
      "IH-IA-O: 0.06201550387596899\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Intensity</th>\n",
       "      <th>Standard</th>\n",
       "      <th>Strong</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Polarity</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Negative</th>\n",
       "      <td>16.020672</td>\n",
       "      <td>2.067183</td>\n",
       "      <td>18.087855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Positive</th>\n",
       "      <td>52.713178</td>\n",
       "      <td>29.198966</td>\n",
       "      <td>81.912145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>68.733850</td>\n",
       "      <td>31.266150</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Intensity   Standard     Strong         All\n",
       "Polarity                                   \n",
       "Negative   16.020672   2.067183   18.087855\n",
       "Positive   52.713178  29.198966   81.912145\n",
       "All        68.733850  31.266150  100.000000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The total count of sentiments with multiple opinions is: 97\n",
      "The percent of sentiments with multiple opinions is: 0.25064599483204136\n"
     ]
    }
   ],
   "source": [
    "json_eda(df_es_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fine Grain F1 scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize.simple import SpaceTokenizer\n",
    "st = SpaceTokenizer()\n",
    "\n",
    "DATA_DIR = './data'\n",
    "id2sentiment = {3: 'Negative', 4: 'Neutral', 5: 'Positive'}\n",
    "\n",
    "\n",
    "class SemEvalEvaluator:\n",
    "\n",
    "    def __init__(self, predicted_tuples, dataset_name, mode=\"manual\"):\n",
    "\n",
    "        self.mode = mode\n",
    "\n",
    "        if mode == \"valid\":\n",
    "            gold_path = DATA_DIR / dataset_name / 'test.json'\n",
    "            self.sent_ids, self.golden_triplets, self.golden_quadtuples = self.load_gold_from_file(gold_path)\n",
    "            #self.predicted_triplets, self.predicted_quadtuples = self.convert_to_submission_tuple_format(predicted_tuples)\n",
    "            self.predicted_triplets, self.predicted_quadtuples = self.convert_to_submission_tuple_format_V2(predicted_tuples)\n",
    "        \n",
    "        elif mode == \"manual\":\n",
    "            gold_path = dataset_name\n",
    "            pred_path = predicted_tuples\n",
    "            self.sent_ids, self.golden_quadtuples = self.load_gold_from_file(gold_path)\n",
    "            self.sent_ids, self.predicted_quadtuples = self.load_preds_from_file(pred_path)\n",
    "            #self.predicted_triplets, self.predicted_quadtuples = self.convert_to_submission_tuple_format(predicted_tuples)\n",
    "            # self.predicted_triplets, self.predicted_quadtuples = self.convert_to_submission_tuple_format_V2(predicted_tuples)\n",
    "\n",
    "\n",
    "    def convert_opinion_to_quadtuples(self, sentence):\n",
    "        text = sentence[\"text\"]\n",
    "        opinions = sentence[\"opinions\"]\n",
    "        opinion_tuples = []\n",
    "        token_offsets = list(st.span_tokenize(text))\n",
    "        #\n",
    "        if len(opinions) > 0:\n",
    "            for opinion in opinions:\n",
    "                holder_char_idxs = opinion[\"Source\"][1]\n",
    "                target_char_idxs = opinion[\"Target\"][1]\n",
    "                exp_char_idxs = opinion[\"Polar_expression\"][1]\n",
    "                polarity = opinion[\"Polarity\"]\n",
    "                #\n",
    "                holder = self.convert_char_offsets_to_token_idxs(holder_char_idxs, token_offsets)\n",
    "                target = self.convert_char_offsets_to_token_idxs(target_char_idxs, token_offsets)\n",
    "                exp = self.convert_char_offsets_to_token_idxs(exp_char_idxs, token_offsets)\n",
    "                opinion_tuples.append((holder, target, exp, polarity))\n",
    "        return opinion_tuples\n",
    "\n",
    "    def convert_char_offsets_to_token_idxs(self, char_offsets, token_offsets):\n",
    "        \"\"\"\n",
    "        char_offsets: list of str\n",
    "        token_offsets: list of tuples\n",
    "\n",
    "        >>> text = \"I think the new uni ( ) is a great idea\"\n",
    "        >>> char_offsets = [\"8:19\"]\n",
    "        >>> token_offsets =\n",
    "        [(0,1), (2,7), (8,11), (12,15), (16,19), (20,21), (22,23), (24,26), (27,28), (29,34), (35,39)]\n",
    "\n",
    "        >>> convert_char_offsets_to_token_idxs(char_offsets, token_offsets)\n",
    "        >>> (2,3,4)\n",
    "        \"\"\"\n",
    "        token_idxs = []\n",
    "        #\n",
    "        for char_offset in char_offsets:\n",
    "            bidx, eidx = char_offset.split(\":\")\n",
    "            bidx, eidx = int(bidx), int(eidx)\n",
    "            intoken = False\n",
    "            for i, (b, e) in enumerate(token_offsets):\n",
    "                if b == bidx:\n",
    "                    intoken = True\n",
    "                if intoken:\n",
    "                    token_idxs.append(i)\n",
    "                if e == eidx:\n",
    "                    intoken = False\n",
    "        return set(token_idxs)\n",
    "\n",
    "    def load_gold_from_file(self, gold_path):\n",
    "\n",
    "        with open(gold_path) as infile:\n",
    "            gold = json.load(infile)\n",
    "\n",
    "        sent_ids = [s[\"sent_id\"] for s in gold] # It is orderly\n",
    "\n",
    "        if self.mode=='valid' or self.mode=='test' or self.mode=='manual':\n",
    "        \n",
    "            golden_quadtuples = dict([(s[\"sent_id\"], self.convert_opinion_to_quadtuples(s)) for s in gold])\n",
    "\n",
    "            return sent_ids, golden_quadtuples\n",
    "\n",
    "        elif self.mode=='predict':\n",
    "\n",
    "            return sent_ids\n",
    "    \n",
    "    def load_preds_from_file(self, pred_path): # DL\n",
    "\n",
    "        with open(pred_path) as infile:\n",
    "            pred = json.load(infile)\n",
    "\n",
    "        sent_ids = [s[\"sent_id\"] for s in pred] # It is orderly\n",
    "\n",
    "        if self.mode=='valid' or self.mode=='test' or self.mode=='manual':\n",
    "        \n",
    "            pred_quadtuples = dict([(s[\"sent_id\"], self.convert_opinion_to_quadtuples(s)) for s in pred])\n",
    "\n",
    "            return sent_ids, pred_quadtuples\n",
    "\n",
    "        elif self.mode=='predict':\n",
    "\n",
    "            return sent_ids\n",
    "\n",
    "\n",
    "    def convert_to_submission_tuple_format_V2(self, tuples): #JR\n",
    "\n",
    "        assert len(self.sent_ids) == len(tuples)\n",
    "\n",
    "        all_converted_triplets = {}\n",
    "        all_converted_quadtuples = {}\n",
    "\n",
    "        for sent_id, i in zip(self.sent_ids, tuples):\n",
    "\n",
    "            converted_triplets = []\n",
    "            converted_quadtuples = []\n",
    "\n",
    "            for tuple in i:\n",
    "                holder_start_idx, holder_end_idx = tuple[0], tuple[1]\n",
    "                aspect_start_idx, aspect_end_idx = tuple[2], tuple[3]\n",
    "                opinion_start_idx, opinion_end_idx  = tuple[4], tuple[5]\n",
    "                \n",
    "                holder_range_set = set(range(holder_start_idx-1, holder_end_idx)) if set(range(holder_start_idx-1, holder_end_idx)) != set([-1]) else set()\n",
    "                aspect_range_set = set(range(aspect_start_idx-1, aspect_end_idx)) if set(range(aspect_start_idx-1, aspect_end_idx)) != set([-1]) else set()\n",
    "                opinion_range_set = set(range(opinion_start_idx-1, opinion_end_idx)) if set(range(opinion_start_idx-1, opinion_end_idx)) != set([-1]) else set()\n",
    "            \n",
    "                converted_triplets.append((aspect_range_set, opinion_range_set, id2sentiment[tuple[6]]))\n",
    "                converted_quadtuples.append((holder_range_set, aspect_range_set, opinion_range_set, id2sentiment[tuple[6]]))\n",
    "\n",
    "            all_converted_triplets[sent_id] = converted_triplets\n",
    "            all_converted_quadtuples[sent_id] = converted_quadtuples\n",
    "\n",
    "        return all_converted_triplets, all_converted_quadtuples\n",
    "\n",
    "\n",
    "    def sent_quadtuples_in_list(self, sent_tuple1, list_of_sent_tuples, keep_polarity=True):\n",
    "        holder1, target1, exp1, pol1 = sent_tuple1\n",
    "        if len(holder1) == 0:\n",
    "            holder1 = frozenset([\"_\"])\n",
    "        if len(target1) == 0:\n",
    "            target1 = frozenset([\"_\"])\n",
    "        for holder2, target2, exp2, pol2 in list_of_sent_tuples:\n",
    "            if len(holder2) == 0:\n",
    "                holder2 = frozenset([\"_\"])\n",
    "            if len(target2) == 0:\n",
    "                target2 = frozenset([\"_\"])\n",
    "            if (\n",
    "                len(holder1.intersection(holder2)) > 0\n",
    "                and len(target1.intersection(target2)) > 0\n",
    "                and len(exp1.intersection(exp2)) > 0\n",
    "            ):\n",
    "                if keep_polarity:\n",
    "                    if pol1 == pol2:\n",
    "                        return True\n",
    "                else:\n",
    "                    return True\n",
    "        return False\n",
    "\n",
    "    def quadtuple_weighted_score(self, sent_tuple1, list_of_sent_tuples):\n",
    "        best_overlap = 0\n",
    "        holder1, target1, exp1, pol1 = sent_tuple1\n",
    "        if len(holder1) == 0:\n",
    "            holder1 = frozenset([\"_\"])\n",
    "        if len(target1) == 0:\n",
    "            target1 = frozenset([\"_\"])\n",
    "        for holder2, target2, exp2, pol2 in list_of_sent_tuples:\n",
    "            if len(holder2) == 0:\n",
    "                holder2 = frozenset([\"_\"])\n",
    "            if len(target2) == 0:\n",
    "                target2 = frozenset([\"_\"])\n",
    "            if (\n",
    "                len(holder2.intersection(holder1)) > 0\n",
    "                and len(target2.intersection(target1)) > 0\n",
    "                and len(exp2.intersection(exp1)) > 0\n",
    "            ):\n",
    "                holder_overlap = len(holder2.intersection(holder1)) / len(holder1)\n",
    "                target_overlap = len(target2.intersection(target1)) / len(target1)\n",
    "                exp_overlap = len(exp2.intersection(exp1)) / len(exp1)\n",
    "                overlap = (holder_overlap + target_overlap + exp_overlap) / 3\n",
    "                if overlap > best_overlap:\n",
    "                    best_overlap = overlap\n",
    "        return best_overlap\n",
    "\n",
    "    def quadtuple_precision(self, gold, pred, keep_polarity=True, weighted=True):\n",
    "        \"\"\"\n",
    "        Weighted true positives / (true positives + false positives)\n",
    "        \"\"\"\n",
    "        weighted_tp = []\n",
    "        tp = []\n",
    "        fp = []\n",
    "        #\n",
    "        for sent_idx in pred.keys():\n",
    "            \n",
    "            atomic_tp = [] # DL\n",
    "            atomic_fp = [] # Dl\n",
    "\n",
    "            ptuples = pred[sent_idx]\n",
    "            gtuples = gold[sent_idx]\n",
    "            for stuple in ptuples:\n",
    "                if self.sent_quadtuples_in_list(stuple, gtuples, keep_polarity):\n",
    "                    if weighted:\n",
    "                        weighted_tp.append(self.quadtuple_weighted_score(stuple, gtuples))\n",
    "                        # print('added weighted_tp') # DL\n",
    "                        tp.append(1)\n",
    "                        atomic_tp.append(1) # DL\n",
    "\n",
    "                    else:\n",
    "                        weighted_tp.append(1)\n",
    "                        tp.append(1)\n",
    "                        atomic_fp.append(1) # DL\n",
    "                else:\n",
    "                    fp.append(1)\n",
    "                    atomic_fp.append(1) # DL\n",
    "                \n",
    "                print(f'sent_idx: {sent_idx}, {sum(weighted_tp) / (sum(atomic_tp) + sum(atomic_fp) + 0.0000000000000001)}' ) # DL\n",
    "\n",
    "        return sum(weighted_tp) / (sum(tp) + sum(fp) + 0.0000000000000001)\n",
    "\n",
    "    def quadtuple_recall(self, gold, pred, keep_polarity=True, weighted=True):\n",
    "        \"\"\"\n",
    "        Weighted true positives / (true positives + false negatives)\n",
    "        \"\"\"\n",
    "        weighted_tp = []\n",
    "        tp = []\n",
    "        fn = []\n",
    "        #\n",
    "        assert len(gold) == len(pred)\n",
    "        #\n",
    "        for sent_idx in pred.keys():\n",
    "            \n",
    "            atomic_weighted_tp = [] # DL\n",
    "            atomic_tp = [] # DL\n",
    "            atomic_fn = [] # Dl\n",
    "\n",
    "\n",
    "            ptuples = pred[sent_idx]\n",
    "            gtuples = gold[sent_idx]\n",
    "            for stuple in gtuples:\n",
    "                if self.sent_quadtuples_in_list(stuple, ptuples, keep_polarity):\n",
    "                    if weighted:\n",
    "                        weighted_tp.append(self.quadtuple_weighted_score(stuple, ptuples))\n",
    "                        tp.append(1)\n",
    "                        atomic_tp.append(1) # DL\n",
    "                    else:\n",
    "                        weighted_tp.append(1)\n",
    "                        tp.append(1)\n",
    "                        atomic_tp.append(1) # DL\n",
    "                else:\n",
    "                    fn.append(1)\n",
    "                    atomic_fn.append(1)\n",
    "\n",
    "                print(f'sent_idx: {sent_idx}, {sum(weighted_tp) / (sum(atomic_tp) + sum(atomic_fp) + 0.0000000000000001)}' ) # DL\n",
    "\n",
    "        return sum(weighted_tp) / (sum(tp) + sum(fn) + 0.0000000000000001)\n",
    "\n",
    "    def get_all_quadtuple_metrics(self, keep_polarity=True, weighted=True):\n",
    "\n",
    "        prec = self.quadtuple_precision(self.golden_quadtuples, self.predicted_quadtuples, keep_polarity, weighted)\n",
    "        rec = self.quadtuple_recall(self.golden_quadtuples, self.predicted_quadtuples, keep_polarity, weighted)\n",
    "\n",
    "        return prec, rec, 2 * (prec * rec) / (prec + rec + 0.00000000000000001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-8, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-4, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-6, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-7, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00029_2a5891717bff19947a0b858a3b1e8230-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00029_2a5891717bff19947a0b858a3b1e8230-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00029_2a5891717bff19947a0b858a3b1e8230-4, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00029_2a5891717bff19947a0b858a3b1e8230-4, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00029_2a5891717bff19947a0b858a3b1e8230-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00029_2a5891717bff19947a0b858a3b1e8230-5, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00029_2a5891717bff19947a0b858a3b1e8230-5, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00029_2a5891717bff19947a0b858a3b1e8230-5, 1.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00106_74e62e551e5b441356b001c5f7ca2070-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00106_74e62e551e5b441356b001c5f7ca2070-1, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00106_74e62e551e5b441356b001c5f7ca2070-1, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00106_74e62e551e5b441356b001c5f7ca2070-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00106_74e62e551e5b441356b001c5f7ca2070-3, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00106_74e62e551e5b441356b001c5f7ca2070-3, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00106_74e62e551e5b441356b001c5f7ca2070-3, 0.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00106_74e62e551e5b441356b001c5f7ca2070-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00106_74e62e551e5b441356b001c5f7ca2070-4, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00106_74e62e551e5b441356b001c5f7ca2070-7, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00106_74e62e551e5b441356b001c5f7ca2070-7, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00106_74e62e551e5b441356b001c5f7ca2070-8, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00106_74e62e551e5b441356b001c5f7ca2070-8, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00106_74e62e551e5b441356b001c5f7ca2070-8, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00106_74e62e551e5b441356b001c5f7ca2070-8, 4e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00223_fe94638c44f4dd789e8a42550e1e6beb-1, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00223_fe94638c44f4dd789e8a42550e1e6beb-1, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00223_fe94638c44f4dd789e8a42550e1e6beb-1, 0.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00223_fe94638c44f4dd789e8a42550e1e6beb-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00223_fe94638c44f4dd789e8a42550e1e6beb-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00223_fe94638c44f4dd789e8a42550e1e6beb-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00223_fe94638c44f4dd789e8a42550e1e6beb-4, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00223_fe94638c44f4dd789e8a42550e1e6beb-6, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00091_64b95794b1aa41efbaabced339b2deff-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00091_64b95794b1aa41efbaabced339b2deff-1, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00091_64b95794b1aa41efbaabced339b2deff-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00091_64b95794b1aa41efbaabced339b2deff-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00091_64b95794b1aa41efbaabced339b2deff-4, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00091_64b95794b1aa41efbaabced339b2deff-4, 1.0\n",
      "sent_idx: opener_en/kaf/hotel/english00091_64b95794b1aa41efbaabced339b2deff-5, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00091_64b95794b1aa41efbaabced339b2deff-5, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00091_64b95794b1aa41efbaabced339b2deff-6, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00091_64b95794b1aa41efbaabced339b2deff-7, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00091_64b95794b1aa41efbaabced339b2deff-7, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00091_64b95794b1aa41efbaabced339b2deff-7, 3e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00091_64b95794b1aa41efbaabced339b2deff-8, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00134_94658e39af0d72b00ead2ac314bf7074-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00134_94658e39af0d72b00ead2ac314bf7074-2, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00134_94658e39af0d72b00ead2ac314bf7074-2, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00134_94658e39af0d72b00ead2ac314bf7074-2, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00164_f47aebce11fc3b76fecdd1db6a13904f-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00164_f47aebce11fc3b76fecdd1db6a13904f-1, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00164_f47aebce11fc3b76fecdd1db6a13904f-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00164_f47aebce11fc3b76fecdd1db6a13904f-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00164_f47aebce11fc3b76fecdd1db6a13904f-4, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00164_f47aebce11fc3b76fecdd1db6a13904f-5, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00164_f47aebce11fc3b76fecdd1db6a13904f-8, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00164_f47aebce11fc3b76fecdd1db6a13904f-10, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00164_f47aebce11fc3b76fecdd1db6a13904f-10, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00141_a1049912305bfe5c61247438bc2866bd-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00141_a1049912305bfe5c61247438bc2866bd-2, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00141_a1049912305bfe5c61247438bc2866bd-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00141_a1049912305bfe5c61247438bc2866bd-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00141_a1049912305bfe5c61247438bc2866bd-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00141_a1049912305bfe5c61247438bc2866bd-8, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00141_a1049912305bfe5c61247438bc2866bd-8, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00057_3c3ad20e93ca57ff88759a290c8aaa88-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00057_3c3ad20e93ca57ff88759a290c8aaa88-1, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00057_3c3ad20e93ca57ff88759a290c8aaa88-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00057_3c3ad20e93ca57ff88759a290c8aaa88-3, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00057_3c3ad20e93ca57ff88759a290c8aaa88-3, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00057_3c3ad20e93ca57ff88759a290c8aaa88-3, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00057_3c3ad20e93ca57ff88759a290c8aaa88-3, 1.5\n",
      "sent_idx: opener_en/kaf/hotel/english00057_3c3ad20e93ca57ff88759a290c8aaa88-4, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00057_3c3ad20e93ca57ff88759a290c8aaa88-4, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00057_3c3ad20e93ca57ff88759a290c8aaa88-4, 0.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00057_3c3ad20e93ca57ff88759a290c8aaa88-4, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00057_3c3ad20e93ca57ff88759a290c8aaa88-7, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00057_3c3ad20e93ca57ff88759a290c8aaa88-7, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00050_495e277eea2f9aa0e88d6122299c3894-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00050_495e277eea2f9aa0e88d6122299c3894-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00050_495e277eea2f9aa0e88d6122299c3894-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00050_495e277eea2f9aa0e88d6122299c3894-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00213_f578dc1efdd03b8b4bbd8c7958a7121a-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00213_f578dc1efdd03b8b4bbd8c7958a7121a-1, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00213_f578dc1efdd03b8b4bbd8c7958a7121a-2, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00213_f578dc1efdd03b8b4bbd8c7958a7121a-2, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00213_f578dc1efdd03b8b4bbd8c7958a7121a-4, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-1, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-2, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-2, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-2, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-4, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-5, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-5, 0.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-5, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-5, 0.75\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-5, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-5, 1.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-5, 1.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-7, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-7, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-8, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-9, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00139_a027bbcb73be1f46b42077b535cf955c-9, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00206_f1df933d1e1de3be91e1d732fcdc9077-1, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00206_f1df933d1e1de3be91e1d732fcdc9077-3, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00206_f1df933d1e1de3be91e1d732fcdc9077-4, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00206_f1df933d1e1de3be91e1d732fcdc9077-4, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00206_f1df933d1e1de3be91e1d732fcdc9077-4, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00206_f1df933d1e1de3be91e1d732fcdc9077-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00206_f1df933d1e1de3be91e1d732fcdc9077-6, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00206_f1df933d1e1de3be91e1d732fcdc9077-6, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00206_f1df933d1e1de3be91e1d732fcdc9077-6, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00206_f1df933d1e1de3be91e1d732fcdc9077-7, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00206_f1df933d1e1de3be91e1d732fcdc9077-7, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00206_f1df933d1e1de3be91e1d732fcdc9077-7, 3e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-1, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-2, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-2, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-2, 0.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-3, 3e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-3, 1.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-3, 2.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-3, 2.5\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-4, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-4, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-5, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-6, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-7, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00065_454ddd5112d4969a9a26218d2cb6d815-8, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00039_2bc0812c3efc22fca59475d1ea37a251-1, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00039_2bc0812c3efc22fca59475d1ea37a251-1, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00039_2bc0812c3efc22fca59475d1ea37a251-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00039_2bc0812c3efc22fca59475d1ea37a251-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00039_2bc0812c3efc22fca59475d1ea37a251-2, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00039_2bc0812c3efc22fca59475d1ea37a251-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00039_2bc0812c3efc22fca59475d1ea37a251-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00039_2bc0812c3efc22fca59475d1ea37a251-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00039_2bc0812c3efc22fca59475d1ea37a251-5, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00039_2bc0812c3efc22fca59475d1ea37a251-5, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00155_e51c6adb2b98c032a78935b3923537a0-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00155_e51c6adb2b98c032a78935b3923537a0-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00155_e51c6adb2b98c032a78935b3923537a0-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00155_e51c6adb2b98c032a78935b3923537a0-4, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00155_e51c6adb2b98c032a78935b3923537a0-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00155_e51c6adb2b98c032a78935b3923537a0-5, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00155_e51c6adb2b98c032a78935b3923537a0-6, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00155_e51c6adb2b98c032a78935b3923537a0-6, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00170_fe19919b11b7ac3480d3fe451afd15a7-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00170_fe19919b11b7ac3480d3fe451afd15a7-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00170_fe19919b11b7ac3480d3fe451afd15a7-3, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00170_fe19919b11b7ac3480d3fe451afd15a7-3, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00170_fe19919b11b7ac3480d3fe451afd15a7-3, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00170_fe19919b11b7ac3480d3fe451afd15a7-3, 1.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00170_fe19919b11b7ac3480d3fe451afd15a7-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00170_fe19919b11b7ac3480d3fe451afd15a7-4, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00097_6ace8a44dd0caa70cdaa62a2cc5cfd54-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00097_6ace8a44dd0caa70cdaa62a2cc5cfd54-1, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00097_6ace8a44dd0caa70cdaa62a2cc5cfd54-3, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00097_6ace8a44dd0caa70cdaa62a2cc5cfd54-3, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00097_6ace8a44dd0caa70cdaa62a2cc5cfd54-3, 0.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00097_6ace8a44dd0caa70cdaa62a2cc5cfd54-3, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00097_6ace8a44dd0caa70cdaa62a2cc5cfd54-3, 0.75\n",
      "sent_idx: opener_en/kaf/hotel/english00097_6ace8a44dd0caa70cdaa62a2cc5cfd54-3, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00097_6ace8a44dd0caa70cdaa62a2cc5cfd54-3, 0.6666666666666666\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00097_6ace8a44dd0caa70cdaa62a2cc5cfd54-3, 0.8333333333333334\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00097_6ace8a44dd0caa70cdaa62a2cc5cfd54-3, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00097_6ace8a44dd0caa70cdaa62a2cc5cfd54-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00083_7b3d49a3481f9f590bab53cb14befd11-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00083_7b3d49a3481f9f590bab53cb14befd11-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00083_7b3d49a3481f9f590bab53cb14befd11-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00083_7b3d49a3481f9f590bab53cb14befd11-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00083_7b3d49a3481f9f590bab53cb14befd11-5, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00083_7b3d49a3481f9f590bab53cb14befd11-5, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00032_274ec81481c2a582f1ad265d7c4b6f22-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00032_274ec81481c2a582f1ad265d7c4b6f22-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00032_274ec81481c2a582f1ad265d7c4b6f22-2, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00032_274ec81481c2a582f1ad265d7c4b6f22-2, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00032_274ec81481c2a582f1ad265d7c4b6f22-3, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00032_274ec81481c2a582f1ad265d7c4b6f22-4, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00032_274ec81481c2a582f1ad265d7c4b6f22-4, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00032_274ec81481c2a582f1ad265d7c4b6f22-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00032_274ec81481c2a582f1ad265d7c4b6f22-5, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00032_274ec81481c2a582f1ad265d7c4b6f22-6, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-1, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-1, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-2, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-2, 4e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-4, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-5, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-6, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-7, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-7, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-8, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00035_340af6662193972260b6e387d1a4e322-9, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00059_572d2abf5b0020974b5e8838c9205793-1, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00059_572d2abf5b0020974b5e8838c9205793-2, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00059_572d2abf5b0020974b5e8838c9205793-3, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00059_572d2abf5b0020974b5e8838c9205793-3, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00059_572d2abf5b0020974b5e8838c9205793-4, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00059_572d2abf5b0020974b5e8838c9205793-4, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00059_572d2abf5b0020974b5e8838c9205793-5, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00059_572d2abf5b0020974b5e8838c9205793-5, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00151_de21bc4aef573cf6ec1065b90fd89128-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00151_de21bc4aef573cf6ec1065b90fd89128-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00151_de21bc4aef573cf6ec1065b90fd89128-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00105_94806e4b92092b3243eeee8623cbd2d5-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00105_94806e4b92092b3243eeee8623cbd2d5-1, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00105_94806e4b92092b3243eeee8623cbd2d5-1, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00105_94806e4b92092b3243eeee8623cbd2d5-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-1, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-2, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-2, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-2, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-2, 1.5\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-2, 0.75\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-2, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-2, 1.25\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-2, 0.8333333333333334\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-2, 1.0\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-2, 0.75\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-2, 0.6\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-2, 0.7\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-3, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-3, 1.0\n",
      "sent_idx: opener_en/kaf/hotel/english00081_74f5522e8f5628a2c6591c3411d1bc7f-4, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00112_7ec44bf4213be4f1226c04cb302f1b7b-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00112_7ec44bf4213be4f1226c04cb302f1b7b-2, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00112_7ec44bf4213be4f1226c04cb302f1b7b-3, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00112_7ec44bf4213be4f1226c04cb302f1b7b-3, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00112_7ec44bf4213be4f1226c04cb302f1b7b-3, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00112_7ec44bf4213be4f1226c04cb302f1b7b-3, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00112_7ec44bf4213be4f1226c04cb302f1b7b-3, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00112_7ec44bf4213be4f1226c04cb302f1b7b-3, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00112_7ec44bf4213be4f1226c04cb302f1b7b-3, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00112_7ec44bf4213be4f1226c04cb302f1b7b-4, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00112_7ec44bf4213be4f1226c04cb302f1b7b-4, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00112_7ec44bf4213be4f1226c04cb302f1b7b-4, 0.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00219_fa68930c28a452677d4f82d9400ddf4a-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00219_fa68930c28a452677d4f82d9400ddf4a-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00219_fa68930c28a452677d4f82d9400ddf4a-5, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00219_fa68930c28a452677d4f82d9400ddf4a-6, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-3, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-5, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-6, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-6, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-7, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-7, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-7, 0.25\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-7, 0.16666666666666666\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-7, 0.3333333333333333\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-9, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-9, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00138_9f942cd30f42d0abad59c036950aef81-10, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00165_c7ecb4fa1c55fb7a9d4cbe384a48efb4-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00165_c7ecb4fa1c55fb7a9d4cbe384a48efb4-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00165_c7ecb4fa1c55fb7a9d4cbe384a48efb4-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00165_c7ecb4fa1c55fb7a9d4cbe384a48efb4-2, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00165_c7ecb4fa1c55fb7a9d4cbe384a48efb4-2, 4e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00165_c7ecb4fa1c55fb7a9d4cbe384a48efb4-2, 5e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00165_c7ecb4fa1c55fb7a9d4cbe384a48efb4-2, 6e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00165_c7ecb4fa1c55fb7a9d4cbe384a48efb4-3, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00165_c7ecb4fa1c55fb7a9d4cbe384a48efb4-4, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00165_c7ecb4fa1c55fb7a9d4cbe384a48efb4-4, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00165_c7ecb4fa1c55fb7a9d4cbe384a48efb4-4, 1.0\n",
      "sent_idx: opener_en/kaf/hotel/english00165_c7ecb4fa1c55fb7a9d4cbe384a48efb4-5, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00162_ef371f2cdf69b4ac91bf97d372159ed5-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00162_ef371f2cdf69b4ac91bf97d372159ed5-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00162_ef371f2cdf69b4ac91bf97d372159ed5-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00162_ef371f2cdf69b4ac91bf97d372159ed5-4, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00162_ef371f2cdf69b4ac91bf97d372159ed5-5, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00162_ef371f2cdf69b4ac91bf97d372159ed5-5, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00162_ef371f2cdf69b4ac91bf97d372159ed5-6, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00162_ef371f2cdf69b4ac91bf97d372159ed5-6, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00162_ef371f2cdf69b4ac91bf97d372159ed5-6, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00162_ef371f2cdf69b4ac91bf97d372159ed5-6, 4e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00162_ef371f2cdf69b4ac91bf97d372159ed5-6, 5e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00162_ef371f2cdf69b4ac91bf97d372159ed5-7, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-3, 3e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-3, 1.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-3, 2.0\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-3, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-3, 1.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-4, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-4, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-4, 4e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-4, 5e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-4, 6e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-5, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00099_6bd92afbc1db2e16e92cc8cf0105c93e-5, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00139_d1d16f606db4a5ea9d19e6af2c969115-1, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00139_d1d16f606db4a5ea9d19e6af2c969115-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00139_d1d16f606db4a5ea9d19e6af2c969115-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00139_d1d16f606db4a5ea9d19e6af2c969115-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00139_d1d16f606db4a5ea9d19e6af2c969115-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00146_dc6ed17346231f40aed08dcb4c1e24ae-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00146_dc6ed17346231f40aed08dcb4c1e24ae-1, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00146_dc6ed17346231f40aed08dcb4c1e24ae-2, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00146_dc6ed17346231f40aed08dcb4c1e24ae-2, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00146_dc6ed17346231f40aed08dcb4c1e24ae-2, 0.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00146_dc6ed17346231f40aed08dcb4c1e24ae-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00003_0726ad8554ba78a391080a2d6c705f2d-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00003_0726ad8554ba78a391080a2d6c705f2d-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00003_0726ad8554ba78a391080a2d6c705f2d-2, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00003_0726ad8554ba78a391080a2d6c705f2d-2, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00003_0726ad8554ba78a391080a2d6c705f2d-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00003_0726ad8554ba78a391080a2d6c705f2d-3, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00003_0726ad8554ba78a391080a2d6c705f2d-4, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00003_0726ad8554ba78a391080a2d6c705f2d-4, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00003_0726ad8554ba78a391080a2d6c705f2d-4, 0.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00135_977b29727cbd452452362a22780ee24a-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00135_977b29727cbd452452362a22780ee24a-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00135_977b29727cbd452452362a22780ee24a-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00135_977b29727cbd452452362a22780ee24a-2, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00135_977b29727cbd452452362a22780ee24a-2, 4e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00135_977b29727cbd452452362a22780ee24a-2, 5e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00135_977b29727cbd452452362a22780ee24a-2, 6e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00135_977b29727cbd452452362a22780ee24a-2, 7e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00135_977b29727cbd452452362a22780ee24a-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00004_077eb74146f2afec4488e171a134f655-1, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00004_077eb74146f2afec4488e171a134f655-1, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00004_077eb74146f2afec4488e171a134f655-2, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00004_077eb74146f2afec4488e171a134f655-2, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00004_077eb74146f2afec4488e171a134f655-3, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00004_077eb74146f2afec4488e171a134f655-3, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00004_077eb74146f2afec4488e171a134f655-4, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00004_077eb74146f2afec4488e171a134f655-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00004_077eb74146f2afec4488e171a134f655-5, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00004_077eb74146f2afec4488e171a134f655-6, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00004_077eb74146f2afec4488e171a134f655-6, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00004_077eb74146f2afec4488e171a134f655-6, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00004_077eb74146f2afec4488e171a134f655-7, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00005_0de4e57838f243c67953cb70e70e693b-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00005_0de4e57838f243c67953cb70e70e693b-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00005_0de4e57838f243c67953cb70e70e693b-3, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00005_0de4e57838f243c67953cb70e70e693b-4, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00005_0de4e57838f243c67953cb70e70e693b-4, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00005_0de4e57838f243c67953cb70e70e693b-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00005_0de4e57838f243c67953cb70e70e693b-5, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00005_0de4e57838f243c67953cb70e70e693b-5, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00005_0de4e57838f243c67953cb70e70e693b-6, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00005_0de4e57838f243c67953cb70e70e693b-6, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00059_3f1e667b4e85710912b3c6b5f2f74982-1, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00059_3f1e667b4e85710912b3c6b5f2f74982-1, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00059_3f1e667b4e85710912b3c6b5f2f74982-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00059_3f1e667b4e85710912b3c6b5f2f74982-4, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00059_3f1e667b4e85710912b3c6b5f2f74982-4, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00059_3f1e667b4e85710912b3c6b5f2f74982-6, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00059_3f1e667b4e85710912b3c6b5f2f74982-7, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00059_3f1e667b4e85710912b3c6b5f2f74982-7, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00059_3f1e667b4e85710912b3c6b5f2f74982-7, 1.0\n",
      "sent_idx: opener_en/kaf/hotel/english00059_3f1e667b4e85710912b3c6b5f2f74982-8, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00059_3f1e667b4e85710912b3c6b5f2f74982-8, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00059_3f1e667b4e85710912b3c6b5f2f74982-8, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00059_3f1e667b4e85710912b3c6b5f2f74982-9, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00059_3f1e667b4e85710912b3c6b5f2f74982-9, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00038_3630f623138397b96ee2a4794334e86f-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00038_3630f623138397b96ee2a4794334e86f-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00038_3630f623138397b96ee2a4794334e86f-5, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00038_3630f623138397b96ee2a4794334e86f-5, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00038_3630f623138397b96ee2a4794334e86f-6, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00038_3630f623138397b96ee2a4794334e86f-6, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00038_3630f623138397b96ee2a4794334e86f-6, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00009_10f983ef52497c1afcc050bedcbda0d4-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00009_10f983ef52497c1afcc050bedcbda0d4-1, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00009_10f983ef52497c1afcc050bedcbda0d4-2, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00009_10f983ef52497c1afcc050bedcbda0d4-2, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00009_10f983ef52497c1afcc050bedcbda0d4-2, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00009_10f983ef52497c1afcc050bedcbda0d4-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00009_10f983ef52497c1afcc050bedcbda0d4-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00009_10f983ef52497c1afcc050bedcbda0d4-4, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00009_10f983ef52497c1afcc050bedcbda0d4-4, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00009_10f983ef52497c1afcc050bedcbda0d4-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00030_2e4f193ed7a254c92854201a12034580-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00030_2e4f193ed7a254c92854201a12034580-1, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00030_2e4f193ed7a254c92854201a12034580-1, 3e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00030_2e4f193ed7a254c92854201a12034580-2, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00030_2e4f193ed7a254c92854201a12034580-2, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00030_2e4f193ed7a254c92854201a12034580-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00030_2e4f193ed7a254c92854201a12034580-3, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00030_2e4f193ed7a254c92854201a12034580-4, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00030_2e4f193ed7a254c92854201a12034580-4, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00030_2e4f193ed7a254c92854201a12034580-4, 0.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00030_2e4f193ed7a254c92854201a12034580-5, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00015_10b768c75ce75f96ee4722d418252686-1, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00015_10b768c75ce75f96ee4722d418252686-1, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00015_10b768c75ce75f96ee4722d418252686-2, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00015_10b768c75ce75f96ee4722d418252686-2, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00015_10b768c75ce75f96ee4722d418252686-2, 0.25\n",
      "sent_idx: opener_en/kaf/hotel/english00015_10b768c75ce75f96ee4722d418252686-3, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00015_10b768c75ce75f96ee4722d418252686-3, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00015_10b768c75ce75f96ee4722d418252686-4, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00015_10b768c75ce75f96ee4722d418252686-5, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00015_10b768c75ce75f96ee4722d418252686-5, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00172_cee4504ab4cc06e93f44e33ed1e02b4f-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00172_cee4504ab4cc06e93f44e33ed1e02b4f-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00172_cee4504ab4cc06e93f44e33ed1e02b4f-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00172_cee4504ab4cc06e93f44e33ed1e02b4f-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00172_cee4504ab4cc06e93f44e33ed1e02b4f-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00172_cee4504ab4cc06e93f44e33ed1e02b4f-5, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00172_cee4504ab4cc06e93f44e33ed1e02b4f-6, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00172_cee4504ab4cc06e93f44e33ed1e02b4f-7, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00172_cee4504ab4cc06e93f44e33ed1e02b4f-8, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00172_cee4504ab4cc06e93f44e33ed1e02b4f-8, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00172_cee4504ab4cc06e93f44e33ed1e02b4f-8, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00198_e8893ad293f6ca6d1000776fba1d4af5-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00198_e8893ad293f6ca6d1000776fba1d4af5-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00198_e8893ad293f6ca6d1000776fba1d4af5-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00198_e8893ad293f6ca6d1000776fba1d4af5-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00198_e8893ad293f6ca6d1000776fba1d4af5-4, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00198_e8893ad293f6ca6d1000776fba1d4af5-5, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00198_e8893ad293f6ca6d1000776fba1d4af5-6, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00198_e8893ad293f6ca6d1000776fba1d4af5-6, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00074_5043dc58494745b461da0773b8d0a648-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00074_5043dc58494745b461da0773b8d0a648-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00074_5043dc58494745b461da0773b8d0a648-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00074_5043dc58494745b461da0773b8d0a648-3, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00074_5043dc58494745b461da0773b8d0a648-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00074_5043dc58494745b461da0773b8d0a648-4, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00074_5043dc58494745b461da0773b8d0a648-4, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00074_5043dc58494745b461da0773b8d0a648-5, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00074_5043dc58494745b461da0773b8d0a648-5, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00074_5043dc58494745b461da0773b8d0a648-5, 0.25\n",
      "sent_idx: opener_en/kaf/hotel/english00074_5043dc58494745b461da0773b8d0a648-5, 0.16666666666666666\n",
      "sent_idx: opener_en/kaf/hotel/english00121_892f5a094070c44e45e64e331624bd21-4, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00121_892f5a094070c44e45e64e331624bd21-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00121_892f5a094070c44e45e64e331624bd21-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00121_892f5a094070c44e45e64e331624bd21-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00121_892f5a094070c44e45e64e331624bd21-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00121_892f5a094070c44e45e64e331624bd21-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00121_892f5a094070c44e45e64e331624bd21-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00121_892f5a094070c44e45e64e331624bd21-5, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00121_892f5a094070c44e45e64e331624bd21-5, 0.07142857142857142\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00054_383c9bd9d83d938ba9cc5d84d2d07b90-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00054_383c9bd9d83d938ba9cc5d84d2d07b90-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00054_383c9bd9d83d938ba9cc5d84d2d07b90-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00054_383c9bd9d83d938ba9cc5d84d2d07b90-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00054_383c9bd9d83d938ba9cc5d84d2d07b90-4, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00054_383c9bd9d83d938ba9cc5d84d2d07b90-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00054_383c9bd9d83d938ba9cc5d84d2d07b90-5, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00054_383c9bd9d83d938ba9cc5d84d2d07b90-5, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00054_383c9bd9d83d938ba9cc5d84d2d07b90-5, 4e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00054_383c9bd9d83d938ba9cc5d84d2d07b90-6, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00054_383c9bd9d83d938ba9cc5d84d2d07b90-6, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00054_383c9bd9d83d938ba9cc5d84d2d07b90-7, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00054_383c9bd9d83d938ba9cc5d84d2d07b90-8, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00014_17224596ebb4a03e814c12ea1c18e41e-1, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00014_17224596ebb4a03e814c12ea1c18e41e-2, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00014_17224596ebb4a03e814c12ea1c18e41e-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00014_17224596ebb4a03e814c12ea1c18e41e-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00014_17224596ebb4a03e814c12ea1c18e41e-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00014_17224596ebb4a03e814c12ea1c18e41e-4, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00014_17224596ebb4a03e814c12ea1c18e41e-4, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00014_17224596ebb4a03e814c12ea1c18e41e-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00014_17224596ebb4a03e814c12ea1c18e41e-5, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00014_17224596ebb4a03e814c12ea1c18e41e-7, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00014_17224596ebb4a03e814c12ea1c18e41e-8, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00014_17224596ebb4a03e814c12ea1c18e41e-9, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00001_039d4b7b17acc9ef4243f32de0e3c70e-2, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00001_039d4b7b17acc9ef4243f32de0e3c70e-2, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00001_039d4b7b17acc9ef4243f32de0e3c70e-3, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00001_039d4b7b17acc9ef4243f32de0e3c70e-3, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00001_039d4b7b17acc9ef4243f32de0e3c70e-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00132_c92dfef90d8b18e771944f1e8ac008f5-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00132_c92dfef90d8b18e771944f1e8ac008f5-1, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00132_c92dfef90d8b18e771944f1e8ac008f5-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00132_c92dfef90d8b18e771944f1e8ac008f5-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00132_c92dfef90d8b18e771944f1e8ac008f5-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00132_c92dfef90d8b18e771944f1e8ac008f5-4, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00060_3f651800e6cfeae4225f40d33646d2e3-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00060_3f651800e6cfeae4225f40d33646d2e3-1, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00060_3f651800e6cfeae4225f40d33646d2e3-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00060_3f651800e6cfeae4225f40d33646d2e3-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00060_3f651800e6cfeae4225f40d33646d2e3-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00060_3f651800e6cfeae4225f40d33646d2e3-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00060_3f651800e6cfeae4225f40d33646d2e3-3, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00060_3f651800e6cfeae4225f40d33646d2e3-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00060_3f651800e6cfeae4225f40d33646d2e3-5, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00147_a4f827045ccca4c837ce365bd870850a-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00147_a4f827045ccca4c837ce365bd870850a-1, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00147_a4f827045ccca4c837ce365bd870850a-2, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00147_a4f827045ccca4c837ce365bd870850a-2, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00147_a4f827045ccca4c837ce365bd870850a-2, 0.25\n",
      "sent_idx: opener_en/kaf/hotel/english00147_a4f827045ccca4c837ce365bd870850a-2, 0.16666666666666666\n",
      "sent_idx: opener_en/kaf/hotel/english00147_a4f827045ccca4c837ce365bd870850a-3, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-1, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-1, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-3, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6, 0.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00214_f731285c2d232cf15e5cdae66ab186b1-6, 0.3333333333333333\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-1, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-1, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-2, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-2, 4e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-4, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-5, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-6, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-7, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-7, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-8, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00161_eed91ac341d52eabf370dd4b6c3c3219-9, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00008_09b68606f282d93380da630fd5e3738a-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00008_09b68606f282d93380da630fd5e3738a-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00008_09b68606f282d93380da630fd5e3738a-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00008_09b68606f282d93380da630fd5e3738a-2, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00008_09b68606f282d93380da630fd5e3738a-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00008_09b68606f282d93380da630fd5e3738a-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00008_09b68606f282d93380da630fd5e3738a-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00008_09b68606f282d93380da630fd5e3738a-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00008_09b68606f282d93380da630fd5e3738a-5, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00008_09b68606f282d93380da630fd5e3738a-5, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00149_aa9eb0dd94281b1d1b5182662c047aff-1, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00149_aa9eb0dd94281b1d1b5182662c047aff-2, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00149_aa9eb0dd94281b1d1b5182662c047aff-2, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00149_aa9eb0dd94281b1d1b5182662c047aff-3, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00149_aa9eb0dd94281b1d1b5182662c047aff-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00149_aa9eb0dd94281b1d1b5182662c047aff-4, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00149_aa9eb0dd94281b1d1b5182662c047aff-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00149_aa9eb0dd94281b1d1b5182662c047aff-5, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00149_aa9eb0dd94281b1d1b5182662c047aff-5, 0.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00149_aa9eb0dd94281b1d1b5182662c047aff-5, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00149_aa9eb0dd94281b1d1b5182662c047aff-5, 0.3333333333333333\n",
      "sent_idx: opener_en/kaf/hotel/english00149_aa9eb0dd94281b1d1b5182662c047aff-9, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00149_aa9eb0dd94281b1d1b5182662c047aff-10, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00149_aa9eb0dd94281b1d1b5182662c047aff-10, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00149_aa9eb0dd94281b1d1b5182662c047aff-11, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00042_3a43c28555e55bbf555b1c021553cb68-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00042_3a43c28555e55bbf555b1c021553cb68-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00042_3a43c28555e55bbf555b1c021553cb68-2, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00042_3a43c28555e55bbf555b1c021553cb68-3, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00042_3a43c28555e55bbf555b1c021553cb68-3, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00045_308abf0d21bd195544447890f464633b-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00045_308abf0d21bd195544447890f464633b-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00045_308abf0d21bd195544447890f464633b-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00045_308abf0d21bd195544447890f464633b-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00045_308abf0d21bd195544447890f464633b-3, 3e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00045_308abf0d21bd195544447890f464633b-4, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00045_308abf0d21bd195544447890f464633b-4, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00045_308abf0d21bd195544447890f464633b-6, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00045_308abf0d21bd195544447890f464633b-6, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00045_308abf0d21bd195544447890f464633b-6, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00045_308abf0d21bd195544447890f464633b-7, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00045_308abf0d21bd195544447890f464633b-8, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00045_308abf0d21bd195544447890f464633b-8, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00045_308abf0d21bd195544447890f464633b-9, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00045_308abf0d21bd195544447890f464633b-9, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00114_a4f827045ccca4c837ce365bd870850a-1, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00114_a4f827045ccca4c837ce365bd870850a-1, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00114_a4f827045ccca4c837ce365bd870850a-2, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00114_a4f827045ccca4c837ce365bd870850a-2, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00114_a4f827045ccca4c837ce365bd870850a-2, 0.25\n",
      "sent_idx: opener_en/kaf/hotel/english00114_a4f827045ccca4c837ce365bd870850a-2, 0.16666666666666666\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00114_a4f827045ccca4c837ce365bd870850a-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00007_0fb7d41dba55088490bdd31b9c850e4b-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00007_0fb7d41dba55088490bdd31b9c850e4b-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00007_0fb7d41dba55088490bdd31b9c850e4b-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00007_0fb7d41dba55088490bdd31b9c850e4b-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00007_0fb7d41dba55088490bdd31b9c850e4b-4, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00007_0fb7d41dba55088490bdd31b9c850e4b-5, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00007_0fb7d41dba55088490bdd31b9c850e4b-6, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-1, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-2, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-3, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-3, 4e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-4, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-5, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-6, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-6, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-6, 3e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-6, 1.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-7, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-7, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00012_13287f4f9c645ff79357fe506a813696-7, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00136_cf8140e17a3855f7a4a38936d6193415-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00136_cf8140e17a3855f7a4a38936d6193415-1, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00136_cf8140e17a3855f7a4a38936d6193415-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00136_cf8140e17a3855f7a4a38936d6193415-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00136_cf8140e17a3855f7a4a38936d6193415-3, 3e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00136_cf8140e17a3855f7a4a38936d6193415-4, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00136_cf8140e17a3855f7a4a38936d6193415-4, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00136_cf8140e17a3855f7a4a38936d6193415-4, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00136_cf8140e17a3855f7a4a38936d6193415-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00136_cf8140e17a3855f7a4a38936d6193415-5, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00136_cf8140e17a3855f7a4a38936d6193415-5, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00136_cf8140e17a3855f7a4a38936d6193415-6, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00136_cf8140e17a3855f7a4a38936d6193415-6, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00136_cf8140e17a3855f7a4a38936d6193415-8, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00136_cf8140e17a3855f7a4a38936d6193415-8, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00125_bce630ab47dd16dfe5a755703c2b9aa8-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00125_bce630ab47dd16dfe5a755703c2b9aa8-2, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00125_bce630ab47dd16dfe5a755703c2b9aa8-2, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00125_bce630ab47dd16dfe5a755703c2b9aa8-2, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00125_bce630ab47dd16dfe5a755703c2b9aa8-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00125_bce630ab47dd16dfe5a755703c2b9aa8-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00125_bce630ab47dd16dfe5a755703c2b9aa8-4, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00125_bce630ab47dd16dfe5a755703c2b9aa8-5, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00125_bce630ab47dd16dfe5a755703c2b9aa8-5, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00125_bce630ab47dd16dfe5a755703c2b9aa8-5, 0.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00125_bce630ab47dd16dfe5a755703c2b9aa8-5, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00125_bce630ab47dd16dfe5a755703c2b9aa8-5, 0.75\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00117_add4bfee14df94e5646b1c7018b6c690-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00117_add4bfee14df94e5646b1c7018b6c690-1, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00117_add4bfee14df94e5646b1c7018b6c690-2, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00117_add4bfee14df94e5646b1c7018b6c690-4, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00117_add4bfee14df94e5646b1c7018b6c690-5, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00130_91c66f32f964fdc62c980526105ffefb-2, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00130_91c66f32f964fdc62c980526105ffefb-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00130_91c66f32f964fdc62c980526105ffefb-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00130_91c66f32f964fdc62c980526105ffefb-4, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00130_91c66f32f964fdc62c980526105ffefb-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00130_91c66f32f964fdc62c980526105ffefb-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00130_91c66f32f964fdc62c980526105ffefb-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00130_91c66f32f964fdc62c980526105ffefb-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00130_91c66f32f964fdc62c980526105ffefb-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00130_91c66f32f964fdc62c980526105ffefb-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00130_91c66f32f964fdc62c980526105ffefb-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00130_91c66f32f964fdc62c980526105ffefb-5, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00216_f7c75ca42432376cd15a2aff7694125b-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00216_f7c75ca42432376cd15a2aff7694125b-1, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00216_f7c75ca42432376cd15a2aff7694125b-1, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00216_f7c75ca42432376cd15a2aff7694125b-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00216_f7c75ca42432376cd15a2aff7694125b-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00216_f7c75ca42432376cd15a2aff7694125b-2, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00216_f7c75ca42432376cd15a2aff7694125b-2, 4e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00216_f7c75ca42432376cd15a2aff7694125b-2, 2.0\n",
      "sent_idx: opener_en/kaf/hotel/english00216_f7c75ca42432376cd15a2aff7694125b-2, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00216_f7c75ca42432376cd15a2aff7694125b-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00023_1aa3c5b4f003f6646e3d6f72045d4750-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00023_1aa3c5b4f003f6646e3d6f72045d4750-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00023_1aa3c5b4f003f6646e3d6f72045d4750-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00023_1aa3c5b4f003f6646e3d6f72045d4750-4, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00023_1aa3c5b4f003f6646e3d6f72045d4750-4, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00023_1aa3c5b4f003f6646e3d6f72045d4750-4, 0.25\n",
      "sent_idx: opener_en/kaf/hotel/english00023_1aa3c5b4f003f6646e3d6f72045d4750-4, 0.16666666666666666\n",
      "sent_idx: opener_en/kaf/hotel/english00023_1aa3c5b4f003f6646e3d6f72045d4750-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00203_eb3bae183b573012e1d929eaa6f6c99d-3, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00203_eb3bae183b573012e1d929eaa6f6c99d-4, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00203_eb3bae183b573012e1d929eaa6f6c99d-4, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00203_eb3bae183b573012e1d929eaa6f6c99d-5, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00203_eb3bae183b573012e1d929eaa6f6c99d-5, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00203_eb3bae183b573012e1d929eaa6f6c99d-5, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00203_eb3bae183b573012e1d929eaa6f6c99d-6, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00203_eb3bae183b573012e1d929eaa6f6c99d-8, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00203_eb3bae183b573012e1d929eaa6f6c99d-8, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00156_bceddd6fce4fdf6cc6c030fee377f4aa-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00156_bceddd6fce4fdf6cc6c030fee377f4aa-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00156_bceddd6fce4fdf6cc6c030fee377f4aa-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00156_bceddd6fce4fdf6cc6c030fee377f4aa-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00156_bceddd6fce4fdf6cc6c030fee377f4aa-4, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00156_bceddd6fce4fdf6cc6c030fee377f4aa-4, 0.5\n",
      "sent_idx: opener_en/kaf/hotel/english00156_bceddd6fce4fdf6cc6c030fee377f4aa-4, 0.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00156_bceddd6fce4fdf6cc6c030fee377f4aa-5, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00156_bceddd6fce4fdf6cc6c030fee377f4aa-6, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00156_bceddd6fce4fdf6cc6c030fee377f4aa-6, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00156_bceddd6fce4fdf6cc6c030fee377f4aa-7, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00156_bceddd6fce4fdf6cc6c030fee377f4aa-7, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00156_bceddd6fce4fdf6cc6c030fee377f4aa-8, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00156_bceddd6fce4fdf6cc6c030fee377f4aa-8, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00156_bceddd6fce4fdf6cc6c030fee377f4aa-9, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00156_bceddd6fce4fdf6cc6c030fee377f4aa-9, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00117_8523187c9a437e85989d599da4a3943a-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00117_8523187c9a437e85989d599da4a3943a-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00117_8523187c9a437e85989d599da4a3943a-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00117_8523187c9a437e85989d599da4a3943a-4, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00117_8523187c9a437e85989d599da4a3943a-4, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00117_8523187c9a437e85989d599da4a3943a-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00117_8523187c9a437e85989d599da4a3943a-5, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00117_8523187c9a437e85989d599da4a3943a-6, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00117_8523187c9a437e85989d599da4a3943a-6, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00117_8523187c9a437e85989d599da4a3943a-7, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00117_8523187c9a437e85989d599da4a3943a-7, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00117_8523187c9a437e85989d599da4a3943a-8, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00117_8523187c9a437e85989d599da4a3943a-8, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00117_8523187c9a437e85989d599da4a3943a-8, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00117_8523187c9a437e85989d599da4a3943a-8, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00117_8523187c9a437e85989d599da4a3943a-8, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00038_2a905390ffe1f23af8075a24535f6fa6-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00038_2a905390ffe1f23af8075a24535f6fa6-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00038_2a905390ffe1f23af8075a24535f6fa6-2, 2e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00038_2a905390ffe1f23af8075a24535f6fa6-3, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00038_2a905390ffe1f23af8075a24535f6fa6-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00038_2a905390ffe1f23af8075a24535f6fa6-4, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00038_2a905390ffe1f23af8075a24535f6fa6-4, 3e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00038_2a905390ffe1f23af8075a24535f6fa6-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00038_2a905390ffe1f23af8075a24535f6fa6-5, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00038_2a905390ffe1f23af8075a24535f6fa6-6, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00038_2a905390ffe1f23af8075a24535f6fa6-6, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-2, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-2, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-2, 1.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-2, 1.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-3, 3e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-3, 1.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-3, 2.0\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-3, 1.0\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-3, 0.6666666666666666\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-3, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-3, 0.625\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-3, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-3, 0.6\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-3, 0.7\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-3, 0.8\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00143_a1ec3eba7f3fa0420d4d2fed1bca5701-3, 0.9\n",
      "sent_idx: opener_en/kaf/hotel/english00075_6ec3813631e401743488b9fcf0d114c0-1, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00075_6ec3813631e401743488b9fcf0d114c0-2, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00075_6ec3813631e401743488b9fcf0d114c0-2, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-1, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-5, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-5, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-6, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-6, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-6, 3e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-7, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-8, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-8, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-8, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-8, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00195_e73656ac387fb1d16eee3f0baafa97d3-8, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00006_081b55dce5238cb8cb60cb2c79fd2c66-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00006_081b55dce5238cb8cb60cb2c79fd2c66-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00006_081b55dce5238cb8cb60cb2c79fd2c66-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00006_081b55dce5238cb8cb60cb2c79fd2c66-3, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00140_a052f3702cce7297bc2aff92662d1db3-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00140_a052f3702cce7297bc2aff92662d1db3-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00140_a052f3702cce7297bc2aff92662d1db3-5, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00140_a052f3702cce7297bc2aff92662d1db3-7, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00140_a052f3702cce7297bc2aff92662d1db3-7, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00140_a052f3702cce7297bc2aff92662d1db3-8, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00140_a052f3702cce7297bc2aff92662d1db3-9, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00140_a052f3702cce7297bc2aff92662d1db3-9, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00140_a052f3702cce7297bc2aff92662d1db3-12, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00140_a052f3702cce7297bc2aff92662d1db3-12, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00220_fa8115ab7f09678324fbeee56a7b1f34-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00220_fa8115ab7f09678324fbeee56a7b1f34-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00220_fa8115ab7f09678324fbeee56a7b1f34-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00220_fa8115ab7f09678324fbeee56a7b1f34-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00220_fa8115ab7f09678324fbeee56a7b1f34-3, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00220_fa8115ab7f09678324fbeee56a7b1f34-3, 4e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00220_fa8115ab7f09678324fbeee56a7b1f34-3, 2.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00220_fa8115ab7f09678324fbeee56a7b1f34-3, 2.5\n",
      "sent_idx: opener_en/kaf/hotel/english00220_fa8115ab7f09678324fbeee56a7b1f34-3, 1.25\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00220_fa8115ab7f09678324fbeee56a7b1f34-3, 1.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00220_fa8115ab7f09678324fbeee56a7b1f34-3, 1.75\n",
      "sent_idx: opener_en/kaf/hotel/english00220_fa8115ab7f09678324fbeee56a7b1f34-4, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00220_fa8115ab7f09678324fbeee56a7b1f34-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00220_fa8115ab7f09678324fbeee56a7b1f34-5, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00222_fd5bd3330c0b96b7ddc0ff8e0482a572-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00222_fd5bd3330c0b96b7ddc0ff8e0482a572-1, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00222_fd5bd3330c0b96b7ddc0ff8e0482a572-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00222_fd5bd3330c0b96b7ddc0ff8e0482a572-3, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00222_fd5bd3330c0b96b7ddc0ff8e0482a572-4, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00222_fd5bd3330c0b96b7ddc0ff8e0482a572-4, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00222_fd5bd3330c0b96b7ddc0ff8e0482a572-4, 0.25\n",
      "sent_idx: opener_en/kaf/hotel/english00222_fd5bd3330c0b96b7ddc0ff8e0482a572-4, 0.16666666666666666\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00222_fd5bd3330c0b96b7ddc0ff8e0482a572-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00120_afc3a95e54291de7a4e3f3d692cec8ee-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00120_afc3a95e54291de7a4e3f3d692cec8ee-2, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00120_afc3a95e54291de7a4e3f3d692cec8ee-2, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00120_afc3a95e54291de7a4e3f3d692cec8ee-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00120_afc3a95e54291de7a4e3f3d692cec8ee-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00120_afc3a95e54291de7a4e3f3d692cec8ee-3, 3e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00120_afc3a95e54291de7a4e3f3d692cec8ee-3, 1.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00120_afc3a95e54291de7a4e3f3d692cec8ee-3, 2.0\n",
      "sent_idx: opener_en/kaf/hotel/english00078_726dab247b93f8e93eeeabc72f74d655-1, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00078_726dab247b93f8e93eeeabc72f74d655-1, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00078_726dab247b93f8e93eeeabc72f74d655-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00078_726dab247b93f8e93eeeabc72f74d655-3, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00078_726dab247b93f8e93eeeabc72f74d655-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00078_726dab247b93f8e93eeeabc72f74d655-5, 0.0\n",
      "sent_idx: opener_en/kaf/hotel/english00078_726dab247b93f8e93eeeabc72f74d655-6, 0.0\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-1, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-2, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-2, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-3, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-3, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-4, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-4, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-4, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-5, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-5, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-5, 3e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-5, 4e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-6, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-6, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-8, 1e+16\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-8, 0.5\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9, 1e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9, 2e+16\n",
      "added weighted_tp\n",
      "sent_idx: opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9, 3e+16\n",
      "0.6820724980299452 0.6370001835030743 0.6587662870896918\n"
     ]
    }
   ],
   "source": [
    "semeval_evaluator = SemEvalEvaluator(predicted_tuples=\"./en_all-roberta-large-v1_pred.json\", dataset_name=\"./data/opener_en/test.json\", mode=\"manual\")\n",
    "_precision, _recall, _f1 = semeval_evaluator.get_all_quadtuple_metrics()\n",
    "print(_precision, _recall, _f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>Source</th>\n",
       "      <th>Source_string</th>\n",
       "      <th>Source_index</th>\n",
       "      <th>Target</th>\n",
       "      <th>Target_string</th>\n",
       "      <th>Target_index</th>\n",
       "      <th>Polar_expression</th>\n",
       "      <th>Polar_expression_string</th>\n",
       "      <th>...</th>\n",
       "      <th>Implicit_target_exclusive</th>\n",
       "      <th>Implicit_holder_exclusive</th>\n",
       "      <th>H-A-O</th>\n",
       "      <th>H-IA-IO</th>\n",
       "      <th>H-A-IO</th>\n",
       "      <th>H-IA-O</th>\n",
       "      <th>IH-A-O</th>\n",
       "      <th>IH-A-IO</th>\n",
       "      <th>IH-IA-O</th>\n",
       "      <th>IH-IA-IO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Thank you for such a wonderful experience .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-8</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[experience], [31:41]]</td>\n",
       "      <td>[experience]</td>\n",
       "      <td>[31:41]</td>\n",
       "      <td>[[wonderful], [21:30]]</td>\n",
       "      <td>[wonderful]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hotel was under construction .</td>\n",
       "      <td>opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-2</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[Hotel], [0:5]]</td>\n",
       "      <td>[Hotel]</td>\n",
       "      <td>[0:5]</td>\n",
       "      <td>[[under construction], [10:28]]</td>\n",
       "      <td>[under construction]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Carpets where very old .</td>\n",
       "      <td>opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-3</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[Carpets], [0:7]]</td>\n",
       "      <td>[Carpets]</td>\n",
       "      <td>[0:7]</td>\n",
       "      <td>[[very old], [14:22]]</td>\n",
       "      <td>[very old]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Staircase only on the outside and not heated .</td>\n",
       "      <td>opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-4</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[Staircase], [0:9]]</td>\n",
       "      <td>[Staircase]</td>\n",
       "      <td>[0:9]</td>\n",
       "      <td>[[only on the outside], [10:29]]</td>\n",
       "      <td>[only on the outside]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Staircase only on the outside and not heated .</td>\n",
       "      <td>opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-4</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[Staircase], [0:9]]</td>\n",
       "      <td>[Staircase]</td>\n",
       "      <td>[0:9]</td>\n",
       "      <td>[[not heated], [34:44]]</td>\n",
       "      <td>[not heated]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>841</th>\n",
       "      <td>Front desk wasn 't the best , quite slow for passport registration and clerk insisted that we pay up for a better room .</td>\n",
       "      <td>opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-8</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[Front desk], [0:10]]</td>\n",
       "      <td>[Front desk]</td>\n",
       "      <td>[0:10]</td>\n",
       "      <td>[[wasn 't the best], [11:27]]</td>\n",
       "      <td>[wasn 't the best]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>842</th>\n",
       "      <td>Front desk wasn 't the best , quite slow for passport registration and clerk insisted that we pay up for a better room .</td>\n",
       "      <td>opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-8</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[better room], [107:118]]</td>\n",
       "      <td>[better room]</td>\n",
       "      <td>[107:118]</td>\n",
       "      <td>[[pay], [94:97]]</td>\n",
       "      <td>[pay]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>843</th>\n",
       "      <td>In · room breakfast was tasty and quite large , awesome .</td>\n",
       "      <td>opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[room breakfast], [5:19]]</td>\n",
       "      <td>[room breakfast]</td>\n",
       "      <td>[5:19]</td>\n",
       "      <td>[[tasty], [24:29]]</td>\n",
       "      <td>[tasty]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>844</th>\n",
       "      <td>In · room breakfast was tasty and quite large , awesome .</td>\n",
       "      <td>opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[room breakfast], [5:19]]</td>\n",
       "      <td>[room breakfast]</td>\n",
       "      <td>[5:19]</td>\n",
       "      <td>[[quite large], [34:45]]</td>\n",
       "      <td>[quite large]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>845</th>\n",
       "      <td>In · room breakfast was tasty and quite large , awesome .</td>\n",
       "      <td>opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[room breakfast], [5:19]]</td>\n",
       "      <td>[room breakfast]</td>\n",
       "      <td>[5:19]</td>\n",
       "      <td>[[awesome], [48:55]]</td>\n",
       "      <td>[awesome]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>846 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                         text  \\\n",
       "0                                                                                 Thank you for such a wonderful experience .   \n",
       "1                                                                                              Hotel was under construction .   \n",
       "2                                                                                                    Carpets where very old .   \n",
       "3                                                                              Staircase only on the outside and not heated .   \n",
       "4                                                                              Staircase only on the outside and not heated .   \n",
       "..                                                                                                                        ...   \n",
       "841  Front desk wasn 't the best , quite slow for passport registration and clerk insisted that we pay up for a better room .   \n",
       "842  Front desk wasn 't the best , quite slow for passport registration and clerk insisted that we pay up for a better room .   \n",
       "843                                                                 In · room breakfast was tasty and quite large , awesome .   \n",
       "844                                                                 In · room breakfast was tasty and quite large , awesome .   \n",
       "845                                                                 In · room breakfast was tasty and quite large , awesome .   \n",
       "\n",
       "                                                                 sent_id  \\\n",
       "0    opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-8   \n",
       "1    opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-2   \n",
       "2    opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-3   \n",
       "3    opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-4   \n",
       "4    opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-4   \n",
       "..                                                                   ...   \n",
       "841  opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-8   \n",
       "842  opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-8   \n",
       "843  opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9   \n",
       "844  opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9   \n",
       "845  opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9   \n",
       "\n",
       "       Source Source_string Source_index                      Target  \\\n",
       "0    [[], []]            []           []     [[experience], [31:41]]   \n",
       "1    [[], []]            []           []            [[Hotel], [0:5]]   \n",
       "2    [[], []]            []           []          [[Carpets], [0:7]]   \n",
       "3    [[], []]            []           []        [[Staircase], [0:9]]   \n",
       "4    [[], []]            []           []        [[Staircase], [0:9]]   \n",
       "..        ...           ...          ...                         ...   \n",
       "841  [[], []]            []           []      [[Front desk], [0:10]]   \n",
       "842  [[], []]            []           []  [[better room], [107:118]]   \n",
       "843  [[], []]            []           []  [[room breakfast], [5:19]]   \n",
       "844  [[], []]            []           []  [[room breakfast], [5:19]]   \n",
       "845  [[], []]            []           []  [[room breakfast], [5:19]]   \n",
       "\n",
       "        Target_string Target_index                  Polar_expression  \\\n",
       "0        [experience]      [31:41]            [[wonderful], [21:30]]   \n",
       "1             [Hotel]        [0:5]   [[under construction], [10:28]]   \n",
       "2           [Carpets]        [0:7]             [[very old], [14:22]]   \n",
       "3         [Staircase]        [0:9]  [[only on the outside], [10:29]]   \n",
       "4         [Staircase]        [0:9]           [[not heated], [34:44]]   \n",
       "..                ...          ...                               ...   \n",
       "841      [Front desk]       [0:10]     [[wasn 't the best], [11:27]]   \n",
       "842     [better room]    [107:118]                  [[pay], [94:97]]   \n",
       "843  [room breakfast]       [5:19]                [[tasty], [24:29]]   \n",
       "844  [room breakfast]       [5:19]          [[quite large], [34:45]]   \n",
       "845  [room breakfast]       [5:19]              [[awesome], [48:55]]   \n",
       "\n",
       "    Polar_expression_string  ... Implicit_target_exclusive  \\\n",
       "0               [wonderful]  ...                     False   \n",
       "1      [under construction]  ...                     False   \n",
       "2                [very old]  ...                     False   \n",
       "3     [only on the outside]  ...                     False   \n",
       "4              [not heated]  ...                     False   \n",
       "..                      ...  ...                       ...   \n",
       "841      [wasn 't the best]  ...                     False   \n",
       "842                   [pay]  ...                     False   \n",
       "843                 [tasty]  ...                     False   \n",
       "844           [quite large]  ...                     False   \n",
       "845               [awesome]  ...                     False   \n",
       "\n",
       "    Implicit_holder_exclusive  H-A-O  H-IA-IO  H-A-IO  H-IA-O  IH-A-O  \\\n",
       "0                        True      0    False   False   False    True   \n",
       "1                        True      0    False   False   False    True   \n",
       "2                        True      0    False   False   False    True   \n",
       "3                        True      0    False   False   False    True   \n",
       "4                        True      0    False   False   False    True   \n",
       "..                        ...    ...      ...     ...     ...     ...   \n",
       "841                      True      0    False   False   False    True   \n",
       "842                      True      0    False   False   False    True   \n",
       "843                      True      0    False   False   False    True   \n",
       "844                      True      0    False   False   False    True   \n",
       "845                      True      0    False   False   False    True   \n",
       "\n",
       "     IH-A-IO  IH-IA-O  IH-IA-IO  \n",
       "0      False    False     False  \n",
       "1      False    False     False  \n",
       "2      False    False     False  \n",
       "3      False    False     False  \n",
       "4      False    False     False  \n",
       "..       ...      ...       ...  \n",
       "841    False    False     False  \n",
       "842    False    False     False  \n",
       "843    False    False     False  \n",
       "844    False    False     False  \n",
       "845    False    False     False  \n",
       "\n",
       "[846 rows x 27 columns]"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_en_roberta_largev1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>Source</th>\n",
       "      <th>Source_string</th>\n",
       "      <th>Source_index</th>\n",
       "      <th>Target</th>\n",
       "      <th>Target_string</th>\n",
       "      <th>Target_index</th>\n",
       "      <th>Polar_expression</th>\n",
       "      <th>Polar_expression_string</th>\n",
       "      <th>...</th>\n",
       "      <th>Implicit_target_exclusive</th>\n",
       "      <th>Implicit_holder_exclusive</th>\n",
       "      <th>H-A-O</th>\n",
       "      <th>H-IA-IO</th>\n",
       "      <th>H-A-IO</th>\n",
       "      <th>H-IA-O</th>\n",
       "      <th>IH-A-O</th>\n",
       "      <th>IH-A-IO</th>\n",
       "      <th>IH-IA-O</th>\n",
       "      <th>IH-IA-IO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>They were there just to keep us happy .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-6</td>\n",
       "      <td>[[us], [29:31]]</td>\n",
       "      <td>[us]</td>\n",
       "      <td>[29:31]</td>\n",
       "      <td>[[They], [0:4]]</td>\n",
       "      <td>[They]</td>\n",
       "      <td>[0:4]</td>\n",
       "      <td>[[to keep, happy], [21:28, 32:37]]</td>\n",
       "      <td>[to keep, happy]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A wonderful place to go and we are planning to return as soon as we can .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-7</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[place], [12:17]]</td>\n",
       "      <td>[place]</td>\n",
       "      <td>[12:17]</td>\n",
       "      <td>[[wonderful], [2:11]]</td>\n",
       "      <td>[wonderful]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A wonderful place to go and we are planning to return as soon as we can .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-7</td>\n",
       "      <td>[[we], [28:30]]</td>\n",
       "      <td>[we]</td>\n",
       "      <td>[28:30]</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[planning to return], [35:53]]</td>\n",
       "      <td>[planning to return]</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Thank you for such a wonderful experience .</td>\n",
       "      <td>opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-8</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[experience], [31:41]]</td>\n",
       "      <td>[experience]</td>\n",
       "      <td>[31:41]</td>\n",
       "      <td>[[wonderful], [21:30]]</td>\n",
       "      <td>[wonderful]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Hotel was under construction .</td>\n",
       "      <td>opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-2</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[Hotel], [0:5]]</td>\n",
       "      <td>[Hotel]</td>\n",
       "      <td>[0:5]</td>\n",
       "      <td>[[under construction], [10:28]]</td>\n",
       "      <td>[under construction]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>860</th>\n",
       "      <td>Front desk wasn 't the best , quite slow for passport registration and clerk insisted that we pay up for a better room .</td>\n",
       "      <td>opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-8</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[Front desk], [0:10]]</td>\n",
       "      <td>[Front desk]</td>\n",
       "      <td>[0:10]</td>\n",
       "      <td>[[quite slow], [30:40]]</td>\n",
       "      <td>[quite slow]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>861</th>\n",
       "      <td>Front desk wasn 't the best , quite slow for passport registration and clerk insisted that we pay up for a better room .</td>\n",
       "      <td>opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-8</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[Front desk], [0:10]]</td>\n",
       "      <td>[Front desk]</td>\n",
       "      <td>[0:10]</td>\n",
       "      <td>[[wasn 't the best], [11:27]]</td>\n",
       "      <td>[wasn 't the best]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>862</th>\n",
       "      <td>In · room breakfast was tasty and quite large , awesome .</td>\n",
       "      <td>opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[breakfast], [10:19]]</td>\n",
       "      <td>[breakfast]</td>\n",
       "      <td>[10:19]</td>\n",
       "      <td>[[quite large], [34:45]]</td>\n",
       "      <td>[quite large]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>863</th>\n",
       "      <td>In · room breakfast was tasty and quite large , awesome .</td>\n",
       "      <td>opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[breakfast], [10:19]]</td>\n",
       "      <td>[breakfast]</td>\n",
       "      <td>[10:19]</td>\n",
       "      <td>[[tasty], [24:29]]</td>\n",
       "      <td>[tasty]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>864</th>\n",
       "      <td>In · room breakfast was tasty and quite large , awesome .</td>\n",
       "      <td>opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9</td>\n",
       "      <td>[[], []]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[[breakfast], [10:19]]</td>\n",
       "      <td>[breakfast]</td>\n",
       "      <td>[10:19]</td>\n",
       "      <td>[[awesome], [48:55]]</td>\n",
       "      <td>[awesome]</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>865 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                         text  \\\n",
       "0                                                                                     They were there just to keep us happy .   \n",
       "1                                                   A wonderful place to go and we are planning to return as soon as we can .   \n",
       "2                                                   A wonderful place to go and we are planning to return as soon as we can .   \n",
       "3                                                                                 Thank you for such a wonderful experience .   \n",
       "4                                                                                              Hotel was under construction .   \n",
       "..                                                                                                                        ...   \n",
       "860  Front desk wasn 't the best , quite slow for passport registration and clerk insisted that we pay up for a better room .   \n",
       "861  Front desk wasn 't the best , quite slow for passport registration and clerk insisted that we pay up for a better room .   \n",
       "862                                                                 In · room breakfast was tasty and quite large , awesome .   \n",
       "863                                                                 In · room breakfast was tasty and quite large , awesome .   \n",
       "864                                                                 In · room breakfast was tasty and quite large , awesome .   \n",
       "\n",
       "                                                                 sent_id  \\\n",
       "0    opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-6   \n",
       "1    opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-7   \n",
       "2    opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-7   \n",
       "3    opener_en/kaf/hotel/english00148_ddb06f1e4ab012d85f9120c394168c48-8   \n",
       "4    opener_en/kaf/hotel/english00028_2a1e03a3e5d6afe2c5024926cb7ff9ab-2   \n",
       "..                                                                   ...   \n",
       "860  opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-8   \n",
       "861  opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-8   \n",
       "862  opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9   \n",
       "863  opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9   \n",
       "864  opener_en/kaf/hotel/english00113_8161bacaf54503f451c7246b073003a5-9   \n",
       "\n",
       "              Source Source_string Source_index                   Target  \\\n",
       "0    [[us], [29:31]]          [us]      [29:31]          [[They], [0:4]]   \n",
       "1           [[], []]            []           []       [[place], [12:17]]   \n",
       "2    [[we], [28:30]]          [we]      [28:30]                 [[], []]   \n",
       "3           [[], []]            []           []  [[experience], [31:41]]   \n",
       "4           [[], []]            []           []         [[Hotel], [0:5]]   \n",
       "..               ...           ...          ...                      ...   \n",
       "860         [[], []]            []           []   [[Front desk], [0:10]]   \n",
       "861         [[], []]            []           []   [[Front desk], [0:10]]   \n",
       "862         [[], []]            []           []   [[breakfast], [10:19]]   \n",
       "863         [[], []]            []           []   [[breakfast], [10:19]]   \n",
       "864         [[], []]            []           []   [[breakfast], [10:19]]   \n",
       "\n",
       "    Target_string Target_index                    Polar_expression  \\\n",
       "0          [They]        [0:4]  [[to keep, happy], [21:28, 32:37]]   \n",
       "1         [place]      [12:17]               [[wonderful], [2:11]]   \n",
       "2              []           []     [[planning to return], [35:53]]   \n",
       "3    [experience]      [31:41]              [[wonderful], [21:30]]   \n",
       "4         [Hotel]        [0:5]     [[under construction], [10:28]]   \n",
       "..            ...          ...                                 ...   \n",
       "860  [Front desk]       [0:10]             [[quite slow], [30:40]]   \n",
       "861  [Front desk]       [0:10]       [[wasn 't the best], [11:27]]   \n",
       "862   [breakfast]      [10:19]            [[quite large], [34:45]]   \n",
       "863   [breakfast]      [10:19]                  [[tasty], [24:29]]   \n",
       "864   [breakfast]      [10:19]                [[awesome], [48:55]]   \n",
       "\n",
       "    Polar_expression_string  ... Implicit_target_exclusive  \\\n",
       "0          [to keep, happy]  ...                     False   \n",
       "1               [wonderful]  ...                     False   \n",
       "2      [planning to return]  ...                      True   \n",
       "3               [wonderful]  ...                     False   \n",
       "4      [under construction]  ...                     False   \n",
       "..                      ...  ...                       ...   \n",
       "860            [quite slow]  ...                     False   \n",
       "861      [wasn 't the best]  ...                     False   \n",
       "862           [quite large]  ...                     False   \n",
       "863                 [tasty]  ...                     False   \n",
       "864               [awesome]  ...                     False   \n",
       "\n",
       "    Implicit_holder_exclusive H-A-O  H-IA-IO  H-A-IO  H-IA-O  IH-A-O  IH-A-IO  \\\n",
       "0                       False     1    False   False   False   False    False   \n",
       "1                        True     0    False   False   False    True    False   \n",
       "2                       False     0    False   False    True   False    False   \n",
       "3                        True     0    False   False   False    True    False   \n",
       "4                        True     0    False   False   False    True    False   \n",
       "..                        ...   ...      ...     ...     ...     ...      ...   \n",
       "860                      True     0    False   False   False    True    False   \n",
       "861                      True     0    False   False   False    True    False   \n",
       "862                      True     0    False   False   False    True    False   \n",
       "863                      True     0    False   False   False    True    False   \n",
       "864                      True     0    False   False   False    True    False   \n",
       "\n",
       "     IH-IA-O  IH-IA-IO  \n",
       "0      False     False  \n",
       "1      False     False  \n",
       "2      False     False  \n",
       "3      False     False  \n",
       "4      False     False  \n",
       "..       ...       ...  \n",
       "860    False     False  \n",
       "861    False     False  \n",
       "862    False     False  \n",
       "863    False     False  \n",
       "864    False     False  \n",
       "\n",
       "[865 rows x 27 columns]"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_opener_en_test"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "SemEval2022-Task10",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}