Learning_To_Rank_Medical_Publications.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Learning To Rank Medical Publications.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [
        "D9OYB7OfcoiL",
        "IJHQCZOHr5qx"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/JellePiepenbrock/information-retrieval-2018/blob/master/Learning_To_Rank_Medical_Publications.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "A3-tuT3US1Rc",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Learning To Rank Medical Publications"
      ]
    },
    {
      "metadata": {
        "id": "wc-Aum2Fa6e-",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "\n",
        "# Resources\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "D9OYB7OfcoiL",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Dataset\n",
        "### Use this link: https://trec.nist.gov/data/t9_filtering.html\n",
        "Useful Links\n",
        "### https://trec.nist.gov/pubs/trec9/papers/filtering_new.pdf\n",
        "### https://www.microsoft.com/en-us/research/project/mslr/"
      ]
    },
    {
      "metadata": {
        "id": "f2mBtAN2cytz",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Drive Mounting"
      ]
    },
    {
      "metadata": {
        "id": "GEm0celoaHsy",
        "colab_type": "code",
        "outputId": "d8050afc-7c88-41ce-d34f-85e7a2ea94ee",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "qxvZUCIN9xgN",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Imports & High-level settings"
      ]
    },
    {
      "metadata": {
        "id": "gcvaIS8p0U3L",
        "colab_type": "code",
        "outputId": "cb571c86-ebdc-45f1-f882-13c432133ea5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 217
        }
      },
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import itertools\n",
        "import re\n",
        "from sklearn.metrics import average_precision_score, roc_auc_score\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from nltk.tokenize import wordpunct_tokenize, sent_tokenize\n",
        "import string\n",
        "from sklearn.preprocessing import scale\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.ensemble import RandomForestClassifier ,AdaBoostClassifier\n",
        "from sklearn import svm\n",
        "import operator\n",
        "from gensim.summarization.bm25 import get_bm25_weights\n",
        "import nltk\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "!pip install stop-words\n",
        "\n",
        "from sklearn.metrics import roc_auc_score\n",
        "  \n",
        "\n",
        "topk = 30"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "Collecting stop-words\n",
            "  Downloading https://files.pythonhosted.org/packages/1c/cb/d58290804b7a4c5daa42abbbe2a93c477ae53e45541b1825e86f0dfaaf63/stop-words-2018.7.23.tar.gz\n",
            "Building wheels for collected packages: stop-words\n",
            "  Running setup.py bdist_wheel for stop-words ... \u001b[?25l-\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/75/37/6a/2b295e03bd07290f0da95c3adb9a74ba95fbc333aa8b0c7c78\n",
            "Successfully built stop-words\n",
            "Installing collected packages: stop-words\n",
            "Successfully installed stop-words-2018.7.23\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "nGVFQG1DKGT6",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Data Loading"
      ]
    },
    {
      "metadata": {
        "id": "CKTJtMUPj_dO",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def merge_cels(row):\n",
        "    return str(row['M']) + ' ' + str(row['T']) + ' ' + str(row['W'])\n",
        "\n",
        "def get_documents(filename):\n",
        "  \n",
        "  database = {}\n",
        "  with open(filename) as f:\n",
        "    lines = f.readlines()\n",
        "\n",
        "    for n, line in enumerate(lines):\n",
        "      if not n == len(lines):\n",
        "        if line[:2] ==\".I\":\n",
        "          entry = int(line[2:])\n",
        "          database[entry] = {}\n",
        "        elif line[:2] ==\".U\":\n",
        "          database[entry]['U'] = lines[n+1].strip()\n",
        "      \n",
        "        elif line[:2] ==\".S\":\n",
        "          database[entry]['S'] = lines[n+1].strip()\n",
        "        elif line[:2] ==\".M\":\n",
        "          database[entry]['M'] = lines[n+1].strip()\n",
        "        elif line[:2] ==\".T\":\n",
        "          database[entry]['T'] = lines[n+1].strip()\n",
        "        elif line[:2] ==\".P\":\n",
        "          database[entry]['P'] = lines[n+1].strip()\n",
        "        elif line[:2] ==\".W\":\n",
        "          database[entry]['W'] = lines[n+1].strip()\n",
        "        elif line[:2] ==\".A\":\n",
        "          database[entry]['A'] = lines[n+1].strip()\n",
        "  \n",
        "  df = pd.DataFrame(database).T.reset_index()\n",
        "  df['merged'] = df.apply(merge_cels, axis=1)  # Merged title, abstract and keywords\n",
        "\n",
        "  return df\n",
        "\n",
        "def get_judgments(filename):\n",
        "  \n",
        "  jud = pd.read_table(filename, header=None)\n",
        "  jud.columns = ['topic','uid', 'relevance']\n",
        "  return jud\n",
        "\n",
        "def get_queries(filename):\n",
        "  \n",
        "  with open(filename, 'r') as file:\n",
        "    lines = file.readlines()\n",
        "    result = {}\n",
        "    for idx, line in enumerate(lines):\n",
        "        if '<num>' in line:\n",
        "            key = line.replace('<num> Number: ', '').strip()\n",
        "            result[key] = []\n",
        "        elif '<title>' in line:\n",
        "            result[key].append(line.replace('<title> ', '').strip())\n",
        "        elif '<desc>' in line:\n",
        "            cnt = 0\n",
        "            desc = True\n",
        "            result[key].append(lines[idx+1].replace('<desc> ', '').strip())\n",
        "\n",
        "  result = pd.DataFrame(result).T.reset_index()\n",
        "  result.columns =['Topic','patient', 'query']\n",
        "                \n",
        "  return result\n",
        "\n",
        "def get_run(filename, top=topk):\n",
        "\n",
        "  run = pd.read_table(filename, header=None)\n",
        "  run.columns = ['Topic', 'Query', 'ui', 'rank', 'score', 'run name']\n",
        "  run = run.groupby(by='Topic').head(top)\n",
        "  return run\n",
        "\n",
        "train_docs = get_documents('drive/My Drive/Information Retrieval/Data/ohsu-trec/pre-test/ohsumed.87')\n",
        "test_docs = get_documents('drive/My Drive/Information Retrieval/Data/ohsu-trec/trec9-test/ohsumed.88-91')\n",
        "queries = get_queries('drive/My Drive/Information Retrieval/Data/ohsu-trec/trec9-train/query.ohsu.1-63')\n",
        "train_judgments = get_judgments('drive/My Drive/Information Retrieval/Data/ohsu-trec/trec9-train/qrels.ohsu.batch.87')\n",
        "test_judgments = get_judgments('drive/My Drive/Information Retrieval/Data/ohsu-trec/trec9-test/qrels.ohsu.88-91')\n",
        "run = get_run('drive/My Drive/Information Retrieval/Runs/input.ok9bf2po.gz')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "lDflIJlK76Gw",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from stop_words import get_stop_words\n",
        "\n",
        "def normalisation(text):\n",
        "  \n",
        "    text = text.lower()\n",
        "\n",
        "    while '  ' in text:\n",
        "        text = text.replace('  ', ' ')\n",
        "\n",
        "\n",
        "    tokens = wordpunct_tokenize(text)\n",
        "    word_list = [w.lower() for w in tokens]\n",
        "    clean_list = [word for word in word_list]\n",
        "\n",
        "    return clean_list"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "MaYzdu0P8Bcy",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Set up TF-IDF**"
      ]
    },
    {
      "metadata": {
        "id": "tHPMb_k21Cro",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "all_docs = list(train_docs['W'].dropna()) + list(test_docs['W'].dropna())\n",
        "all_keyw = list(train_docs['M'].dropna()) + list(test_docs['M'].dropna())\n",
        "all_titles = list(train_docs['T'].dropna()) + list(test_docs['T'].dropna())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "GXj2Cr0A8Mpk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer\n",
        "\n",
        "def tfidf_features(X_train):\n",
        "    \"\"\"\n",
        "        X_train, X_test — samples        \n",
        "        return TF-IDF vectorized representation of each sample and vocabulary\n",
        "    \"\"\"\n",
        "   \n",
        "    tfidf_vectorizer = TfidfVectorizer(analyzer='word', max_df=0.5, min_df=1) # , ngram_range=(1, 2), stop_words='english', \n",
        "    tfidf_vectorizer.fit_transform(X_train)\n",
        "    \n",
        "    return tfidf_vectorizer\n",
        "\n",
        "tfidf_vectorizer_a = tfidf_features(all_docs)\n",
        "tfidf_vectorizer_k = tfidf_features(all_keyw)\n",
        "tfidf_vectorizer_t = tfidf_features(all_titles)\n",
        "\n",
        "\n",
        "def get_tfidf(text, tfidf_vectorizer):\n",
        "    return list(tfidf_vectorizer.transform([text]).toarray())[0]\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "R-1lcYtt8oCF",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Get Idf Dictionary**"
      ]
    },
    {
      "metadata": {
        "id": "g0kbyQ7_8hPp",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from collections import defaultdict\n",
        "word_idf = defaultdict(lambda: 0)\n",
        "\n",
        "def get_idf(stream):\n",
        "  for doc in stream:\n",
        "      words = set(normalisation(doc)) # tokenization?\n",
        "      for word in words:\n",
        "          word_idf[word] += 1\n",
        "          \n",
        "  return word_idf\n",
        "\n",
        "word_idf_all = get_idf(all_docs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "mlHaPtjT8v5y",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Get features and their index for the K-folding**"
      ]
    },
    {
      "metadata": {
        "id": "lXQvoJHrSbjW",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# mapping from feature groups to feature indices\n",
        "from collections import OrderedDict\n",
        "\n",
        "featuremap = OrderedDict({\"KW - Query Term Coverage\": [0, 1], \"KW General Statistics\": [2,3], \"KW IDF\":[4], \"KW Partial TF\":list(range(5,15)), \"KW TF\":list(range(15,25)), \"KW TF-IDF\":list(range(25,30)), \"KW Cosine\":[30],\n",
        "              \"KW Character Counts\":list(range(31,122)), \"KW POS Tags\":list(range(122, 152)),\n",
        "             \"T - Query Term Coverage\": [152, 153], \"T General Statistics\": [154,155], \"T IDF\":[156], \"T Partial TF\":list(range(157, 167)), \"T TF\":list(range(167,177)), \"T TF-IDF\":list(range(177,182)), \"T Cosine\":[182],\n",
        "              \"T Character Counts\":list(range(183, 274)), \"T POS Tags\":list(range(274, 304)),\n",
        "             \"ABS - Query Term Coverage\": [304, 305], \"ABS General Statistics\": [306, 307], \"ABS IDF\":[308], \"ABS Partial TF\":list(range(309, 319)), \"ABS TF\":list(range(319,329)), \"ABS TF-IDF\":list(range(329,334)), \"ABS Cosine\":[334],\n",
        "              \"ABS Character Counts\":list(range(335, 426)), \"ABS POS Tags\":list(range(426, 456))\n",
        "             \n",
        "             })\n",
        "sum([len(k) for k in featuremap.values()])\n",
        "totallist = []\n",
        "for k in featuremap.values():\n",
        "  totallist += k\n",
        "  \n",
        "assert sum(list(range(456))) - sum(totallist) == 0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "owKDbcB9aiKu",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Large k-folding process"
      ]
    },
    {
      "metadata": {
        "id": "sOeYKIZdWX6P",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Functions and imports"
      ]
    },
    {
      "metadata": {
        "id": "fKuJmLUdaqmk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from random import shuffle\n",
        "from sklearn.model_selection import KFold\n",
        "from stop_words import get_stop_words\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import roc_auc_score \n",
        "from sklearn.metrics import average_precision_score\n",
        "from sklearn.metrics import recall_score\n",
        "from itertools import chain, combinations\n",
        "\n",
        "def create_test_pairs(run, queries, test_docs, judgments):\n",
        "    \"\"\" Returns a list of (query, document) pairs.\"\"\"\n",
        "    \n",
        "    pairlist = []\n",
        "    rel_list = []\n",
        "    for query in queries:\n",
        "#         print(query)\n",
        "        topic = run.loc[run['Topic'] == query]\n",
        "        top30 = list(topic['ui'])\n",
        "        \n",
        "        relevancy_df = pd.merge(topic, judgments, left_on='ui', right_on='uid', how='left')\n",
        "        relevancy_df['relevance'] = relevancy_df['relevance'].fillna(0)\n",
        "        relevancy_df['relevance'] = relevancy_df['relevance'].apply(lambda x: 1.0 if (x == 2 or x == 1) else 0)\n",
        "        rel_sublist = list(relevancy_df['relevance'])\n",
        "        \n",
        "        for e, document in enumerate(top30):\n",
        "            pairlist.append((query, document))\n",
        "            rel_list.append(rel_sublist[e])\n",
        "    return pairlist, rel_list\n",
        "\n",
        "  \n",
        "def create_X(pairs, rel_list, queries, test_docs):\n",
        "    print(\"Pairs: \", len(pairs))\n",
        "    print(\"Rellist: \", len(rel_list))\n",
        "\n",
        "    X_abs = []\n",
        "    X_keywords = []\n",
        "    X_titles = []\n",
        "    X_queries = []\n",
        "    relevance = []\n",
        "\n",
        "    for pair in range(len(pairs)):\n",
        "\n",
        "        q = queries[queries['Topic'] == pairs[pair][0]]['query'].values[0]\n",
        "#         d = test_docs[test_docs['U'] == str(pairs[pair][1])]['merged'].values[0]\n",
        "        d_keywords = test_docs[test_docs['U'] == str(pairs[pair][1])]['M'].values[0]\n",
        "        d_title = test_docs[test_docs['U'] == str(pairs[pair][1])]['T'].values[0]\n",
        "        d_abstract = test_docs[test_docs['U'] == str(pairs[pair][1])]['W'].values[0]\n",
        "        # d = test_docs[test_docs['U'] == str(pairs[pair][1])]['W'].values[0]\n",
        "        r = rel_list[pair]\n",
        "\n",
        "#         if type(d)==str and type(q) == str:\n",
        "        X_queries.append(q)\n",
        "        if pd.isna(d_abstract):\n",
        "#           print(\"nan wtf\")\n",
        "          X_abs.append(\"No abstract provided\")\n",
        "        else:\n",
        "          X_abs.append(d_abstract)\n",
        "          \n",
        "        X_keywords.append(d_keywords)\n",
        "        X_titles.append(d_title)\n",
        "        \n",
        "        relevance.append(r)\n",
        "\n",
        "    print('Length')\n",
        "    print(len(X_queries))\n",
        "#     print(len(X_docs))\n",
        "    print(len(relevance))\n",
        "        \n",
        "    return X_abs, X_keywords, X_titles, X_queries, relevance\n",
        "\n",
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "from scipy.spatial.distance import euclidean\n",
        "import nltk\n",
        "\n",
        "# nltk.download('averaged_perceptron_tagger')\n",
        "alphabet  = string.ascii_letters + string.whitespace + string.punctuation\n",
        "pos_tags = ['CC','CD','DT','EX','IN','JJ','JJR','JJS','LS','MD','NN','NNP','NNS','PDT','POS','PRP','RB','RBR','RBS','RP','TO','UH','VB','VBD','VBG','VBN','VBP','VBZ','WDT','WP','WRB']\n",
        "    \n",
        "# keywords, title\n",
        "\n",
        "def features_stream_keywords(X_kw, X_queries):\n",
        "    \n",
        "  \n",
        "    res = []\n",
        "    \n",
        "    for i in range(len(X_kw)):\n",
        "      \n",
        "        if i % 250 == 0:\n",
        "          print(i)\n",
        "          \n",
        "        features = []\n",
        "        q = X_queries[i]\n",
        "        d = X_kw[i]\n",
        "\n",
        "        # Remove stopwords\n",
        "        qbag = wordpunct_tokenize(q) #normalisation(q)\n",
        "        dbag = wordpunct_tokenize(d) #normalisation(d)\n",
        "        \n",
        "        d_pos = nltk.pos_tag(dbag)\n",
        "        \n",
        "        q_length = len(qbag)\n",
        "        d_length = len(dbag)\n",
        "        \n",
        "        features = []                \n",
        "        \n",
        "        # Query term number features\n",
        "        features.append(sum(1 for q_word in qbag if q_word in dbag)) # covered query term number\n",
        "        features.append(sum(1 for q_word in qbag if q_word in dbag)/q_length)  # covered query term ratio\n",
        "        \n",
        "        # Stats\n",
        "        features.append(q_length)  # stream length\n",
        "        features.append(d_length)  # stream length\n",
        "        \n",
        "        # IDF of query terms\n",
        "        features.append(sum(1/word_idf_all[q] if q in word_idf_all  else 0 for q in qbag ))  # IDF (inverse document frequency)\n",
        "        # Partial matching.\n",
        "        \n",
        "  \n",
        "        partial = [sum(1 for d_word in dbag if (q_word in d_word) or (d_word in q_word)) for q_word in qbag] # list of counts of each query term in the document\n",
        "        \n",
        "          \n",
        "        features.append(sum(partial)) # sum of term frequency\n",
        "        features.append(min(partial)) # min of term frequency\n",
        "        features.append(max(partial)) # max of term frequency\n",
        "        features.append(np.mean(partial)) # mean of term frequency\n",
        "        features.append(np.var(partial)) # varience of term frequency\n",
        "\n",
        "        features.append(sum(partial) / d_length) # normalized  sum of stream length\n",
        "        features.append(min(partial) / d_length) # normalized min of stream length\n",
        "        features.append(max(partial) / d_length) # normalized max of stream length\n",
        "        features.append(np.mean(partial) / d_length) # normalized mean of stream length\n",
        "        features.append(np.var(partial) / d_length) # normalized varience| of stream length\n",
        "  \n",
        "        # Term frequency \n",
        "        TF = [sum(1 for d_word in dbag if d_word == q_word) for q_word in qbag]\n",
        "        features.append(sum(TF)) # sum of term frequency\n",
        "        features.append(min(TF)) # min of term frequency\n",
        "        features.append(max(TF)) # max of term frequency\n",
        "        features.append(np.mean(TF)) # mean of term frequency\n",
        "        features.append(np.var(TF)) # varience of term frequency\n",
        "\n",
        "        # Normalized term frequency\n",
        "        features.append(sum(TF) / d_length) # normalized  sum of stream length\n",
        "        features.append(min(TF) / d_length) # normalized min of stream length\n",
        "        features.append(max(TF) / d_length) # normalized max of stream length\n",
        "        features.append(np.mean(TF) / d_length) # normalized mean of stream length\n",
        "        features.append(np.var(TF) / d_length) # normalized varience| of stream length\n",
        "        \n",
        "        # TFIDF\n",
        "        TFIDF = get_tfidf(d, tfidf_vectorizer_k)\n",
        "        features.append(sum(TFIDF)) # sum of tf*idf\n",
        "        features.append(min(TFIDF)) # min of tf*idf\n",
        "        features.append(max(TFIDF)) # max of tf*idf\n",
        "        features.append(np.mean(TFIDF)) # mean of tf*idf\n",
        "        features.append(np.var(TFIDF)) # varience of tf*idf\n",
        "      \n",
        "  \n",
        "        # Distance\n",
        "\n",
        "        # Cosine distance between document and query tf-idf vectors      \n",
        "       \n",
        "        features.append(cosine_similarity([get_tfidf(q, tfidf_vectorizer_k)], [get_tfidf(d, tfidf_vectorizer_k)]).tolist()[0][0])\n",
        "        \n",
        "      \n",
        "#         Character Counts\n",
        "        character_counts = [0 for k in alphabet]\n",
        "        for word in dbag:\n",
        "            for e, k in enumerate(alphabet):\n",
        "                character_counts[e] += word.count(k)\n",
        "\n",
        "        features += character_counts \n",
        "        \n",
        "        # POS tag features\n",
        "        for pos_tag in pos_tags:\n",
        "            count = 0\n",
        "            for word in d_pos:\n",
        "                if word[1] == pos_tag:\n",
        "                    count += 1\n",
        "            features.append(count)\n",
        "\n",
        "        \n",
        "  \n",
        "        res.append(features)\n",
        "    return res\n",
        "\n",
        "def features_stream_title(X_title, X_queries):\n",
        "    res = []\n",
        "    \n",
        "    \n",
        "    \n",
        "    for i in range(len(X_title)):\n",
        "      \n",
        "        if i % 250 == 0:\n",
        "          print(i)\n",
        "          \n",
        "        features = []\n",
        "        q = X_queries[i]\n",
        "        d = X_title[i]\n",
        "\n",
        "        # Remove stopwords\n",
        "        qbag = wordpunct_tokenize(q)#normalisation(q)\n",
        "        dbag = wordpunct_tokenize(d)#normalisation(d)\n",
        "        \n",
        "        d_pos = nltk.pos_tag(dbag)\n",
        "        \n",
        "        q_length = len(qbag)\n",
        "        d_length = len(dbag)\n",
        "        \n",
        "        features = []       \n",
        "\n",
        "        # Query term number features\n",
        "        features.append(sum(1 for q_word in qbag if q_word in dbag)) # covered query term number\n",
        "        features.append(sum(1 for q_word in qbag if q_word in dbag)/q_length)  # covered query term ratio\n",
        "\n",
        "        # Stats\n",
        "        features.append(q_length)  # stream length\n",
        "        features.append(d_length)  # stream length\n",
        "        \n",
        "        # IDF of query terms\n",
        "        features.append(sum(1/word_idf_all[q] if q in word_idf_all  else 0 for q in qbag ))  # IDF (inverse document frequency)\n",
        "        \n",
        "#         # Partial matching.\n",
        "        partial = [sum(1 for d_word in dbag if (q_word in d_word) or (d_word in q_word)) for q_word in qbag] # list of counts of each query term in the document\n",
        "       \n",
        "        features.append(sum(partial)) # sum of term frequency\n",
        "        features.append(min(partial)) # min of term frequency\n",
        "        features.append(max(partial)) # max of term frequency\n",
        "        features.append(np.mean(partial)) # mean of term frequency\n",
        "        features.append(np.var(partial)) # varience of term frequency\n",
        "\n",
        "        features.append(sum(partial) / d_length) # normalized  sum of stream length\n",
        "        features.append(min(partial) / d_length) # normalized min of stream length\n",
        "        features.append(max(partial) / d_length) # normalized max of stream length\n",
        "        features.append(np.mean(partial) / d_length) # normalized mean of stream length\n",
        "        features.append(np.var(partial) / d_length) # normalized varience| of stream length\n",
        "\n",
        "        # Term Frequency\n",
        "        TF = [sum(1 for d_word in dbag if d_word == q_word) for q_word in qbag]\n",
        "        features.append(sum(TF)) # sum of term frequency\n",
        "        features.append(min(TF)) # min of term frequency\n",
        "        features.append(max(TF)) # max of term frequency\n",
        "        features.append(np.mean(TF)) # mean of term frequency\n",
        "        features.append(np.var(TF)) # varience of term frequency\n",
        "\n",
        "        # Normalized Term Frequency\n",
        "        features.append(sum(TF) / d_length) # normalized  sum of stream length\n",
        "        features.append(min(TF) / d_length) # normalized min of stream length\n",
        "        features.append(max(TF) / d_length) # normalized max of stream length\n",
        "        features.append(np.mean(TF) / d_length) # normalized mean of stream length\n",
        "        features.append(np.var(TF) / d_length) # normalized varience| of stream length\n",
        "\n",
        "        # TFIDF\n",
        "  \n",
        "        TFIDF = get_tfidf(d, tfidf_vectorizer_t)\n",
        "        features.append(sum(TFIDF)) # sum of tf*idf\n",
        "        features.append(min(TFIDF)) # min of tf*idf\n",
        "        features.append(max(TFIDF)) # max of tf*idf\n",
        "        features.append(np.mean(TFIDF)) # mean of tf*idf\n",
        "        features.append(np.var(TFIDF)) # varience of tf*idf\n",
        "\n",
        "        # Distance\n",
        "        # Cosine distance between document and query tf-idf vectors\n",
        "\n",
        "#         Add distance measures between qbag and TFIDF  \n",
        "\n",
        "        features.append(cosine_similarity([get_tfidf(q, tfidf_vectorizer_t)], [get_tfidf(d, tfidf_vectorizer_t)]).tolist()[0][0])\n",
        "        \n",
        "        # Character Counts\n",
        "        character_counts = [0 for k in alphabet]\n",
        "        for word in dbag:\n",
        "            for e, k in enumerate(alphabet):\n",
        "                character_counts[e] += word.count(k)\n",
        "\n",
        "        features += character_counts \n",
        "\n",
        "        # POS tag features\n",
        "        for pos_tag in pos_tags:\n",
        "            count = 0\n",
        "            for word in d_pos:\n",
        "                if word[1] == pos_tag:\n",
        "                    count += 1\n",
        "            features.append(count)\n",
        "        \n",
        "        res.append(features)\n",
        "    \n",
        "    return res\n",
        "\n",
        "\n",
        "def features_stream_abs(X_abs, X_queries):\n",
        "  \n",
        "    \"\"\" https://arxiv.org/pdf/1803.05127.pdf page 4. \"\"\"\n",
        "    \n",
        "    res = []\n",
        "    \n",
        "    \n",
        "    for i in range(len(X_abs)):\n",
        "      \n",
        "        if i % 250 == 0:\n",
        "          print(i)\n",
        "          \n",
        "        features = []\n",
        "        q = X_queries[i]\n",
        "        d = X_abs[i]\n",
        "\n",
        "        # Remove stopwords\n",
        "        qbag = wordpunct_tokenize(q)#normalisation(q)\n",
        "        dbag = wordpunct_tokenize(d)#normalisation(d)\n",
        "        \n",
        "        d_pos = nltk.pos_tag(dbag)\n",
        "        \n",
        "        q_length = len(qbag)\n",
        "        d_length = len(dbag)\n",
        "        \n",
        "        features = []\n",
        "        \n",
        "\n",
        "        dbag_lower = [k.lower() for k in dbag]\n",
        "        qbag_lower = [k.lower() for k in qbag]\n",
        "        \n",
        "        # Query term number features\n",
        "        features.append(sum(1 for q_word in qbag if q_word in dbag)) # covered query term number\n",
        "        features.append(sum(1 for q_word in qbag if q_word in dbag)/q_length)  # covered query term ratio\n",
        "\n",
        "        # Stats\n",
        "        features.append(q_length)  # stream length\n",
        "        features.append(d_length)  # stream length\n",
        "        \n",
        "        # IDF of query terms\n",
        "        features.append(sum(1/word_idf_all[q] if q in word_idf_all  else 0 for q in qbag ))  # IDF (inverse document frequency)\n",
        "\n",
        "        # Partial matching.\n",
        "        partial = [sum(1 for d_word in dbag if (q_word in d_word) or (d_word in q_word)) for q_word in qbag] # list of counts of each query term in the document\n",
        "        \n",
        "        features.append(sum(partial)) # sum of term frequency\n",
        "        features.append(min(partial)) # min of term frequency\n",
        "        features.append(max(partial)) # max of term frequency\n",
        "        features.append(np.mean(partial)) # mean of term frequency\n",
        "        features.append(np.var(partial)) # varience of term frequency\n",
        "\n",
        "        features.append(sum(partial) / d_length) # normalized  sum of stream length\n",
        "        features.append(min(partial) / d_length) # normalized min of stream length\n",
        "        features.append(max(partial) / d_length) # normalized max of stream length\n",
        "        features.append(np.mean(partial) / d_length) # normalized mean of stream length\n",
        "        features.append(np.var(partial) / d_length) # normalized varience| of stream length\n",
        "        \n",
        "#         \n",
        "        # Term frequency\n",
        "        TF = [sum(1 for d_word in dbag if d_word == q_word) for q_word in qbag]\n",
        "        features.append(sum(TF)) # sum of term frequency\n",
        "        features.append(min(TF)) # min of term frequency\n",
        "        features.append(max(TF)) # max of term frequency\n",
        "        features.append(np.mean(TF)) # mean of term frequency\n",
        "        features.append(np.var(TF)) # varience of term frequency\n",
        "  \n",
        "        # Normalized term frequency\n",
        "        features.append(sum(TF) / d_length) # normalized  sum of stream length\n",
        "        features.append(min(TF) / d_length) # normalized min of stream length\n",
        "        features.append(max(TF) / d_length) # normalized max of stream length\n",
        "        features.append(np.mean(TF) / d_length) # normalized mean of stream length\n",
        "        features.append(np.var(TF) / d_length) # normalized varience| of stream length\n",
        "\n",
        "        # TFIDF\n",
        "        TFIDF = get_tfidf(d, tfidf_vectorizer_a)\n",
        "        features.append(sum(TFIDF)) # sum of tf*idf\n",
        "        features.append(min(TFIDF)) # min of tf*idf\n",
        "        features.append(max(TFIDF)) # max of tf*idf\n",
        "        features.append(np.mean(TFIDF)) # mean of tf*idf\n",
        "        features.append(np.var(TFIDF)) # varience of tf*idf\n",
        "\n",
        "        # Distance\n",
        "        # Cosine distance between document and query tf-idf vectors\n",
        "\n",
        "        features.append(cosine_similarity([get_tfidf(q, tfidf_vectorizer_a)], [get_tfidf(d, tfidf_vectorizer_a)]).tolist()[0][0])       \n",
        "        \n",
        "#         Character counts\n",
        "        \n",
        "        character_counts = [0 for k in alphabet]\n",
        "        for word in dbag:\n",
        "            for e, k in enumerate(alphabet):\n",
        "                character_counts[e] += word.count(k)\n",
        "\n",
        "        features += character_counts     \n",
        "\n",
        "        # POS tag features\n",
        "        for pos_tag in pos_tags:\n",
        "            count = 0\n",
        "            for word in d_pos:\n",
        "                if word[1] == pos_tag:\n",
        "                    count += 1\n",
        "            features.append(count)\n",
        "\n",
        "\n",
        "        res.append(features)\n",
        "\n",
        "    return res\n",
        "\n",
        "def print_evaluation_scores(y_val, predicted):\n",
        "    print('Accuracy: ', accuracy_score(y_val, predicted))\n",
        "    print('F1-score', f1_score(y_val, predicted))\n",
        "    print('Precision ', average_precision_score(y_val, predicted))\n",
        "    \n",
        "def logistic_regression(X_train, X_val, y_train, y_val):\n",
        "  \n",
        "    clf = LogisticRegression(C=0.8, penalty='l2')  \n",
        "    clf.fit(X_train, y_train)\n",
        "    preds = clf.predict_proba(X_val)[:, 1]\n",
        "    print_evaluation_scores(y_val, clf.predict(X_val))\n",
        "    return preds, roc_auc_score(y_val, preds), clf.coef_ ,clf\n",
        "  \n",
        "def chunks(l, n):\n",
        "    \"\"\"Yield successive n-sized chunks from l.\"\"\"\n",
        "    for i in range(0, len(l), n):\n",
        "        yield l[i:i + n]\n",
        "        \n",
        "def pred2rank(preds):\n",
        "  \n",
        "  predrank = list(zip(range(topk), preds))  \n",
        "  return predrank\n",
        "\n",
        "def get_ranking(topic, run, judgments):\n",
        "  ranks = run.loc[run['Topic'] == topic]\n",
        "  ranks['inverted_rank'] = 1/ranks['rank']\n",
        "  ranks = pd.merge(ranks, judgments, left_on=['Topic','ui'], right_on=['topic','uid'], how='left')\n",
        "  ranks['relevance'] = ranks['relevance'].fillna(0)\n",
        "  ranks['relevance'] = ranks['relevance'].apply(lambda x: 1.0 if (x == 2 or x == 1) else 0)\n",
        "  return ranks['relevance']\n",
        "\n",
        "def get_inv_rank(topic, run):\n",
        "\n",
        "  ranks = run.loc[run['Topic'] == topic]\n",
        "  ranks['inverted_rank'] = 1/ranks['rank']\n",
        "  return ranks['inverted_rank']\n",
        "\n",
        "def powerset(iterable):\n",
        "      \"powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)\"\n",
        "      s = list(iterable)\n",
        "      return chain.from_iterable(combinations(s, r) for r in range(len(s)+1))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "K8CVIPE5Wiak",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Split the dataset and find the best feature groups using Logistic Regression Approach"
      ]
    },
    {
      "metadata": {
        "id": "sajlxmhYam6P",
        "colab_type": "code",
        "outputId": "623391f7-29e5-4b79-bc78-3aa44e8ab56a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 11872
        }
      },
      "cell_type": "code",
      "source": [
        "import pickle\n",
        "import random\n",
        "random.seed(4711)\n",
        "qu = [ \"OHSU\" + str(k)for k in range(1,64)]\n",
        "shuffle(qu)\n",
        "test = qu[50:]\n",
        "trainval = qu[:50]\n",
        "\n",
        "with open('data_split.pkl', 'wb') as output:\n",
        "  pickle.dump(qu, output, pickle.HIGHEST_PROTOCOL)\n",
        "  \n",
        "kf = KFold(n_splits=5,  random_state=4711)\n",
        "kf.get_n_splits(trainval)\n",
        "top_level_single_groups = []\n",
        "top_level_powerset = []\n",
        "top_level_baseline_val = np.zeros(5)\n",
        "\n",
        "for fold, (train_ind, val_ind) in enumerate(kf.split(trainval)):\n",
        "  print(train_ind, trainval)\n",
        "  train = [trainval[k] for k in train_ind]\n",
        "  val = [trainval[k] for k in val_ind]\n",
        "  \n",
        "  train_pairs, train_rel_list = create_test_pairs(run, train, test_docs, test_judgments)\n",
        "  val_pairs, val_rel_list = create_test_pairs(run, val, test_docs, test_judgments)\n",
        "  test_pairs, test_rel_list = create_test_pairs(run, test, test_docs, test_judgments)\n",
        "  print('Training set', len(train_pairs))\n",
        "  print('Training set', len(train_rel_list))\n",
        "  print('Test set', len(test_pairs))\n",
        "  print('Test set', len(test_rel_list))\n",
        "  print('Validation set', len(val_pairs))\n",
        "  print('Validation set', len(val_rel_list))\n",
        "\n",
        "  print('# of unique train documents', len(set([v[1] for v in train_pairs])))\n",
        "  print('# of unique train topics', len(set([v[0] for v in train_pairs])))\n",
        "  print('# of unique val documents', len(set([v[1] for v in val_pairs])))\n",
        "  print('# of unique train topics', len(set([v[0] for v in val_pairs])))\n",
        "  print('# of unique test documents', len(set([v[1] for v in test_pairs])))\n",
        "  print('# of unique train topics', len(set([v[0] for v in test_pairs])))\n",
        "  \n",
        "  X_train_abs, X_train_keywords, X_train_titles,  X_train_queries, train_rel_list = create_X(train_pairs, train_rel_list, queries, test_docs)\n",
        "  X_val_abs, X_val_keywords, X_val_titles,  X_val_queries, val_rel_list = create_X(val_pairs, val_rel_list, queries, test_docs)\n",
        "  X_test_abs, X_test_keywords, X_test_titles, X_test_queries, test_rel_list = create_X(test_pairs, test_rel_list, queries, test_docs) \n",
        "  \n",
        "\n",
        "  train_features = np.hstack( [features_stream_abs(X_train_abs, X_train_queries),features_stream_keywords(X_train_keywords, X_train_queries), features_stream_title(X_train_titles, X_train_queries)])\n",
        "  val_features = np.hstack( [features_stream_abs(X_val_abs, X_val_queries),features_stream_keywords(X_val_keywords, X_val_queries), features_stream_title(X_val_titles, X_val_queries)])\n",
        "  test_features = np.hstack([features_stream_abs(X_test_abs, X_test_queries), features_stream_keywords(X_test_keywords, X_test_queries), features_stream_title(X_test_titles, X_test_queries)])\n",
        "\n",
        "  scaler = StandardScaler()\n",
        "  print(scaler.fit(train_features))\n",
        "  StandardScaler(copy=True, with_mean=True, with_std=True)\n",
        "  train_features = scaler.transform(train_features)\n",
        "  val_features = scaler.transform(val_features)\n",
        "  test_features = scaler.transform(test_features)\n",
        "  \n",
        "  rank_performance_list = []\n",
        "\n",
        "  for group in featuremap:\n",
        "    train_features_subset = train_features[:, featuremap[group]]\n",
        "    val_features_subset = val_features[:, featuremap[group]]\n",
        "    print(group, train_features_subset.shape)\n",
        "    preds, auc, _, _ = logistic_regression(train_features_subset, val_features_subset, train_rel_list, val_rel_list)\n",
        "\n",
        "    logreglist = []\n",
        "    for chunk in chunks(preds, topk):\n",
        "      newrank = pd.DataFrame(pred2rank(chunk)).iloc[:, 1]\n",
        "\n",
        "      logreglist.append(newrank)\n",
        "\n",
        "    relevancelist = []\n",
        "    ranklist = []\n",
        "    for e, ohsu in enumerate(val):\n",
        "      relevance = get_ranking(ohsu, run, test_judgments)  \n",
        "      relevancelist.append(relevance)\n",
        "\n",
        "      ranking = get_inv_rank(ohsu, run)\n",
        "      ranklist.append(ranking)       \n",
        "\n",
        "    microsoft_aps = []\n",
        "\n",
        "    for (a, b) in zip(relevancelist, ranklist):\n",
        "      ap = (average_precision_score(a,b))\n",
        "      if pd.isna(ap):\n",
        "    \n",
        "        microsoft_aps.append(0)\n",
        "      else:\n",
        "    \n",
        "        microsoft_aps.append(ap)\n",
        "\n",
        "    logistic_aps = []\n",
        "    for (a, b) in zip(relevancelist, logreglist):\n",
        "      ap = (average_precision_score(a,b))\n",
        "      if pd.isna(ap):\n",
        "    \n",
        "        logistic_aps.append(0)\n",
        "      else:\n",
        "    \n",
        "        logistic_aps.append(ap)\n",
        "    rank_performance_list.append(np.mean(logistic_aps))\n",
        "    print(\"MAP Microsoft: \", np.mean(microsoft_aps))\n",
        "    print(\"MAP Logistic Rerank: \", np.mean(logistic_aps))\n",
        "    top_level_baseline_val[fold] = np.mean(microsoft_aps)\n",
        "    \n",
        "  results_df = pd.DataFrame()\n",
        "  results_df['Feature Group'] = list(featuremap.keys())\n",
        "  results_df['Rerank MAP'] = rank_performance_list\n",
        "  sorted_results = results_df.sort_values(by='Rerank MAP', ascending=False)\n",
        "  top10_fg = sorted_results['Feature Group'][:10]\n",
        "  print(sorted_results)\n",
        "  sorted_results.to_csv('drive/My Drive/Information Retrieval/results_groups_text_normalisation_19_dec_kfold_j_{}.csv'.format(fold))\n",
        "  top_level_single_groups.append(sorted_results)\n",
        "\n"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33\n",
            " 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49] ['OHSU41', 'OHSU38', 'OHSU13', 'OHSU19', 'OHSU10', 'OHSU6', 'OHSU37', 'OHSU17', 'OHSU49', 'OHSU59', 'OHSU40', 'OHSU61', 'OHSU34', 'OHSU14', 'OHSU21', 'OHSU63', 'OHSU54', 'OHSU16', 'OHSU26', 'OHSU30', 'OHSU8', 'OHSU62', 'OHSU29', 'OHSU46', 'OHSU51', 'OHSU31', 'OHSU5', 'OHSU25', 'OHSU1', 'OHSU39', 'OHSU27', 'OHSU18', 'OHSU15', 'OHSU7', 'OHSU48', 'OHSU42', 'OHSU24', 'OHSU33', 'OHSU44', 'OHSU3', 'OHSU11', 'OHSU12', 'OHSU47', 'OHSU35', 'OHSU57', 'OHSU60', 'OHSU53', 'OHSU28', 'OHSU58', 'OHSU20']\n",
            "Training set 1200\n",
            "Training set 1200\n",
            "Test set 390\n",
            "Test set 390\n",
            "Validation set 300\n",
            "Validation set 300\n",
            "# of unique train documents 1126\n",
            "# of unique train topics 40\n",
            "# of unique val documents 300\n",
            "# of unique train topics 10\n",
            "# of unique test documents 390\n",
            "# of unique train topics 13\n",
            "Pairs:  1200\n",
            "Rellist:  1200\n",
            "Length\n",
            "1200\n",
            "1200\n",
            "Pairs:  300\n",
            "Rellist:  300\n",
            "Length\n",
            "300\n",
            "300\n",
            "Pairs:  390\n",
            "Rellist:  390\n",
            "Length\n",
            "390\n",
            "390\n",
            "0\n",
            "250\n",
            "500\n",
            "750\n",
            "1000\n",
            "0\n",
            "250\n",
            "500\n",
            "750\n",
            "1000\n",
            "0\n",
            "250\n",
            "500\n",
            "750\n",
            "1000\n",
            "0\n",
            "250\n",
            "0\n",
            "250\n",
            "0\n",
            "250\n",
            "0\n",
            "250\n",
            "0\n",
            "250\n",
            "0\n",
            "250\n",
            "StandardScaler(copy=True, with_mean=True, with_std=True)\n",
            "KW - Query Term Coverage (1200, 2)\n",
            "Accuracy:  0.64\n",
            "F1-score 0.43157894736842106\n",
            "Precision  0.42908882783882785\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.4896477382023076\n",
            "KW General Statistics (1200, 2)\n",
            "Accuracy:  0.6266666666666667\n",
            "F1-score 0.0\n",
            "Precision  0.37333333333333335\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
            "  'precision', 'predicted', average, warn_for)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.46525783739007054\n",
            "KW IDF (1200, 1)\n",
            "Accuracy:  0.6266666666666667\n",
            "F1-score 0.0\n",
            "Precision  0.37333333333333335\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
            "  'precision', 'predicted', average, warn_for)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.33333333333333337\n",
            "KW Partial TF (1200, 10)\n",
            "Accuracy:  0.66\n",
            "F1-score 0.2272727272727273\n",
            "Precision  0.42377976190476196\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.4752290380857234\n",
            "KW TF (1200, 10)\n",
            "Accuracy:  0.6066666666666667\n",
            "F1-score 0.1917808219178082\n",
            "Precision  0.37813725490196076\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.5184099172789461\n",
            "KW TF-IDF (1200, 5)\n",
            "Accuracy:  0.6266666666666667\n",
            "F1-score 0.0\n",
            "Precision  0.37333333333333335\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
            "  'precision', 'predicted', average, warn_for)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.46584147133236264\n",
            "KW Cosine (1200, 1)\n",
            "Accuracy:  0.6566666666666666\n",
            "F1-score 0.3905325443786982\n",
            "Precision  0.43391604010025064\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.5233012170817498\n",
            "KW Character Counts (1200, 91)\n",
            "Accuracy:  0.6266666666666667\n",
            "F1-score 0.2911392405063291\n",
            "Precision  0.39934523809523814\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.46932092921960233\n",
            "KW POS Tags (1200, 30)\n",
            "Accuracy:  0.6433333333333333\n",
            "F1-score 0.218978102189781\n",
            "Precision  0.4036904761904762\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.5023876380195388\n",
            "T - Query Term Coverage (1200, 2)\n",
            "Accuracy:  0.6266666666666667\n",
            "F1-score 0.0\n",
            "Precision  0.37333333333333335\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
            "  'precision', 'predicted', average, warn_for)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.35220391294169484\n",
            "T General Statistics (1200, 2)\n",
            "Accuracy:  0.6266666666666667\n",
            "F1-score 0.0\n",
            "Precision  0.37333333333333335\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
            "  'precision', 'predicted', average, warn_for)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.419187491625269\n",
            "T IDF (1200, 1)\n",
            "Accuracy:  0.6266666666666667\n",
            "F1-score 0.0\n",
            "Precision  0.37333333333333335\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
            "  'precision', 'predicted', average, warn_for)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.33333333333333337\n",
            "T Partial TF (1200, 10)\n",
            "Accuracy:  0.6133333333333333\n",
            "F1-score 0.0\n",
            "Precision  0.37333333333333335\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.44036146829079764\n",
            "T TF (1200, 10)\n",
            "Accuracy:  0.6333333333333333\n",
            "F1-score 0.03508771929824561\n",
            "Precision  0.3845238095238095\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.39068089882871515\n",
            "T TF-IDF (1200, 5)\n",
            "Accuracy:  0.6266666666666667\n",
            "F1-score 0.0\n",
            "Precision  0.37333333333333335\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
            "  'precision', 'predicted', average, warn_for)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.37772203216838596\n",
            "T Cosine (1200, 1)\n",
            "Accuracy:  0.6266666666666667\n",
            "F1-score 0.16417910447761194\n",
            "Precision  0.3857738095238095\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.5191426916110395\n",
            "T Character Counts (1200, 91)\n",
            "Accuracy:  0.6166666666666667\n",
            "F1-score 0.267515923566879\n",
            "Precision  0.39083333333333337\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.4138725443367989\n",
            "T POS Tags (1200, 30)\n",
            "Accuracy:  0.62\n",
            "F1-score 0.049999999999999996\n",
            "Precision  0.3733779761904762\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.40843359913370536\n",
            "ABS - Query Term Coverage (1200, 2)\n",
            "Accuracy:  0.55\n",
            "F1-score 0.28571428571428575\n",
            "Precision  0.3678648732220161\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.4766340081330373\n",
            "ABS General Statistics (1200, 2)\n",
            "Accuracy:  0.6333333333333333\n",
            "F1-score 0.03508771929824561\n",
            "Precision  0.3845238095238095\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.43704325142834843\n",
            "ABS IDF (1200, 1)\n",
            "Accuracy:  0.6266666666666667\n",
            "F1-score 0.0\n",
            "Precision  0.37333333333333335\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
            "  'precision', 'predicted', average, warn_for)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.33333333333333337\n",
            "ABS Partial TF (1200, 10)\n",
            "Accuracy:  0.61\n",
            "F1-score 0.2641509433962264\n",
            "Precision  0.3871099290780142\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.5009574224800435\n",
            "ABS TF (1200, 10)\n",
            "Accuracy:  0.5466666666666666\n",
            "F1-score 0.29896907216494845\n",
            "Precision  0.3682389663182346\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.5743170068522907\n",
            "ABS TF-IDF (1200, 5)\n",
            "Accuracy:  0.6233333333333333\n",
            "F1-score 0.017391304347826087\n",
            "Precision  0.3729761904761905\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.42522563878203157\n",
            "ABS Cosine (1200, 1)\n",
            "Accuracy:  0.6233333333333333\n",
            "F1-score 0.3891891891891892\n",
            "Precision  0.4118460534898891\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.6463568003743632\n",
            "ABS Character Counts (1200, 91)\n",
            "Accuracy:  0.5533333333333333\n",
            "F1-score 0.15189873417721517\n",
            "Precision  0.3612836438923396\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.419294323344854\n",
            "ABS POS Tags (1200, 30)\n",
            "Accuracy:  0.5933333333333334\n",
            "F1-score 0.046875\n",
            "Precision  0.3683556547619048\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:424: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:433: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.4179353337844164\n",
            "MAP Logistic Rerank:  0.40004086997715305\n",
            "                Feature Group  Rerank MAP\n",
            "24                 ABS Cosine    0.646357\n",
            "22                     ABS TF    0.574317\n",
            "6                   KW Cosine    0.523301\n",
            "15                   T Cosine    0.519143\n",
            "4                       KW TF    0.518410\n",
            "8                 KW POS Tags    0.502388\n",
            "21             ABS Partial TF    0.500957\n",
            "0    KW - Query Term Coverage    0.489648\n",
            "18  ABS - Query Term Coverage    0.476634\n",
            "3               KW Partial TF    0.475229\n",
            "7         KW Character Counts    0.469321\n",
            "5                   KW TF-IDF    0.465841\n",
            "1       KW General Statistics    0.465258\n",
            "12               T Partial TF    0.440361\n",
            "19     ABS General Statistics    0.437043\n",
            "23                 ABS TF-IDF    0.425226\n",
            "25       ABS Character Counts    0.419294\n",
            "10       T General Statistics    0.419187\n",
            "16         T Character Counts    0.413873\n",
            "17                 T POS Tags    0.408434\n",
            "26               ABS POS Tags    0.400041\n",
            "13                       T TF    0.390681\n",
            "14                   T TF-IDF    0.377722\n",
            "9     T - Query Term Coverage    0.352204\n",
            "20                    ABS IDF    0.333333\n",
            "11                      T IDF    0.333333\n",
            "2                      KW IDF    0.333333\n",
            "[ 0  1  2  3  4  5  6  7  8  9 20 21 22 23 24 25 26 27 28 29 30 31 32 33\n",
            " 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49] ['OHSU41', 'OHSU38', 'OHSU13', 'OHSU19', 'OHSU10', 'OHSU6', 'OHSU37', 'OHSU17', 'OHSU49', 'OHSU59', 'OHSU40', 'OHSU61', 'OHSU34', 'OHSU14', 'OHSU21', 'OHSU63', 'OHSU54', 'OHSU16', 'OHSU26', 'OHSU30', 'OHSU8', 'OHSU62', 'OHSU29', 'OHSU46', 'OHSU51', 'OHSU31', 'OHSU5', 'OHSU25', 'OHSU1', 'OHSU39', 'OHSU27', 'OHSU18', 'OHSU15', 'OHSU7', 'OHSU48', 'OHSU42', 'OHSU24', 'OHSU33', 'OHSU44', 'OHSU3', 'OHSU11', 'OHSU12', 'OHSU47', 'OHSU35', 'OHSU57', 'OHSU60', 'OHSU53', 'OHSU28', 'OHSU58', 'OHSU20']\n",
            "Training set 1200\n",
            "Training set 1200\n",
            "Test set 390\n",
            "Test set 390\n",
            "Validation set 300\n",
            "Validation set 300\n",
            "# of unique train documents 1139\n",
            "# of unique train topics 40\n",
            "# of unique val documents 299\n",
            "# of unique train topics 10\n",
            "# of unique test documents 390\n",
            "# of unique train topics 13\n",
            "Pairs:  1200\n",
            "Rellist:  1200\n",
            "Length\n",
            "1200\n",
            "1200\n",
            "Pairs:  300\n",
            "Rellist:  300\n",
            "Length\n",
            "300\n",
            "300\n",
            "Pairs:  390\n",
            "Rellist:  390\n",
            "Length\n",
            "390\n",
            "390\n",
            "0\n",
            "250\n",
            "500\n",
            "750\n",
            "1000\n",
            "0\n",
            "250\n",
            "500\n",
            "750\n",
            "1000\n",
            "0\n",
            "250\n",
            "500\n",
            "750\n",
            "1000\n",
            "0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-13-c5938f815fec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     44\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     45\u001b[0m   \u001b[0mtrain_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhstack\u001b[0m\u001b[0;34m(\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfeatures_stream_abs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train_abs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_train_queries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfeatures_stream_keywords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train_keywords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_train_queries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeatures_stream_title\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train_titles\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_train_queries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 46\u001b[0;31m   \u001b[0mval_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhstack\u001b[0m\u001b[0;34m(\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfeatures_stream_abs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_val_abs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_val_queries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfeatures_stream_keywords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_val_keywords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_val_queries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeatures_stream_title\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_val_titles\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_val_queries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     47\u001b[0m   \u001b[0mtest_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfeatures_stream_abs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test_abs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test_queries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeatures_stream_keywords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test_keywords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test_queries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeatures_stream_title\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test_titles\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test_queries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-12-0b355561a28c>\u001b[0m in \u001b[0;36mfeatures_stream_abs\u001b[0;34m(X_abs, X_queries)\u001b[0m\n\u001b[1;32m    364\u001b[0m         \u001b[0;31m# TFIDF\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    365\u001b[0m         \u001b[0mTFIDF\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_tfidf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtfidf_vectorizer_a\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 366\u001b[0;31m         \u001b[0mfeatures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTFIDF\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# sum of tf*idf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    367\u001b[0m         \u001b[0mfeatures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTFIDF\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# min of tf*idf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    368\u001b[0m         \u001b[0mfeatures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTFIDF\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# max of tf*idf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "E4C3f7H0bPIR",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "##  Print best feature groups"
      ]
    },
    {
      "metadata": {
        "id": "YmFRl7xQSewl",
        "colab_type": "code",
        "outputId": "9cdd815b-f1bc-4418-afde-8c2164184238",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 476
        }
      },
      "cell_type": "code",
      "source": [
        "\n",
        "dataframelist = []\n",
        "\n",
        "for f in range(5):\n",
        "  \n",
        "    df = pd.read_csv('drive/My Drive/Information Retrieval/results_groups_text_normalisation_19_dec_kfold_{}.csv'.format(f), index_col=0)\n",
        "#     print(df)\n",
        "    dataframelist.append(df)\n",
        "    \n",
        "comb = dataframelist[0]\n",
        "for df in dataframelist[1:]:\n",
        "    comb = pd.merge(comb, df, left_on='Feature Group', right_on='Feature Group')\n",
        "\n",
        "\n",
        "comb.columns = ['Feature Group', '1', '2', '3', '4', '5']\n",
        "comb['mean'] = (comb['1'] + comb['2'] + comb['3'] + comb['4'] + comb['5']) / 5\n",
        "sorted_kfolded = comb.sort_values(by='mean', ascending=False)\n",
        "printcols = ['Feature Group', 'mean']\n",
        "print(sorted_kfolded[printcols].head(20).to_latex(index=False))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\\begin{tabular}{lr}\n",
            "\\toprule\n",
            "             Feature Group &      mean \\\\\n",
            "\\midrule\n",
            " ABS Cosine &  0.549897 \\\\\n",
            " KW Cosine &  0.527671 \\\\\n",
            " KW TF &  0.522050 \\\\\n",
            " ABS TF &  0.491927 \\\\\n",
            " KW - Query Term Coverage &  0.474476 \\\\\n",
            " T Cosine &  0.460525 \\\\\n",
            " ABS Partial TF &  0.437935 \\\\\n",
            " ABS - Query Term Coverage &  0.429816 \\\\\n",
            " KW Partial TF &  0.426497 \\\\\n",
            " ABS TF-IDF &  0.406383 \\\\\n",
            " ABS POS Tags &  0.401065 \\\\\n",
            " KW Character Counts &  0.398235 \\\\\n",
            " KW POS Tags &  0.396493 \\\\\n",
            " KW General Statistics &  0.396014 \\\\\n",
            " KW TF-IDF &  0.393549 \\\\\n",
            " T Character Counts &  0.389345 \\\\\n",
            " ABS General Statistics &  0.388998 \\\\\n",
            " ABS Character Counts &  0.368723 \\\\\n",
            " T General Statistics &  0.364600 \\\\\n",
            " T TF-IDF &  0.349383 \\\\\n",
            "\\bottomrule\n",
            "\\end{tabular}\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "B_nOWnj7ZPbQ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Get best feature group combinations (takes 4 hours)"
      ]
    },
    {
      "metadata": {
        "id": "KJMrjuYLJWE2",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "for fold, (train_ind, val_ind) in enumerate(kf.split(trainval)):\n",
        "  print(train_ind, trainval)\n",
        "  train = [trainval[k] for k in train_ind]\n",
        "  val = [trainval[k] for k in val_ind]\n",
        "  \n",
        "  train_pairs, train_rel_list = create_test_pairs(run, train, test_docs, test_judgments)\n",
        "  val_pairs, val_rel_list = create_test_pairs(run, val, test_docs, test_judgments)\n",
        "  test_pairs, test_rel_list = create_test_pairs(run, test, test_docs, test_judgments)\n",
        "  print('Training set', len(train_pairs))\n",
        "  print('Training set', len(train_rel_list))\n",
        "  print('Test set', len(test_pairs))\n",
        "  print('Test set', len(test_rel_list))\n",
        "  print('Validation set', len(val_pairs))\n",
        "  print('Validation set', len(val_rel_list))\n",
        "\n",
        "  print('# of unique train documents', len(set([v[1] for v in train_pairs])))\n",
        "  print('# of unique train topics', len(set([v[0] for v in train_pairs])))\n",
        "  print('# of unique val documents', len(set([v[1] for v in val_pairs])))\n",
        "  print('# of unique train topics', len(set([v[0] for v in val_pairs])))\n",
        "  print('# of unique test documents', len(set([v[1] for v in test_pairs])))\n",
        "  print('# of unique train topics', len(set([v[0] for v in test_pairs])))\n",
        "  \n",
        "  X_train_abs, X_train_keywords, X_train_titles,  X_train_queries, train_rel_list = create_X(train_pairs, train_rel_list, queries, test_docs)\n",
        "  X_val_abs, X_val_keywords, X_val_titles,  X_val_queries, val_rel_list = create_X(val_pairs, val_rel_list, queries, test_docs)\n",
        "  X_test_abs, X_test_keywords, X_test_titles, X_test_queries, test_rel_list = create_X(test_pairs, test_rel_list, queries, test_docs) \n",
        "  \n",
        "\n",
        "  train_features = np.hstack( [features_stream_abs(X_train_abs, X_train_queries),features_stream_keywords(X_train_keywords, X_train_queries), features_stream_title(X_train_titles, X_train_queries)])\n",
        "  val_features = np.hstack( [features_stream_abs(X_val_abs, X_val_queries),features_stream_keywords(X_val_keywords, X_val_queries), features_stream_title(X_val_titles, X_val_queries)])\n",
        "  test_features = np.hstack([features_stream_abs(X_test_abs, X_test_queries), features_stream_keywords(X_test_keywords, X_test_queries), features_stream_title(X_test_titles, X_test_queries)])\n",
        "\n",
        "  scaler = StandardScaler()\n",
        "  print(scaler.fit(train_features))\n",
        "  StandardScaler(copy=True, with_mean=True, with_std=True)\n",
        "  train_features = scaler.transform(train_features)\n",
        "  val_features = scaler.transform(val_features)\n",
        "  test_features = scaler.transform(test_features)  \n",
        " \n",
        "  feature_groups_intervals = [] \n",
        " \n",
        "  subsets = list(powerset(list(top10_fg)))[1:]\n",
        "  subsets = [k for k in subsets if len(k) > 1]\n",
        "  \n",
        "  feat_subsets = []\n",
        "  for a in subsets:\n",
        "    feats = []\n",
        "    for group in a:\n",
        "      feats += featuremap[group]\n",
        "\n",
        "    feat_subsets.append(feats)\n",
        "\n",
        "\n",
        "  print(feat_subsets)\n",
        "  rank_performance_list = []\n",
        "\n",
        "  for group in feat_subsets:\n",
        "    print(group)\n",
        "    train_features_subset = train_features[:, group]\n",
        "    val_features_subset = val_features[:, group]\n",
        "    print(group, train_features_subset.shape)\n",
        "    preds, auc, coefs, clf = logistic_regression(train_features_subset, val_features_subset, train_rel_list, val_rel_list)\n",
        "\n",
        "    logreglist = []\n",
        "    for chunk in chunks(preds, topk):\n",
        "      newrank = pd.DataFrame(pred2rank(chunk)).iloc[:, 1]\n",
        "      logreglist.append(newrank)\n",
        "\n",
        "    relevancelist = []\n",
        "    ranklist = []\n",
        "    for e, ohsu in enumerate(val):\n",
        "      relevance = get_ranking(ohsu, run, test_judgments)  \n",
        "      relevancelist.append(relevance)\n",
        "\n",
        "      ranking = get_inv_rank(ohsu, run)\n",
        "      ranklist.append(ranking)       \n",
        "\n",
        "    microsoft_aps = []\n",
        "\n",
        "    for (a, b) in zip(relevancelist, ranklist):\n",
        "      ap = (average_precision_score(a,b))\n",
        "      if pd.isna(ap):\n",
        "    \n",
        "        microsoft_aps.append(0)\n",
        "      else:\n",
        "    \n",
        "        microsoft_aps.append(ap)\n",
        "\n",
        "    logistic_aps = []\n",
        "    for (a, b) in zip(relevancelist, logreglist):\n",
        "      ap = (average_precision_score(a,b))\n",
        "      if pd.isna(ap):    \n",
        "        logistic_aps.append(0)\n",
        "      else:    \n",
        "        logistic_aps.append(ap)\n",
        "    rank_performance_list.append(np.mean(logistic_aps))\n",
        "    print(\"MAP Microsoft: \", np.mean(microsoft_aps))\n",
        "    print(\"MAP Logistic Rerank: \", np.mean(logistic_aps))\n",
        "\n",
        "  results_df = pd.DataFrame()\n",
        "  results_df['Feature Group'] = list(subsets)\n",
        "  results_df['Rerank MAP'] = rank_performance_list\n",
        "  sorted_results = results_df.sort_values(by='Rerank MAP', ascending=False)\n",
        "#   top10_fg_powerset = sorted_results['Feature Group']\n",
        "\n",
        "  print(sorted_results)\n",
        "  sorted_results.to_csv('drive/My Drive/Information Retrieval/results_powersets_text_normalization_19dec_kfold_{}.csv'.format(fold))\n",
        "#   top_level_powerset.append(sorted_results)\n",
        "\n",
        "  \n",
        "# with open('singlegroup_results.pkl', 'wb') as output:\n",
        "#   pickle.dump(top_level_single_groups, output, pickle.HIGHEST_PROTOCOL)\n",
        "  \n",
        "# with open('powerset_results.pkl', 'wb') as output:\n",
        "#   pickle.dump(top_level_powerset, output, pickle.HIGHEST_PROTOCOL)\n",
        "\n",
        "with open('drive/My Drive/Information Retrieval/baselines.pkl', 'wb') as output:\n",
        "  pickle.dump(top_level_baseline_val, output, pickle.HIGHEST_PROTOCOL)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "EtmOOPxXZZBK",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Get the optimal set of features"
      ]
    },
    {
      "metadata": {
        "id": "lEFDbiT7SwRe",
        "colab_type": "code",
        "outputId": "46f406cd-30a9-4c58-bb99-a7da1e8d2d95",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 476
        }
      },
      "cell_type": "code",
      "source": [
        "## Get the results and the optimal feature set\n",
        "pd.set_option('display.max_colwidth', -1)\n",
        "# filelist = []\n",
        "# for f in range(5):\n",
        "#   filelist.append('drive/My Drive/Information Retrieval/results_powersets_text_normalization_19dec_kfold_{}.csv'.format(f))\n",
        "  \n",
        "dataframelist = []\n",
        "\n",
        "for f in range(5):\n",
        "  \n",
        "    df = pd.read_csv('drive/My Drive/Information Retrieval/results_powersets_text_normalization_19dec_kfold_{}.csv'.format(f), index_col=0)\n",
        "#     print(df)\n",
        "    dataframelist.append(df)\n",
        "    \n",
        "comb = dataframelist[0]\n",
        "for df in dataframelist[1:]:\n",
        "    comb = pd.merge(comb, df, left_on='Feature Group', right_on='Feature Group')\n",
        "\n",
        "\n",
        "comb.columns = ['Feature Group', '1', '2', '3', '4', '5']\n",
        "comb['mean'] = (comb['1'] + comb['2'] + comb['3'] + comb['4'] + comb['5']) / 5\n",
        "sorted_kfolded = comb.sort_values(by='mean', ascending=False)\n",
        "printcols = ['Feature Group', 'mean']\n",
        "print(sorted_kfolded[printcols].head(20).to_latex(index=False))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\\begin{tabular}{lr}\n",
            "\\toprule\n",
            "                                                                                                        Feature Group &      mean \\\\\n",
            "\\midrule\n",
            " ('ABS Cosine', 'KW Cosine', 'KW - Query Term Coverage', 'T Cosine', 'ABS Partial TF', 'ABS - Query Term Coverage') &  0.579328 \\\\\n",
            " ('ABS Cosine', 'KW TF', 'KW - Query Term Coverage', 'T Cosine') &  0.578810 \\\\\n",
            " ('ABS Cosine', 'KW Cosine', 'KW - Query Term Coverage', 'ABS Partial TF', 'ABS - Query Term Coverage', 'ABS TF-IDF') &  0.577741 \\\\\n",
            " ('ABS Cosine', 'KW - Query Term Coverage', 'T Cosine') &  0.577499 \\\\\n",
            " ('ABS Cosine', 'KW - Query Term Coverage', 'T Cosine', 'ABS TF-IDF') &  0.576175 \\\\\n",
            " ('ABS Cosine', 'KW Cosine', 'KW - Query Term Coverage') &  0.576139 \\\\\n",
            " ('ABS Cosine', 'KW TF', 'T Cosine') &  0.575641 \\\\\n",
            " ('ABS Cosine', 'KW Cosine', 'KW - Query Term Coverage', 'ABS Partial TF', 'ABS - Query Term Coverage') &  0.575631 \\\\\n",
            " ('ABS Cosine', 'KW TF', 'KW - Query Term Coverage') &  0.575200 \\\\\n",
            " ('ABS Cosine', 'KW TF') &  0.574649 \\\\\n",
            " ('ABS Cosine', 'KW Cosine', 'KW - Query Term Coverage', 'T Cosine') &  0.574387 \\\\\n",
            " ('KW Cosine', 'KW - Query Term Coverage', 'T Cosine', 'ABS - Query Term Coverage') &  0.574293 \\\\\n",
            " ('ABS Cosine', 'KW Cosine', 'KW - Query Term Coverage', 'ABS - Query Term Coverage') &  0.572945 \\\\\n",
            " ('ABS Cosine', 'KW TF', 'KW - Query Term Coverage', 'ABS Partial TF', 'ABS - Query Term Coverage') &  0.572673 \\\\\n",
            " ('ABS Cosine', 'T Cosine', 'KW Partial TF') &  0.572609 \\\\\n",
            " ('ABS Cosine', 'KW Cosine', 'KW TF', 'KW - Query Term Coverage', 'ABS - Query Term Coverage') &  0.572468 \\\\\n",
            " ('ABS Cosine', 'KW Cosine', 'KW TF', 'KW - Query Term Coverage', 'ABS Partial TF', 'ABS - Query Term Coverage') &  0.571725 \\\\\n",
            " ('ABS Cosine', 'KW TF', 'KW - Query Term Coverage', 'T Cosine', 'ABS - Query Term Coverage') &  0.571143 \\\\\n",
            " ('ABS Cosine', 'KW Cosine', 'KW TF', 'KW - Query Term Coverage', 'T Cosine') &  0.571087 \\\\\n",
            " ('ABS Cosine', 'KW TF', 'KW - Query Term Coverage', 'T Cosine', 'ABS Partial TF') &  0.570656 \\\\\n",
            "\\bottomrule\n",
            "\\end{tabular}\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "acxP7SFdZrce",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Predict on test set with model trained on best feature set"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "id": "ENeUrvGuZou_",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "## Test set\n",
        "\n",
        "best_set = ('ABS Cosine', 'KW Cosine', 'KW - Query Term Coverage', 'T Cosine', 'ABS Partial TF', 'ABS - Query Term Coverage')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "kXyBJtnmky67",
        "colab_type": "code",
        "outputId": "22a1b76a-590b-4eca-e625-c1ec59c17971",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1591
        }
      },
      "cell_type": "code",
      "source": [
        "\n",
        "print(train_ind, trainval)\n",
        "\n",
        "\n",
        "train_pairs, train_rel_list = create_test_pairs(run, trainval, test_docs, test_judgments)\n",
        "test_pairs, test_rel_list = create_test_pairs(run, test, test_docs, test_judgments)\n",
        "print('Training set', len(train_pairs))\n",
        "print('Training set', len(train_rel_list))\n",
        "print('Test set', len(test_pairs))\n",
        "print('Test set', len(test_rel_list))\n",
        "# print('Validation set', len(val_pairs))\n",
        "# print('Validation set', len(val_rel_list))\n",
        "\n",
        "print('# of unique train documents', len(set([v[1] for v in train_pairs])))\n",
        "print('# of unique train topics', len(set([v[0] for v in train_pairs])))\n",
        "# print('# of unique val documents', len(set([v[1] for v in val_pairs])))\n",
        "# print('# of unique train topics', len(set([v[0] for v in val_pairs])))\n",
        "print('# of unique test documents', len(set([v[1] for v in test_pairs])))\n",
        "print('# of unique train topics', len(set([v[0] for v in test_pairs])))\n",
        "\n",
        "X_train_abs, X_train_keywords, X_train_titles,  X_train_queries, train_rel_list = create_X(train_pairs, train_rel_list, queries, test_docs)\n",
        "# X_val_abs, X_val_keywords, X_val_titles,  X_val_queries, val_rel_list = create_X(val_pairs, val_rel_list, queries, test_docs)\n",
        "X_test_abs, X_test_keywords, X_test_titles, X_test_queries, test_rel_list = create_X(test_pairs, test_rel_list, queries, test_docs) \n",
        "\n",
        "\n",
        "train_features = np.hstack( [features_stream_abs(X_train_abs, X_train_queries),features_stream_keywords(X_train_keywords, X_train_queries), features_stream_title(X_train_titles, X_train_queries)])\n",
        "# val_features = np.hstack( [features_stream_abs(X_val_abs, X_val_queries),features_stream_keywords(X_val_keywords, X_val_queries), features_stream_title(X_val_titles, X_val_queries)])\n",
        "test_features = np.hstack([features_stream_abs(X_test_abs, X_test_queries), features_stream_keywords(X_test_keywords, X_test_queries), features_stream_title(X_test_titles, X_test_queries)])\n",
        "\n",
        "scaler = StandardScaler()\n",
        "print(scaler.fit(train_features))\n",
        "StandardScaler(copy=True, with_mean=True, with_std=True)\n",
        "train_features = scaler.transform(train_features)\n",
        "# val_features = scaler.transform(val_features)\n",
        "test_features = scaler.transform(test_features)  \n",
        "\n",
        "feature_groups_intervals = [] \n",
        "\n",
        "subsets = [best_set]\n",
        "\n",
        "feat_subsets = []\n",
        "for a in subsets:\n",
        "  feats = []\n",
        "  for group in a:\n",
        "    feats += featuremap[group]\n",
        "\n",
        "  feat_subsets.append(feats)\n",
        "\n",
        "\n",
        "print(feat_subsets)\n",
        "rank_performance_list = []\n",
        "\n",
        "for group in feat_subsets:\n",
        "  print(group)\n",
        "  train_features_subset = train_features[:, group]\n",
        "#   val_features_subset = val_features[:, group]\n",
        "  test_features_subset = test_features[:, group]\n",
        "  print(group, train_features_subset.shape)\n",
        "  preds, auc, coefs, clf = logistic_regression(train_features_subset, test_features_subset, train_rel_list, test_rel_list)\n",
        "#   sns.barplot(coefs)\n",
        "  logreglist = []\n",
        "  for chunk in chunks(preds, topk):\n",
        "    newrank = pd.DataFrame(pred2rank(chunk)).iloc[:, 1]\n",
        "    logreglist.append(newrank)\n",
        "\n",
        "  relevancelist = []\n",
        "  ranklist = []\n",
        "  for e, ohsu in enumerate(test):\n",
        "    relevance = get_ranking(ohsu, run, test_judgments)  \n",
        "    relevancelist.append(relevance)\n",
        "\n",
        "    ranking = get_inv_rank(ohsu, run)\n",
        "    ranklist.append(ranking)       \n",
        "\n",
        "  microsoft_aps = []\n",
        "\n",
        "  for (a, b) in zip(relevancelist, ranklist):\n",
        "    ap = (average_precision_score(a,b))\n",
        "    if pd.isna(ap):\n",
        "\n",
        "      microsoft_aps.append(0)\n",
        "    else:\n",
        "\n",
        "      microsoft_aps.append(ap)\n",
        "\n",
        "  logistic_aps = []\n",
        "  for (a, b) in zip(relevancelist, logreglist):\n",
        "    ap = (average_precision_score(a,b))\n",
        "    if pd.isna(ap):    \n",
        "      logistic_aps.append(0)\n",
        "    else:    \n",
        "      logistic_aps.append(ap)\n",
        "  rank_performance_list.append(np.mean(logistic_aps))\n",
        "  print(\"MAP Microsoft: \", np.mean(microsoft_aps))\n",
        "  print(\"MAP Logistic Rerank: \", np.mean(logistic_aps))\n",
        "\n",
        "results_df = pd.DataFrame()\n",
        "results_df['Feature Group'] = list(subsets)\n",
        "results_df['Rerank MAP'] = rank_performance_list\n",
        "sorted_results = results_df.sort_values(by='Rerank MAP', ascending=False)\n",
        "#   top10_fg_powerset = sorted_results['Feature Group']\n",
        "\n",
        "print(sorted_results)\n",
        "sorted_results.to_csv('drive/My Drive/Information Retrieval/results_powersets_text_normalization_20dec_test_set_fulltrainingdata.csv')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23\n",
            " 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39] ['OHSU41', 'OHSU38', 'OHSU13', 'OHSU19', 'OHSU10', 'OHSU6', 'OHSU37', 'OHSU17', 'OHSU49', 'OHSU59', 'OHSU40', 'OHSU61', 'OHSU34', 'OHSU14', 'OHSU21', 'OHSU63', 'OHSU54', 'OHSU16', 'OHSU26', 'OHSU30', 'OHSU8', 'OHSU62', 'OHSU29', 'OHSU46', 'OHSU51', 'OHSU31', 'OHSU5', 'OHSU25', 'OHSU1', 'OHSU39', 'OHSU27', 'OHSU18', 'OHSU15', 'OHSU7', 'OHSU48', 'OHSU42', 'OHSU24', 'OHSU33', 'OHSU44', 'OHSU3', 'OHSU11', 'OHSU12', 'OHSU47', 'OHSU35', 'OHSU57', 'OHSU60', 'OHSU53', 'OHSU28', 'OHSU58', 'OHSU20']\n",
            "Training set 1500\n",
            "Training set 1500\n",
            "Test set 390\n",
            "Test set 390\n",
            "# of unique train documents 1409\n",
            "# of unique train topics 50\n",
            "# of unique test documents 390\n",
            "# of unique train topics 13\n",
            "Pairs:  1500\n",
            "Rellist:  1500\n",
            "Length\n",
            "1500\n",
            "1500\n",
            "Pairs:  390\n",
            "Rellist:  390\n",
            "Length\n",
            "390\n",
            "390\n",
            "0\n",
            "250\n",
            "500\n",
            "750\n",
            "1000\n",
            "1250\n",
            "0\n",
            "250\n",
            "500\n",
            "750\n",
            "1000\n",
            "1250\n",
            "0\n",
            "250\n",
            "500\n",
            "750\n",
            "1000\n",
            "1250\n",
            "0\n",
            "250\n",
            "0\n",
            "250\n",
            "0\n",
            "250\n",
            "StandardScaler(copy=True, with_mean=True, with_std=True)\n",
            "[[334, 30, 0, 1, 182, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 304, 305]]\n",
            "[334, 30, 0, 1, 182, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 304, 305]\n",
            "[334, 30, 0, 1, 182, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 304, 305] (1500, 17)\n",
            "Accuracy:  0.7\n",
            "F1-score 0.5551330798479087\n",
            "Precision  0.5054099249751424\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.6/dist-packages/seaborn/categorical.py:1428: FutureWarning: remove_na is deprecated and is a private function. Do not use.\n",
            "  stat_data = remove_na(group_data)\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:439: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:448: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "MAP Microsoft:  0.412973099101576\n",
            "MAP Logistic Rerank:  0.6751135204904107\n",
            "                                                                                            Feature Group  \\\n",
            "0  (ABS Cosine, KW Cosine, KW - Query Term Coverage, T Cosine, ABS Partial TF, ABS - Query Term Coverage)   \n",
            "\n",
            "   Rerank MAP  \n",
            "0  0.675114    \n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcwAAAFKCAYAAACQMm9DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADpVJREFUeJzt3X+slnX9x/HXOQcQyOMvBIYzVmta\nbmaz6Vy6JDPO0gzHnGx+NdbSWSvUufkF/BG0tqTQXBY1I7G5RbNg/sEWXyKnf7g6sYpGRVviH02c\nv/BXBwQVjtf3D8aZ2Dn45gD3uc/Z47Hxx7mvc933571rH57nvu6jdDRN0wQAOKTOkV4AAIwGggkA\nBYIJAAWCCQAFggkABYIJAAXjDnVwx46drVrHETv55Ml57bXdI72Mo8Y87W+szWSe9jfWZmrXeaZO\n7R708THzDnPcuK6RXsJRZZ72N9ZmMk/7G2szjbZ5xkwwAeBYEkwAKBBMACgQTAAoEEwAKBBMACgQ\nTAAoEEwAKBBMACgQTAAoEEwAKBBMACgQTAAoEEwAKBBMACgQTAAoEEwAKBBMACgQTAAoEEwAKBBM\nACgQTAAoEEwAKBBMACgQTAAoEEwAKBBMACgQTAAoGDfSC2Bw/7Nw9UgvgSNw///OGeklAEeZd5gA\nUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQ\nIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAg\nmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCY\nAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgA\nUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQ\nIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAg\nmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCY\nAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgA\nUCCYAFAwrpUv9uabe7J3795j8twTJryTnTt3HZPnHgnv7HtzpJfAEdi5s2+kl3DYxtoeGmvzJGNv\npqMxz/jx4zNx4qSjtKJD62iaphnq4I4dO4/aCz300Mr87nf/l0O8HAAclo6Ozsye/fl85Ss3HrXn\nnDq1e9DHW3ZLduPG9WIJwFHVNO9k48b1LXmtlgWzp+fydHT4yBSAo6ejozM9PZe35rVadUs2Obaf\nYU6ZcnxeeWXs3Nu/Y8WGkV4CR+DuBZ8f6SUctrG2h8baPMnYm+lozHMsPsMc6pZsS3/pZ+LEScfs\nw9kTT+zO22+PnXewneMmjvQSOALd3SeM9BIO21jbQ2NtnmTszTTa5hk9KwWAESSYAFAgmABQIJgA\nUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQ\nIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAg\nmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCY\nAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgA\nUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQ\nIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAg\nmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCY\nAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAgmABQIJgAUCCYAFAwbqQXwOB+\nufza7Nixc6SXcdRMndo9puZJxuZMwNC8wwSAAsEEgALBBIACwQSAAsEEgALBBIACwQSAAsEEgALB\nBIACwQSAAsEEgALBBIACwQSAAsEEgALBBIACwQSAAsEEgALBBIACwQSAAsEEgALBBIACwQSAAsEE\ngALBBIACwQSAAsEEgALBBIACwQSAAsEEgIKOpmmakV4EALQ77zABoEAwAaBAMAGgQDABoEAwAaBA\nMAGgYNxIL+BQ9u7dm8WLF+e5555LV1dXli1blg9+8IMHfc+6devy8MMPp7OzM/PmzcvVV1+dffv2\n5c4778wzzzyT/v7+LFy4MOedd16+9KUvZffu3Zk8eXKSZNGiRTn77LNbNs/dd9+dLVu2pKOjI3fc\ncUfOOeecgWN/+MMfct9996WrqysXX3xxvvGNbwx5zvPPP5+FCxemv78/U6dOzT333JMJEya0bI4j\nmWf58uX5y1/+kn379uWrX/1qenp6snjx4mzdujUnnXRSkuT666/PZz7zmbafZ9OmTbnllltyxhln\nJEnOPPPMfPOb32yb6zOcmdasWZN169YNfM8//vGP/PWvfx3xvVOZ56233sqSJUuybdu2PProo4c8\nZ7Rco6FmGq37aLB5RsM+GtC0sUcffbT51re+1TRN0zz55JPNLbfcctDxN954o+np6Wn6+vqaPXv2\nNF/4whea1157rVm7dm2zdOnSpmma5qmnnmquuuqqpmma5rrrrmv+9a9/tXSGAzZt2tTceOONTdM0\nzdNPP93MmzfvoOOXXXZZ89xzzzX9/f3NNddc02zbtm3IcxYvXtysX7++aZqm+f73v9+sXr26hZPs\nN5x5ent7mxtuuKFpmqZ59dVXm1mzZjVN0zSLFi1qHn/88Zau/72GM88f//jH5qabbvqv52qH69M0\nw5vpvecf2H8juXfevZ5DzfPtb3+7+fnPf97MnTv3fc8ZLddosJlG8z4abJ5230fv1ta3ZHt7ezN7\n9uwkyYUXXpjNmzcfdHzLli35+Mc/nu7u7kycODGf/OQns3nz5syZMye33357kuSUU07J66+/3vK1\nv1dvb28+97nPJUk+8pGP5D//+U927dqVJNm+fXtOPPHEzJgxI52dnZk1a1Z6e3uHPGfTpk259NJL\nkySXXHJJent7R8U8559/fu6///4kyQknnJA9e/akv7+/5WsfzHDmGUo7XJ/kyGf68Y9/nK9//est\nX/dQDjVPktx6660Dx9/vnNFwjZLBZxqt+ygZfJ6htMs1ere2DubLL7+cU045JUnS2dmZjo6OvP32\n24MeT/bHcceOHRk/fnyOO+64JMnDDz+cK664YuB7fvjDH+baa6/NkiVL8uabb7Zokv1rPfnkk/9r\nrUmyY8eOQecY6pw9e/YM3JqYMmXKwPO00nDm6erqGrilt3bt2lx88cXp6upKkvziF7/I/Pnzc+ut\nt+bVV19t4ST7DWeeJHn66afzta99Lddcc01+//vfJ0lbXJ9k+DMlyd/+9rfMmDEjU6dOHXhspPbO\nAYeaJ0mOP/748jmj4Rolg880WvdRMvg8SXvvo3drm88w16xZkzVr1hz02JYtWw76unmf/4vfe4+v\nXr06W7duzQMPPJAkmT9/fj760Y9m5syZWbp0aVavXp3rr7/+KKz+8L3fLNVzhvM8x8LhrOOxxx7L\n2rVr89BDDyVJrrzyypx00kk566yzsnLlyqxYsSJLliw5VkstqczzoQ99KAsWLMhll12W7du3Z/78\n+dm4ceNhP0+rHM5a1q5dm7lz5w583U5754CxtocS+6jd91HbvMO8+uqr8+tf//qgP3Pnzh34qWLv\n3r1pmuagD32nTZuWl19+eeDrl156KdOmTUuyP8CPP/54fvKTn2T8+PFJktmzZ2fmzJlJks9+9rN5\n6qmnWjXeoGs98NP7e4+9+OKLmTZt2pDnTJ48eeAn/APf22rDmSdJnnzyyTzwwAP52c9+lu7u7iTJ\npz71qZx11llJWn9dDhjOPNOnT8/ll1+ejo6OzJw5M6eeempefPHFtrg+g627eo2S/bfDzj333IGv\nR3LvHHCoeQ73nNFwjQ5lNO6jobT7Pnq3tgnmYC666KJs2LAhSfLEE0/kggsuOOj4Jz7xifz9739P\nX19f3njjjWzevDnnnXdetm/fnkceeSQrVqwYuDXbNE2+/OUvp6+vL8n+vxAO/FZWq2b57W9/myTZ\nunVrpk2bNnB74vTTT8+uXbvy7LPPZt++fXniiSdy0UUXDXnOhRdeOPD4xo0b8+lPf7plcxzJPDt3\n7szy5cvz05/+dOA3+ZLkpptuyvbt25O0/rocyTzr1q3LqlWrkuy/xfnKK69k+vTpbXF9hjtTsv8v\npw984AMDP5yO9N6pzHO454yGazSU0bqPhtLu++jd2vpfK+nv789dd92Vf//735kwYUK++93vZsaM\nGVm5cmXOP//8nHvuudmwYUNWrVqVjo6OXHfddZkzZ07uu+++/OY3v8lpp5028FyrVq3KY489lgcf\nfDCTJk3K9OnT853vfCeTJk1q2Tz33ntv/vznP6ejoyNLly7NP//5z3R3d2f27Nn505/+lHvvvTdJ\n0tPTM3C7673nfOxjH8tLL72URYsW5a233sppp52WZcuWDbyLbqXDnedXv/pVfvSjH+XDH/7wwHN8\n73vfyzPPPJN77rknkyZNyuTJk7Ns2bJMmTKl7efZtWtXbrvttvT19WXv3r1ZsGBBZs2a1TbXZzgz\nJfv/U5If/OAHefDBBweeZ/369SO6dyrz3HzzzXnhhReybdu2nH322Zk3b16++MUvtvUeGs5Mu3fv\nHrX7aLB5LrnkkrbfRwe0dTABoF209S1ZAGgXggkABYIJAAWCCQAFggkABYIJAAWCCQAFggkABf8P\nJQZyIUFLKb8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f1c123bafd0>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "4K1zwQ9TZ5nn",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Plot coefficients of best model"
      ]
    },
    {
      "metadata": {
        "id": "iSiOSaBDytn1",
        "colab_type": "code",
        "outputId": "dfb9ae2f-edc0-4be3-fe65-a538c5e55bcd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 422
        }
      },
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "print(len(feat_subsets[0]), len(coefs[0]))\n",
        "best_feats = ['KW Query Term Coverage', 'KW Normalized QTC', 'KW Cosine', 'T Cosine','ABS Query Term Coverage', 'ABS Normalized QTC', 'ABS Partial TF Sum', 'ABS Partial TF Min',\n",
        "              'ABS Partial TF Max', 'ABS Partial TF Mean', 'ABS Partial TF Variance', 'ABS Partial TF Sum Normalized', 'ABS Partial TF Min Normalized',\n",
        "              'ABS Partial TF Max Normalized', 'ABS Partial TF Mean Normalized', 'ABS Partial TF Variance Normalized',  'ABS Cosine']\n",
        "g = sns.barplot(best_feats, coefs[0])\n",
        "plt.setp(g.get_xticklabels(), rotation=-90)\n",
        "len(best_feats)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "17 17\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/seaborn/categorical.py:1428: FutureWarning: remove_na is deprecated and is a private function. Do not use.\n",
            "  stat_data = remove_na(group_data)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "17"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 97
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAecAAAFRCAYAAABOnmU8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAHMlJREFUeJzt3X9QlXX+9/HX4ZdpoMJ2Thl5b45Z\nNKglrusg3uYq1t5OVl8MxRSzLNdVa/PGHVm6V9pGwNvEmTV33C87tZmVkUW1a6bOzteZdpXC0rDY\nar+UsJoKB0MEQfDHdf/heG5ZTNyLD/Dh8HzMMHOuH7zfnwsuzovrus65jsdxHEcAAMAaId09AAAA\n0BrhDACAZQhnAAAsQzgDAGAZwhkAAMsQzgAAWCasuwdwkd9f391DAACgy3i9Ud+7jCNnAAAsQzgD\nAGAZwhkAAMsQzgAAWIZwBgDAMoQzAACWIZwBALAM4QwAgGUIZwAALEM4AwBgGcIZAADLEM4AAFiG\ncAYAwDKuP5UqNzdXpaWl8ng8ysrK0siRI9usk5+fr08//VSbNm3q0CABAOgOzuslxmt60n7c7jqu\njpxLSkpUWVmpwsJC5eTkKCcnp8065eXl2rt3r5vyAAD0aq7Cubi4WMnJyZKkoUOHqq6uTg0NDa3W\nWbVqlZYuXdrxEQIA0Mu4CueamhpFR0cHpmNiYuT3+wPTRUVF+vGPf6zY2NiOjxAAgF7G9TXnSzmO\nE3h84sQJFRUV6Y9//KOqqqquukZ0dD+FhYWaGA4AAEZUd0JNrzeq3XVchbPP51NNTU1gurq6Wl6v\nV5L04Ycf6rvvvtPs2bPV0tKif/7zn8rNzVVWVtYVa9bWNroZCgAAPYrfXy/pyiHt6rR2UlKSduzY\nIUkqKyuTz+dTZGSkJOmnP/2ptm3bpjfeeEPr169XfHx8u8EMAAD+P1dHzgkJCYqPj1daWpo8Ho+y\ns7NVVFSkqKgoTZkyxfQYAQDoVTzOpReMu9HFw3wAAGzRme9zNn5aGwAAdB7CGQAAyxDOAABYhnAG\nAMAyhDMAAJYhnAEAsAzhDACAZQhnAAAsQzgDAGAZwhkAAMsY+chI495813zNB+83XxMAgE7AkTMA\nAJYhnAEAsIydp7UBoBst+eCo8ZrrJwwyXhPBiyNnAAAsQzgDAGAZwhkAAMsQzgAAWIZwBgDAMoQz\nAACWIZwBALAM4QwAgGUIZwAALEM4AwBgGcIZAADLEM4AAFiGcAYAwDKEMwAAliGcAQCwDOEMAIBl\nCGcAACxDOAMAYBnCGQAAy4S5/cbc3FyVlpbK4/EoKytLI0eODCx744039OabbyokJERxcXHKzs6W\nx+MxMmAAAIKdqyPnkpISVVZWqrCwUDk5OcrJyQksa2pq0nvvvadXX31Vr7/+ur755hvt37/f2IAB\nAAh2rsK5uLhYycnJkqShQ4eqrq5ODQ0NkqS+fftq48aNCg8PV1NTkxoaGuT1es2NGACAIOfqtHZN\nTY3i4+MD0zExMfL7/YqMjAzMKygo0Msvv6y5c+dq8ODB7daMju6nsLBQSZLfzaDa4fVGdUJVAMHp\nqPGKPAf1TNWdUPNq9gXX15wv5ThOm3kLFizQ3Llz9fjjj2v06NEaPXr0FWvU1jaaGMr38vvrO7U+\nAFwJz0G46OK+cKWQdnVa2+fzqaamJjBdXV0dOHV94sQJ7d27V5J0zTXXaMKECdq3b5+bNgAA9Equ\nwjkpKUk7duyQJJWVlcnn8wVOaZ89e1aZmZk6deqUJOmzzz7TkCFDDA0XAIDg5+q0dkJCguLj45WW\nliaPx6Ps7GwVFRUpKipKU6ZM0eLFizV37lyFhYXptttu0+TJk02PGwCAoOX6mvOyZctaTcfFxQUe\np6SkKCUlxf2oAADoxbhDGAAAliGcAQCwDOEMAIBlCGcAACxDOAMAYBnCGQAAyxDOAABYhnAGAMAy\nhDMAAJYhnAEAsAzhDACAZQhnAAAsQzgDAGAZwhkAAMsQzgAAWIZwBgDAMoQzAACWIZwBALAM4QwA\ngGUIZwAALEM4AwBgGcIZAADLEM4AAFiGcAYAwDKEMwAAlgnr7gH0Boffesh4zZumv2a8po3+878e\nNF7zZ5PeNF4TAEziyBkAAMsQzgAAWIZwBgDAMoQzAACWIZwBALAM4QwAgGVcv5UqNzdXpaWl8ng8\nysrK0siRIwPLPvzwQ61du1YhISEaMmSIcnJyFBLC/wEAAFwNV4lZUlKiyspKFRYWKicnRzk5Oa2W\nr1ixQuvWrdPrr7+uU6dO6a9//auRwQIA0Bu4Cufi4mIlJydLkoYOHaq6ujo1NDQElhcVFemGG26Q\nJMXExKi2ttbAUAEA6B1chXNNTY2io6MD0zExMfL7/YHpyMhISVJ1dbV2796tu+66q4PDBACg9zBy\n+07HcdrMO378uBYuXKjs7OxWQf59oqP7KSwsVJLkb2ddN7zeqE6oenUOd0LN7tyeno6fHdp31HhF\n9rueqboTal7NvuAqnH0+n2pqagLT1dXV8nq9gemGhgY9/vjjeuqppzR+/Pirqllb2+hmKFfN76/v\n1PpdLdi2pyvxs0N3YL/DRRf3hSuFtKvT2klJSdqxY4ckqaysTD6fL3AqW5JWrVqlhx9+WBMmTHBT\nHgCAXs3VkXNCQoLi4+OVlpYmj8ej7OxsFRUVKSoqSuPHj9c777yjyspKvfnmhU//uffeezVz5kyj\nAwcAIFi5vua8bNmyVtNxcXGBx59//rn7EQEA0MtxZxAAACxDOAMAYBnCGQAAyxDOAABYhnAGAMAy\nhDMAAJYhnAEAsAzhDACAZQhnAAAsQzgDAGAZwhkAAMsQzgAAWIZwBgDAMoQzAACWIZwBALAM4QwA\ngGUIZwAALEM4AwBgGcIZAADLEM4AAFiGcAYAwDKEMwAAliGcAQCwDOEMAIBlCGcAACxDOAMAYBnC\nGQAAyxDOAABYhnAGAMAyhDMAAJYhnAEAsExYdw+gO53dssp4zbDUTOM1AQC9C0fOAABYxnU45+bm\naubMmUpLS9OBAwdaLWtubtby5cuVkpLS4QECANDbuArnkpISVVZWqrCwUDk5OcrJyWm1fPXq1br9\n9tuNDBAAgN7GVTgXFxcrOTlZkjR06FDV1dWpoaEhsHzp0qWB5QAA4N/jKpxramoUHR0dmI6JiZHf\n7w9MR0ZGdnxkAAD0UkZere04TodrREf3U1hYqCTJ3866bni9UW3mHe2iPoe7qA+uDj87tM/8swP7\nXc9U3Qk1r2ZfcBXOPp9PNTU1genq6mp5vV43pQJqaxs79P3t8fvrO7V+sPYJRr3lZ/fIB28Yr/nH\nCTOM1+wtest+h/Zd3BeuFNKuTmsnJSVpx44dkqSysjL5fD5OZQMAYIirI+eEhATFx8crLS1NHo9H\n2dnZKioqUlRUlKZMmaInn3xSx44d08GDB5Wenq4ZM2Zo2rRppscOAEBQcn3NedmyZa2m4+LiAo/X\nrVvnfkQAAPRy3CEMAADLEM4AAFiGcAYAwDKEMwAAliGcAQCwDOEMAIBlCGcAACxDOAMAYBnCGQAA\nyxDOAABYhnAGAMAyRj7PGXbY+6eZxmuOua/QeE0AwJVx5AwAgGUIZwAALEM4AwBgGcIZAADLEM4A\nAFiGcAYAwDKEMwAAliGcAQCwDOEMAIBlCGcAACzD7TsB9AgLPvjYeM2CCT8yXhMwgSNnAAAsQzgD\nAGAZwhkAAMsQzgAAWIZwBgDAMoQzAACWIZwBALAM4QwAgGUIZwAALEM4AwBgGde378zNzVVpaak8\nHo+ysrI0cuTIwLI9e/Zo7dq1Cg0N1YQJE7R48WIjgwWAYPL2X5uM1/yP/9nXeE10PVdHziUlJaqs\nrFRhYaFycnKUk5PTavnKlSv1/PPPa/Pmzdq9e7fKy8uNDBYAgN7AVTgXFxcrOTlZkjR06FDV1dWp\noaFBknTo0CENGDBAgwYNUkhIiO666y4VFxebGzEAAEHOVTjX1NQoOjo6MB0TEyO/3y9J8vv9iomJ\nuewyAADQPiMfGek4TodrREf3U1hY6IWJn8/pcL2rsiin/XUM8C78c5f0mTp/W5f0kaQXN95tvOaj\nD+9sM+//zNxhvM/l/K93Zxmv+f79my87f+rb/9d4r23/sbzNvK3T5xvvcznT3nzLeM0/Pzi9zby3\np//EeJ/vUzg9qkv6LEjpmj7/9ar5A6RJs71t5v33+irjfYYtuf6y84+tLTPe64b/Hd925hOTjfe5\nGq7C2efzqaamJjBdXV0tr9d72WVVVVXy+Xzt1qytbXQzFAQRv7++u4dgVFduDz87dLWu+h0F89+R\n1/v9/5y5Oq2dlJSkHTsuHNGUlZXJ5/MpMjJSknTTTTepoaFBhw8f1tmzZ7Vr1y4lJSW5aQMAQK/k\n6sg5ISFB8fHxSktLk8fjUXZ2toqKihQVFaUpU6bomWeeUUZGhiRp6tSpGjJkiNFBAwAQzFxfc162\nbFmr6bi4uMDjMWPGqLCw0P2oAADoxbhDGAAAliGcAQCwDOEMAIBlCGcAACxDOAMAYBnCGQAAyxDO\nAABYxsi9tQEAGDizX3cPIWhw5AwAgGUIZwAALEM4AwBgGcIZAADLEM4AAFiGcAYAwDKEMwAAliGc\nAQCwDOEMAIBlCGcAACxDOAMAYBnCGQAAyxDOAABYhnAGAMAyfGQk0MU2jl/U3UMAYDmOnAEAsAzh\nDACAZQhnAAAsQzgDAGAZwhkAAMsQzgAAWIZwBgDAMoQzAACWIZwBALAM4QwAgGVchfOZM2eUkZGh\nWbNmac6cOTp06FCbderq6jR//nw9+eSTHR4kAAC9iatw3rp1q/r376/Nmzdr4cKFys/Pb7NOdna2\nRo8e3eEBAgDQ27gK5+LiYk2ZMkWSNG7cOO3bt6/NOitXriScAQBwwVU419TUKCYm5kKBkBB5PB61\ntLS0WicyMrLjowMAoBdq9yMjt2zZoi1btrSaV1pa2mracZwODyQ6up/CwkI7XAc9l9cb1d1DMCrY\ntqcr8bMz7bTxit39OzrWCTW7e5su1W44p6amKjU1tdW8zMxM+f1+xcXF6cyZM3IcRxERER0aSG1t\nY4e+Hz2f31/f3UMwKti2pyvxs7NfMP6OunqbrvTPgKvT2klJSdq+fbskadeuXRo7dqy7kQEAgDba\nPXK+nKlTp2rPnj2aNWuWIiIitGrVKklSQUGBxowZo5EjR2revHk6efKkqqqqlJ6erkWLFikxMdHo\n4AEACEauwjk0NFR5eXlt5i9YsCDweNOmTe5HBaDHePGuu7t7CEDQ4Q5hAABYhnAGAMAyhDMAAJYh\nnAEAsAzhDACAZQhnAAAsQzgDAGAZwhkAAMsQzgAAWIZwBgDAMoQzAACWIZwBALAM4QwAgGUIZwAA\nLEM4AwBgGcIZAADLEM4AAFiGcAYAwDKEMwAAliGcAQCwDOEMAIBlCGcAACxDOAMAYBnCGQAAy4R1\n9wAAG7w8rqC7hwDg3xCa/j+6ewidiiNnAAAsQzgDAGAZwhkAAMsQzgAAWIZwBgDAMoQzAACWIZwB\nALAM4QwAgGVc3YTkzJkzyszM1JEjRxQaGqq8vDwNHjy41Trbtm3Tiy++qJCQECUmJmrp0qVGBgwA\nQLBzdeS8detW9e/fX5s3b9bChQuVn5/fanlTU5PWrFmjl156SYWFhdqzZ4/Ky8uNDBgAgGDnKpyL\ni4s1ZcoUSdK4ceO0b9++Vsv79u2rP/3pT4qMjJTH49HAgQN14sSJjo8WAIBewNVp7ZqaGsXExEiS\nQkJC5PF41NLSooiIiMA6kZGRkqSvvvpK3377re64444r1oyO7qewsFA3w0GQ8HqjunsIQFCaNJu/\nrZ6m3XDesmWLtmzZ0mpeaWlpq2nHcS77vRUVFVq2bJny8/MVHh5+xT61tY3tDQVBzu+v7+4hAECX\nudIBSbvhnJqaqtTU1FbzMjMz5ff7FRcXpzNnzshxnFZHzZJ07NgxLV68WKtXr9btt9/ucugAAPQ+\nrq45JyUlafv27ZKkXbt2aezYsW3Wefrpp/XMM88oPj6+YyMEAKCXcXXNeerUqdqzZ49mzZqliIgI\nrVq1SpJUUFCgMWPGaODAgfr444+1bt26wPfMmzdPkydPNjNqAACCmMf5vgvGXYzrjT3Ln7dNN15z\n2tS3jNcEAFtd6ZozdwgDAMAyhDMAAJYhnAEAsAzhDACAZQhnAAAsQzgDAGAZwhkAAMsQzgAAWIZw\nBgDAMoQzAACWIZwBALAM99YGAKAbcG9tAAB6EMIZAADLEM4AAFiGcAYAwDKEMwAAliGcAQCwDOEM\nAIBlCGcAACxDOAMAYBnCGQAAyxDOAABYhnAGAMAyhDMAAJax5lOpAADABRw5AwBgGcIZAADLEM4A\nAFiGcAYAwDKEMwAAliGcAQCwDOEMAIBlenQ479+/X+vXr9eKFSu0YsUKrVu3Tnv37jXao6ysrM28\n8+fPq6mpyWif5uZmvfTSS236bdu2TadPnzbaK9i2iT7296KP/b3oY1evHhvOa9eu1fr16+Xz+TRx\n4kRNnDhRN954o1544QXl5eUZ67Nq1ao2886ePauf/exn+sc//mGsT1ZWlo4cOaIf/OAHreYfPXpU\nmZmZxvpIwbdN9LG/F33s70Ufy3o5PdTMmTNdLft3TZo0yXn++efbfP361792kpOTjfV56KGHWk1f\nug1z5swx1sdxgm+b6GN/L/rY34s+dvXqsUfOoaGhlz09+/XXX8vj8RjrExYWpujo6DZft912m+bN\nm2esT3Nzs2pqaiRJX375paqrq/Xxxx/ryJEjamlpMdZHCr5too/9vehjfy/62NUrzOQAu9KKFSv0\n7LPPqqqqSj6fTx6PR7W1terfv7+efvppY32uv/56zZ4921i97/PEE08oNTVV/fr1U0hIiDZs2KC8\nvDwdPXpUv/rVr4z2CrZtCrY+S5Ys0YwZM9S3b1+Fhobq97//vXJzc3X06FFlZWUZ69OVvehjfy/6\nuHfpc8O/9nL93NChY3kLnDp1yqmoqHAqKiqchoYG4/Xr6uqM17wcv98fOK2cmJjojBs3zpk8ebKT\nmZnpHD161Giv/fv3O/fdd5+TmJjozJs3zykvL3cee+wxJyUlxSkpKTHWp6u2Kdj6bNiwwXEcxzl+\n/Hhg3qlTp5x33nnHWI+u7kUf+3vRx73PP//cmTZtmrNt27ZW83/xi184+/fvd1Wzx4dzsHjkkUec\nd99912lubg7Ma2lpcbZv3278+khaWppTXl7uOM6FoE5MTHT27dvnVFRUOLNmzTLWp6u2Kdj6fF+t\ntWvXOhs3bjTWpyt70cf+XvRxb+bMmc4XX3wRmP7ggw8cx3Gcw4cPu35O7bGntYNNS0uL7rvvvlbz\nwsPDdc8992jTpk1Ge4WHh2vo0KGSpDvvvFMDBgzQqFGjJF24Hm1KV21TsPU5cOCAJk2aJI/HI+df\nPtG1urpac+fO7XG96GN/L/q4Fx4erri4uMD0ypUrtX37dsXGxrp+TiWcLXHjjTdq5cqVuueee3T9\n9dfL4/Ho+PHj2rVrl7xer9FeAwcO1IYNGzRixAjt2bNHAwYM0OrVqzVgwAD169fPWJ+u2qZg6zNi\nxAj94Q9/MFbPhl70sb8Xfdzr06ePXnnlFY0cOVI7d+7ULbfcouzsbPl8PvXp08dVTcLZEnl5eXr7\n7bf1yiuvqLq6WpLk9Xo1btw4LVmyxGiv3Nxcbdq0SVu3blV8fLxee+01FRUVqba2Vs8995yxPl21\nTcHW584771Tfvn2N1bOhF33s70Uf91avXq2CggJ98MEHGj16tH73u9/p3XffVVVVlVavXu2qpsf5\n1+N9BL3du3crMTFRISGt30l36NAhDR482Hi/l19+WXfddZd++MMfGq990cGDBzVkyJBOq3/RuXPn\nFBoaKkk6efKkKioqdNNNNykmJsZon+bmZpWWlgb+Cbjuuut0xx13GH+yOX/+fJv9oDNUVFTo5ptv\nbjP/9OnTuuaaa4z2+vvf/66YmBjdcMMNgXnffvutYmNjjfaRpI0bN2rixImdum9LXbd/S9Lhw4d1\n/PhxeTwexcbGtrmxRmcqKytTfHy80XqVlZUaMWKEBg8erGPHjqm+vl7Dhg0z1uNSJp/rCOdeaOzY\nsYqNjVVubm7gOklTU5MeeughPfDAA3r44YeN9rv//vs1aNAgDR8+XAsWLFBERITR+vn5+Tpy5Ijy\n8/NVWFioVatWqU+fPvJ4PDp58uRl3w/vxmuvvab3339fmzZt0o4dO7R69Wrdeuut+uabb/TII48o\nLS3NSJ+dO3cqPz9fw4cPD5wu/+6773TgwAEtWrSozXXvjkhPT29zvbyxsVFPP/20fvOb36h///5G\n+owaNUoJCQnKyckJhGZjY6OmTZumjIwMTZ061UifgoIC7dy5U88884yGDx8emP/UU0/plltuMX4W\nqrP3bUlas2aNjh492un794EDB5SXl6drr71W5eXlio2N1cmTJ9W3b189++yzra6pdpbL7Y9uPf/8\n89q9e7eGDx+uAwcOKDk5Wdu3b1dERITGjh2rpUuXGulzKaP7g9tXp6HnSk9Pdw4ePOjce++9rV7m\nf/78eefBBx803u/i3XLee+89Z86cOc7f/vY3o/WnT58eeNzY2OhkZGQEptPT0431eeCBBwKv1J45\nc6ZTW1vrOM6FV2xfOoaOSk1Nderr69vMb2hoMNrHcS5s04cffuh89NFHrb42bNjgzJ8/31if9PR0\nZ+/evc69997rHDx4MDC/sbHR6D6XmprqnDt3LjC9cuXKwGOTdw7815qdtW87Ttft36mpqYG3jtbV\n1TmZmZmO4zjOF1984aSkpBjrM27cOOfuu+92MjIynK1btzqnT58OLDO5PZf+vpubm52kpKTA329a\nWpqxPpfraWJ/4JpzL+Q4jm6++WYVFBRo4cKF+uUvf6nx48fr7NmzRvuUlJRIkurr6/XRRx/puuuu\n06OPPqrf/va3euutt7R27VojfUJDQ3X27NnAqyKPHDkSWOYYPDHk8XhUV1cnr9crn8+n8PDwQA+T\nR0whISGXvctdWFhY4JS6KVVVVXr11Vcv2+/aa6811sdxHP3oRz/Sc889pyeeeEJr1qzRbbfdFrhB\nhEkXf+f19fV6//339fOf/1wDBw402qOr9m2p6/bvkJAQRUZGSpIiIiJUWVkpSYqLizN618Xdu3er\nsbFRX375pfbs2aM5c+YoKSlJ8+fPN/73eubMGYWHh6ulpUXnz5/XuXPn5Fx4C7GxPlLn7A+Ecy90\n8Q9t0KBBKigo0LJly5SXl6empiY99thjxvq8/fbbkqTa2lq98847gfkX38ZlyuTJk5Wenq6EhAQV\nFxe3Ou1r8kll+fLlmjdvnuLi4tSnTx/Nnj1bo0aN0hdffKFHH33UWJ/09HRNnz5do0aNCrwq/Lvv\nvtO+ffuMX3K45ZZbtG7dOqM1L+fi7yEuLk5r1qxRRkaGbrzxRp04cUKJiYnG+kycOFGzZs1SfHy8\n9u3bp+XLl2v27Nk6f/68HnjgAWN9umrflrpu/x4/frxmz56t22+/XZ988olSUlIkSRkZGZo0aZKx\nPpLUr18/JSQkKCEhQUuWLNFf/vIXLVq0SF9//bWxHlOnTlVKSopuvfVWffXVV1q8eLFmzJihc+fO\nBbbNlM7YH7jm3At99tlnGjFiRKt5VVVVuvbaawP/OZv0wgsvaP78+cbrXuqzzz5TeXm5br311lYv\nKHEcx+gT2Pnz5/X555/r8OHD8ng8gRdqmb7W2NDQoE8++aTVq8ITEhKMXQO+6KOPPtLYsWON1ryc\nXbt26Sc/+Ulg+ty5c/r00081cOBA44H25ZdfqqKiQiNGjFBsbKyamprU0NBg/C2JUtfs21LX7d9f\nffWVDh48qGHDhgV+L83Nza7fDvTv+vTTT3XnnXcaq3fkyBEdOnRIw4YNU0xMjE6dOqXTp0932ovc\nTO4PhDMAAJbpsZ9KBQBAsCKcAQCwDOEMAIBlCGcAACxDOAMAYBnCGQAAy/w/0IvbsbGFlc8AAAAA\nSUVORK5CYII=\n",
            "text/plain": [
              "<matplotlib.figure.Figure at 0x7f1c07929400>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "metadata": {
        "id": "3F06mim47qFE",
        "colab_type": "code",
        "outputId": "e7d11f25-3068-4a6d-d034-83f971da50d4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        }
      },
      "cell_type": "code",
      "source": [
        "feat_subsets[0]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[334,\n",
              " 30,\n",
              " 0,\n",
              " 1,\n",
              " 182,\n",
              " 309,\n",
              " 310,\n",
              " 311,\n",
              " 312,\n",
              " 313,\n",
              " 314,\n",
              " 315,\n",
              " 316,\n",
              " 317,\n",
              " 318,\n",
              " 304,\n",
              " 305]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 96
        }
      ]
    },
    {
      "metadata": {
        "id": "AIlwtzcPNkQz",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# RankSVM"
      ]
    },
    {
      "metadata": {
        "id": "7PYheGbPc6aN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 726
        },
        "outputId": "d05b7ab9-c800-448b-e696-b253030f5d17"
      },
      "cell_type": "code",
      "source": [
        "\n",
        "train_pairs, train_rel_list = create_test_pairs(run, trainval, test_docs, test_judgments)\n",
        "test_pairs, test_rel_list = create_test_pairs(run, test, test_docs, test_judgments)\n",
        "\n",
        "\n",
        "print('# of unique train documents', len(set([v[1] for v in train_pairs])))\n",
        "print('# of unique train topics', len(set([v[0] for v in train_pairs])))\n",
        "\n",
        "print('# of unique test documents', len(set([v[1] for v in test_pairs])))\n",
        "print('# of unique train topics', len(set([v[0] for v in test_pairs])))\n",
        "\n",
        "X_train_abs, X_train_keywords, X_train_titles,  X_train_queries, train_rel_list = create_X(train_pairs, train_rel_list, queries, test_docs)\n",
        "\n",
        "X_test_abs, X_test_keywords, X_test_titles, X_test_queries, test_rel_list = create_X(test_pairs, test_rel_list, queries, test_docs) \n",
        "\n",
        "\n",
        "train_features = np.hstack( [features_stream_abs(X_train_abs, X_train_queries),features_stream_keywords(X_train_keywords, X_train_queries), features_stream_title(X_train_titles, X_train_queries)])\n",
        "test_features = np.hstack([features_stream_abs(X_test_abs, X_test_queries), features_stream_keywords(X_test_keywords, X_test_queries), features_stream_title(X_test_titles, X_test_queries)])\n",
        "\n",
        "scaler = StandardScaler()\n",
        "print(scaler.fit(train_features))\n",
        "StandardScaler(copy=True, with_mean=True, with_std=True)\n",
        "train_features = scaler.transform(train_features)\n",
        "test_features = scaler.transform(test_features) "
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "# of unique train documents 1409\n",
            "# of unique train topics 50\n",
            "# of unique test documents 390\n",
            "# of unique train topics 13\n",
            "Pairs:  1500\n",
            "Rellist:  1500\n",
            "Length\n",
            "1500\n",
            "1500\n",
            "Pairs:  390\n",
            "Rellist:  390\n",
            "Length\n",
            "390\n",
            "390\n",
            "0\n",
            "250\n",
            "500\n",
            "750\n",
            "1000\n",
            "1250\n",
            "0\n",
            "250\n",
            "500\n",
            "750\n",
            "1000\n",
            "1250\n",
            "0\n",
            "250\n",
            "500\n",
            "750\n",
            "1000\n",
            "1250\n",
            "0\n",
            "250\n",
            "0\n",
            "250\n",
            "0\n",
            "250\n",
            "StandardScaler(copy=True, with_mean=True, with_std=True)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Ko3-jliqAPw_",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# SUBSET features for RANKSVM\n",
        "subs = np.sort([334,\n",
        " 30,\n",
        " 0,\n",
        " 1,\n",
        " 182,\n",
        " 309,\n",
        " 310,\n",
        " 311,\n",
        " 312,\n",
        " 313,\n",
        " 314,\n",
        " 315,\n",
        " 316,\n",
        " 317,\n",
        " 318,\n",
        " 304,\n",
        " 305])\n",
        "\n",
        "train_features = train_features[:, subs]\n",
        "test_features = test_features[:, subs]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "6iSFnSM3sHN5",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Create query, document and corresponding feature pairs for test and training set"
      ]
    },
    {
      "metadata": {
        "id": "Bz_2MtZCNf0h",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Relate features to documents\n",
        "query_documents_train = {}\n",
        "query_documents_test = {}\n",
        "cnt = 0\n",
        "\n",
        "for idx, topic in enumerate(run['Topic'][run['Topic'].isin(trainval)]):\n",
        "    if topic not in query_documents_train.keys():\n",
        "        query_documents_train[topic] = []\n",
        "    uid = {run.iloc[idx, 2] : train_features[idx]}\n",
        "    query_documents_train[topic].append(uid)\n",
        "\n",
        "for idx, topic in enumerate(run['Topic'][run['Topic'].isin(test)]):\n",
        "    if topic not in query_documents_test.keys():\n",
        "        query_documents_test[topic] = []\n",
        "    uid = {run.iloc[idx, 2] : test_features[idx]}\n",
        "    query_documents_test[topic].append(uid)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZVzjx3pxr2KC",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Get training pairs and train SVM"
      ]
    },
    {
      "metadata": {
        "id": "sM33RD8L2iMh",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# RankedSVM\n",
        "from sklearn import svm\n",
        "import operator\n",
        "\n",
        "def create_pairs(queries_doc_dict):\n",
        "    '''\n",
        "    Create pairs of documents.\n",
        "    '''\n",
        "    keys = queries_doc_dict.keys()\n",
        "    query_pairs = {}\n",
        "    class_pairs = {}\n",
        "    for q in keys:\n",
        "        classes = []\n",
        "        query_documents = [list(x.keys())[0] for x in queries_doc_dict[q]]\n",
        "        pairs = []\n",
        "        for doc in query_documents:\n",
        "            for doc2 in query_documents:\n",
        "                if doc2 != doc:\n",
        "                    pairs.append([doc, doc2])\n",
        "\n",
        "                    first = list([x for x in query_documents \n",
        "                             if x in [doc, doc2]])[0]\n",
        "                    classes.append(1 if doc == first else 0)\n",
        "        class_pairs[q] = classes\n",
        "        query_pairs[q] = pairs\n",
        "    return query_pairs, class_pairs\n",
        "\n",
        "query_pairs_train, query_train_classes = create_pairs(query_documents_train)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "J1ODNcYeH2Yp",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "X_train, y_train = [], []\n",
        "\n",
        "\n",
        "for key in query_pairs_train.keys():\n",
        "    '''\n",
        "    Concatenate features for documents so that we can train our SVM.\n",
        "    '''\n",
        "    query_dict = query_documents_train[key]\n",
        "    for e, dp in enumerate(query_pairs_train[key]):\n",
        "        feature = []\n",
        "        #sanity\n",
        "        res1 = [list(x.values())[0] for x in query_dict if list(x.keys()) == dp[0]][0]\n",
        "        res2 = [list(x.values())[0] for x in query_dict if list(x.keys()) == dp[1]][0]\n",
        "        feature = res1 - res2\n",
        "        X_train.append(feature)\n",
        "    y_train.extend(query_train_classes[key])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "W_PMV5_pUxhF",
        "colab_type": "code",
        "outputId": "f6f0dd2f-3fdd-4042-ef54-44d04297ec04",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        }
      },
      "cell_type": "code",
      "source": [
        "# Train normal SVM on query pairs and binary labels\n",
        "model2 = svm.SVC(kernel='linear')#(C=0.01, kernel='linear')#, probability=True)#svm.SVC(kernel='linear')\n",
        "model2.fit(X_train, y_train)\n"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
              "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
              "  kernel='linear', max_iter=-1, probability=True, random_state=None,\n",
              "  shrinking=True, tol=0.001, verbose=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "metadata": {
        "id": "yyohQfSJpTem",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "query_pairs_test, query_test_classes = create_pairs(query_documents_test)\n",
        "X_test, y_test = [], []\n",
        "for key in query_pairs_test.keys():\n",
        "    '''\n",
        "    Concatenate features for documents so that we can train our SVM.\n",
        "    '''\n",
        "    query_dict = query_documents_test[key]\n",
        "    for e, dp in enumerate(query_pairs_test[key]):\n",
        "        feature = []\n",
        "        res = [list(x.values())[0] for x in query_dict if list(x.keys()) in dp]\n",
        "        feature = np.concatenate((res[0], res[1]), axis=0)\n",
        "        feature = res[0] - res[1]\n",
        "\n",
        "        X_test.append(feature)\n",
        "    y_test.extend(query_test_classes[key])\n",
        "#print_evaluation_scores(y_test, model.predict(X_test))\n",
        "# query_pairs_test"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "LuZm1hGIEjif",
        "colab_type": "code",
        "outputId": "d64bedb6-c13d-4b3f-87dc-bb03ae990cd4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "roc_auc_score(y_test, model2.predict(X_test)), roc_auc_score(y_train, model2.predict(X_train))"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.5, 0.5420689655172414)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "metadata": {
        "id": "f83D7fbkE7RB",
        "colab_type": "code",
        "outputId": "74e07724-41a0-4146-d250-5fb5e580934e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "print_evaluation_scores(y_train, model2.predict(X_train))"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy:  0.5420689655172414\n",
            "F1-score 0.5436635205717951\n",
            "Precision  0.5227919982114686\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "wehSj5vMFCwg",
        "colab_type": "code",
        "outputId": "54a40adb-889f-438b-ed25-972875b7a23b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "cell_type": "code",
      "source": [
        "print_evaluation_scores(y_test, model2.predict(X_test))"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy:  0.5\n",
            "F1-score 0.5057250240363604\n",
            "Precision  0.5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "IJHQCZOHr5qx",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Get query pairs for testing"
      ]
    },
    {
      "metadata": {
        "id": "ew53u9fMr-10",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## Determine ranking for test pairs by using pair wise ranking"
      ]
    },
    {
      "metadata": {
        "id": "uAUKU7mm7WII",
        "colab_type": "code",
        "outputId": "4da83ce2-43e0-4861-f52d-e3f22aa0e848",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 475
        }
      },
      "cell_type": "code",
      "source": [
        "def pair_wise_ranking(query_pairs, query_dict, model):\n",
        "    '''\n",
        "    Compare all query pairs.\n",
        "    Input (doc1, doc2) if doc1 is higher than doc 2, return 1\n",
        "    Input (doc1, doc3) if doc1 is lower than doc 3, return -1\n",
        "    Sum all results for the documents, which gives the rank.\n",
        "    ''' \n",
        "    results = {}\n",
        "\n",
        "    for dp in query_pairs:\n",
        "        if dp[0] not in results.keys():\n",
        "            results[dp[0]] = 0\n",
        "\n",
        "        # Get feature for documents.\n",
        "        res1 = [list(x.values())[0] for x in query_dict if list(x.keys()) == dp[0]][0]\n",
        "        res2 = [list(x.values())[0] for x in query_dict if list(x.keys()) == dp[1]][0]\n",
        "        #feature = np.concatenate((res[0], res[1]), axis=0)\n",
        "        feature = res1 - res2   \n",
        "        #score = model.predict_proba([feature])[:,0][0]\n",
        "        #print(score)\n",
        "        #results[dp[0]] += score\n",
        "        score = model.predict([feature])[0]\n",
        "        if score == 1:\n",
        "            # Signal that ranking is correct.\n",
        "            results[dp[0]] += 1\n",
        "        else:\n",
        "            # Signal that ranking is incorrect.\n",
        "            results[dp[0]] -= 1\n",
        "    \n",
        "    return list(results.items())\n",
        "\n",
        "\n",
        "ranking_queries = {}\n",
        "for query in list(query_pairs_test.keys()):\n",
        "    ranking_queries[query] = pair_wise_ranking(query_pairs_test[query],\n",
        "                                               query_documents_test[query], \n",
        "                                               model2)\n",
        "    print(ranking_queries[query])\n",
        "    print('Processed ranking for query {}, amount of docs for this query {}'.format(query, len(ranking_queries[query])))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[(88002743, -27), (88095668, 3), (88103805, -19), (88181296, -7), (88196514, 5), (88196515, -3), (88201371, -9), (88238703, 9), (88238705, 25), (88238706, 21), (88283372, -21), (89015550, 11), (89015552, 15), (89023354, 23), (89078872, 1), (89153654, -23), (89178991, 13), (89182887, -25), (89184080, 17), (89184093, 29), (89201699, -5), (89201700, 27), (89213723, -15), (89225293, -13), (89228983, -11), (89244867, 19), (89244872, -1), (89244873, -17), (89270656, 7), (89277954, -29)]\n",
            "Processed ranking for query OHSU2, amount of docs for this query 30\n",
            "[(88012687, -13), (88045917, -5), (88046825, 23), (88048250, 13), (88070865, 15), (88071146, 5), (88099036, 1), (88102808, -27), (88145328, -23), (88154613, -3), (88159008, 9), (88161153, 17), (88162311, -9), (88252600, 11), (88274205, -7), (88280216, 7), (89043526, -19), (89116245, 19), (89127813, -17), (89148176, -25), (89149259, -11), (89271792, -29), (89271810, 21), (89309709, 29), (89309717, 3), (89355395, 25), (90022649, -15), (90042765, -1), (90055106, 27), (90113624, -21)]\n",
            "Processed ranking for query OHSU22, amount of docs for this query 30\n",
            "[(88001916, 11), (88052442, -1), (88109215, -7), (88142064, 13), (88150661, 7), (88164699, -21), (88187751, -15), (88229816, 29), (88253129, 9), (88265683, -9), (88332382, 21), (88336473, -27), (89010880, 17), (89028085, -13), (89051712, 27), (89087280, -23), (89089506, 15), (89106065, -3), (89128008, 5), (89150903, -25), (89177425, -19), (89194988, -11), (89203794, 25), (89215950, -5), (89274993, -29), (89303745, 1), (89336620, 19), (89376276, -17), (90002740, 3), (90050820, 23)]\n",
            "Processed ranking for query OHSU23, amount of docs for this query 30\n",
            "[(88008229, 21), (88094531, -21), (88133144, -27), (88160773, 1), (88186120, -5), (88210998, 11), (88242135, -17), (88273595, 9), (88283564, -25), (88317800, 5), (88326593, 3), (89003913, -13), (89004916, 29), (89022817, 29), (89064610, 17), (89105344, -19), (89116184, -15), (89165392, -29), (89207160, -23), (89283795, -7), (89289603, 23), (89338239, 7), (89338252, 19), (89339634, -9), (89340902, 15), (90005169, -3), (90005198, -1), (90015007, -11), (90056730, 25), (90059800, 13)]\n",
            "Processed ranking for query OHSU32, amount of docs for this query 30\n",
            "[(88024291, 1), (88059684, 23), (88059685, 25), (88094571, -17), (88161066, 19), (88174097, 21), (88199356, -27), (88205421, -13), (88217423, -25), (88222637, 9), (88260438, 3), (88269244, 11), (89001178, 29), (89009548, 5), (89019530, -7), (89040110, 7), (89062182, -11), (89062749, -15), (89230949, -5), (89239871, 15), (89239887, -19), (89239892, -23), (89239896, 13), (89239897, -3), (89239898, 17), (89239899, -21), (89239900, -9), (89255639, -1), (89266789, 27), (89272176, -29)]\n",
            "Processed ranking for query OHSU36, amount of docs for this query 30\n",
            "[(88046867, -29), (88046868, -5), (88047008, -5), (88099742, -21), (88103662, -21), (88131198, -3), (88148217, 25), (88150688, 17), (88171096, 17), (88217189, 7), (88261455, 29), (88302882, -27), (88337036, -13), (88337325, 1), (89047596, -17), (89132713, -17), (89133267, 27), (89146830, -15), (89181883, 13), (89235749, -1), (89257225, 5), (89258628, 5), (89263010, 11), (89267409, 11), (89268695, -25), (89284544, 23), (89301782, 21), (90028709, 21), (90039554, -9), (90055473, -9)]\n",
            "Processed ranking for query OHSU4, amount of docs for this query 30\n",
            "[(88021421, -9), (88078936, 9), (88123727, -15), (88131337, -29), (88155701, 23), (88160885, 21), (88218778, 25), (88220552, -7), (88252663, 5), (88319307, -13), (89001922, 29), (89051337, -25), (89052153, -27), (89054667, -5), (89087246, 7), (89124833, -23), (89144637, -21), (89164729, 15), (89178357, -3), (89178893, 11), (89184063, 19), (89205336, 17), (89220101, -17), (89248825, -1), (89248828, -19), (89314948, 1), (89331080, 3), (89346923, 27), (89348244, 13), (90025673, -11)]\n",
            "Processed ranking for query OHSU43, amount of docs for this query 30\n",
            "[(88011546, -3), (88071546, -7), (88100715, -9), (88104427, 3), (88119320, 7), (88127532, 5), (88156864, -1), (88223081, -13), (88243891, -11), (89012276, -5), (89012283, -23), (89012295, 9), (89034388, 1), (89040051, -19), (89044996, -15), (89048063, 17), (89070804, 17), (89077330, 19), (89141976, 13), (89148583, -29), (89162203, 29), (89162622, 23), (89318394, 27), (89369520, -17), (89380403, 11), (90013422, 21), (90020377, -21), (90024696, -25), (90030863, -27), (90037152, 25)]\n",
            "Processed ranking for query OHSU45, amount of docs for this query 30\n",
            "[(88018207, -21), (88054165, -29), (88054188, -19), (88071366, 9), (88136533, 15), (88155696, -25), (88181043, 7), (88190910, 19), (88195479, 3), (88224404, 25), (88271160, 1), (88311912, -17), (89029978, -27), (89049603, 17), (89075409, -7), (89096134, 5), (89129324, 11), (89169734, 27), (89209934, -1), (89220458, 21), (89226674, -13), (89231583, -23), (89237620, -5), (89249291, 13), (89340893, -11), (89349832, -9), (89349835, 29), (90004349, 23), (90004534, -3), (90023534, -15)]\n",
            "Processed ranking for query OHSU50, amount of docs for this query 30\n",
            "[(88001951, -29), (88016824, 17), (88026647, -27), (88037826, 29), (88056768, -23), (88081984, 21), (88132171, -21), (88150697, -9), (88192320, -5), (88226461, 27), (88270332, 23), (88272122, 7), (88310729, 19), (89002693, -25), (89028129, -19), (89053236, 25), (89065595, -1), (89086594, 11), (89089486, -7), (89168080, -17), (89212437, -13), (89252604, 13), (89260624, 5), (89298622, 1), (89350331, -15), (89373578, 15), (90004785, 9), (90030104, -3), (90030144, -11), (90042701, 3)]\n",
            "Processed ranking for query OHSU52, amount of docs for this query 30\n",
            "[(88009758, 29), (88035892, -5), (88049838, -21), (88074412, -11), (88099740, -7), (88118645, 1), (88123738, -19), (88162513, -25), (88162941, -15), (88189945, 7), (88211078, 11), (88212192, 23), (88220893, -29), (88220900, -17), (88230660, 17), (88239997, 9), (88247524, 5), (88262791, -9), (88268408, -27), (88279312, 25), (88279814, 21), (88288988, 3), (88303058, 27), (88332874, -13), (88335370, 15), (89009416, -23), (89036995, 19), (89061790, -3), (89068612, -1), (89077200, 13)]\n",
            "Processed ranking for query OHSU55, amount of docs for this query 30\n",
            "[(88003515, -21), (88023104, -17), (88034663, 27), (88091170, -7), (88170074, 3), (88239909, 1), (88307990, -1), (88313704, -11), (88331045, -15), (89071147, -3), (89140238, 25), (89140661, 11), (89149416, 23), (89164223, 5), (89192618, -13), (89200632, 15), (89284544, 17), (89301189, -9), (89314292, -5), (89317157, 29), (89317174, 19), (89324500, -27), (89337223, -29), (89342550, -23), (89344774, -25), (89366992, 7), (90023512, 21), (90140817, 13), (90141373, 9), (90142831, -19)]\n",
            "Processed ranking for query OHSU56, amount of docs for this query 30\n",
            "[(88102789, 1), (88110902, 27), (88195757, 25), (88226107, 29), (88242380, 21), (88250216, 19), (88271095, 3), (89006116, 23), (89019795, 15), (89064470, 17), (89065432, -23), (89069697, 9), (89078872, 5), (89091929, -29), (89121386, 11), (89154018, -15), (89231311, -25), (89232866, 7), (89272296, -13), (89292543, -19), (89305160, -3), (89324905, -3), (89371682, -3), (90006552, -21), (90059604, -17), (90062939, -9), (90098432, 13), (90119565, -27), (90154690, -1), (90164383, -9)]\n",
            "Processed ranking for query OHSU9, amount of docs for this query 30\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "qs1-U5kytwQw",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "newranklist[8]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "KNLG3xWDfj1v",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "svmlist = []\n",
        "for key, values in ranking_queries.items():\n",
        "  #print(key, len(values))\n",
        "  #print(ranking_queries)\n",
        "  vals = [k[1] for k in values]\n",
        "  svmlist.append(pd.DataFrame(vals))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "6Gk8sF67a0kD",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "  ranks = run.loc[run['Topic'] == topic]\n",
        "  ranks['inverted_rank'] = 1/ranks['rank']\n",
        "  ranks = pd.merge(ranks, judgments, left_on=['Topic','ui'], right_on=['topic','uid'], how='left')\n",
        "  ranks['relevance'] = ranks['relevance'].fillna(0)\n",
        "  ranks['relevance'] = ranks['relevance'].apply(lambda x: 1.0 if (x == 2 or x == 1) else 0)\n",
        "  return ranks['relevance']\n",
        "\n",
        "def get_inv_rank(topic, run):\n",
        "\n",
        "  ranks = run.loc[run['Topic'] == topic]\n",
        "  ranks['inverted_rank'] = 1/ranks['rank']\n",
        "  return ranks['inverted_rank']\n",
        "\n",
        "relevancelist = []\n",
        "ranklist = []\n",
        "for e, ohsu in enumerate(test):\n",
        "  relevance = get_ranking(ohsu, run, test_judgments)  \n",
        "  relevancelist.append(relevance)\n",
        "                \n",
        "  ranking = get_inv_rank(ohsu, run)\n",
        "  ranklist.append(ranking)       \n",
        "   \n",
        "microsoft_aps = []\n",
        "\n",
        "for (a, b) in zip(relevancelist, ranklist):\n",
        "  ap = (average_precision_score(a,b))\n",
        "  if pd.isna(ap):\n",
        "#     print(0)\n",
        "    microsoft_aps.append(0)\n",
        "  else:\n",
        "#     print(ap)\n",
        "    microsoft_aps.append(ap)\n",
        "  \n",
        "\n",
        "svm_aps = []\n",
        "for (a, b) in zip(relevancelist, svmlist):\n",
        "  ap = (average_precision_score(a,b))\n",
        "  if pd.isna(ap):\n",
        "#     print(0)\n",
        "    svm_aps.append(0)\n",
        "  else:\n",
        "#     print(ap)\n",
        "    svm_aps.append(ap)\n",
        "\n",
        "  \n",
        "print(\"MAP Microsoft: \", np.mean(microsoft_aps))\n",
        "print(\"MAP SVM Rerank: \", np.mean(svm_aps))"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}