cosine_similarity.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import cross_validation\n",
    "from sklearn.metrics import mean_squared_error,r2_score\n",
    "import fileinput\n",
    "import operator\n",
    "import random\n",
    "\n",
    "#Fit the code to the screen\n",
    "from IPython.core.display import HTML\n",
    "HTML(\"<style>.container { width:100% !important; }</style>\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading User Movie and Ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tank/home/adif/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:9: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators; you can avoid this warning by specifying engine='python'.\n"
     ]
    }
   ],
   "source": [
    "#Reading the movies data\n",
    "my_path = !pwd\n",
    "movies_filename = \"/features/movie_features.pik\"\n",
    "user_filename = \"/features/user_features_advanced.pik\"\n",
    "ratings_filename = \"/ml-1m/ratings.dat\"\n",
    "movie_features = pickle.load(open(my_path[0]+movies_filename, 'rb'))\n",
    "user_features = pickle.load(open(my_path[0]+user_filename, 'rb'))\n",
    "user_features['gender'] = user_features['gender'].map({'F':0,'M':1})\n",
    "dfratings = pd.read_csv(my_path[0]+ratings_filename, index_col=None, sep='::', header=None)\n",
    "dfratings.columns = [\"userId\", \"movieId\", \"rating\", \"timestamp\"]\n",
    "#dfratings.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 1.48291571754 -0.25333959004\n",
      "2 1.01025056948 -0.27116773905\n",
      "3 0.902556314857 -0.33933298313\n",
      "4 0.853929384966 -0.349503735692\n",
      "5 0.799635535308 -0.378532237854\n",
      "6 0.777145026576 -0.389450296171\n",
      "7 0.770117614244 -0.436176654013\n",
      "8 0.754093109339 -0.419425624428\n",
      "9 0.753227031131 -0.481007291566\n"
     ]
    }
   ],
   "source": [
    "#USER SIMILARITY_FUNCTIONS\n",
    "\n",
    "N_MOST_SIMILAR=10\n",
    "def cosine_similarity(user_features,userId1,userId2,similarity_features):\n",
    "    user1 = user_features[user_features[\"userId\"] == userId1][similarity_features]\n",
    "    user2 = user_features[user_features[\"userId\"] == userId2][similarity_features]\n",
    "    sim = np.dot(user1,user2.T)/np.linalg.norm(user1)/np.linalg.norm(user2)\n",
    "    return sim[0][0]\n",
    "    \n",
    "\n",
    "#returns the 'n' most similar users to 'userId' according to 'similarity_features'. \n",
    "#If 'predicted_movieId' is not 'None' we restrict the users to chose from only to users that rated the movie \n",
    "def n_most_similar(n,userId,user_features,similarity_features,predicted_movieId=None):\n",
    "    #print user_features[user_features[\"userId\"]==5530]\n",
    "    #print user_features[user_features[\"userId\"]==2106]\n",
    "    users_to_chose_from = user_features\n",
    "    if predicted_movieId!=None:\n",
    "        users_to_chose_from = dfratings[dfratings[\"movieId\"]==predicted_movieId]\n",
    "    similarity_dict = dict()\n",
    "    for simliarity_userId in users_to_chose_from[\"userId\"].unique():\n",
    "        if userId!=simliarity_userId:\n",
    "            similarity_dict[simliarity_userId] = cosine_similarity(user_features,userId,simliarity_userId,similarity_features)\n",
    "            #print simliarity_userId, similarity_dict[simliarity_userId]\n",
    "    sorted_dict = sorted(similarity_dict.items(), key=operator.itemgetter(1)) \n",
    "    sorted_dict.reverse()\n",
    "    ids =[]\n",
    "    for items in sorted_dict[:min(n,len(sorted_dict)-1)]:\n",
    "        ids.append(items[0])\n",
    "    return ids\n",
    "\n",
    "\n",
    "user_genre_pref_filename = '/explore/user_genre_pref.pik'\n",
    "user_genre_pref = pickle.load(open(my_path[0]+user_genre_pref_filename, 'rb'))\n",
    "similarity_features = list(user_genre_pref.columns)\n",
    "similarity_feaures.append(\"rate_num\")\n",
    "similarity_feaures.append(\"rate_avg\")\n",
    "similarity_feaures.append(\"rate_std\")  \n",
    "similarity_feaures.append(\"avg_score_diff\")\n",
    "random_seed = 42\n",
    "#print user_features[\"userId\"][0]\n",
    "MAX_SIMILAR_USERS=10\n",
    "predictions = []\n",
    "\n",
    "tested_userIds = [457,2768,1980,402,4126,3418,1112,3119,1737,1322]\n",
    "#tested_userIds = [457,2768,1980]\n",
    "ratings_pred = []\n",
    "ratings_real = []\n",
    "for n in range(1,MAX_SIMILAR_USERS):\n",
    "    ratings_pred.append([])\n",
    "    ratings_real.append([])    \n",
    "\n",
    "for n in range(1,MAX_SIMILAR_USERS):\n",
    "    for dummy_train_indices,dfratings_test_indices in dfratings_shuffle_indices:\n",
    "        for tested_userId in tested_userIds:\n",
    "            user_ratings = dfratings[dfratings[\"userId\"]==tested_userId]\n",
    "            user_ratings.reset_index(inplace=True)\n",
    "            user_ratings_shuffle_indices = cross_validation.ShuffleSplit(len(user_ratings), n_iter=1, test_size=0.1, random_state=random_seed)\n",
    "            for dummy_train_indices,user_ratings_test_indices in user_ratings_shuffle_indices:\n",
    "                predicted_movieIds = (user_ratings[\"movieId\"])[user_ratings_test_indices]\n",
    "                for predicted_movieId in predicted_movieIds:\n",
    "                    n_similar_userIds = n_most_similar(n,tested_userId,user_features,similarity_features,predicted_movieId)\n",
    "                    similar_ratings = []\n",
    "                    for similar_userId in n_similar_userIds:\n",
    "                        similar_user_ratings = dfratings[dfratings[\"userId\"]==similar_userId]\n",
    "                        similar_movie_rating = similar_user_ratings[similar_user_ratings[\"movieId\"] == predicted_movieId][\"rating\"]\n",
    "                        similar_ratings.append(similar_movie_rating.item())\n",
    "                    #user_rating = dfratings[dfratings[\"userId\"]==tested_userId]\n",
    "                    #user_movie_rating = user_rating[user_rating[\"movieId\"] == predicted_movieId][\"rating\"]                    \n",
    "                    predict = float(sum(similar_ratings))/float(len(similar_ratings))\n",
    "                    ratings_pred[n-1].append(predict)\n",
    "                    #ratings_real[n-1].append(user_movie_rating.item())\n",
    "        with open(my_path[0]+'/ratings_pred{0}.pik'.format(n), 'wb') as handle:\n",
    "            pickle.dump(ratings_pred, handle, protocol=pickle.HIGHEST_PROTOCOL)                    \n",
    "        #with open(my_path[0]+'/ratings_real{0}.pik'.format(n), 'wb') as handle:\n",
    "            #pickle.dump(ratings_real, handle, protocol=pickle.HIGHEST_PROTOCOL)                                            \n",
    "        print n,mean_squared_error(ratings_pred[n-1],ratings_real[0]),r2_score(ratings_pred[n-1],ratings_real[0])\n",
    "                    \n",
    "                    \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "random 3.69248291572 -0.779901852889\n"
     ]
    }
   ],
   "source": [
    "ratings_rand=[]\n",
    "for i in range(len(ratings_real[0])):\n",
    "    ratings_rand.append(random.randint(1,5))\n",
    "print \"random\",mean_squared_error(ratings_rand,ratings_real[0]),r2_score(ratings_rand,ratings_real[0])    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-216-8544e1b2a8db>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     12\u001b[0m                 \u001b[0mpredicted_movieIds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0muser_ratings\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"movieId\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0muser_ratings_test_indices\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m                 \u001b[1;32mfor\u001b[0m \u001b[0mpredicted_movieId\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mpredicted_movieIds\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 14\u001b[1;33m                     \u001b[0mn_similar_userIds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mn_most_similar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtested_userId\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0muser_features\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msimilarity_features\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpredicted_movieId\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     15\u001b[0m                     \u001b[0msimilar_ratings\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m                     \u001b[1;32mfor\u001b[0m \u001b[0msimilar_userId\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mn_similar_userIds\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-211-d3fd6f298c42>\u001b[0m in \u001b[0;36mn_most_similar\u001b[1;34m(n, userId, user_features, similarity_features, predicted_movieId)\u001b[0m\n\u001b[0;32m     20\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0msimliarity_userId\u001b[0m \u001b[1;32min\u001b[0m \u001b[0musers_to_chose_from\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"userId\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munique\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     21\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0muserId\u001b[0m\u001b[1;33m!=\u001b[0m\u001b[0msimliarity_userId\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 22\u001b[1;33m             \u001b[0msimilarity_dict\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0msimliarity_userId\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcosine_similarity\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0muser_features\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0muserId\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msimliarity_userId\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msimilarity_features\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     23\u001b[0m             \u001b[1;31m#print simliarity_userId, similarity_dict[simliarity_userId]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m     \u001b[0msorted_dict\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msorted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msimilarity_dict\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moperator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitemgetter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-211-d3fd6f298c42>\u001b[0m in \u001b[0;36mcosine_similarity\u001b[1;34m(user_features, userId1, userId2, similarity_features)\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mN_MOST_SIMILAR\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mcosine_similarity\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0muser_features\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0muserId1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0muserId2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msimilarity_features\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m     \u001b[0muser1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0muser_features\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0muser_features\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"userId\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0muserId1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0msimilarity_features\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m     \u001b[0muser2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0muser_features\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0muser_features\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"userId\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0muserId2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0msimilarity_features\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m     \u001b[0msim\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0muser1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0muser2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mT\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlinalg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0muser1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlinalg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0muser2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/adif/anaconda2/lib/python2.7/site-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   1961\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mSeries\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mIndex\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1962\u001b[0m             \u001b[1;31m# either boolean or fancy integer index\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1963\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1964\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1965\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_frame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/adif/anaconda2/lib/python2.7/site-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m_getitem_array\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   2003\u001b[0m             \u001b[0mkey\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck_bool_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2004\u001b[0m             \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnonzero\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2005\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtake\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2006\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2007\u001b[0m             \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mix\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_convert_to_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/adif/anaconda2/lib/python2.7/site-packages/pandas/core/generic.pyc\u001b[0m in \u001b[0;36mtake\u001b[1;34m(self, indices, axis, convert, is_copy)\u001b[0m\n\u001b[0;32m   1369\u001b[0m         new_data = self._data.take(indices,\n\u001b[0;32m   1370\u001b[0m                                    \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_block_manager_axis\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1371\u001b[1;33m                                    convert=True, verify=True)\n\u001b[0m\u001b[0;32m   1372\u001b[0m         \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_constructor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__finalize__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1373\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/adif/anaconda2/lib/python2.7/site-packages/pandas/core/internals.pyc\u001b[0m in \u001b[0;36mtake\u001b[1;34m(self, indexer, axis, verify, convert)\u001b[0m\n\u001b[0;32m   3617\u001b[0m         \u001b[0mn\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3618\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mconvert\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3619\u001b[1;33m             \u001b[0mindexer\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmaybe_convert_indices\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3620\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3621\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mverify\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/adif/anaconda2/lib/python2.7/site-packages/pandas/core/indexing.pyc\u001b[0m in \u001b[0;36mmaybe_convert_indices\u001b[1;34m(indices, n)\u001b[0m\n\u001b[0;32m   1743\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mint_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1744\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1745\u001b[1;33m     \u001b[0mmask\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mindices\u001b[0m \u001b[1;33m<\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1746\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mmask\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0many\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1747\u001b[0m         \u001b[0mindices\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mmask\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mn\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for n in range(10,15):\n",
    "    ratings_pred.append([])\n",
    "   \n",
    "\n",
    "for n in range(10,15):\n",
    "    for dummy_train_indices,dfratings_test_indices in dfratings_shuffle_indices:\n",
    "        for tested_userId in tested_userIds:\n",
    "            user_ratings = dfratings[dfratings[\"userId\"]==tested_userId]\n",
    "            user_ratings.reset_index(inplace=True)\n",
    "            user_ratings_shuffle_indices = cross_validation.ShuffleSplit(len(user_ratings), n_iter=1, test_size=0.1, random_state=random_seed)\n",
    "            for dummy_train_indices,user_ratings_test_indices in user_ratings_shuffle_indices:\n",
    "                predicted_movieIds = (user_ratings[\"movieId\"])[user_ratings_test_indices]\n",
    "                for predicted_movieId in predicted_movieIds:\n",
    "                    n_similar_userIds = n_most_similar(n,tested_userId,user_features,similarity_features,predicted_movieId)\n",
    "                    similar_ratings = []\n",
    "                    for similar_userId in n_similar_userIds:\n",
    "                        similar_user_ratings = dfratings[dfratings[\"userId\"]==similar_userId]\n",
    "                        similar_movie_rating = similar_user_ratings[similar_user_ratings[\"movieId\"] == predicted_movieId][\"rating\"]\n",
    "                        similar_ratings.append(similar_movie_rating.item())\n",
    "                    user_rating = dfratings[dfratings[\"userId\"]==tested_userId]\n",
    "                    user_movie_rating = user_rating[user_rating[\"movieId\"] == predicted_movieId][\"rating\"]                    \n",
    "                    predict = float(sum(similar_ratings))/float(len(similar_ratings))\n",
    "                    ratings_pred[n-1].append(predict)\n",
    "                    #ratings_real[n-1].append(user_movie_rating.item())\n",
    "        with open(my_path[0]+'/ratings_pred{0}.pik'.format(n), 'wb') as handle:\n",
    "            pickle.dump(ratings_pred, handle, protocol=pickle.HIGHEST_PROTOCOL)                    \n",
    "        #with open(my_path[0]+'/ratings_real{0}.pik'.format(n), 'wb') as handle:\n",
    "            p#ickle.dump(ratings_real, handle, protocol=pickle.HIGHEST_PROTOCOL)                                            \n",
    "        print n,mean_squared_error(ratings_pred[n-1],ratings_real[0]),r2_score(ratings_pred[n-1],ratings_real[0])\n",
    "      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.83116883117\n"
     ]
    }
   ],
   "source": [
    "ratings = dfratings[\"rating\"][dfratings_test_indices]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading ratings data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anat/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:3: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators; you can avoid this warning by specifying engine='python'.\n",
      "  app.launch_new_instance()\n"
     ]
    }
   ],
   "source": [
    "#Reading the ratings data\n",
    "ratings_filename = \"/ml-1m/ratings.dat\"\n",
    "dfratings = pd.read_csv(my_path[0]+ratings_filename, index_col=None, sep='::', header=None)\n",
    "dfratings.columns = [\"userId\", \"movieId\", \"rating\", \"timestamp\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating genres vector for each movie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Disable false positive Pandas warning for 'SettingWithCopyWarning'\n",
    "pd.options.mode.chained_assignment = None\n",
    "\n",
    "#Adding genres vector to each movie\n",
    "all_genres = ['Mystery', 'Sci-Fi', 'Crime', 'Drama', 'Animation', 'IMAX', 'Action', 'Comedy', 'Documentary', 'War', 'Romance', 'Horror', \n",
    "              'Film-Noir', 'Musical', 'Fantasy', 'Adventure', 'Children', 'Thriller', 'Western']\n",
    "for genre in all_genres:\n",
    "    dfmovies[genre] = dfmovies['genres'].apply(lambda x: x.find(genre)>=0).astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create users genre preference vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n",
      "5000\n",
      "6000\n"
     ]
    }
   ],
   "source": [
    "users = (dfratings[\"userId\"]).unique()\n",
    "genres_pref = []\n",
    "for userId in users:\n",
    "    user_ratings = dfratings[dfratings[\"userId\"] == userId]\n",
    "    movies_total = 0\n",
    "    geners_rating_score = [0] * len(all_genres)\n",
    "    for index, rating in user_ratings.iterrows():\n",
    "        movieId = rating[\"movieId\"]\n",
    "        score = rating[\"rating\"]\n",
    "        movie_data = dfmovies.loc[dfmovies[\"movieId\"] == movieId]\n",
    "        for genre in all_genres:\n",
    "            if (movie_data[genre].item() == 1):\n",
    "                geners_rating_score[all_genres.index(genre)] += score\n",
    "    genres_pref.append(geners_rating_score)\n",
    "    if (userId % 1000 == 0):\n",
    "        print userId"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Save the genres prefrence to dataframe and to pickle\n",
    "user_pref_df = pd.DataFrame(genres_pref)\n",
    "user_pref_df.columns = all_genres\n",
    "with open(my_path[0]+'/explore/user_genre_pref.pik', 'wb') as handle:\n",
    "    pickle.dump(user_pref_df, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open(my_path[0]+'/explore/user_genre_pref.pik', 'rb') as handle:\n",
    "    user_pref_df = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unify all the users features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading the users data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anat/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:2: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators; you can avoid this warning by specifying engine='python'.\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    }
   ],
   "source": [
    "users_filename = \"/ml-1m/users.dat\"\n",
    "dfusers = pd.read_csv(my_path[0]+users_filename, sep='::', header=None)\n",
    "dfusers.columns = [\"userId\",\"gender\", \"age\", \"occupation\", \"zip-code\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Extracting gender, age and occupation features\n",
    "del dfusers[\"zip-code\"]\n",
    "user_features = dfusers.join(user_pref_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Calculating number of ratings and average rating per user\n",
    "users = (dfratings[\"userId\"]).unique()\n",
    "user_rate_num = []\n",
    "user_rate_avg = []\n",
    "user_rate_std = []\n",
    "for userId in users:\n",
    "    user_ratings = dfratings[dfratings[\"userId\"] == userId]\n",
    "    user_rate_num.append(user_ratings.shape[0])\n",
    "    user_rate_avg.append(np.average(user_ratings[\"rating\"]))\n",
    "    user_rate_std.append(np.std(user_ratings[\"rating\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "user_features[\"rate_num\"] = user_rate_num\n",
    "user_features[\"rate_avg\"] = user_rate_avg\n",
    "user_features[\"rate_std\"] = user_rate_std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open(my_path[0]+'/features/user_features.pik', 'wb') as handle:\n",
    "    pickle.dump(user_features, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(my_path[0]+'/features/user_features.pik', 'rb') as handle:\n",
    "    user_features = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Calculate movies features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anat/anaconda2/lib/python2.7/site-packages/numpy/core/_methods.py:59: RuntimeWarning: Mean of empty slice.\n",
      "  warnings.warn(\"Mean of empty slice.\", RuntimeWarning)\n"
     ]
    }
   ],
   "source": [
    "#Calculate average rating and number of ratings\n",
    "all_movies = dfmovies[\"movieId\"]\n",
    "avg_rate = []\n",
    "num_rate = []\n",
    "std_rate = []\n",
    "for movie in all_movies:\n",
    "    movie_data = dfratings[dfratings[\"movieId\"] == movie]\n",
    "    num_rate.append(movie_data.shape[0])\n",
    "    avg_rate.append(np.average(movie_data[\"rating\"]))\n",
    "    std_rate.append(np.std(movie_data[\"rating\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Removing movies with 0 reviews\n",
    "movies_features = dfmovies.copy(deep=True)\n",
    "exclude_movies = []\n",
    "for i in sorted(all_movies.index, reverse=True):\n",
    "    if (num_rate[i] == 0):\n",
    "        exclude_movies.append(all_movies[i])\n",
    "        del avg_rate[i]\n",
    "        del num_rate[i]\n",
    "        movies_features.drop(movies_features.index[i], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unify all movies features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "del movies_features[\"genres\"]\n",
    "movies_features[\"avg_rate\"] = avg_rate\n",
    "movies_features[\"num_rate\"] = num_rate\n",
    "movies_features[\"std_rate\"] = std_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(my_path[0]+'/features/movie_features.pik', 'wb') as handle:\n",
    "    pickle.dump(movies_features, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ShuffleSplit(6040, n_iter=1, test_size=0.01, random_state=42)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_validation.ShuffleSplit(len(user_features), n_iter=1, test_size=.01, random_state=random_seed)\n",
    "user_features[\"userId\"][userId_shuffle_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "def set_global_similarity_dict(userId,user_features,similarity_features):\n",
    "    #print user_features[user_features[\"userId\"]==5530]\n",
    "    #print user_features[user_features[\"userId\"]==2106]\n",
    "    similarity_dict=dict()\n",
    "    users_to_chose_from = user_features\n",
    "    for simliarity_userId in users_to_chose_from[\"userId\"].unique():\n",
    "        if userId!=simliarity_userId:\n",
    "            similarity_dict[simliarity_userId] = cosine_similarity(user_features,userId,simliarity_userId,similarity_features)\n",
    "    sorted_dict = sorted(similarity_dict.items(), key=operator.itemgetter(1)) \n",
    "    sorted_dict.reverse()\n",
    "    ids =[]\n",
    "    for items in sorted_dict:\n",
    "        ids.append(items[0])\n",
    "    global_similarity_dict[userId] = ids\n",
    "tested_userIds = [457,2768,1980,402,4126,3418,1112,3119,1737,1322]\n",
    "for tested_userId in tested_userIds:\n",
    "    set_global_similarity_dict(tested_userId,user_features,similarity_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def n_most_similar_global(n,userId,user_features,similarity_features,predicted_movieId=None):\n",
    "    #print user_features[user_features[\"userId\"]==5530]\n",
    "    #print user_features[user_features[\"userId\"]==2106]\n",
    "    all_similar_ids = global_similarity_dict[userId]\n",
    "    users_to_chose_from = user_features\n",
    "    if predicted_movieId!=None:\n",
    "        users_to_chose_from = dfratings[dfratings[\"movieId\"]==predicted_movieId]\n",
    "    valid_user_ids = users_to_chose_from[\"userId\"].unique()        \n",
    "    ids =[]\n",
    "    i=0\n",
    "    for similar_id in all_similar_ids:\n",
    "        if similar_id in valid_user_ids:\n",
    "            i+=1\n",
    "            if i>n:\n",
    "                break\n",
    "            ids.append(similar_id)\n",
    "    return ids\n",
    "ratings_pred = []\n",
    "for n in [64,128,256,512]:\n",
    "    while n >= len(ratings_pred):\n",
    "        ratings_pred.append([])\n",
    "    for tested_userId in tested_userIds:\n",
    "            user_ratings = dfratings[dfratings[\"userId\"]==tested_userId]\n",
    "            user_ratings.reset_index(inplace=True)\n",
    "            \n",
    "            user_ratings_shuffle_indices = cross_validation.ShuffleSplit(len(user_ratings), n_iter=10, test_size=0.1, random_state=random_seed)\n",
    "            ratings_pred[n-1]=[]\n",
    "            ratings_real = []\n",
    "            for dummy_train_indices,user_ratings_test_indices in user_ratings_shuffle_indices:\n",
    "                predicted_movieIds = (user_ratings[\"movieId\"])[user_ratings_test_indices]\n",
    "                for predicted_movieId in predicted_movieIds:\n",
    "                    n_similar_userIds = n_most_similar_global(n,tested_userId,user_features,similarity_features,predicted_movieId)\n",
    "                    similar_ratings = []\n",
    "                    for similar_userId in n_similar_userIds:\n",
    "                        similar_user_ratings = dfratings[dfratings[\"userId\"]==similar_userId]\n",
    "                        similar_movie_rating = similar_user_ratings[similar_user_ratings[\"movieId\"] == predicted_movieId][\"rating\"]\n",
    "                        similar_ratings.append(similar_movie_rating.item())\n",
    "                    user_rating = dfratings[dfratings[\"userId\"]==tested_userId]\n",
    "                    user_movie_rating = user_rating[user_rating[\"movieId\"] == predicted_movieId][\"rating\"]\n",
    "                    predict = float(sum(similar_ratings))/float(len(similar_ratings))\n",
    "                    ratings_pred[n-1].append(predict)\n",
    "                    ratings_real.append(user_movie_rating.item())\n",
    "            with open(my_path[0]+'/ratings_pred{0}.pik'.format(n), 'wb') as handle:\n",
    "                pickle.dump(ratings_pred, handle, protocol=pickle.HIGHEST_PROTOCOL)                    \n",
    "                print n,tested_userId,len(ratings_pred[n-1]),mean_squared_error(ratings_pred[n-1],ratings_real),r2_score(ratings_pred[n-1],ratings_real)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}