Feature_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "import fileinput\n",
    "\n",
    "#Fit the code to the screen\n",
    "from IPython.core.display import HTML\n",
    "HTML(\"<style>.container { width:100% !important; }</style>\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading movies data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anat/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:4: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators; you can avoid this warning by specifying engine='python'.\n"
     ]
    }
   ],
   "source": [
    "#Reading the movies data\n",
    "my_path = !pwd\n",
    "movies_filename = \"/ml-1m/movies.dat\"\n",
    "dfmovies = pd.read_csv(my_path[0]+movies_filename, sep='::', header=None)\n",
    "dfmovies.columns = [\"movieId\",\"title\", \"genres\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Extract the year from the title\n",
    "dfmovies[\"year\"] = dfmovies['title'].apply(lambda x: x[x.rfind(\"(\")+1:x.rfind(\")\")])\n",
    "dfmovies[\"name\"] = dfmovies['title'].apply(lambda x: x[:x.rfind(\"(\")-1])\n",
    "del dfmovies['title']\n",
    "#Get the columns to the original order\n",
    "cols = dfmovies.columns.tolist()\n",
    "cols = cols[:1] + cols[-1:] + cols[-2:-1] + cols[1:2]\n",
    "dfmovies = dfmovies[cols]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading ratings data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anat/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:3: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators; you can avoid this warning by specifying engine='python'.\n",
      "  app.launch_new_instance()\n"
     ]
    }
   ],
   "source": [
    "#Reading the ratings data\n",
    "ratings_filename = \"/ml-1m/ratings.dat\"\n",
    "dfratings = pd.read_csv(my_path[0]+ratings_filename, index_col=None, sep='::', header=None)\n",
    "dfratings.columns = [\"userId\", \"movieId\", \"rating\", \"timestamp\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating genres vector for each movie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Disable false positive Pandas warning for 'SettingWithCopyWarning'\n",
    "pd.options.mode.chained_assignment = None\n",
    "\n",
    "#Adding genres vector to each movie\n",
    "all_genres = ['Mystery', 'Sci-Fi', 'Crime', 'Drama', 'Animation', 'IMAX', 'Action', 'Comedy', 'Documentary', 'War', 'Romance', 'Horror', \n",
    "              'Film-Noir', 'Musical', 'Fantasy', 'Adventure', 'Children', 'Thriller', 'Western']\n",
    "for genre in all_genres:\n",
    "    dfmovies[genre] = dfmovies['genres'].apply(lambda x: x.find(genre)>=0).astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create users genre preference vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n",
      "5000\n",
      "6000\n"
     ]
    }
   ],
   "source": [
    "users = (dfratings[\"userId\"]).unique()\n",
    "genres_pref = []\n",
    "for userId in users:\n",
    "    user_ratings = dfratings[dfratings[\"userId\"] == userId]\n",
    "    movies_total = 0\n",
    "    geners_rating_score = [0] * len(all_genres)\n",
    "    for index, rating in user_ratings.iterrows():\n",
    "        movieId = rating[\"movieId\"]\n",
    "        score = rating[\"rating\"]\n",
    "        movie_data = dfmovies.loc[dfmovies[\"movieId\"] == movieId]\n",
    "        for genre in all_genres:\n",
    "            if (movie_data[genre].item() == 1):\n",
    "                geners_rating_score[all_genres.index(genre)] += score\n",
    "    genres_pref.append(geners_rating_score)\n",
    "    if (userId % 1000 == 0):\n",
    "        print userId"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Save the genres prefrence to dataframe and to pickle\n",
    "user_pref_df = pd.DataFrame(genres_pref)\n",
    "user_pref_df.columns = all_genres\n",
    "with open(my_path[0]+'/explore/user_genre_pref.pik', 'wb') as handle:\n",
    "    pickle.dump(user_pref_df, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open(my_path[0]+'/explore/user_genre_pref.pik', 'rb') as handle:\n",
    "    user_pref_df = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unify all the users features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading the users data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anat/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:2: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators; you can avoid this warning by specifying engine='python'.\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    }
   ],
   "source": [
    "users_filename = \"/ml-1m/users.dat\"\n",
    "dfusers = pd.read_csv(my_path[0]+users_filename, sep='::', header=None)\n",
    "dfusers.columns = [\"userId\",\"gender\", \"age\", \"occupation\", \"zip-code\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Extracting gender, age and occupation features\n",
    "del dfusers[\"zip-code\"]\n",
    "user_features = dfusers.join(user_pref_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Calculating number of ratings and average rating per user\n",
    "users = (dfratings[\"userId\"]).unique()\n",
    "user_rate_num = []\n",
    "user_rate_avg = []\n",
    "user_rate_std = []\n",
    "for userId in users:\n",
    "    user_ratings = dfratings[dfratings[\"userId\"] == userId]\n",
    "    user_rate_num.append(user_ratings.shape[0])\n",
    "    user_rate_avg.append(np.average(user_ratings[\"rating\"]))\n",
    "    user_rate_std.append(np.std(user_ratings[\"rating\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "user_features[\"rate_num\"] = user_rate_num\n",
    "user_features[\"rate_avg\"] = user_rate_avg\n",
    "user_features[\"rate_std\"] = user_rate_std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open(my_path[0]+'/features/user_features.pik', 'wb') as handle:\n",
    "    pickle.dump(user_features, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(my_path[0]+'/features/user_features.pik', 'rb') as handle:\n",
    "    user_features = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Calculate movies features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anat/anaconda2/lib/python2.7/site-packages/numpy/core/_methods.py:59: RuntimeWarning: Mean of empty slice.\n",
      "  warnings.warn(\"Mean of empty slice.\", RuntimeWarning)\n"
     ]
    }
   ],
   "source": [
    "#Calculate average rating and number of ratings\n",
    "all_movies = dfmovies[\"movieId\"]\n",
    "avg_rate = []\n",
    "num_rate = []\n",
    "std_rate = []\n",
    "for movie in all_movies:\n",
    "    movie_data = dfratings[dfratings[\"movieId\"] == movie]\n",
    "    num_rate.append(movie_data.shape[0])\n",
    "    avg_rate.append(np.average(movie_data[\"rating\"]))\n",
    "    std_rate.append(np.std(movie_data[\"rating\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Removing movies with 0 reviews\n",
    "movies_features = dfmovies.copy(deep=True)\n",
    "exclude_movies = []\n",
    "for i in sorted(all_movies.index, reverse=True):\n",
    "    if (num_rate[i] == 0):\n",
    "        exclude_movies.append(all_movies[i])\n",
    "        del avg_rate[i]\n",
    "        del num_rate[i]\n",
    "        movies_features.drop(movies_features.index[i], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Unify all movies features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "del movies_features[\"genres\"]\n",
    "movies_features[\"avg_rate\"] = avg_rate\n",
    "movies_features[\"num_rate\"] = num_rate\n",
    "movies_features[\"std_rate\"] = std_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(my_path[0]+'/features/movie_features.pik', 'wb') as handle:\n",
    "    pickle.dump(movies_features, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open(my_path[0]+'/features/movie_features.pik', 'rb') as handle:\n",
    "    movies_features = pickle.load(handle)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}