nba-data-pull.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "import numpy as np\n",
    "import yaml\n",
    "import json\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## NBA Data Pull"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "outputs": [],
   "source": [
    "r = requests.get('https://www.balldontlie.io/api/v1/players', params={'per_page': 100})\n",
    "j = json.loads(r.content.decode())\n",
    "pd.DataFrame(j['data'])\n",
    "\n",
    "r_list = [requests.get('https://www.balldontlie.io/api/v1/players', params={'per_page': 100, 'page': n}) for n in range(0, j['meta']['total_pages'])]\n",
    "\n",
    "df_players = pd.concat([pd.json_normalize(json.loads(m.content.decode())['data']) for m in r_list])\n",
    "df_players = df_players.sort_values('id').drop_duplicates().set_index('id')\n",
    "\n",
    "df_players.to_csv('player_list.csv')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "outputs": [],
   "source": [
    "r = requests.get('https://www.balldontlie.io/api/v1/teams', params={'per_page': 100})\n",
    "j = json.loads(r.content.decode())\n",
    "pd.DataFrame(j['data'])\n",
    "\n",
    "r_list = [requests.get('https://www.balldontlie.io/api/v1/teams', params={'per_page': 100, 'page': n}) for n in range(0, j['meta']['total_pages'])]\n",
    "\n",
    "df_teams = pd.concat([pd.json_normalize(json.loads(m.content.decode())['data']) for m in r_list])\n",
    "df_teams = df_teams.sort_values('id').drop_duplicates().set_index('id')\n",
    "\n",
    "df_teams.to_csv('team_list.csv')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "outputs": [],
   "source": [
    "r = requests.get('https://www.balldontlie.io/api/v1/games', params={'per_page': 100})\n",
    "j = json.loads(r.content.decode())\n",
    "pd.DataFrame(j['data'])\n",
    "\n",
    "#rate limit hecks you up here if you don't pace urself\n",
    "\n",
    "r_list = []\n",
    "\n",
    "for i in range(0, j['meta']['total_pages']):\n",
    "    time.sleep(1)\n",
    "    r_list.append(\n",
    "        requests.get('https://www.balldontlie.io/api/v1/games',\n",
    "                     params={'per_page': 100, 'page': i}\n",
    "                     )\n",
    "    )\n",
    "\n",
    "df_games = pd.concat([pd.json_normalize(json.loads(m.content.decode())['data']) for m in r_list])\n",
    "df_games = df_games.sort_values('id').drop_duplicates().set_index('id')\n",
    "\n",
    "df_games.to_csv('game_list.csv')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "outputs": [
    {
     "data": {
      "text/plain": "{'total_pages': 11824,\n 'current_page': 1,\n 'next_page': 2,\n 'per_page': 100,\n 'total_count': 1182335}"
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = requests.get('https://www.balldontlie.io/api/v1/stats', params={'per_page': 100})\n",
    "j = json.loads(r.content.decode())\n",
    "pd.json_normalize(j['data'])\n",
    "j['meta']\n",
    "#rate limit hecks you up here if you don't pace urself"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### yes we are actually unironically scraping everything all the way back to 1979. Sorry @hi_im_danny"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "outputs": [],
   "source": [
    "r_list = []\n",
    "\n",
    "for i in range(1, j['meta']['total_pages']):\n",
    "    time.sleep(1)\n",
    "    r = requests.get('https://www.balldontlie.io/api/v1/stats',\n",
    "                     params={'per_page': 100, 'page': i}\n",
    "                     )\n",
    "    r_list.append(r)\n",
    "    with open('stats_json/'+ str(i) + \".json\", \"w\") as file:\n",
    "        file.write(r.content.decode())\n",
    "\n",
    "df_stats = pd.concat([pd.json_normalize(json.loads(m.content.decode())['data']) for m in r_list])\n",
    "df_stats = df_stats.sort_values('id').drop_duplicates().set_index('id')\n",
    "\n",
    "df_stats.to_csv('stats_list.csv')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "Cell \u001B[0;32mIn [93], line 9\u001B[0m\n\u001B[1;32m      6\u001B[0m         r \u001B[38;5;241m=\u001B[39m file\u001B[38;5;241m.\u001B[39mread()\n\u001B[1;32m      7\u001B[0m         c_list\u001B[38;5;241m.\u001B[39mappend(r)\n\u001B[0;32m----> 9\u001B[0m df_stats \u001B[38;5;241m=\u001B[39m pd\u001B[38;5;241m.\u001B[39mconcat([df_stats, pd\u001B[38;5;241m.\u001B[39mconcat([pd\u001B[38;5;241m.\u001B[39mjson_normalize(json\u001B[38;5;241m.\u001B[39mloads(m)[\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mdata\u001B[39m\u001B[38;5;124m'\u001B[39m]) \u001B[38;5;28;01mfor\u001B[39;00m m \u001B[38;5;129;01min\u001B[39;00m c_list])])\n",
      "Cell \u001B[0;32mIn [93], line 9\u001B[0m, in \u001B[0;36m<listcomp>\u001B[0;34m(.0)\u001B[0m\n\u001B[1;32m      6\u001B[0m         r \u001B[38;5;241m=\u001B[39m file\u001B[38;5;241m.\u001B[39mread()\n\u001B[1;32m      7\u001B[0m         c_list\u001B[38;5;241m.\u001B[39mappend(r)\n\u001B[0;32m----> 9\u001B[0m df_stats \u001B[38;5;241m=\u001B[39m pd\u001B[38;5;241m.\u001B[39mconcat([df_stats, pd\u001B[38;5;241m.\u001B[39mconcat([\u001B[43mpd\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mjson_normalize\u001B[49m\u001B[43m(\u001B[49m\u001B[43mjson\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mloads\u001B[49m\u001B[43m(\u001B[49m\u001B[43mm\u001B[49m\u001B[43m)\u001B[49m\u001B[43m[\u001B[49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mdata\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m]\u001B[49m\u001B[43m)\u001B[49m \u001B[38;5;28;01mfor\u001B[39;00m m \u001B[38;5;129;01min\u001B[39;00m c_list])])\n",
      "File \u001B[0;32m~/miniforge3/envs/basketball/lib/python3.10/site-packages/pandas/io/json/_normalize.py:458\u001B[0m, in \u001B[0;36m_json_normalize\u001B[0;34m(data, record_path, meta, meta_prefix, record_prefix, errors, sep, max_level)\u001B[0m\n\u001B[1;32m    448\u001B[0m \u001B[38;5;66;03m# check to see if a simple recursive function is possible to\u001B[39;00m\n\u001B[1;32m    449\u001B[0m \u001B[38;5;66;03m# improve performance (see #15621) but only for cases such\u001B[39;00m\n\u001B[1;32m    450\u001B[0m \u001B[38;5;66;03m# as pd.Dataframe(data) or pd.Dataframe(data, sep)\u001B[39;00m\n\u001B[1;32m    451\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m (\n\u001B[1;32m    452\u001B[0m     record_path \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[1;32m    453\u001B[0m     \u001B[38;5;129;01mand\u001B[39;00m meta \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m    456\u001B[0m     \u001B[38;5;129;01mand\u001B[39;00m max_level \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[1;32m    457\u001B[0m ):\n\u001B[0;32m--> 458\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mDataFrame\u001B[49m\u001B[43m(\u001B[49m\u001B[43m_simple_json_normalize\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdata\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43msep\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msep\u001B[49m\u001B[43m)\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    460\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m record_path \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m    461\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28many\u001B[39m([\u001B[38;5;28misinstance\u001B[39m(x, \u001B[38;5;28mdict\u001B[39m) \u001B[38;5;28;01mfor\u001B[39;00m x \u001B[38;5;129;01min\u001B[39;00m y\u001B[38;5;241m.\u001B[39mvalues()] \u001B[38;5;28;01mfor\u001B[39;00m y \u001B[38;5;129;01min\u001B[39;00m data):\n\u001B[1;32m    462\u001B[0m         \u001B[38;5;66;03m# naive normalization, this is idempotent for flat records\u001B[39;00m\n\u001B[1;32m    463\u001B[0m         \u001B[38;5;66;03m# and potentially will inflate the data considerably for\u001B[39;00m\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m    467\u001B[0m         \u001B[38;5;66;03m# TODO: handle record value which are lists, at least error\u001B[39;00m\n\u001B[1;32m    468\u001B[0m         \u001B[38;5;66;03m#       reasonably\u001B[39;00m\n",
      "File \u001B[0;32m~/miniforge3/envs/basketball/lib/python3.10/site-packages/pandas/core/frame.py:753\u001B[0m, in \u001B[0;36mDataFrame.__init__\u001B[0;34m(self, data, index, columns, dtype, copy)\u001B[0m\n\u001B[1;32m    744\u001B[0m         columns \u001B[38;5;241m=\u001B[39m ensure_index(columns)\n\u001B[1;32m    745\u001B[0m     arrays, columns, index \u001B[38;5;241m=\u001B[39m nested_data_to_arrays(\n\u001B[1;32m    746\u001B[0m         \u001B[38;5;66;03m# error: Argument 3 to \"nested_data_to_arrays\" has incompatible\u001B[39;00m\n\u001B[1;32m    747\u001B[0m         \u001B[38;5;66;03m# type \"Optional[Collection[Any]]\"; expected \"Optional[Index]\"\u001B[39;00m\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m    751\u001B[0m         dtype,\n\u001B[1;32m    752\u001B[0m     )\n\u001B[0;32m--> 753\u001B[0m     mgr \u001B[38;5;241m=\u001B[39m \u001B[43marrays_to_mgr\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m    754\u001B[0m \u001B[43m        \u001B[49m\u001B[43marrays\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    755\u001B[0m \u001B[43m        \u001B[49m\u001B[43mcolumns\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    756\u001B[0m \u001B[43m        \u001B[49m\u001B[43mindex\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    757\u001B[0m \u001B[43m        \u001B[49m\u001B[43mdtype\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mdtype\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    758\u001B[0m \u001B[43m        \u001B[49m\u001B[43mtyp\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmanager\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m    759\u001B[0m \u001B[43m    \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    760\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m    761\u001B[0m     mgr \u001B[38;5;241m=\u001B[39m ndarray_to_mgr(\n\u001B[1;32m    762\u001B[0m         data,\n\u001B[1;32m    763\u001B[0m         index,\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m    767\u001B[0m         typ\u001B[38;5;241m=\u001B[39mmanager,\n\u001B[1;32m    768\u001B[0m     )\n",
      "File \u001B[0;32m~/miniforge3/envs/basketball/lib/python3.10/site-packages/pandas/core/internals/construction.py:124\u001B[0m, in \u001B[0;36marrays_to_mgr\u001B[0;34m(arrays, columns, index, dtype, verify_integrity, typ, consolidate)\u001B[0m\n\u001B[1;32m    121\u001B[0m         index \u001B[38;5;241m=\u001B[39m ensure_index(index)\n\u001B[1;32m    123\u001B[0m     \u001B[38;5;66;03m# don't force copy because getting jammed in an ndarray anyway\u001B[39;00m\n\u001B[0;32m--> 124\u001B[0m     arrays \u001B[38;5;241m=\u001B[39m \u001B[43m_homogenize\u001B[49m\u001B[43m(\u001B[49m\u001B[43marrays\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mindex\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdtype\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    125\u001B[0m     \u001B[38;5;66;03m# _homogenize ensures\u001B[39;00m\n\u001B[1;32m    126\u001B[0m     \u001B[38;5;66;03m#  - all(len(x) == len(index) for x in arrays)\u001B[39;00m\n\u001B[1;32m    127\u001B[0m     \u001B[38;5;66;03m#  - all(x.ndim == 1 for x in arrays)\u001B[39;00m\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m    130\u001B[0m \n\u001B[1;32m    131\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m    132\u001B[0m     index \u001B[38;5;241m=\u001B[39m ensure_index(index)\n",
      "File \u001B[0;32m~/miniforge3/envs/basketball/lib/python3.10/site-packages/pandas/core/internals/construction.py:618\u001B[0m, in \u001B[0;36m_homogenize\u001B[0;34m(data, index, dtype)\u001B[0m\n\u001B[1;32m    615\u001B[0m             val \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mdict\u001B[39m(val)\n\u001B[1;32m    616\u001B[0m         val \u001B[38;5;241m=\u001B[39m lib\u001B[38;5;241m.\u001B[39mfast_multiget(val, oindex\u001B[38;5;241m.\u001B[39m_values, default\u001B[38;5;241m=\u001B[39mnp\u001B[38;5;241m.\u001B[39mnan)\n\u001B[0;32m--> 618\u001B[0m     val \u001B[38;5;241m=\u001B[39m \u001B[43msanitize_array\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m    619\u001B[0m \u001B[43m        \u001B[49m\u001B[43mval\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mindex\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdtype\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mdtype\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcopy\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mraise_cast_failure\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\n\u001B[1;32m    620\u001B[0m \u001B[43m    \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    621\u001B[0m     com\u001B[38;5;241m.\u001B[39mrequire_length_match(val, index)\n\u001B[1;32m    623\u001B[0m homogenized\u001B[38;5;241m.\u001B[39mappend(val)\n",
      "File \u001B[0;32m~/miniforge3/envs/basketball/lib/python3.10/site-packages/pandas/core/construction.py:598\u001B[0m, in \u001B[0;36msanitize_array\u001B[0;34m(data, index, dtype, copy, raise_cast_failure, allow_2d)\u001B[0m\n\u001B[1;32m    595\u001B[0m             subarr \u001B[38;5;241m=\u001B[39m np\u001B[38;5;241m.\u001B[39marray(data, copy\u001B[38;5;241m=\u001B[39mcopy)\n\u001B[1;32m    596\u001B[0m     \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m    597\u001B[0m         \u001B[38;5;66;03m# we will try to copy by-definition here\u001B[39;00m\n\u001B[0;32m--> 598\u001B[0m         subarr \u001B[38;5;241m=\u001B[39m \u001B[43m_try_cast\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdata\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdtype\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mcopy\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mraise_cast_failure\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    600\u001B[0m \u001B[38;5;28;01melif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(data, ABCExtensionArray):\n\u001B[1;32m    601\u001B[0m     \u001B[38;5;66;03m# it is already ensured above this is not a PandasArray\u001B[39;00m\n\u001B[1;32m    602\u001B[0m     subarr \u001B[38;5;241m=\u001B[39m data\n",
      "File \u001B[0;32m~/miniforge3/envs/basketball/lib/python3.10/site-packages/pandas/core/construction.py:778\u001B[0m, in \u001B[0;36m_try_cast\u001B[0;34m(arr, dtype, copy, raise_cast_failure)\u001B[0m\n\u001B[1;32m    775\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m arr\u001B[38;5;241m.\u001B[39mdtype \u001B[38;5;241m!=\u001B[39m \u001B[38;5;28mobject\u001B[39m:\n\u001B[1;32m    776\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m sanitize_to_nanoseconds(arr, copy\u001B[38;5;241m=\u001B[39mcopy)\n\u001B[0;32m--> 778\u001B[0m out \u001B[38;5;241m=\u001B[39m \u001B[43mmaybe_infer_to_datetimelike\u001B[49m\u001B[43m(\u001B[49m\u001B[43marr\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    779\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m out \u001B[38;5;129;01mis\u001B[39;00m arr \u001B[38;5;129;01mand\u001B[39;00m copy:\n\u001B[1;32m    780\u001B[0m     out \u001B[38;5;241m=\u001B[39m out\u001B[38;5;241m.\u001B[39mcopy()\n",
      "File \u001B[0;32m~/miniforge3/envs/basketball/lib/python3.10/site-packages/pandas/core/dtypes/cast.py:1244\u001B[0m, in \u001B[0;36mmaybe_infer_to_datetimelike\u001B[0;34m(value)\u001B[0m\n\u001B[1;32m   1241\u001B[0m     \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m   1242\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m td_values\u001B[38;5;241m.\u001B[39mreshape(shape)\n\u001B[0;32m-> 1244\u001B[0m inferred_type, seen_str \u001B[38;5;241m=\u001B[39m \u001B[43mlib\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43minfer_datetimelike_array\u001B[49m\u001B[43m(\u001B[49m\u001B[43mensure_object\u001B[49m\u001B[43m(\u001B[49m\u001B[43mv\u001B[49m\u001B[43m)\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1245\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m inferred_type \u001B[38;5;129;01min\u001B[39;00m [\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mperiod\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124minterval\u001B[39m\u001B[38;5;124m\"\u001B[39m]:\n\u001B[1;32m   1246\u001B[0m     \u001B[38;5;66;03m# Incompatible return value type (got \"Union[ExtensionArray, ndarray]\",\u001B[39;00m\n\u001B[1;32m   1247\u001B[0m     \u001B[38;5;66;03m# expected \"Union[ndarray, DatetimeArray, TimedeltaArray, PeriodArray,\u001B[39;00m\n\u001B[1;32m   1248\u001B[0m     \u001B[38;5;66;03m# IntervalArray]\")\u001B[39;00m\n\u001B[1;32m   1249\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m lib\u001B[38;5;241m.\u001B[39mmaybe_convert_objects(  \u001B[38;5;66;03m# type: ignore[return-value]\u001B[39;00m\n\u001B[1;32m   1250\u001B[0m         v, convert_period\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m, convert_interval\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m\n\u001B[1;32m   1251\u001B[0m     )\n",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "source": [
    "# if you had to break up your requests, this will read in the cached data\n",
    "\n",
    "c_list=[]\n",
    "for i in range(1, 7929):\n",
    "    with open('stats_json/'+ str(i) + '.json', 'r') as file:\n",
    "        r = file.read()\n",
    "        c_list.append(r)\n",
    "\n",
    "df_stats = pd.concat([df_stats, pd.concat([pd.json_normalize(json.loads(m)['data']) for m in c_list])])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "outputs": [],
   "source": [
    "df_stats = df_stats.convert_dtypes()\n",
    "df_stats = df_stats.dropna(subset=['player.first_name'])\n",
    "df_stats = df_stats.dropna(axis=1, how='all')\n",
    "df_stats = df_stats.dropna()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "outputs": [],
   "source": [
    "df_stats.to_csv('stats_list.csv')"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}