Adding proxy support

stats/args.py

@@ -65,4 +65,10 @@ def create_parser(parser):
         action='store_true',
         help='Setting to define stdout logging level. If set, only "ok" will be printed if ran successfully. This currently only applies to refreshing a db, and not loading one.')
 
+    parser.add_argument(
+        '--proxy',
+        dest='proxy',
+        default=None,
+        help='This will set a proxy for every request. This is useful if you want to run this from one of the major cloud providers. stats.nba.com does not allow traffic from them directly. Example value: "http://127.0.0.1:8888" if you are running a local proxy on port 8888.')
+
     return parser

stats/general_requester.py

@@ -1,7 +1,4 @@
-import requests
-
-from utils import get_rowset_mapping, column_names_from_table
-from constants import headers
+from utils import get_rowset_mapping, column_names_from_table, get_request
 from db_utils import insert_many
 
 
@@ -31,7 +28,7 @@ def generate_rows(self, params):
         """
 
         # json response
-        response = requests.get(url=self.url, headers=headers, params=params).json()
+        response = get_request(self.url, params, self.settings)
 
         result_sets = response['resultSets'][0]
         rowset = result_sets['rowSet']

stats/nba_sql.py

@@ -273,6 +273,7 @@ def main(args):
     seasons = args.seasons
     skip_tables = args.skip_tables
     quiet = args.quiet
+    proxy = args.proxy
 
     if not quiet:
         print(f"Loading seasons: {seasons}.")
@@ -285,7 +286,8 @@ def main(args):
         args.database_host,
         args.batch_size,
         args.sqlite_path,
-        args.quiet)
+        args.quiet,
+        args.proxy)
 
     if default_mode_set:
         default_mode(settings, create_schema, request_gap, seasons, skip_tables)

stats/play_by_play.py

@@ -1,9 +1,8 @@
-import requests
 import urllib.parse
 
 from models import PlayByPlay
-from constants import headers
 from db_utils import insert_many
+from utils import get_request
 
 
 class PlayByPlayRequester:
@@ -30,7 +29,7 @@ def fetch_game(self, game_id):
         # Encode without safe '+', apparently the NBA likes unsafe url params.
         params_str = urllib.parse.urlencode(params, safe=':+')
 
-        response = requests.get(url=self.url, headers=headers, params=params_str).json()
+        response = get_request(self.url, params, self.settings)
 
         # pulling just the data we want
         player_info = response['resultSets'][0]['rowSet']

stats/player_game_log.py

@@ -1,12 +1,10 @@
-import requests
 import urllib.parse
 
 from db_utils import insert_many
 from utils import get_rowset_mapping, column_names_from_table, season_id_to_int
 from models import PlayerGameLog, PlayerGameLogTemp
 from game import GameEntry
 from general_requester import GenericRequester
-from constants import headers
 
 
 class PlayerGameLogRequester(GenericRequester):
@@ -106,7 +104,7 @@ def fetch_season(self, season_id):
         # Encode without safe '+', apparently the NBA likes unsafe url params.
         params_str = urllib.parse.urlencode(params, safe=':+')
 
-        response = requests.get(url=self.url, headers=headers, params=params_str).json()
+        response = get_request(self.url, params, self.settings)
 
         result_sets = response['resultSets'][0]
         rowset = result_sets['rowSet']

stats/player_general_traditional_total.py

@@ -1,10 +1,9 @@
-import requests
 import urllib.parse
 
 from utils import get_rowset_mapping, column_names_from_table, season_id_to_int
 from models import PlayerGeneralTraditionalTotal
 from general_requester import GenericRequester
-from constants import headers
+from utils import get_request
 
 
 class PlayerGeneralTraditionalTotalRequester(GenericRequester):
@@ -31,7 +30,7 @@ def generate_rows(self, season_id):
         params_str = urllib.parse.urlencode(params, safe=':+')
 
         # json response
-        response = requests.get(url=self.url, headers=headers, params=params_str).json()
+        response = get_request(self.url, params, self.settings)
 
         result_sets = response['resultSets'][0]
         rowset = result_sets['rowSet']

stats/player_season.py

@@ -1,10 +1,8 @@
-import requests
 import urllib.parse
 
-from utils import get_rowset_mapping, column_names_from_table, season_id_to_int
+from utils import get_rowset_mapping, column_names_from_table, season_id_to_int, get_request
 from models import PlayerSeason
 from general_requester import GenericRequester
-from constants import headers
 
 
 class PlayerSeasonRequester(GenericRequester):
@@ -31,7 +29,7 @@ def populate_season(self, season_id):
         params_str = urllib.parse.urlencode(params, safe=':+')
 
         # json response
-        response = requests.get(url=self.url, headers=headers, params=params_str).json()
+        response = get_request(self.url, params, self.settings)
 
         result_sets = response['resultSets'][0]
         rowset = result_sets['rowSet']

stats/settings.py

@@ -16,7 +16,7 @@
 
 class Settings:
 
-    def __init__(self, database_type, database_name, database_user, database_password, database_host, batch_size, sqlite_path, quiet):
+    def __init__(self, database_type, database_name, database_user, database_password, database_host, batch_size, sqlite_path, quiet, proxy):
 
         self.user_agent = (
             "Mozilla/5.0 (X11; Linux x86_64) "
@@ -31,6 +31,7 @@ def __init__(self, database_type, database_name, database_user, database_passwor
         password = DB_PASSWORD
         host = DB_HOST
         self.batch_size = batch_size
+        self.proxy = proxy
 
         if database_name is not None:
             name = database_name

stats/team.py

@@ -4,7 +4,7 @@
 
 class TeamRequester(GenericRequester):
 
-    team_details_url = 'https://stats.nba.com/stats/teamdetails'
+    team_details_url = 'http://stats.nba.com/stats/teamdetails'
 
     def __init__(self, settings):
         """

stats/utils.py

@@ -2,7 +2,9 @@
 Misc utilities.
 """
 
+import requests
 import datetime
+from constants import headers
 
 
 def season_id_to_int(season_id):
@@ -108,3 +110,14 @@ def generate_valid_season(season):
     next_year = next_year_rev[::-1]
     return f"{season}-{next_year}"
 
+def get_request(url, params, settings):
+
+    if settings.proxy:
+        proxies = {
+            'http': settings.proxy,
+            'https': settings.proxy,
+        }
+    else:
+        proxies = {}
+
+    return requests.get(url=url, headers=headers, params=params, proxies=proxies).json()