<commit_before>from pandas.io.json import json_normalize
import json
import pandas as pd
import requests
import time

class WNBAScraper:

    headers = {'user-agent': ('Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'),
               'Dnt': ('1'),
               'Accept-Encoding': ('gzip, deflate, sdch'),
               'Accept-Language': ('en'),
               'origin': ('http://stats.nba.com'),
               'Connection': 'close'}
    id_prefix = 'http://data.wnba.com/data/5s/v2015/json/mobile_teams/wnba/'
    log_prefix = 'http://stats.nba.com/stats/playergamelog?LeagueID=10&playerid='

    log_columns = ['season', 'pid', 'gid', 'date', 'matchup',
                   'wl', 'min', 'fgm', 'fga', 'fg_pct',
                   'fg3m', 'fg3a', 'fg3_pct', 'ftm', 'fta',
                   'ft_pct', 'oreb', 'dreb', 'reb', 'ast',
                   'stl', 'blk', 'tov', 'pf', 'pts', 'pm',
                   'vid_available']

    scraped_logs = pd.DataFrame()

    def get_player_info(self, year):
        response = requests.get(self.id_prefix + '/' + str(year) + '/players/10_player_info.json', headers=self.headers, stream=False)
        data = json.loads(response.content)
        response.connection.close()
        df = pd.DataFrame(data['pls']['pl'])
        df = df.drop_duplicates(subset='pid')
        return df

    def get_player_log_headers(self, df):
        if df.empty is False:
            df.columns = self.log_columns
            df['pid'] = df.pid.astype(int)
            return df
        else:
            return df

    def get_player_log(self, year, pid):
        if year == 2013:
            response = requests.get(self.log_prefix + str(pid) + '&Season=2013-14&SeasonType=Regular+Season', headers=self.headers)
            data = json.loads(response.content)
            df = json_normalize(data, ['resultSets', 'rowSet'])
            df = self.get_player_log_headers(df)
            return df
        elif year == 2014:
            response = requests.get(self.log_prefix + str(pid) + '&Season=2014-15&SeasonType=Regular+Season', headers=self.headers)
            data = json.loads(response.content)
            df = json_normalize(data, ['resultSets', 'rowSet'])
            df = self.get_player_log_headers(df)
            return df
        elif year == 2015:
            response = requests.get(self.log_prefix + str(pid) + '&Season=2015-16&SeasonType=Regular+Season', headers=self.headers)
            data = json.loads(response.content)
            df = json_normalize(data, ['resultSets', 'rowSet'])
            df = self.get_player_log_headers(df)
            return df
        elif year == 2016:
            response = requests.get(self.log_prefix + str(pid) + '&Season=2016-17&SeasonType=Regular+Season', headers=self.headers)
            data = json.loads(response.content)
            df = json_normalize(data, ['resultSets', 'rowSet'])
            df = self.get_player_log_headers(df)
            return df
        else:
            response = requests.get(self.log_prefix + str(pid) + '&Season=2017-18&SeasonType=Regular+Season', headers=self.headers)
            data = json.loads(response.content)
            response.connection.close()
            df = json_normalize(data, ['resultSets', 'rowSet'])
            df = self.get_player_log_headers(df)
            return df

    def run(self, years, outfile):
        for year in years:
            print(year)
            season_logs = pd.DataFrame(columns=self.log_columns)
            player_info = self.get_player_info(year)
            player_info['player'] = player_info.fn + ' ' + player_info.ln
            player_ids = player_info['pid'].values.tolist()
            for pid in player_ids:
                df = self.get_player_log(year, pid)
                print(df)
                time.sleep(1)
                season_logs = season_logs.append(df, ignore_index=True)

            season_logs = season_logs.merge(player_info, on='pid', how='left')
            self.scraped_logs = self.scraped_logs.append(season_logs, ignore_index=True)

        self.scraped_logs.to_csv(outfile, index=False)

if __name__ == '__main__':
    print("Scraping game logs...")
    scraper = WNBAScraper()
    scraper.run(years=[2013, 2014, 2015, 2016, 2017], outfile='./gamelogs.csv')<commit_msg>Streamlined game log workflow<commit_after>from pandas.io.json import json_normalize
import json
import pandas as pd
import requests

class WNBAScraper:

    headers = {'user-agent': ('Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36'),
               'Dnt': ('1'),
               'Accept-Encoding': ('gzip, deflate, sdch'),
               'Accept-Language': ('en'),
               'origin': ('http://stats.nba.com'),
               'Connection': 'close'}
    id_prefix = 'http://data.wnba.com/data/5s/v2015/json/mobile_teams/wnba/'
    log_prefix = 'http://stats.nba.com/stats/playergamelog?LeagueID=10&playerid='

    log_columns = ['season', 'pid', 'gid', 'date', 'matchup',
                   'wl', 'min', 'fgm', 'fga', 'fg_pct',
                   'fg3m', 'fg3a', 'fg3_pct', 'ftm', 'fta',
                   'ft_pct', 'oreb', 'dreb', 'reb', 'ast',
                   'stl', 'blk', 'tov', 'pf', 'pts', 'pm',
                   'vid_available']

    scraped_logs = pd.DataFrame()

    def get_player_info(self, year):
        response = requests.get(self.id_prefix + '/' + str(year) + '/players/10_player_info.json', headers=self.headers, stream=False)
        data = json.loads(response.content)
        response.connection.close()
        df = pd.DataFrame(data['pls']['pl'])
        df = df.drop_duplicates(subset='pid')
        return df

    def get_player_log_headers(self, df):
        if df.empty is False:
            df.columns = self.log_columns
            df['pid'] = df.pid.astype(int)
            return df
        else:
            return df

    def get_player_log(self, year, pid):
        response = requests.get(self.log_prefix + str(pid) + '&Season=' + str(year) + '-' + str(int(str(year)[2:4]) + 1) + '&SeasonType=Regular+Season', headers=self.headers)
        data = json.loads(response.content)
        df = json_normalize(data, ['resultSets', 'rowSet'])
        df = self.get_player_log_headers(df)
        return df

    def run(self, years, outfile):
        for year in years:
            print('Scraping game logs from ' + str(year))
            season_logs = pd.DataFrame(columns=self.log_columns)
            player_info = self.get_player_info(year)
            player_info['player'] = player_info.fn + ' ' + player_info.ln
            player_ids = player_info['pid'].values.tolist()
            for pid in player_ids:
                df = self.get_player_log(year, pid)
                season_logs = season_logs.append(df, ignore_index=True)

            season_logs = season_logs.merge(player_info, on='pid', how='left')
            self.scraped_logs = self.scraped_logs.append(season_logs, ignore_index=True)

        self.scraped_logs.to_csv(outfile, index=False)

if __name__ == '__main__':
    scraper = WNBAScraper()
    years = input('Enter desired years in a comma separated list to scrape data:')
    scraper.run(years=years.split(','), outfile='./gamelogs.csv')
{'fraction_non_alphanumeric': 0.1441663247161811, 'fraction_numerical': 0.020243468745725617, 'mean_word_length': 2.4056823474615743, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2515215', 'n_tokens_mistral': 2487, 'n_tokens_neox': 2316, 'n_words': 447}