<jupyter_start><jupyter_text>Cleaning and Merging match and point-by-point Dataframes<jupyter_code>from helper_functions import *
from data_functions import *
import pandas as pd
import numpy as np
import re
import math
import copy

TOUR = 'atp'
COUNT = False
START_YEAR = 2010
ONLY_PBP = 1
pd.options.mode.chained_assignment = None
atp_year_list = []
for i in xrange(1968,2018):
    atp_year_list.append(pd.read_csv("../tennis_data/atp/atp_matches_{0}.csv".format(i)))
df = pd.concat(atp_year_list, ignore_index = True)

# these may be changes specific to atp dataframe; normalize_name() is specific to atp/wta...
df = df.rename(columns={'winner_name':'w_name','loser_name':'l_name','tourney_id':'tny_id',\
                        'tourney_name':'tny_name','tourney_date':'tny_date'})
df['w_name'] = [normalize_name(x,tour=TOUR) for x in df['w_name']]
df['l_name'] = [normalize_name(x,tour=TOUR) for x in df['l_name']]
df['tny_name'] = ['Davis Cup' if 'Davis Cup' in s else s for s in df['tny_name']]
df['tny_name'] = [s.replace('Australian Chps.','Australian Open').replace('Australian Open-2',\
            'Australian Open').replace('U.S. National Chps.','US Open') for s in df['tny_name']]

ret_strings = ('ABN','DEF','In Progress','RET','W/O',' RET',' W/O','nan','walkover')
abd_strings = ('abandoned','ABN','ABD','DEF','def','unfinished','Walkover')
atp_all_matches = format_match_df(df,ret_strings=ret_strings,abd_strings=abd_strings)
print atp_all_matches.shape
# find the overall serve/return percentages; also by tournament
sv_pct = np.sum(atp_all_matches['w_swon']+atp_all_matches['l_swon'])/np.sum(atp_all_matches['w_svpt']+atp_all_matches['l_svpt'])
print 'atp avg serve pct: ', sv_pct
print 'atp avg return pct: ', 1-sv_pct
# generate tourney stats from one year behind START_DATE for stats_52
# get elo with constant and dynamic K
start_ind = atp_all_matches[atp_all_matches['match_year']>=START_YEAR-1].index[0]
atp_all_matches = generate_elo(atp_all_matches,0)
atp_all_matches = generate_elo(atp_all_matches,1)
atp_all_matches = generate_52_stats(atp_all_matches,start_ind)
atp_all_matches = generate_tny_stats(atp_all_matches,start_ind)
# START_YEAR = 2013
# start_ind = atp_all_matches[atp_all_matches['match_year']>=START_YEAR-1].index[0]
# sub_df = generate_52_stats(atp_all_matches,start_ind)
#rankings = sorted(zip(players_elo.keys(),players_elo.values()),key=lambda x: x[1].value,reverse=True)
#rankings
# Combine all the matches that have pbp (point by point) information into one dataframe
# and clean up columns in preparation for merging with all_atp_matches
pbp_matches_archive = pd.read_csv("../tennis_data/pbp_matches_atp_main_archive.csv")
pbp_matches_archive_old = pd.read_csv("../tennis_data/pbp_matches_atp_main_archive_old.csv")
pbp_matches_current = pd.read_csv("../tennis_data/pbp_matches_atp_main_current.csv")
pbp_matches = pd.concat([pbp_matches_archive_old.loc[:932],pbp_matches_archive,pbp_matches_current])
pbp_matches.winner = pbp_matches.winner - 1
pbp_matches = pbp_matches.reset_index(); del pbp_matches['index']
pbp_matches = format_pbp_df(pbp_matches,tour=TOUR)

# there's a few duplicate rows, but it doesn't matter since we use hashing to connect
#print pbp_matches.shape
#print len(set(pbp_matches['pbp'].values))
# dictionary with each key as 'w_name'+'l_name'+'match_year'+'score' to connect pbp 
# strings to atp_all_matches (I removed parentheses terms from tb scores)
# adding match_month loses 800-900 matches; replacing dashes with spaces saves over 1000 matches
# other discrepancies due to tiebreak scores, or qualies
duplicates = ['<NAME> 2011 6-4 6-4','<NAME> <NAME> 2011 6-3 7-6',
        '<NAME> 2011 6-4 6-4','<NAME> 2011 6-4 6-2',
        '<NAME> <NAME> 2012 6-1 6-4','<NAME> 2012 2-6 6-4 6-3',
        '<NAME> 2012 6-3 6-3','<NAME> 2012 6-3 6-4',
        '<NAME> <NAME> 2013 6-2 6-4','<NAME> <NAME> 2013 6-3 7-6',
        '<NAME> 2013 6-3 6-4','<NAME> 2015 6-1 6-4',
        '<NAME> 2015 6-3 6-3']
collision_d = dict(zip(duplicates,[0]*len(duplicates)))

# connects the two dataframes on match keys and reformats columns fro w/l to p0/p1
cols = ['_name','_elo','_sf_elo','_elo_538','_sf_elo_538','_52_swon','_52_svpt','_52_rwon',\
        '_52_rpt','_sf_52_swon','_sf_52_svpt','_sf_52_rwon','_sf_52_rpt']
df = connect_df(match_df=atp_all_matches,pbp_df=pbp_matches,col_d=collision_d,player_cols=cols,\
                start_year=2000)
df['elo_diff'] = [df['p0_elo'][i] - df['p1_elo'][i] for i in xrange(len(df))]
df['sf_elo_diff'] = [df['p0_sf_elo'][i] - df['p1_sf_elo'][i] for i in xrange(len(df))]
df['elo_diff_538'] = [df['p0_elo_538'][i] - df['p1_elo_538'][i] for i in xrange(len(df))]
df['sf_elo_diff_538'] = [df['p0_sf_elo_538'][i] - df['p1_sf_elo_538'][i] for i in xrange(len(df))]

# dataframe with only matches that have pbp
if ONLY_PBP:
    df = df[df['pbp']!='None']
else:
    df = df[df['winner']!='None']
df = df.reset_index(drop=True)
# TO DO: check that these are correct...something is WRONG with the betas and sigma2s...
p_hat,df = generate_JS_stats(df)
print p_hat,1-p_hat
# alter columns...
df = df[['tny_id','tny_name','surface','tny_date','match_year','match_month',
         u'p0_name', u'p1_name', u'p0_elo',
         u'p1_elo', u'p0_sf_elo', u'p1_sf_elo', u'p0_elo_538', u'p1_elo_538',
         u'p0_sf_elo_538', u'p1_sf_elo_538', u'p0_52_swon',u'p0_52_svpt', 
         u'p1_52_swon', u'p1_52_svpt', u'p0_52_rwon', u'p0_52_rpt',
         u'p1_52_rwon', u'p1_52_rpt', 
         u'elo_diff', u'sf_elo_diff',
         u'elo_diff_538', u'sf_elo_diff_538',
         u'p0_s_pct', u'p0_r_pct', u'p1_s_pct', u'p1_r_pct', 
         u'p0_s_pct_JS', u'p1_s_pct_JS', u'p0_r_pct_JS', u'p1_r_pct_JS',
         u'p0_sf_52_swon', u'p0_sf_52_svpt',u'p1_sf_52_swon', u'p1_sf_52_svpt', 
         u'p0_sf_52_rwon', u'p0_sf_52_rpt', u'p1_sf_52_rwon', u'p1_sf_52_rpt',
         u'p0_sf_s_pct', u'p0_sf_r_pct', u'p1_sf_s_pct', u'p1_sf_r_pct', 
         u'p0_sf_s_pct_JS', u'p1_sf_s_pct_JS', u'p0_sf_r_pct_JS', u'p1_sf_r_pct_JS',
         u'avg_52_s', u'avg_52_r', u'sf_avg_52_s', u'sf_avg_52_r',
         'tny_stats','best_of','score','pbp','winner']]

# binary indicator for whether player 0 won (reverse this for pbp_matches and randomly
# generated 0s and 1s)
df['winner'] = [1-winner for winner in df['winner']]
f_av = 0.646775
g_av = 1 - f_av

df['match_id'] = range(len(df))
df['tny_stats'] = [f_av if stat==0 else stat for stat in df['tny_stats']]
df['p0_s_kls'] = df['tny_stats']+(df['p0_s_pct']-df['avg_52_s']) - (df['p1_r_pct']-df['avg_52_r'])
df['p1_s_kls'] = df['tny_stats']+(df['p1_s_pct']-df['avg_52_s']) - (df['p0_r_pct']-df['avg_52_r'])
df['p0_s_kls_JS'] = df['tny_stats']+(df['p0_s_pct_JS']-df['avg_52_s']) - (df['p1_r_pct_JS']-df['avg_52_r'])
df['p1_s_kls_JS'] = df['tny_stats']+(df['p1_s_pct_JS']-df['avg_52_s']) - (df['p0_r_pct_JS']-df['avg_52_r'])
df['p0_sf_s_kls'] = df['tny_stats']+(df['p0_sf_s_pct']-df['sf_avg_52_s']) - (df['p1_sf_r_pct']-df['sf_avg_52_r'])
df['p1_sf_s_kls'] = df['tny_stats']+(df['p1_sf_s_pct']-df['sf_avg_52_s']) - (df['p0_sf_r_pct']-df['sf_avg_52_r'])
df['p0_sf_s_kls_JS'] = df['tny_stats']+(df['p0_sf_s_pct_JS']-df['sf_avg_52_s']) - (df['p1_sf_r_pct_JS']-df['sf_avg_52_r'])
df['p1_sf_s_kls_JS'] = df['tny_stats']+(df['p1_sf_s_pct_JS']-df['sf_avg_52_s']) - (df['p0_sf_r_pct_JS']-df['sf_avg_52_r'])
df.head()
#df.to_csv('../my_data/elo_atp_matches_21st_century_9_11.csv')
df.to_csv('../my_data/elo_pbp_with_surface_9_11.csv')<jupyter_output><empty_output><jupyter_text>Enumerating point-by-point dataframe<jupyter_code>import time

df = pd.read_csv('../my_data/elo_pbp_with_surface_8_27.csv')
del df['Unnamed: 0']

# append . to end of pbp string to signify end of match if not marked '0' means player0 won;
# '1' means player1 won (originally player1, player2); small fix for dead rubbers
df['pbp'] = [s + '.' if s[-1]!= '.' else s for s in df['pbp']]

tid_dict = dict(zip(df['match_id'],df['tourney_id']))
df.rename(columns={'tourney_name':'tny_name'},inplace=True)
df['tny_name'] = [df['tny_name'][i]+' (3)' if df['tny_name'][i]=='<NAME>' and \
                  df['best_of'][i]==3 else df['tny_name'][i] for i in xrange(len(df))]
# split this df into best-of-three and best-of-five matches
gs_tourneys = ['Australian Open', 'Davis Cup', '<NAME>', 'US Open', 'Wimbledon']
df_pbp3 = df[~df['tny_name'].isin(gs_tourneys)]
df_pbp5 = df[df['tny_name'].isin(gs_tourneys)]
df_pbp3=df_pbp3.reset_index();df_pbp5=df_pbp5.reset_index()
del df_pbp3['index']; del df_pbp5['index']

final_set_d = Counter()
for gs in gs_tourneys: final_set_d[gs]=1 if gs!='US Open' else 0
set_order_d = dict(zip(range(len(df)),[get_set_order(df['pbp'][i]) for i in xrange(len(df))]))

start = time.clock()
cols = ['match_id','surface','elo_diff','s_elo_diff','winner','p0_s_pct','p0_s_pct_JS',\
        'p1_s_pct','p1_s_pct_JS','p0_r_pct','p0_r_pct_JS','p1_r_pct','p1_r_pct_JS', \
        'tny_stats','best_of']
df_pred = generate_df_2(df_pbp3,cols,0)
print 'time: ', time.clock()-start
breaks = [0]*len(df_pred)
for i in xrange(len(df_pred)):
    breaks[i] = break_point(df_pred['score'][i])
df_pred['up_break_point'] = [a[0] for a in breaks]
df_pred['down_break_point'] = [a[1] for a in breaks]
df_pred['break_adv'] = [service_breaks(df_pred['score'][i]) for i in xrange(len(df_pred))]
f_av = 0.646775
g_av = 1 - f_av

df_pred['tny_stats'] = [f_av if stat==0 else stat for stat in df_pred['tny_stats']]
df_pred['p0_s_kls'] = df_pred['tny_stats'] + (df_pred['p0_s_pct']-f_av) - (df_pred['p1_r_pct']-g_av)
df_pred['p1_s_kls'] = df_pred['tny_stats'] + (df_pred['p1_s_pct']-f_av) - (df_pred['p0_r_pct']-g_av)
df_pred['p0_s_kls_JS'] = df_pred['tny_stats'] + (df_pred['p0_s_pct_JS']-f_av) - (df_pred['p1_r_pct_JS']-g_av)
df_pred['p1_s_kls_JS'] = df_pred['tny_stats'] + (df_pred['p1_s_pct_JS']-f_av) - (df_pred['p0_r_pct_JS']-g_av)
df_pred['tny_id'] = [tid_dict[m_id] for m_id in df_pred['match_id']]
df_pred['server'] = 1 - df_pred['server']
df_pred.head(2)
# add a heuristic lead function
df_pred['lead_margin'] = df_pred['sets_0']-df_pred['sets_1'] + \
                        (df_pred['games_0']-df_pred['games_1'])/6. + \
                        (df_pred['points_0']-df_pred['points_1'])/24.
df_pred.to_csv('../my_data/feature_df_pbp3_9_6.csv')<jupyter_output><empty_output><jupyter_text>Additional Features<jupyter_code>df_pred = pd.read_csv('../my_data/feature_df_pbp3_9_6.csv')
del df_pred['Unnamed: 0']
df_pred['sv_points_pct_0'] = np.nan_to_num(np.divide(df_pred['p0_swp'],df_pred['p0_sp'].astype('float')))
df_pred['sv_points_pct_1'] = np.nan_to_num(np.divide(df_pred['p1_swp'],df_pred['p1_sp'].astype('float')))
def reduce_p(points):
    while points[0]+points[1] > 6:
        points = points[0]-1,points[1]-1
    return points
# adding score features
set_scores = ['set_00','set_01','set_10','set_11']
nums = ['0','1','2','3','4','5']
game_scores = ['game_'+num_a+num_b for num_a in nums for num_b in nums] + \
                ['game_65','game_56','game_66']
point_scores = ['point_'+num_a+num_b for num_a in nums[:4] for num_b in nums[:4]]
point_d = dict(zip(point_scores,[1]*len(point_scores)))

# then, create columns for each with np.zeros()...
for col in set_scores+game_scores+point_scores:
    df_pred[col] = np.zeros(len(df_pred))
for col in ['sets_0','sets_1','games_0','games_1','points_0','points_1']:
    df_pred[col] = df_pred[col].astype(int)
for i in xrange(len(df_pred)):
    if i%100000==0:
        print i
    df_pred['set_'+str(df_pred['sets_0'][i])+str(df_pred['sets_1'][i])][i] = 1
    df_pred['game_'+str(df_pred['games_0'][i])+str(df_pred['games_1'][i])][i] = 1
    points = reduce_p((df_pred['points_0'][i],df_pred['points_1'][i]))
    if 'point_'+str(points[0])+str(points[1]) in point_d:
        df_pred['point_'+str(points[0])+str(points[1])][i] = 1
df_pred.to_csv('../my_data/feature_df_pbp3_9_6.csv')
# NEXT: FIX THE INPUT TO matchProb()
import pandas as pd
import numpy as np
f_av = 0.646775
g_av = 1 - f_av
import sys
sys.path.insert(0,'/Users/jacobgollub/Desktop/college (current)/research/pbp_explorations/scripts/sackmann')
import tennisGameProbability,tennisMatchProbability,tennisSetProbability,tennisTiebreakProbability
from tennisMatchProbability import matchProb

df_pred = pd.read_csv('../my_data/feature_df_pbp3_9_6.csv')
del df_pred['Unnamed: 0']
# can run beta experiments based off of p(s)=.64 as well as klaassen and JS serve percentages
alphas = [1,10,100,200,300,400,500,600,1000]
for alpha in alphas:
    df_pred['p0_s_kls_a_'+str(alpha)] = (df_pred['p0_s_kls']*alpha+df_pred['p0_swp'])/(alpha+df_pred['p0_sp'])
    df_pred['p1_s_kls_a_'+str(alpha)] = (df_pred['p1_s_kls']*alpha+df_pred['p1_swp'])/(alpha+df_pred['p1_sp'])
import time
start = time.clock()

probs = np.zeros([10,len(df_pred)])
for i,row in df_pred.iterrows():
    if row['server']:
        probs[0][i] = matchProb(f_av,g_av,mv=int(row['sets_0']),mw=int(row['sets_1']),sv=int(row['games_0']), \
                                sw=int(row['games_1']),gv=int(row['points_0']),gw=int(row['points_1']))
        probs[1][i] = matchProb(row['p0_s_kls'],1-row['p1_s_kls'],mv=int(row['sets_0']),mw=int(row['sets_1']), \
                                sv=int(row['games_0']),sw=int(row['games_1']),gv=int(row['points_0']),gw=int(row['points_1']))
        probs[2][i] = matchProb(row['p0_s_kls_JS'],1-row['p1_s_kls_JS'],mv=int(row['sets_0']),mw=int(row['sets_1']), \
                                sv=int(row['games_0']),sw=int(row['games_1']),gv=int(row['points_0']),gw=int(row['points_1']))
        for k,alph in enumerate(alphas[2:]):
            probs[k+3][i] = matchProb(row['p0_s_kls_a_'+str(alph)],1-row['p1_s_kls_a_'+str(alph)],mv=int(row['sets_0']),mw=int(row['sets_1']), \
                            sv=int(row['games_0']),sw=int(row['games_1']),gv=int(row['points_0']),gw=int(row['points_1']))
    else:
        probs[0][i] = 1-matchProb(f_av,g_av,mv=int(row['sets_1']),mw=int(row['sets_0']),sv=int(row['games_1']), \
                                sw=int(row['games_0']),gv=int(row['points_1']),gw=int(row['points_0']))
        probs[1][i] = 1-matchProb(row['p1_s_kls'],1-row['p0_s_kls'],mv=int(row['sets_1']),mw=int(row['sets_0']), \
                                sv=int(row['games_1']),sw=int(row['games_0']),gv=int(row['points_1']),gw=int(row['points_0']))
        probs[2][i] = 1-matchProb(row['p1_s_kls_JS'],1-row['p0_s_kls_JS'],mv=int(row['sets_1']),mw=int(row['sets_0']), \
                                sv=int(row['games_1']),sw=int(row['games_0']),gv=int(row['points_1']),gw=int(row['points_0']))
        for k,alph in enumerate(alphas[2:]):
            probs[k+3][i] = 1-matchProb(row['p1_s_kls_a_'+str(alph)],1-row['p0_s_kls_a_'+str(alph)],mv=int(row['sets_1']),mw=int(row['sets_0']), \
                            sv=int(row['games_1']),sw=int(row['games_0']),gv=int(row['points_1']),gw=int(row['points_0']))
    
    if i%100000==0:
        print i

df_pred['sackmann_prob'],df_pred['klaassen_prob'],df_pred['klaassen_prob_JS'] = probs[0],probs[1],probs[2]
df_pred['klaassen_prob_a_100'] = probs[3]
df_pred['klaassen_prob_a_200'], df_pred['klaassen_prob_a_300'] = probs[4],probs[5]
df_pred['klaassen_prob_a_400'], df_pred['klaassen_prob_a_500'] = probs[6],probs[7]
df_pred['klaassen_prob_a_600'], df_pred['klaassen_prob_a_1000'] = probs[8],probs[9]
print time.clock()-start
#df_pred.to_csv('../my_data/feature_df_pbp3_9_6_alphas.csv')
# 7/26 update: seem to have fixed some glitches in my previous adaptation of 538's elo method
# everything is very simple in elo_538.py; also fixed the match order within tournaments in atp_all_matches

# note: we improved from 9563 to 10000 matches with string manipulations
# my elo scores seem close to <NAME>'s when I set k=32 as constant; when using the 538 style of "cooling" k-factor,
# those elos are going to be inflated in comparison (but keep in mind, that's only different across methods)
# tournament serving averages at the Australian Open
for key in sorted(tourney_stats['580'].historical_avgs.keys()):
    print key,': ',tourney_stats['580'].historical_avgs[key][0]/tourney_stats['580'].historical_avgs[key][1]
# an example of a qualifying match in pbp_matches that will not be exported in elo_pbp.csv
z = pbp_matches[pbp_matches['server2']=='<NAME>']
z[z['server1']=='<NAME>']
below = df[df['elo_diff']<0]
above = df[df['elo_diff']>=0]

print (len(below[below['winner']==1])+len(above[above['winner']==0]))/float(len(df))

below = df[df['s_elo_diff']<0]
above = df[df['s_elo_diff']>=0]

print (len(below[below['winner']==1])+len(above[above['winner']==0]))/float(len(df))
# # accept dates in (year,month); last_year contains last 12 month stats, most recent to least
# # each month has a 4x1 row, containing 2x1 for serve and 2x1 for return
# class stats_52():
#     def __init__(self,date):
#         self.most_recent = date
#         self.last_year = np.zeros([12,4])
        
#     def time_diff(self,new_date,old_date):
#         return 12*(new_date[0]-old_date[0])+(new_date[1]-old_date[1])
    
#     def update(self,match_date,match_stats):
#         diff = self.time_diff(match_date,self.most_recent)
#         if diff>=12:
#             self.last_year = np.zeros([12,4])
#         elif diff>0:
#             self.last_year[diff:] = self.last_year[:12-diff]; self.last_year[:diff] = 0
#         self.last_year[0] = self.last_year[0]+match_stats
#         self.most_recent = match_date
# ### Make a dict of players with elo ratings ###
# # now, adding three more surface-specific dictionaries for clay, grass, hard
# players_list = np.union1d(atp_all_matches.w_name, atp_all_matches.l_name)
# players_elo = dict(zip(list(players_list), [elo.Rating() for __ in range(len(players_list))]))
# surface_elo = {}
# for surface in ('Hard','Clay','Grass'):
#     surface_elo[surface] = dict(zip(list(players_list), [elo.Rating() for __ in range(len(players_list))])) 

# elo_1s, elo_2s = [],[]
# surface_elo_1s, surface_elo_2s = [],[]
# elo_obj = elo.Elo_Rater()

# # update player elo from every recorded atp match
# for i, row in atp_all_matches.iterrows():
#     surface = row['surface']; is_gs = row['is_gs']
#     # append elos, rate, update
#     w_elo,l_elo = players_elo[row['w_name']],players_elo[row['l_name']]
#     elo_1s.append(w_elo.value);elo_2s.append(l_elo.value)    
#     elo_obj.rate_1vs1(w_elo,l_elo,is_gs,counts=COUNT)

    
#     surface_elo_1s.append(surface_elo[surface][row['w_name']].value if surface in ('Hard','Clay','Grass') else w_elo.value)
#     surface_elo_2s.append(surface_elo[surface][row['l_name']].value if surface in ('Hard','Clay','Grass') else l_elo.value)
#     if surface in ('Hard','Clay','Grass'):
#         new_elo1, new_elo2 = elo_obj.rate_1vs1(surface_elo[surface][row['w_name']],surface_elo[surface][row['l_name']],is_gs,counts=COUNT)

# # add columns
# atp_all_matches['w_elo'], atp_all_matches['l_elo'] = elo_1s, elo_2s
# atp_all_matches['w_s_elo'], atp_all_matches['l_s_elo'] = surface_elo_1s, surface_elo_2s
# # track player match stats for every match since 2009 (we only need these for pbp matches)
# start_ind = 136146; atp_l = len(atp_all_matches)
# players_stats = {}
# # an array containing 2x1 arrays for winner and loser's previous 12-month serve performance over all matches in df
# match_52_stats = np.zeros([2,atp_l,4])
# w_l = ['w','l']
# for i, row in atp_all_matches.loc[start_ind:].iterrows():    
#     date = row['match_year'],row['match_month']           
#     for k,label in enumerate(w_l):
#         if row[label+'_name'] not in players_stats:
#             players_stats[row[label+'_name']] = stats_52(date)
#         # store serving stats prior to match
#         match_52_stats[k][i] = np.sum(players_stats[row[label+'_name']].last_year,axis=0)
#         # update serving stats if no
#         if row[label+'_swon']==row[label+'_swon'] and row[label+'_svpt']==row[label+'_svpt']:    
#             match_stats = (row[label+'_swon'],row[label+'_svpt'],row[w_l[1-k]+'_svpt']-row[w_l[1-k]+'_swon'],row[w_l[1-k]+'_svpt'])
#             players_stats[row[label+'_name']].update(date,match_stats)
#         match_52_stats[k][i] = np.sum(players_stats[row[label+'_name']].last_year,axis=0)

# for k,label in enumerate(w_l):
#     atp_all_matches[label+'_52_swon'] = match_52_stats[k][:,0]
#     atp_all_matches[label+'_52_svpt'] = match_52_stats[k][:,1]
#     atp_all_matches[label+'_52_rwon'] = match_52_stats[k][:,2]
#     atp_all_matches[label+'_52_rpt'] = match_52_stats[k][:,3]
# tourney_stats = {}
# tourney_52_stats = np.zeros(len(atp_all_matches))
# for i, row in atp_all_matches.loc[start_ind:].iterrows():
#     if row['tourney_name']=='<NAME>' or row['w_swon']!=row['w_swon']:
#         continue
    
#     year,t_id = row['tourney_id'].split('-')
#     year = int(year)
#     match_stats = (row['w_swon']+row['l_swon'],row['w_svpt']+row['l_svpt'])
#     if row['w_swon']!=row['w_swon']:
#         tourney_52_stats[i] = 0 if tourney_stats[t_id].tourney_stats[1][1]==0 else \
#                             tourney_stats[t_id].tourney_stats[1][0]/float(self.tourney_stats[1][1])
        
#     elif t_id in tourney_stats:
#         tourney_52_stats[i] = tourney_stats[t_id].update(year,match_stats)
#     else:
#         tourney_stats[t_id] = tourney_52(year)
#         tourney_52_stats[i] = tourney_stats[t_id].update(year,match_stats)
    
# atp_all_matches['tourney_stats'] = tourney_52_stats
# # Set up the pbp (point by point) column that will contain the pbp information for matches we have info on
# atp_all_matches['pbp'] = [None]*len(atp_all_matches)
# atp_all_matches = atp_all_matches.rename(columns={'winner_name':'w_name','loser_name':'l_name'})
# atp_all_matches['w_name'] = [normalize_name(x) for x in atp_all_matches['w_name']]
# atp_all_matches['l_name'] = [normalize_name(x) for x in atp_all_matches['l_name']]
# atp_all_matches['tourney_name'] = ['<NAME>' if '<NAME>' in s else s for s in atp_all_matches['tourney_name']]
# atp_all_matches['tourney_name'] = [s.replace('Australian Chps.',AO).replace('Australian Open-2',AO).replace('U.S. National Chps.',USO) \
#                                    for s in atp_all_matches['tourney_name']]
# grand_slam_d = dict(zip(['Australian Open','Roland Garros','Wimbledon','US Open'],[1]*4))
# atp_all_matches['is_gs'] = [name in grand_slam_d for name in atp_all_matches['tourney_name']]

# # Get dates into the same format
# atp_all_matches['tourney_date'] = [datetime.datetime.strptime(str(x), "%Y%m%d").date() for x in atp_all_matches['tourney_date']]
# atp_all_matches['match_year'] = [x.year for x in atp_all_matches['tourney_date']]
# atp_all_matches['match_month'] = [x.month for x in atp_all_matches['tourney_date']]
# atp_all_matches['score'] = [re.sub(r"[\(\[].*?[\)\]]", "", str(s)) for s in atp_all_matches['score']] # str(s) fixes any nans
# atp_all_matches['score'] = ['RET' if 'RET' in s else s for s in atp_all_matches['score']]
# atp_all_matches['w_swon'] = [atp_all_matches['w_1stWon'][i]+atp_all_matches['w_2ndWon'][i] for i in xrange(len(atp_all_matches))]
# atp_all_matches['l_swon'] = [atp_all_matches['l_1stWon'][i]+atp_all_matches['l_2ndWon'][i] for i in xrange(len(atp_all_matches))]
# atp_all_matches['w_rwon'] = atp_all_matches['l_svpt'] - atp_all_matches['l_swon']
# atp_all_matches['l_rwon'] = atp_all_matches['w_svpt'] - atp_all_matches['w_swon']
# atp_all_matches['w_rpt'] = atp_all_matches['l_svpt']
# atp_all_matches['l_rpt'] = atp_all_matches['w_svpt']

# # get rid of leading 0s in tourney_id
# atp_all_matches['tourney_id'] = ['-'.join(t.split('-')[:-1] + [t.split('-')[-1][1:]]) if t.split('-')[-1][0]=='0' else t \
#                                 for t in atp_all_matches['tourney_id']]

# # get rid of matches involving a retirement
# atp_all_matches['score'] = ['ABN' if score.split(' ')[-1] in ('abandoned','ABN','ABD','DEF','def','unfinished','Walkover') \
#                             else score for score in atp_all_matches['score']]
# ret_d = dict(zip(ret_strings,[1]*len(ret_strings)))
# atp_all_matches = atp_all_matches.loc[[i for i in range(len(atp_all_matches)) if atp_all_matches['score'][i] not in ret_d]]
# atp_all_matches = atp_all_matches.sort_values(by=['tourney_date','tourney_name','match_num'], ascending=True).reset_index()
# del atp_all_matches['index']
# pbp_matches['w_name'] = np.where(pbp_matches['winner'] == 0, pbp_matches['server1'], pbp_matches['server2'])
# pbp_matches['l_name'] = np.where(pbp_matches['winner'] == 0, pbp_matches['server2'], pbp_matches['server1'])
# pbp_matches['w_name'] = [normalize_name(x,tour=TOUR) for x in pbp_matches['w_name']]
# pbp_matches['l_name'] = [normalize_name(x,tour=TOUR) for x in pbp_matches['l_name']]
# pbp_matches['date'] = pd.to_datetime(pbp_matches['date'])
# pbp_matches['match_year'] = [x.year for x in pbp_matches['date']]
# pbp_matches['match_month'] = [x.month for x in pbp_matches['date']]
# pbp_matches['date'] = [x.date() for x in pbp_matches['date']]
# pbp_matches['score'] = [re.sub(r"[\(\[].*?[\)\]]", "", s) for s in pbp_matches['score']]
# pbp_dict = {}; winner_dict = {}
# for i in xrange(len(pbp_matches)):
#     key = pbp_matches['w_name'][i] +' ' +  pbp_matches['l_name'][i] + ' ' \
#         + str(pbp_matches['match_year'][i]) + ' ' + pbp_matches['score'][i]
#     key = key+' '+str(pbp_matches['match_month'][i]) if key in collision_d else key
#     if key in pbp_dict:
#         continue
#     pbp_dict[key] = pbp_matches['pbp'][i]
#     winner_dict[key] = pbp_matches['winner'][i]

# # in case of a collision (about 10 cases), I only take the first match with that key
# c = 0
# pbps = []
# winners = []
# info = {}

# for i in xrange(len(atp_all_matches)):
#     key = atp_all_matches['w_name'][i] +' ' +  atp_all_matches['l_name'][i] + ' ' \
#         +str(atp_all_matches['match_year'][i])+' '+atp_all_matches['score'][i]
#     key = key+' '+str(atp_all_matches['match_month'][i]) if key in collision_d else key
#     if key in pbp_dict:
#         c += 1
#         pbps.append(pbp_dict[key])
#         winners.append(winner_dict[key])
#         if key in info:
#             pbps[-1] = 'NA'; winners[-1] = 'NA'
#             print 'collision'; print key + ' ' + str(atp_all_matches['match_month'][i])
#         info[key] = 1
#     else:
#         pbps.append('NA')
#         winners.append('NA')
# print c
# atp_all_matches['pbp'] = pbps
# atp_all_matches['winner'] = winners<jupyter_output><empty_output><jupyter_text>Rating Systems 1. http://gobase.org/studying/articles/elo/2. http://www.gamefaqs.com/boards/610657-dota-2/679946463. https://github.com/sublee/elo/blob/master/elo.py4. http://stephenwan.net/thoughts/2012/10/02/elo-rating-system.html5. https://deltadata.wordpress.com/2014/01/11/glicko-2-for-tennis-part-2-the-model/comment-page-1/comment-1926. https://github.com/sublee/glicko2/blob/master/glicko2.py7. http://trueskill.org/8. https://pypi.python.org/pypi/trueskill<jupyter_code># df = atp_all_matches[atp_all_matches['pbp']!='NA']
# cols = df.columns.drop(['loser_id','winner_id'])
# df = df[cols]
# df = df.reset_index()
# del df['index']

# # SET UP LOOP TO CHANGE W,L TO P0,P1
# for col in ['_name','_elo','_s_elo','_52_swon','_52_svpt','_52_rwon','_52_rpt']:
#     df['p0'+col] = [df['l'+col][i] if df['winner'][i] else df['w'+col][i] for i in xrange(len(df))]
#     df['p1'+col] = [df['w'+col][i] if df['winner'][i] else df['l'+col][i] for i in xrange(len(df))]

# df['elo_diff'] = [df['p0_elo'][i] - df['p1_elo'][i] for i in xrange(len(df))]
# df['s_elo_diff'] = [df['p0_s_elo'][i] - df['p1_s_elo'][i] for i in xrange(len(df))]
# df['tourney_name'] = [s if s==s else '<NAME>' for s in df['tourney_name']]
# #### James-Stein estimators for 52-week serve and return percentages ####
# # calculate B_i coefficients for each player in terms of service points
# p_hat = np.sum([df['p0_52_swon'],df['p1_52_swon']])/np.sum([df['p0_52_svpt'],df['p1_52_svpt']])
# for label in ['p0','p1']:
#     df[label+'_s_pct'] = [p_hat if x==0 else x for x in np.nan_to_num(df[label+'_52_swon']/df[label+'_52_svpt'])]
#     df[label+'_r_pct'] = [1-p_hat if x==0 else x for x in np.nan_to_num(df[label+'_52_rwon']/df[label+'_52_rpt'])]

# # fix this name later with find and replace...
# s_history = np.concatenate([df['p0_52_swon']/df['p0_52_svpt'],df['p1_52_swon']/df['p1_52_svpt']],axis=0)
# n = len(s_history)/2
# group_var = np.var(s_history)
# s_points = np.concatenate([df['p0_52_svpt'],df['p1_52_svpt']])
# sigma2_i = np.divide(p_hat*(1-p_hat),s_points,where=s_points>0)
# #sigma2_i = np.nan_to_num(p_hat*(1-p_hat)/np.concatenate([df['p0_52_svpt'],df['p1_52_svpt']]))
# tau2_hat = np.nanvar(s_history)
# B_i = sigma2_i/(tau2_hat+sigma2_i)
# df['B_i0_sv'],df['B_i1_sv'] = B_i[:n],B_i[n:]

# s_history[s_history!=s_history] = p_hat
# group_var = np.var(s_history)
# df['p0_s_pct_JS'] = df['p0_s_pct']+df['B_i0_sv']*(p_hat-df['p0_s_pct'])
# df['p1_s_pct_JS'] = df['p1_s_pct']+df['B_i1_sv']*(p_hat-df['p1_s_pct'])

# # repeat for return averages (slightly different tau^2 value)
# r_history = np.concatenate([df['p0_52_rwon']/df['p0_52_rpt'],df['p1_52_rwon']/df['p1_52_rpt']],axis=0)
# r_points = np.concatenate([df['p0_52_rpt'],df['p1_52_rpt']])
# sigma2_i = np.divide((1-p_hat)*p_hat,r_points,where=r_points>0)
# tau2_hat = np.nanvar(r_history)
# B_i = sigma2_i/(tau2_hat+sigma2_i)
# df['B_i0_r'],df['B_i1_r'] = B_i[:n],B_i[n:]

# r_history[r_history!=r_history] = 1-p_hat
# df['p0_r_pct_JS'] = r_history[:n]+df['B_i0_r']*((1-p_hat)-r_history[:n])
# df['p1_r_pct_JS'] = r_history[n:]+df['B_i1_r']*((1-p_hat)-r_history[n:])
# #demo of match-keys I created 
# a lot of the inconsistencies are challenger-level events or even exhibition matches (see 2017)
# for i in xrange(len(pbp_matches)):
#     key = pbp_matches['w_name'][i] +' ' +  pbp_matches['l_name'][i] + ' ' \
#         + str(pbp_matches['match_year'][i])+ ' ' + pbp_matches['score'][i]
#     if key not in info:
#         print i,key<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19032009900658928, 'fraction_numerical': 0.03565575141318527, 'mean_word_length': 5.4560570071258905, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 52, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '634791', 'license': 'MIT', 'n_tokens_mistral': 13269, 'n_tokens_neox': 12690, 'n_words': 1919, 'path': 'build_datasets/.ipynb_checkpoints/Dataframe Construction-checkpoint.ipynb', 'repo_name': 'JingyaXun/tennis_match_prediction', 'size': '30208'}