<jupyter_start><jupyter_text>Create training data with winning and losing median reg. season stats containing regular and inverted results<jupyter_code>df_win_train = pd.DataFrame(columns=['AFGP','BFGP','AFGP2','BFGP2','AFGP3','BFGP3','AFTP','BFTP','AOR','BOR','ADR','BDR','AAST','BAST','ATO','BTO','ASTL','BSTL','ABLK','BBLK','APF','BPF','AWP','BWP','SDDF','Result'])
df_los_train = pd.DataFrame(columns=['AFGP','BFGP','AFGP2','BFGP2','AFGP3','BFGP3','AFTP','BFTP','AOR','BOR','ADR','BDR','AAST','BAST','ATO','BTO','ASTL','BSTL','ABLK','BBLK','APF','BPF','AWP','BWP','SDDF','Result'])

i = 0
for ii, row in df_ncaa_tourney_compact[(df_ncaa_tourney_compact.Season >= 2003) & (df_ncaa_tourney_compact.Season < 2018)].iterrows():
    t1_fgp = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.WTeamID)].FGP.median()
    t2_fgp = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.LTeamID)].FGP.median()

    t1_fgp2 = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.WTeamID)].FGP2.median()
    t2_fgp2 = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.LTeamID)].FGP2.median()
    
    t1_fgp3 = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.WTeamID)].FGP3.median()
    t2_fgp3 = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.LTeamID)].FGP3.median()
    
    t1_ftp = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.WTeamID)].FTP.median()
    t2_ftp = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.LTeamID)].FTP.median()
    
    t1_or = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.WTeamID)].OR.median()
    t2_or = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.LTeamID)].OR.median()
    
    t1_dr = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.WTeamID)].DR.median()
    t2_dr = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.LTeamID)].DR.median()
    
    t1_ast = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.WTeamID)].AST.median()
    t2_ast = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.LTeamID)].AST.median()
    
    t1_to = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.WTeamID)].TO.median()
    t2_to = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.LTeamID)].TO.median()
    
    t1_stl = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.WTeamID)].STL.median()
    t2_stl = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.LTeamID)].STL.median()
    
    t1_blk = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.WTeamID)].BLK.median()
    t2_blk = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.LTeamID)].BLK.median()
    
    t1_pf = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.WTeamID)].PF.median()
    t2_pf = team_stats[(team_stats.Season == row.Season) & (team_stats.TeamID == row.LTeamID)].PF.median()
    
    t1_wp = win_stats[(win_stats.TeamID == row.WTeamID) & (win_stats.Season == row.Season)].shape[0] / (win_stats[(win_stats.TeamID == row.WTeamID) & (win_stats.Season == row.Season)].shape[0] + los_stats[(los_stats.TeamID == row.WTeamID) & (los_stats.Season == row.Season)].shape[0])
    t2_wp = win_stats[(win_stats.TeamID == row.LTeamID) & (win_stats.Season == row.Season)].shape[0] / (win_stats[(win_stats.TeamID == row.LTeamID) & (win_stats.Season == row.Season)].shape[0] + los_stats[(los_stats.TeamID == row.LTeamID) & (los_stats.Season == row.Season)].shape[0])
               
    seed_diff = get_seed_diff(row.WTeamID,row.LTeamID,row.Season)
        
    df_win_train.loc[i] = [t1_fgp,t2_fgp,t1_fgp2,t2_fgp2,t1_fgp3,t2_fgp3,t1_ftp,t2_ftp,t1_or,t2_or,t1_dr,t2_dr,t1_ast,t2_ast,t1_to,t2_to,t1_stl,t2_stl,t1_blk,t2_blk,t1_pf,t2_pf,t1_wp,t2_wp,seed_diff,1]
    df_los_train.loc[i] = [t2_fgp,t1_fgp,t2_fgp2,t1_fgp2,t2_fgp3,t1_fgp3,t2_ftp,t1_ftp,t2_or,t1_or,t2_dr,t1_dr,t2_ast,t1_ast,t2_to,t1_to,t2_stl,t1_stl,t2_blk,t1_blk,t2_pf,t1_pf,t2_wp,t1_wp,-seed_diff,0]
    
    i=i+1<jupyter_output><empty_output><jupyter_text>Create test data using 2018 reg. season stats and team in submission file<jupyter_code>X_test = pd.DataFrame(columns=['AFGP','BFGP','AFGP2','BFGP2','AFGP3','BFGP3','AFTP','BFTP','AOR','BOR','ADR','BDR','AAST','BAST','ATO','BTO','ASTL','BSTL','ABLK','BBLK','APF','BPF','AWP','BWP','SDDF'])
y_test = pd.DataFrame(columns=['Result'])

i = 0
for ii, row in df_sample_sub2.iterrows():
    year, t1, t2 = get_year_t1_t2(row.ID)
    
    t1_fgp = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t1)].FGP.median()
    t2_fgp = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t2)].FGP.median()
    
    t1_fgp2 = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t1)].FGP2.median()
    t2_fgp2 = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t2)].FGP2.median()
    
    t1_fgp3 = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t1)].FGP3.median()
    t2_fgp3 = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t2)].FGP3.median()
    
    t1_ftp = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t1)].FTP.median()
    t2_ftp = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t2)].FTP.median()
    
    t1_or = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t1)].OR.median()
    t2_or = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t2)].OR.median()
    
    t1_dr = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t1)].DR.median()
    t2_dr = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t2)].DR.median()
    
    t1_ast = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t1)].AST.median()
    t2_ast = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t2)].AST.median()
    
    t1_to = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t1)].TO.median()
    t2_to = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t2)].TO.median()
    
    t1_stl = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t1)].STL.median()
    t2_stl = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t2)].STL.median()
    
    t1_blk = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t1)].BLK.median()
    t2_blk = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t2)].BLK.median()
    
    t1_pf = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t1)].PF.median()
    t2_pf = team_stats[(team_stats.Season == year) & (team_stats.TeamID == t2)].PF.median()
    
    t1_wp = win_stats[(win_stats.TeamID == t1) & (win_stats.Season == year)].shape[0] / (win_stats[(win_stats.TeamID == t1) & (win_stats.Season == year)].shape[0] + los_stats[(los_stats.TeamID == t1) & (los_stats.Season == year)].shape[0])
    t2_wp = win_stats[(win_stats.TeamID == t2) & (win_stats.Season == year)].shape[0] / (win_stats[(win_stats.TeamID == t2) & (win_stats.Season == year)].shape[0] + los_stats[(los_stats.TeamID == t2) & (los_stats.Season == year)].shape[0])
        
    seed_diff = get_seed_diff(t1,t2,year)
        
    X_test.loc[i] = [t1_fgp,t2_fgp,t1_fgp2,t2_fgp2,t1_fgp3,t2_fgp3,t1_ftp,t2_ftp,t1_or,t2_or,t1_dr,t2_dr,t1_ast,t2_ast,t1_to,t2_to,t1_stl,t2_stl,t1_blk,t2_blk,t1_pf,t2_pf,t1_wp,t2_wp,seed_diff]
    y_test.loc[i] = 0.5
    
    i=i+1    
df_predictions = pd.concat((df_win_train, df_los_train))

X_train = df_predictions.loc[:,df_predictions.columns != 'Result']
y_train = df_predictions.Result.values.astype('float')
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV

logreg = LogisticRegression(random_state=0)

params = {'C' : np.logspace(start=-20, stop=20, num=20)}
clf = GridSearchCV(logreg, params, scoring='neg_log_loss', refit=True)
clf.fit(X_train, y_train)
print('Best log_loss: {:.4}, with best C: {}'.format(clf.best_score_, clf.best_params_['C']))
preds = clf.predict_proba(X_test)[:,1]

clipped_preds = np.clip(preds, 0.05, 0.95)

df_sample_sub2.Pred = clipped_preds
#df_sample_sub2.tail()

df_sample_sub2.to_csv('my_2018_submission.csv', index=False)
plt.hist(clipped_preds, bins=50, color='c', edgecolor='black');
df_sample_sub2.Pred.plot.kde();
df_results = pd.DataFrame(columns=['WTeam','LTeam','Prob'])

i = 0
for ii, row in df_sample_sub2.iterrows():
    year, t1, t2 = get_year_t1_t2(row.ID)
    
    df_results.loc[i] = [df_teams[df_teams.TeamID == t1].TeamName.values[0],df_teams[df_teams.TeamID == t2].TeamName.values[0],row.Pred]
    i=i+1
df_results.to_csv('second_log_reg_results2.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1892591758612925, 'fraction_numerical': 0.02972303535239811, 'mean_word_length': 5.229312762973352, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '547597', 'license': 'MIT', 'n_tokens_mistral': 3939, 'n_tokens_neox': 3701, 'n_words': 415, 'path': 'notebooks/second_attempt.ipynb', 'repo_name': 'bobcurrie/2018_ncaa_tourney_men', 'size': '8882'}