<jupyter_start><jupyter_text>Rankability Predicting Sensitivity March Madness DatasetLook for new features:    Existing features:    Diversity of P metrics    Graph measures of tournament matrix as Lapacian    Eigenvalues of tournament matrix    Datasets:   Lichess:       API: https://berserk.readthedocs.io/en/master/        Look for tournaments, grab games played in that time frame and create D matricies       API is pretty simple we just need to build a scraping script.      Sumo:   Data: https://data.world/cervus/sumo-results    It's literally just CSVs, so grab to PANDAS and build D from columns   Bad news: Have to make an account to download data :( /s  Outline of the Experiment 1. Load in the game-by-game data for each year from 2002-2018 2. For each year, construct multiple D matrices using increasing percentages of the games for that year._(For instance in the simplest case, construct a D matrix using only the first half of the season, then a D matrix with the full season.)_ 2. Produce Massey rankings for each D matrix and calculate the Kendall tau between rankings from the same year _(These kendall taus represent the amount that the ranking changed when more data was included)_ 3. For each year, measure features of the restricted dataset (in the simple case, D constructed from 50% of the games) and create a dataset of these early-measurable features. 4. Evaluate whether these early-measurable features can be used to predict the amount that rankings changed after including more data (Kendall taus).<jupyter_code>%load_ext autoreload
%autoreload 2
%matplotlib inline
import copy
import os
import pandas as pd
import numpy as np
import networkx as nx
from scipy.stats import pearsonr
from scipy.stats import skew
from tqdm import tqdm
import matplotlib.pyplot as plt
from pathlib import Path
home = str(Path.home())
home
import sys
sys.path.insert(0,"%s/rankability_toolbox_dev"%home)
import pyrankability
sys.path.insert(0,"%s/sensitivity_study/src"%home)
from sensitivity_tests import *
from utilities import *
from base import *
games={}
remaining_games={}
years = ["2002","2003","2004","2005","2006","2007","2008","2009","2010","2011","2012","2013","2014","2015","2016","2017","2018"]
for year in years:
    games[year],remaining_games[year] = read_data('../data/%steams.txt'%year,'../data/%sgames.txt'%year,'../data/%sMadnessTeams.txt'%year)
     
print(year)
games[year]
remaining_games[year]
# Note to future self: Parameters from FODS paper but might need to be optimized
direct_thress = [0,1,2]
spread_thress = [0,3,5]
weight_indirects = [0.25,0.5,1]
Ds = {}
# fracs represent how much of the data to include
fracs = [0.5,0.75] # 25% of total data added per step
#fracs = [0.64, 0.8, 1.] # 25% of current data added per step
pairs = list(zip(fracs[:-1], fracs[1:]))
#pair_to_predict = pairs[-1]
import itertools
import joblib
load = False
from joblib import Parallel, delayed

if not load:
    for year in tqdm(games.keys()):
        Ds[year] = {}
        madness_teams = np.unique(list(games[year].team1_name.loc[games[year].team1_madness == 1]) + list(games[year].team2_name.loc[games[year].team2_madness == 1]))
        game_list = list(games[year].index)

        game_df = pd.DataFrame({"team1_name":games[year]['team1_name'],
                                "team1_score":games[year]['points1'],
                                "team1_H_A_N": games[year]['H_A_N1'],
                                "team2_name":games[year]['team2_name'],
                                "team2_score":games[year]['points2'],
                                "team2_H_A_N": games[year]['H_A_N1'],
                                "date": games[year]['date']
                               }).sort_values(by='date').drop('date',axis=1)
        
        keys = list(itertools.product(fracs,direct_thress,spread_thress,weight_indirects))
        
        def compute(frac,direct_thres,spread_thres,weight_indirect):
            upper = int(len(game_df)*frac)
            game_df_sample = game_df.iloc[:upper,:]
            map_func = lambda linked: pyrankability.construct.support_map_vectorized_direct_indirect_weighted(linked,direct_thres=direct_thres,spread_thres=spread_thres,weight_indirect=weight_indirect)
            return pyrankability.construct.V_count_vectorized(game_df_sample,map_func).loc[madness_teams,madness_teams]
        
        results = Parallel(n_jobs=-1)(delayed(compute)(frac,direct_thres,spread_thres,weight_indirect) for frac,direct_thres,spread_thres,weight_indirect in keys)
        for i,key in enumerate(keys):
            frac,direct_thres,spread_thres,weight_indirect = key
            Ds[year][(frac,direct_thres,spread_thres,weight_indirect)] = results[i]
if not load:
    massey_rankings = {}
    massey_taus = {}
    massey_results = pd.DataFrame(columns=pairs+["Year"]).set_index("Year")
    colley_rankings = {}
    colley_taus = {}
    colley_results = pd.DataFrame(columns=pairs+["Year"]).set_index("Year")

    massey_rankings = {}
    colley_rankings = {}

    for year in tqdm(Ds.keys()):
        madness_teams = list(np.unique(list(games[year].team1_name.loc[games[year].team1_madness == 1]) + list(games[year].team2_name.loc[games[year].team2_madness == 1])))
        game_list = list(games[year].index)

        game_df = pd.DataFrame({"team1_name":games[year]['team1_name'],
                                "team1_score":games[year]['points1'],
                                "team1_H_A_N": games[year]['H_A_N1'],
                                "team2_name":games[year]['team2_name'],
                                "team2_score":games[year]['points2'],
                                "team2_H_A_N": games[year]['H_A_N1'],
                                "date": games[year]['date']
                               }).sort_values(by='date').drop('date',axis=1)
        
        massey_rankings[year] = pd.DataFrame(columns=["frac","direct_thres","spread_thres","weight_indirect"]+madness_teams)
        colley_rankings[year] = pd.DataFrame(columns=["frac","direct_thres","spread_thres","weight_indirect"]+madness_teams)
        
        keys = list(itertools.product(fracs,direct_thress,spread_thress,weight_indirects))
        
        def compute(frac,direct_thres,spread_thres,weight_indirect):
            upper = int(len(game_df)*frac)
            game_df_sample = game_df.iloc[:upper,:]
                        
            map_func = lambda linked: pyrankability.construct.colley_matrices(linked,direct_thres=direct_thres,spread_thres=spread_thres,weight_indirect=weight_indirect)
            colley_matrix,colley_b = pyrankability.construct.map_vectorized(game_df_sample,map_func)
            colley_matrix = colley_matrix.loc[madness_teams,madness_teams]
            colley_b = colley_b.loc[madness_teams]
            ranking1,r1 = pyrankability.construct.ranking_from_matrices(colley_matrix.fillna(0),colley_b)
            
            map_func = lambda linked: pyrankability.construct.massey_matrices(linked,direct_thres=direct_thres,spread_thres=spread_thres,weight_indirect=weight_indirect)
            massey_matrix,massey_b = pyrankability.construct.map_vectorized(game_df_sample,map_func)
            massey_matrix = massey_matrix.loc[madness_teams,madness_teams]
            massey_b = massey_b.loc[madness_teams]
            ranking2,r2 = pyrankability.construct.ranking_from_matrices(massey_matrix.fillna(0),massey_b.fillna(0))
            
            return pd.Series([frac,direct_thres,spread_thres,weight_indirect]+list(ranking1),index=colley_rankings[year].columns),pd.Series([frac,direct_thres,spread_thres,weight_indirect]+list(ranking2),index=massey_rankings[year].columns)
 
        #frac,direct_thres,spread_thres,weight_indirect = keys[0]
        #compute(frac,direct_thres,spread_thres,weight_indirect)
    
        results = Parallel(n_jobs=-1)(delayed(compute)(frac,direct_thres,spread_thres,weight_indirect) for frac,direct_thres,spread_thres,weight_indirect in keys)
        
        c = 0
        for i,key in enumerate(keys):
            frac,direct_thres,spread_thres,weight_indirect = key
            massey,colley = results[i]
            massey.name = c
            colley.name = c
            massey_rankings[year] = massey_rankings[year].append(massey)
            colley_rankings[year] = colley_rankings[year].append(colley)
            c+=1
            
        """
        c=0
        for frac,direct_thres,spread_thres,weight_indirect in Ds[year].keys():
            upper = int(len(game_df)*frac)
            game_df_sample = game_df.iloc[:upper,:]
                        
            map_func = lambda linked: pyrankability.construct.colley_matrices(linked,direct_thres=direct_thres,spread_thres=spread_thres,weight_indirect=weight_indirect)
            colley_matrix,colley_b = pyrankability.construct.map_vectorized(game_df_sample,map_func)
            colley_matrix = colley_matrix.loc[madness_teams,madness_teams]
            colley_b = colley_b.loc[madness_teams]
            ranking1,r1 = pyrankability.construct.ranking_from_matrices(colley_matrix.fillna(0),colley_b)
            
            map_func = lambda linked: pyrankability.construct.massey_matrices(linked,direct_thres=direct_thres,spread_thres=spread_thres,weight_indirect=weight_indirect)
            massey_matrix,massey_b = pyrankability.construct.map_vectorized(game_df_sample,map_func)
            massey_matrix = massey_matrix.loc[madness_teams,madness_teams]
            massey_b = massey_b.loc[madness_teams]
            ranking2,r2 = pyrankability.construct.ranking_from_matrices(massey_matrix.fillna(0),massey_b.fillna(0))
            
            #ranking1 = MasseyRankingAlgorithm().rank(D1.fillna(0).values)
            #ranking2 = ColleyRankingAlgorithm().rank(D1.fillna(0).values)
            massey_rankings[year] = massey_rankings[year].append(pd.Series([frac,direct_thres,spread_thres,weight_indirect]+list(ranking1),name=c,index=massey_rankings[year].columns))
            colley_rankings[year] = colley_rankings[year].append(pd.Series([frac,direct_thres,spread_thres,weight_indirect]+list(ranking2),name=c,index=colley_rankings[year].columns))
            c+=1
            """
    joblib.dump({"colley_rankings":colley_rankings,"massey_rankings":massey_rankings,"Ds":Ds},"checkpoint1.joblib.z");
if load:
    loaded = joblib.load('checkpoint1.joblib.z')
    colley_rankings=loaded['colley_rankings']
    massey_rankings=loaded['massey_rankings']
    Ds=loaded['Ds']
def predictability(perm,game_df_sample):
    r = -1*perm.argsort()
    
    numberCorrectPredictions = 0
    numGames = len(game_df_sample)
    c = 0
    for i in game_df_sample.index:
        team1ID = game_df_sample.loc[i,"team1_name"]
        team1Score = game_df_sample.loc[i,"points1"]

        team2ID = game_df_sample.loc[i,"team2_name"]
        team2Score = game_df_sample.loc[i,"points2"]
        
        if team1ID in r.index and team2ID in r.index:
            c+=1
            if team1Score > team2Score and r.loc[team1ID] > r.loc[team2ID]:
                numberCorrectPredictions += 1
            elif team2Score > team1Score and r.loc[team2ID] > r.loc[team1ID]:
                numberCorrectPredictions += 1
            elif team1Score == team2Score and r.loc[team1ID] == r.loc[team2ID]:
                numberCorrectPredictions += 1

    return numberCorrectPredictions/c*100

for year in tqdm(Ds.keys()):  
    test_games = remaining_games[year]
    massey_results = massey_rankings[year].set_index(['frac','direct_thres','spread_thres','weight_indirect'])
    colley_results = colley_rankings[year].set_index(['frac','direct_thres','spread_thres','weight_indirect'])
    keys = list(itertools.product(fracs,direct_thress,spread_thress,weight_indirects))
    accs = pd.Series(index=massey_results.index)
    for index in massey_results.index:
        frac,direct_thres,spread_thres,weight_indirect = index
        perm = massey_results.loc[(frac,direct_thres,spread_thres,weight_indirect)]
        accs.loc[index] = predictability(perm,test_games)
    massey_results['predictability'] = accs

    accs = pd.Series(index=colley_results.index)
    for index in colley_results.index:
        frac,direct_thres,spread_thres,weight_indirect = index
        perm = colley_results.loc[(frac,direct_thres,spread_thres,weight_indirect)]
        accs.loc[index] = predictability(perm,test_games)
    colley_results['predictability'] = accs

    colley_rankings[year]=colley_results.reset_index()
    massey_rankings[year]=massey_results.reset_index()

predictability_results = pd.DataFrame(columns=['Year','Method']+['frac','direct_thres','spread_thres','weight_indirect']+['Predictability'])

c = 0
for year in tqdm(Ds.keys()):
    for index in massey_rankings[year].index:
        dt = [year,'Massey'] + list(massey_rankings[year].loc[index,['frac','direct_thres','spread_thres','weight_indirect']]) + list([massey_rankings[year].loc[index,'predictability']])
        predictability_results = predictability_results.append(pd.Series(dt,index=predictability_results.columns,name=c))
        c+=1
    for index in colley_rankings[year].index:    
        dt = [year,'Colley'] + list(colley_rankings[year].loc[index,['frac','direct_thres','spread_thres','weight_indirect']]) + list([colley_rankings[year].loc[index,'predictability']])
        predictability_results = predictability_results.append(pd.Series(dt,index=predictability_results.columns,name=c))
        c+=1
predictability_results
predictability_results.groupby(['frac','Method'])['Predictability'].mean()
import altair as alt

alt.Chart(predictability_results.groupby(['frac','Method'])['Predictability'].mean().to_frame().reset_index()).mark_bar().encode(
    x='Method',
    y=alt.Y('Predictability',title="Mean Predictability"),
    color='Method'#,
    #column='Year:N'
).properties(
    width=180,
    height=180
).facet(column='frac')
import altair as alt

alt.Chart(predictability_results.groupby(['frac','Method'])['Predictability'].min().to_frame().reset_index()).mark_bar().encode(
    x='Method',
    y=alt.Y('Predictability',title="Min Predictability"),
    color='Method'#,
    #column='Year:N'
).properties(
    width=180,
    height=180
).facet(column='frac')
import altair as alt

alt.Chart(predictability_results.groupby(['frac','Method'])['Predictability'].max().to_frame().reset_index()).mark_bar().encode(
    x='Method',
    y=alt.Y('Predictability',title="Min Predictability"),
    color='Method'#,
    #column='Year:N'
).properties(
    width=180,
    height=180
).facet(column='frac')
import altair as alt

alt.Chart(predictability_results.groupby(['frac','Method'])['Predictability'].std().to_frame().reset_index()).mark_bar().encode(
    x='Method',
    y=alt.Y('Predictability',title="Stdev Predictability"),
    color='Method'#,
    #column='Year:N'
).properties(
    width=180,
    height=180
).facet(column='frac')
import altair as alt

alt.Chart(predictability_results).mark_bar().encode(
    x='Method',
    y=alt.Y('Predictability',scale=alt.Scale(domain=(0, 100))),
    color='Method'#,
    #column='Year:N'
).properties(
    width=180,
    height=180
).facet(row='Year:N',column='frac')<jupyter_output><empty_output><jupyter_text>Compute Kendall W for each year<jupyter_code>import kendall_w as kw

def results_to_frame(results,method):
    t = results.to_frame()
    t.columns=['w']
    t['Method'] = method
    t['Year']=year
    t.reset_index(inplace=True)
    return t

ws = pd.DataFrame(columns=['frac','w','Method','Year'])
for year in tqdm(Ds.keys()):
    madness_teams = list(np.unique(list(games[year].team1_name.loc[games[year].team1_madness == 1]) + list(games[year].team2_name.loc[games[year].team2_madness == 1])))
    results = colley_rankings[year].groupby(['frac']).apply(lambda df: kw.compute_w(df[madness_teams].astype(int).T.values.tolist()))
    ws = ws.append(results_to_frame(results,'Colley'))
    results = massey_rankings[year].groupby(['frac']).apply(lambda df: kw.compute_w(df[madness_teams].astype(int).T.values.tolist()))
    ws = ws.append(results_to_frame(results,'Massey'))
ws
import altair as alt

alt.Chart(ws).mark_bar().encode(
    x='Method',
    y=alt.Y('w',scale=alt.Scale(0,1)),
    color='Method'#,
    #column='Year:N'
).properties(
    width=180,
    height=180
).facet(row='Year:N',column='frac')
ws.groupby(['frac','Method'])['w'].std().to_frame().reset_index()
import altair as alt

alt.Chart(ws.groupby(['frac','Method'])['w'].std().to_frame().reset_index()).mark_bar().encode(
    x='Method',
    y=alt.Y('w',title="Stdev w"),
    color='Method'#,
    #column='Year:N'
).properties(
    width=180,
    height=180
).facet(column='frac')
import altair as alt

alt.Chart(ws.groupby(['frac','Method'])['w'].mean().to_frame().reset_index()).mark_bar().encode(
    x='Method',
    y=alt.Y('w',title="Mean w"),
    color='Method'#,
    #column='Year:N'
).properties(
    width=180,
    height=180
).facet(column='frac')
"""
col_names = [
    'Year',
    '# X* frac',
    'k',
    '# X* frac top 40',
    'kendall_w',
    'p_lowerbound',
    'max_L2_dist',
    'mean_L2_dist',
    'min_tau',
    'mean_tau',
    'max_eigenval',
    'min_eigenval',
    'Pair'
]

all_score_df = pd.DataFrame(columns=col_names)

c=0
for year in tqdm(games.keys()):
    # the support matrix for a pair for a given year
    V = Ds[year][pair_to_predict[0]]
    # print(V.fillna(0.0))
    vals, vecs = np.linalg.eig(V.fillna(0.0).to_numpy())
    determinant = np.prod(vals)
    trace = np.sum(vals)
    max_eigenval = np.real(np.max(vals))
    min_eigenval = np.real(np.min(vals))
    dsGraph = nx.from_numpy_matrix(V.fillna(0.0).to_numpy())
    
    # nodesranked = nx.voterank(dsGraph)
    # print(nodesranked)
    
    rresults = rankability_results.iloc[c,:]
    k = rresults['k']
    details = df_details[c]
    x = pd.DataFrame(details['x'],index=V.index,columns=V.columns)
    r = x.sum(axis=0)
    order = np.argsort(r)
    xstar = x.iloc[order,:].iloc[:,order]
    xstar.loc[:,:] = pyrankability.common.threshold_x(xstar.values)
    inxs = np.triu_indices(len(xstar),k=1)
    xstar_upper = xstar.values[inxs[0],inxs[1]]
    nfrac_upper = sum((xstar_upper > 0) & (xstar_upper < 1))
    flat_frac = ((xstar > 0) & (xstar < 1)).sum(axis=0)
    nfrac_top_40 = flat_frac.iloc[:40].sum()
    entry_data = [
        year,
        nfrac_upper*2,
        k,
        nfrac_top_40,
        rresults["kendall_w"],
        rresults["p_lowerbound"],
        rresults["max_L2_dist"],
        rresults["mean_L2_dist"],
        rresults["min_tau"],
        rresults["mean_tau"],
        max_eigenval, 
        min_eigenval,
        pair
    ]
    """
summaries = pd.DataFrame(columns=["k_best","k_worst","k_best/k_worst","frac","threshold","nfrac_upper*2","nfrac_top_40","max_eigenval","min_eigenval"])

def summarize_hillside(D,details,name,frac,threshold):
    perm = np.array(details['P'][0])
    C = pyrankability.construct.C_count(D,threshold=0)
    k_worst = np.sum(np.triu(C.iloc[perm[::-1],:].iloc[:,perm[::-1]]))
    C = pyrankability.construct.C_count(D,threshold=threshold)
    k_best = np.sum(np.triu(C.iloc[perm,:].iloc[:,perm]))
    
    V = D
    vals, vecs = np.linalg.eig(V.fillna(0.0).to_numpy())
    determinant = np.prod(vals)
    trace = np.sum(vals)
    max_eigenval = np.real(np.max(vals))
    min_eigenval = np.real(np.min(vals))
    dsGraph = nx.from_numpy_matrix(V.fillna(0.0).to_numpy())
    
    # nodesranked = nx.voterank(dsGraph)
    # print(nodesranked)
    
    x = pd.DataFrame(details['x'],index=V.index,columns=V.columns)
    r = x.sum(axis=0)
    order = np.argsort(r)
    xstar = x.iloc[order,:].iloc[:,order]
    xstar.loc[:,:] = pyrankability.common.threshold_x(xstar.values)
    inxs = np.triu_indices(len(xstar),k=1)
    xstar_upper = xstar.values[inxs[0],inxs[1]]
    nfrac_upper = sum((xstar_upper > 0) & (xstar_upper < 1))
    flat_frac = ((xstar > 0) & (xstar < 1)).sum(axis=0)
    nfrac_top_40 = flat_frac.iloc[:40].sum()
    entry_data = [
        nfrac_upper*2,
        nfrac_top_40,
        max_eigenval, 
        min_eigenval
    ]
    
    s = pd.Series([k_best,k_worst,k_best/k_worst,frac,threshold]+entry_data,index=summaries.columns,name=name)
    return s

exp_ks = {}
exp_details = {}
thresholds = [0,0.5,1]
for year in tqdm(Ds.keys()):
    for frac in fracs:
        D = Ds[year][frac,0,0,0.25]
        exp_ks[year],exp_details[year] = pyrankability.rank.solve(D,method='hillside', lazy=False, cont=True)
        for threshold in thresholds:
            summaries = summaries.append(summarize_hillside(D,exp_details[year],year,frac,threshold))
summaries.index.name="Year"
summaries.reset_index(inplace=True)
summaries
summaries.head()
results = summaries.reset_index().set_index(['Year','frac']).join(ws.set_index(['Year','frac']))
results
k=10
results.reset_index().groupby(['threshold','frac','Method']).apply(lambda df: set(df.sort_values(by='k_best/k_worst',ascending=False)['Year'][:k]).intersection(df.sort_values(by='w')['Year'][:k]))
results.reset_index().groupby(['threshold','frac','Method']).apply(lambda df: len(set(df.sort_values(by='k_best/k_worst',ascending=False)['Year'][:k]).intersection(df.sort_values(by='w')['Year'][:k]))/k)

results.reset_index().groupby(['threshold','frac','Method']).apply(lambda df: df.sort_values(by='k_best/k_worst',ascending=False)['Year'][:5])
results.reset_index().groupby(['threshold','frac','Method']).apply(lambda df: df.sort_values(by='w')['Year'][:5])
from scipy.stats import pearsonr

fields_to_test = ["k_best","k_worst","k_best/k_worst","nfrac_upper*2","nfrac_top_40","max_eigenval","min_eigenval"]
corr_results = {}
for field in fields_to_test:
    corr_results[field] = pd.DataFrame(results.reset_index().groupby(['threshold','frac','Method']).apply(lambda df: pd.Series(pearsonr(df['w'], df[field]),index=['r','pval'])))

from IPython.display import display, Markdown, Latex

for field in corr_results.keys():
    display(Markdown(f'## {field}'))
    display(corr_results[field])
slopes = pd.DataFrame(results.dropna().reset_index().groupby(['frac','Method','Year']).apply(lambda df: np.polyfit(df['threshold'], df['k_best/k_worst'],deg=1)))
slopes
slopes = pd.DataFrame(results.dropna().reset_index().groupby(['frac','Method','Year']).apply(lambda df: np.polyfit(df['threshold'], df['k_best/k_worst'], 1)[0]))
slopes.columns = ['k_best/k_worst slope']
slopes
slopes.join(results.dropna().reset_index().set_index(['frac','Method','Year'])[['w']]).drop_duplicates().reset_index().groupby(['frac','Method']).apply(
    lambda df: pd.Series(pearsonr(df['w'], df['k_best/k_worst slope']),index=['r','pval']))
results.reset_index().set_index('threshold').loc[0].groupby(['frac','Method']).apply(
    lambda df: pearsonr(df['w'], df['k_best/k_worst']))
results.reset_index().set_index('threshold').loc[0].groupby(['frac','Method']).apply(
    lambda df: pearsonr(df['w'], df['(k_worst-k_best)/k_worst']))
pearsonr(summaries['w'], summaries['k_best/k_worst'])
results = pd.DataFrame({"w":ws.loc[0.5],"k":pd.Series(exp_ks)})
results.corr(method='kendall')<jupyter_output><empty_output><jupyter_text>Digging into specific years<jupyter_code>results[(0.5,0.75)].plot.hist()
idx_min = results[(0.5,0.75)].idxmin()
idx_max = results[(0.5,0.75)].idxmax()
idx_min,idx_max
Ds[idx_min][0.5].stack().plot.hist()
Ds[idx_max][0.5].stack().plot.hist()
idx_min
k_max,details_max = pyrankability.rank.solve(Ds[idx_max][0.5],method='hillside', lazy=False, cont=False)
k_min,details_min = pyrankability.rank.solve(Ds[idx_min][0.5],method='hillside', lazy=False, cont=False)
k_max,k_min
Ds[idx_min][0.5].shape
def summarize_hillside(D,perm):
    C = pyrankability.construct.C_count(D,threshold=0)
    k_best = np.sum(np.triu(C.iloc[perm,:].iloc[:,perm]))
    k_worst = np.sum(np.triu(C.iloc[perm[::-1],:].iloc[:,perm[::-1]]))
    return k_best,k_worst,k_best/k_worst
summarize_hillside(Ds[idx_min][0.5],list(details_min['P'][0]))
summarize_hillside(Ds[idx_max][0.5],list(details_max['P'][0]))
Dordered_min = Ds[idx_min][0.5].iloc[list(details_min['P'][0]),:].iloc[:,list(details_min['P'][0])]
Dordered_min
pyrankability.plot.show_hillside(Ds[idx_min][0.5],details_min['P'][0])
pyrankability.plot.show_hillside(Ds[idx_max][0.5],details_max['P'][0])<jupyter_output><empty_output><jupyter_text>Back to general studies<jupyter_code># Note: what do the contents of this matrix mean??
# Ds['2018'][1.0]
df_details = []
# This function constructs a dataframe of features
# (collected from the first D matrix of each pair)
# To be used to predict the movement of the pair (kendall tau of rankings)
def get_rankability_results(n_restarts=250):
    df_ks = []
    df_years = []
    df_fracs = []
    df_p_stats = {}
    for year in tqdm(games.keys()):
        D = Ds[year][pair_to_predict[0]].fillna(0)
        k,details = pyrankability.rank.solve(D,method='lop', num_random_restarts=n_restarts, lazy=False, cont=True)
        p_stats = get_P_stats(details["P"])
        for name, val in p_stats.items():
            if name not in df_p_stats:
                df_p_stats[name] = []
            df_p_stats[name].append(val)
        df_ks.append(k)
        df_years.append(year)
        df_fracs.append(pair_to_predict[0])
        df_details.append(details)

    results_temp = {"k":df_ks,"Year":df_years,"Fraction":df_fracs}
    for key, val in df_p_stats.items():
        if key in results_temp:
            raise ValueError("Duplicate column name! Fix collision before moving on!")
        results_temp[key] = val

    return pd.DataFrame(results_temp)
rankability_results = get_rankability_results(n_restarts=2)
rankability_results
for_corr = rankability_results.loc[rankability_results.Fraction==pair_to_predict[0]].set_index('Year').join(results)
for_corr
for_corr.corr()
pearsonr(for_corr['k'],for_corr[pair_to_predict])<jupyter_output><empty_output><jupyter_text>Notes Ethan 9/27/20Determinant and trace of Ds not good features Max and min eigenvalue not bad Attempted betweenness centrality features, none worthwhile `betweennesses = nx.betweenness_centrality(dsGraph)avg_bt_centrality = np.average(np.array(list(betweennesses.values())))var_bt_centrality = np.sqrt(np.var(np.array(list(betweennesses.values()))))print(avg_bt_centrality, var_bt_centrality)`<jupyter_code>col_names = [
    'Year',
    '# X* frac',
    'k',
    '# X* frac top 40',
    'kendall_w',
    'p_lowerbound',
    'max_L2_dist',
    'mean_L2_dist',
    'min_tau',
    'mean_tau',
    'max_eigenval',
    'min_eigenval',
    'Pair'
]

all_score_df = pd.DataFrame(columns=col_names)

c=0
for year in tqdm(games.keys()):
    # the support matrix for a pair for a given year
    V = Ds[year][pair_to_predict[0]]
    # print(V.fillna(0.0))
    vals, vecs = np.linalg.eig(V.fillna(0.0).to_numpy())
    determinant = np.prod(vals)
    trace = np.sum(vals)
    max_eigenval = np.real(np.max(vals))
    min_eigenval = np.real(np.min(vals))
    dsGraph = nx.from_numpy_matrix(V.fillna(0.0).to_numpy())
    
    # nodesranked = nx.voterank(dsGraph)
    # print(nodesranked)
    
    rresults = rankability_results.iloc[c,:]
    k = rresults['k']
    details = df_details[c]
    x = pd.DataFrame(details['x'],index=V.index,columns=V.columns)
    r = x.sum(axis=0)
    order = np.argsort(r)
    xstar = x.iloc[order,:].iloc[:,order]
    xstar.loc[:,:] = pyrankability.common.threshold_x(xstar.values)
    inxs = np.triu_indices(len(xstar),k=1)
    xstar_upper = xstar.values[inxs[0],inxs[1]]
    nfrac_upper = sum((xstar_upper > 0) & (xstar_upper < 1))
    flat_frac = ((xstar > 0) & (xstar < 1)).sum(axis=0)
    nfrac_top_40 = flat_frac.iloc[:40].sum()
    entry_data = [
        year,
        nfrac_upper*2,
        k,
        nfrac_top_40,
        rresults["kendall_w"],
        rresults["p_lowerbound"],
        rresults["max_L2_dist"],
        rresults["mean_L2_dist"],
        rresults["min_tau"],
        rresults["mean_tau"],
        max_eigenval, 
        min_eigenval,
        pair
    ]
    entry = pd.Series(entry_data,col_names,name=c)
    c+=1
    all_score_df = all_score_df.append(entry)
all_score_df.set_index("Year",inplace=True)
all_score_df = all_score_df.loc[all_score_df.Pair == pair_to_predict].drop('Pair',axis=1).join(results)
# By this point, all_score_df includes all features that will be used to predict the sensitivity to new information
all_score_df
all_score_df.to_csv("all_score_df.csv")
all_score_df = pd.read_csv("all_score_df.csv")
all_score_df = all_score_df.set_index("Year")

# All the pairs have become strings after being read in
pair_to_predict = str(pair_to_predict)
pairs = [str(p) for p in pairs]
all_score_df.columns
cdf = all_score_df.fillna(0.0).corr()
print(cdf.columns)
cdf = all_score_df.fillna(0.0).corr()
print(cdf.columns)
for col in all_score_df.columns:
    if col not in pairs:
        all_score_df.plot.scatter(col, pair_to_predict, title="Final Sensitivity vs " + col)
        print("Correlation between", pair_to_predict, "and", col, cdf[pair_to_predict].loc[col])
import sklearn.linear_model as skl_lm
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import LeaveOneOut
from sklearn.svm import SVR
from sklearn.dummy import DummyRegressor
from sklearn.model_selection import GridSearchCV

# List of all features to use to predict
# (drops the quantity to be predicted)
all_feature_cols = [c for c in all_score_df.columns if c not in pairs]

# Evaluates a regression model attempting to regress "pred_col" using leave-one-out
#     cross validation. If the model has parameters values to search over, each unique
#     parameter setting will be evaluated using 3-fold cross validation on top of the LOO CV.
# Reported statistics are [mean of absolute error] and [std of absolute error] over all LOO folds.
def evaluate(df,pred_col=pair_to_predict,feature_cols=all_feature_cols,model=DummyRegressor(),param_grid={}):
    loo = LeaveOneOut()
    y = df[pred_col]
    X = df[feature_cols]
    
    grid = GridSearchCV(model,param_grid,refit=True,verbose=0, cv=3, iid=True, n_jobs=-1)
    scores = cross_val_score(grid, X, y, scoring="neg_mean_absolute_error", cv=loo, n_jobs=1)
    return pd.Series([len(scores),np.mean(np.abs(scores)),np.std(scores)],index=["Folds","MAE","STD"])<jupyter_output><empty_output><jupyter_text>Same as above but with exhaustive feature selectionBelow:Seems to be a bug with feature subset selection. All scores come out the same.<jupyter_code>from sklearn.linear_model import LinearRegression
from itertools import chain, combinations

# Evaluates a regression model attempting to regress "pred_col" using leave-one-out
#     cross validation. If the model has parameters values to search over, each unique
#     parameter setting will be evaluated using 3-fold cross validation on top of the LOO CV.
# Reported statistics are [mean of absolute error] and [std of absolute error] over all LOO folds.
def evaluate_exhaustive(df,
                        pred_col=pair_to_predict,
                        feature_cols=all_feature_cols,
                        model=DummyRegressor(),
                        param_grid={},
                        print_best_params=False):
    exhaustive = {}
    y = df[pred_col]
    X = df[feature_cols]
    
    # run on subsets of features
    exhaustive_feat_select = list(chain.from_iterable(combinations(list(range(len(X.columns))), r) for r in range(len(X.columns))))[1:]
    # only 10 feature subsets (out of 2^n) for debug purposes
    best_score = np.Inf
    best_features = None
    for ps in tqdm(exhaustive_feat_select):
        features = X.iloc[:, list(ps)]
        grid = GridSearchCV(model,param_grid,refit=True,verbose=0, cv=3, iid=True, n_jobs=-1)
        exhaustive[ps] = np.mean(np.abs(cross_val_score(grid, features, y, scoring="neg_mean_absolute_error", cv=LeaveOneOut(), n_jobs=1)))
        if exhaustive[ps] < best_score:
            best_score = exhaustive[ps]
            best_features = ps
    
    # print(scores)
    return {"MAE": best_score, "best_feature_subset": [X.columns[f] for f in best_features]}
svr_model_results = evaluate(all_score_df,model=SVR(),param_grid = {'C': [0.1,1,10,100], 'epsilon': [0.1,0.5,1],'kernel': ['linear', 'rbf']})
svr_model_results
lr_model_results = evaluate_exhaustive(all_score_df,model=LinearRegression(),param_grid = {'fit_intercept': [True, False]})
lr_model_results
dummy_model_results = evaluate(all_score_df)
dummy_model_results
x = ['Baseline', 'SVR', "Linear Regression"]
maes = [dummy_model_results["MAE"], svr_model_results["MAE"], lr_model_results["MAE"]]
x_pos = [i for i, _ in enumerate(x)]

plt.bar(x_pos, maes)
plt.xlabel("Model")
plt.ylabel("Error")
plt.title("Mean Absolute Error of Regression Models")

plt.xticks(x_pos, x)

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1453135021225911, 'fraction_numerical': 0.014903948935650368, 'mean_word_length': 4.270239819732819, 'pattern_counts': {'":': 24, '<': 32, '<?xml version=': 0, '>': 35, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '610169', 'license': 'MIT', 'n_tokens_mistral': 11912, 'n_tokens_neox': 11408, 'n_words': 1814, 'path': 'experiments/backupMarchMadnessHyperparameters.ipynb', 'repo_name': 'IGARDS/sensitivity_study', 'size': '32743'}