<jupyter_start><jupyter_text>RPLib Problem 0001 - BaselineProvides the baseline version to rankability problem 0001.<jupyter_code>%load_ext autoreload
%autoreload 2
%matplotlib inline
import copy
import os

import pandas as pd
import numpy as np

from scipy.stats import pearsonr

from tqdm import tqdm
from joblib import Parallel, delayed
import joblib
import itertools
from pathlib import Path
from sklearn.pipeline import Pipeline

from IPython.display import display, Markdown, Latex<jupyter_output><empty_output><jupyter_text>**All packages are relative to the home directory of the user**<jupyter_code>home = str(Path.home())<jupyter_output><empty_output><jupyter_text>**Import the main rankability package**<jupyter_code>import sys
sys.path.insert(0,"%s/rankability_toolbox_dev"%home)
import pyrankability
sys.path.insert(0,"%s/RPLib"%home)
import pyrplib<jupyter_output><empty_output><jupyter_text>**Load the problem information**<jupyter_code>problem = joblib.load("generate.joblib.z")<jupyter_output><empty_output><jupyter_text>Explore and setup the problem<jupyter_code>problem.keys()
print(problem["description"])
problem['target']
problem['data'].keys()
problem['data']['2002'].keys()<jupyter_output><empty_output><jupyter_text>**Create easier to reference variables**<jupyter_code>years = list(problem['data'].keys())
days_to_subtract_keys = list(problem['data'][years[0]].keys())
remaining_games = problem['other']['remaining_games']
madness_teams = problem['other']['madness_teams']
best_df = problem['other']['best_df']
top_k = problem['other']['top_k']
#target_column = f"top{top_k}_intersection"
feature_names = problem['other']['feature_names']
days_to_subtract_keys
target = problem['target']
target<jupyter_output><empty_output><jupyter_text>Create D matrices<jupyter_code>best_df
#Ds = process(problem['data'],problem['target'],best_df)
#Ds
#Ds.iloc[[0,-1]]
#Ds.loc['2002',"D"][0][0]
#Ds.loc['2002',"D"][0][1]
#Ds.index.names
rankings = {}
dt = target.set_index(['days_to_subtract2','Year','direct_thres','spread_thres','weight_indirect','range','Method'])
for index in dt.index:
    rankings[index] = dt.loc[index].iloc[0]['rankings2']
dt = target.set_index(['days_to_subtract1','Year','direct_thres','spread_thres','weight_indirect','range','Method'])
for index in dt.index:
    rankings[index] = dt.loc[index].iloc[0]['rankings1']
rankings_df = pd.DataFrame(rankings).T<jupyter_output><empty_output><jupyter_text>Compute the features<jupyter_code>index_cols = ["Year","days_to_subtract_key","direct_thres","spread_thres","weight_indirect","range","Method"]
feature_columns = ["delta_lop","delta_hillside","nfrac_xstar_lop","nfrac_xstar_hillside","diameter_lop","diameter_hillside"]
process_feature_pipe = Pipeline([('compute_D', pyrplib.transformers.ProcessTransformer(index_cols, days_to_subtract_keys, years, 'madness', madness_teams, best_df)),
                                ('create_features', pyrplib.transformers.CreateFeaturesTransformer(feature_columns, rankings_df, top_k))])
process_feature_pipe.fit(problem['data'])
X = process_feature_pipe.transform(problem['data'])
# X = create_features(Ds,rankings_df,top_k)
X<jupyter_output><empty_output><jupyter_text>Refine the target dataset<jupyter_code>target = problem['target'].groupby(['days_to_subtract1','days_to_subtract2','Method','Year','direct_thres','spread_thres','weight_indirect'])[feature_names].mean()
target
target.corr()
X_for_join = X.copy().reset_index()
X_for_join['days_to_subtract1']= X_for_join['days_to_subtract_key'].str.replace("days_to_subtract=","").astype(float)
X_for_join.weight_indirect=0.1
X_for_join
target
Xy = target.reset_index().set_index(['Method','days_to_subtract1','Year','direct_thres','spread_thres','weight_indirect']).join(X_for_join.set_index(['Method','days_to_subtract1','Year','direct_thres','spread_thres','weight_indirect'])).dropna()
Xy = Xy.reset_index()
Xy<jupyter_output><empty_output><jupyter_text>Process results<jupyter_code>#pairs_by_width = {}
#for f1,f2 in itertools.combinations(X_for_join['days_to_subtract1'].unique().astype(int),2):
#    if f2 < f1:
#        f1,f2 = f2,f1
#    width = f2-f1#round(100*(f2-f1))
#    if width not in pairs_by_width:
#        pairs_by_width[width] = []
#    pairs_by_width[width].append((f1,f2))
pairs_by_width = pyrplib.utils.get_pairs_by_width(X_for_join)
pairs_by_width
["days_to_subtract1","days_to_subtract2","Method","Construction"]+feature_columns
#!sudo pip install pandas --upgrade
#import altair as alt
#
#index_cols = ['Method', 'Construction', 'days_to_subtract1', 'days_to_subtract2','width']
#graph_dfs = {}
#for target_column in feature_names:
#    graph_df = pd.DataFrame(columns=index_cols+feature_columns).set_index(index_cols)
#
#    for width in pairs_by_width.keys():
#        summary = None
#        for pair in pairs_by_width[width]:
#            data = Xy.set_index(['days_to_subtract1','days_to_subtract2']).loc[pair].reset_index()
#            for_corr = data.set_index(['Method','Construction',"days_to_subtract1","days_to_subtract2"])
#            if summary is None:
#                summary = pd.DataFrame(columns=["days_to_subtract1","days_to_subtract2","Method","Construction"]+feature_columns).set_index(list(for_corr.index.names))
#            for ix in for_corr.index.unique():
#                corr_results = for_corr.loc[ix][[target_column]+feature_columns].corr()
#                target_corr_results = corr_results.loc[target_column].drop(target_column)
#                target_corr_results.name = ix
#                summary = summary.append(target_corr_results)
#
#        graph_df1 = summary.reset_index()
#        graph_df1['width'] = width
#        graph_df1 = graph_df1.set_index(index_cols)
#        graph_df = graph_df.append(graph_df1)
#    graph_dfs[target_column]=graph_df
#for key in graph_dfs.keys():
#    graph_dfs[key] = graph_dfs[key].reset_index()
index_cols = ['Method', 'Construction', 'days_to_subtract1', 'days_to_subtract2','width']
graph_dfs = pyrplib.utils.get_graph_dfs(index_cols, feature_columns, feature_names, pairs_by_width, Xy)
graph_dfs['top10_tau'].head()
#graph_dfs[key].head()
pyrplib.utils.display_graph_dfs(graph_dfs, feature_columns, index_cols)
#for key in graph_dfs.keys():
#    display(Markdown(f'## {key}'))
#    graph_df = graph_dfs[key].melt(value_vars=feature_columns,id_vars=index_cols,value_name='Value',var_name='Feature')
#
#    display(Markdown('### Colley'))
#    g = alt.Chart(graph_df.set_index('Method').loc['Colley']).mark_bar().encode(
#        x='width:N',
#        y=alt.Y('average(Value)',scale=alt.Scale(domain=[-.6, .6])),
#        row='Feature:N',
#        color='Construction:N',
#        column='Construction:N'
#    )
#    display(g)
#    
#    display(Markdown('### Massey'))
#    g = alt.Chart(graph_df.set_index('Method').loc['Massey']).mark_bar().encode(
#        x='width:N',
#        y=alt.Y('average(Value)',scale=alt.Scale(domain=[-.6, .6])),
#        row='Feature:N',
#        color='Construction:N',
#        column='Construction:N'
#    )
#    display(g)<jupyter_output><empty_output><jupyter_text>Old below this line  Colley<jupyter_code>g = alt.Chart(graph_df.set_index('Method').loc['Massey']).mark_bar().encode(
    x='width:N',
    y=alt.Y('average(Value)',scale=alt.Scale(domain=[-.6, .6])),
    row='Feature:N',
    color='Construction:N',
    column='Construction:N'
)<jupyter_output><empty_output><jupyter_text>Massey<jupyter_code>g<jupyter_output><empty_output><jupyter_text>old below  0.6 to 0.7 (this was already broken)<jupyter_code>data = Xy.set_index(['frac1','frac2']).loc[(0.6,0.7)].reset_index()
for_corr = data.set_index(['Method','Construction'])
for ix in for_corr.index.unique():
    display(pd.Series(ix,index=for_corr.index.names))
    display(for_corr.loc[ix][[target_column]+feature_columns].corr())<jupyter_output><empty_output><jupyter_text>0.7 to 0.8<jupyter_code>data = Xy.set_index(['frac1','frac2']).loc[(0.7,0.8)].reset_index()
for_corr = data.set_index(['Method'])
for ix in for_corr.index.unique():
    display(pd.Series(ix,index=for_corr.index.names))
    display(for_corr.loc[ix][[target_column]+feature_columns].corr())<jupyter_output><empty_output><jupyter_text>0.8 to 0.9<jupyter_code>data = Xy.set_index(['frac1','frac2']).loc[(0.8,0.9)].reset_index()
for_corr = data.set_index(['Method'])
for ix in for_corr.index.unique():
    display(pd.Series(ix,index=for_corr.index.names))
    display(for_corr.loc[ix][[target_column]+feature_columns].corr())<jupyter_output><empty_output><jupyter_text>0.9 to 1.<jupyter_code>data = Xy.set_index(['frac1','frac2']).loc[(0.9,1.)].reset_index()
for_corr = data.set_index(['Method'])
for ix in for_corr.index.unique():
    display(pd.Series(ix,index=for_corr.index.names))
    display(for_corr.loc[ix][[target_column]+feature_columns].corr())
for_corr = data.set_index(['Method','direct_thres','spread_thres','weight_indirect'])
for_display = pd.DataFrame(columns=feature_columns+list(for_corr.index.names))
for_display.set_index(list(for_corr.index.names),inplace=True)
for ix in for_corr.index.unique():
    dt = for_corr.loc[ix][[target_column]+feature_columns].corr().loc[target_column,feature_columns]
    dt.name = ix
    for_display = for_display.append(dt)
for_display.T
print(for_display.T.to_latex())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16614756739340564, 'fraction_numerical': 0.013424981205026313, 'mean_word_length': 7.118570183086312, 'pattern_counts': {'":': 0, '<': 66, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '472247', 'license': 'MIT', 'n_tokens_mistral': 3503, 'n_tokens_neox': 3363, 'n_words': 401, 'path': 'bak/problems/ACC_NCAA_Men_Basketball/baseline.ipynb', 'repo_name': 'IGARDS/RPLib', 'size': '9311'}