<jupyter_start><jupyter_text>Developing a classification Audit trendIn first iteration, this will only work on datasets that already have two or more binary classification variables included.We will need additional metadata: role options of being predictions or ground truths.<jupyter_code>import numpy as np
import pandas as pd
import wiggum as wg<jupyter_output><empty_output><jupyter_text>First, we will need a dataset that we can work with<jupyter_code>dataset = '../data/rateSPdataDeptRace/'
labeled_df = wg.LabeledDataFrame(dataset)
labeled_df.df.head()<jupyter_output><empty_output><jupyter_text>To test, we need create additional decision columns, to make it interesting, we will make them correlated with the true decision and the race, gender or dept.  We will start by modifying only the dataframe.  We will add the metadata and make a new labeled df after.<jupyter_code>test_df = labeled_df.df
def add_decision_rand(row,p_flip):
    """
    a function that can be applied to a df using apply that flips 'decision' with prob p, indep
    """
    cur = row['decision']
    opp = int(not(cur))
    return np.random.choice([cur, opp],p=[p_flip, 1-p_flip])
    
    
decision_90 =lambda r: add_decision_rand(r,.9)
cur_new = 'decision_noise'
test_df[cur_new] = test_df.apply(decision_90,axis=1)
test_df.head()
test_df.groupby(['gender','department'])['decision'].mean().unstack()
test_df.groupby(['gender','department'])[cur_new].mean().unstack()
test_df.groupby('gender')['decision'].mean()
test_df.groupby('gender')[cur_new].mean()
def add_acc(row,true_col,pred_col):
    label_mat = [['TN','FP'],['FN','TP']]
    return label_mat[row[true_col]][row[pred_col]]
add_acc_cur = lambda row: add_acc(row,'decision',cur_new)
test_df[cur_new+'test'] = test_df.apply(add_acc_cur,axis=1)
test_df.head()
confusion = test_df['decision_noisetest'].value_counts()
confusion
(confusion['TP'] + confusion['TN'])/sum(confusion)
if not('FT' in confusion.keys()):
    print('yes')
def compute_conf_stats(df,col):
    confusion = df[col].value_counts()
    return (confusion['TP'] + confusion['TN'])/sum(confusion)
compute_conf_stats(test_df,'decision_noisetest')
for l,gdf in test_df.groupby('gender'):
    print(compute_conf_stats(gdf,'decision_noisetest'))
confusion = test_df.groupby('gender')['decision_noisetest'].value_counts()
confusion
(confusion['F','TP'] + confusion['F','TN'])/sum(confusion['F'])
(confusion['M','TP'] + confusion['M','TN'])/sum(confusion['M'])<jupyter_output><empty_output><jupyter_text>Now, we make it correlated with somthing<jupyter_code>def add_decision_corr(row,corr_col,p_flips):
    """
    a function that can be applied to a df using apply that flips 'decision' with prob p, indep
    """
    cur = row['decision']
    opp = int(not(cur))
    p_cur = p_flips[row[corr_col]]
    return np.random.choice([cur, opp],p=[p_cur, 1-p_cur])
    
    
p_gender_corr = {'F':.75,'M':.9}
decision_gender =lambda r: add_decision_corr(r,'gender',p_gender_corr)
cur_new = 'decision_gender'
test_df[cur_new] = test_df.apply(decision_gender,axis=1)
add_acc_cur = lambda row: add_acc(row,'decision',cur_new)
test_df[cur_new+'test'] = test_df.apply(add_acc_cur,axis=1)
confusion = test_df.groupby('gender')[cur_new +'test'].value_counts()
confusion
(confusion['F','TP'] + confusion['F','TN'])/sum(confusion['F'])
(confusion['M','TP'] + confusion['M','TN'])/sum(confusion['M'])
test_df.groupby(['department','gender'])[cur_new +'test'].value_counts()

p_race_corr = {'B':.65,'W':.9,'H':.8}
decision_race =lambda r: add_decision_corr(r,'race',p_race_corr)
cur_new = 'decision_race'
test_df[cur_new] = test_df.apply(decision_race,axis=1)
test_df.head()
test_df.drop(columns=['decision_noisetest','decision_gendertest'],inplace=True)
labeled_df.meta_df
multi_decision_labeled_df = wg.LabeledDataFrame(test_df)


roles = {'department':['groupby', 'trend'],'gender':['groupby', 'trend'],'race':['groupby', 'trend'],
         'decision_noise':['trend','prediction'],'decision':['trend','groundtruth'],
         'decision_gender':['trend','prediction'],'decision_race':['trend','prediction']}
var_types = {'department':'categorical','gender':'categorical'}
# is_count = {c:False for c in test_df.columns}

multi_decision_labeled_df.infer_var_types()
multi_decision_labeled_df.set_roles(roles)
multi_decision_labeled_df.set_var_types(var_types)
multi_decision_labeled_df.meta_df
multi_decision_labeled_df.to_csvs('../data/multi_decision_admisions/')
acc_trend = wg.Binary_Accuracy_Trend()
tpr_trend = wg.Binary_TPR_Trend()
ppv_trend = wg.Binary_PPV_Trend()
multi_decision_labeled_df.get_subgroup_trends_1lev([acc_trend,tpr_trend,ppv_trend])
multi_decision_labeled_df.get_SP_rows(thresh=.2)
multi_decision_labeled_df.df.head()
var_list = list(multi_decision_labeled_df.get_vars_per_type('categorical'))
var_list

new_name = '_'.join(var_list)

mergerow =  lambda row: '_'.join([str(v) for v in row[var_list].values])
# multi_decision_labeled_df[var_list[0]].str.cat()
multi_decision_labeled_df.df[new_name] = multi_decision_labeled_df.df.apply(mergerow,axis=1)
multi_decision_labeled_df.df.head()
multi_decision_labeled_df.result_df.head()
view_vars = ['feat1','feat2','group_feat']
score_col = 'subgroup_trend'
multi_decision_labeled_df.result_df.groupby(by=view_vars)[score_col].var()
pair_range = lambda x: np.abs(x.max()-x.min())
multi_decision_labeled_df.result_df.groupby(by=view_vars)[score_col].apply(pair_range)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15583700440528633, 'fraction_numerical': 0.004588839941262849, 'mean_word_length': 8.03648424543947, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '219061', 'license': 'MIT', 'n_tokens_mistral': 2095, 'n_tokens_neox': 1968, 'n_words': 324, 'path': 'research_notebooks/dev_classification_audit_trend.ipynb', 'repo_name': 'carolinesadlerr/wiggum', 'size': '5448'}