<jupyter_start><jupyter_text>Imports<jupyter_code># Import packages
import os
import numpy as np 
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.naive_bayes import GaussianNB, MultinomialNB
from sklearn.metrics import accuracy_score, confusion_matrix, normalized_mutual_info_score
from sklearn.linear_model import LogisticRegression
from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import scale, normalize
from sklearn.model_selection import train_test_split, KFold, cross_val_predict, cross_validate, cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
%matplotlib qt<jupyter_output><empty_output><jupyter_text>Exploration and Preprocessing of the dataset<jupyter_code>adult_train_path = os.path.join(os.getcwd(), 'Datasets', 'adult_train.csv') 
adult_test_path = os.path.join(os.getcwd(), 'Datasets', 'adult_test.csv')
column_names = ['age', 'workclass', 'fnlwgt', 'education', 'education_num', 'marital_status', 'occupation', 'relationship', 'race', 'sex', 'capital_gain', 'capital_loss', 'hours_per_week', 'native_country', 'Income']
def prep_data(path, columns):

    data = []

    for line in open(path):
        line = line.split(',')

        if line[13] != 'United-States':
            line[13] = 'Non-US'
        else:
            line[13] = 'US'

        if line[3] in ['Preschool', '1st-4th', '5th-6th', '7th-8th']:
            line[3] = 'prim-middle-school'
        elif line[3] in ['9th', '10th', '11th', '12th']:
            line[3] = "high-school"

        line[-1] = line[-1][:-1]

        to_int = lambda a : int(line[a])
        for i in [0,2,4,10,11,12]:
            line[i] = to_int(i)

        data.append(line)
        
    return pd.DataFrame(data, columns=column_names)
adult_train_data = prep_data(adult_train_path, columns=column_names)
adult_test_data = prep_data(adult_test_path, columns=column_names)
print('Adult train dataset info:')
display(adult_train_data.info())
print('Adult test dataset info:')
display(adult_test_data.info())
print('Adult train dataset statistical information')
display(adult_train_data.describe())
print('Adult test dataset statistical information')
display(adult_test_data.describe())
print('Adult train dataset head:')
display(adult_train_data.head(10))
print('\nAdult train dataset tail:')
display(adult_train_data.tail(10))
print('Adult test dataset head:')
display(adult_test_data.head(10))
print('\nAdult test dataset tail:')
display(adult_test_data.tail(10))
adult_train_data['sex'].value_counts()
adult_test_data['sex'].value_counts()
ax = sns.countplot(x='sex', data=adult_train_data)
ax.set_xticklabels(['Male', 'Female'])
plt.xlabel('')
plt.show()
ax = sns.countplot(x='sex', data=adult_test_data)
ax.set_xticklabels(['Male', 'Female'])
plt.xlabel('')
plt.show()
adult_train_data['Income'].value_counts()
adult_test_data['Income'].value_counts()
ax = sns.countplot(x='Income', data=adult_train_data)
ax.set_xticklabels(['<=50K', '>50K'])
plt.xlabel('')
plt.show()
ax = sns.countplot(x='Income', data=adult_test_data)
ax.set_xticklabels(['<=50K', '>50K'])
plt.xlabel('')
plt.show()
pd.crosstab(adult_train_data['sex'], adult_train_data['Income']).plot(kind='bar')
plt.title('Income per personal sex')
plt.xlabel('Sex')
plt.ylabel('Income')
plt.show()
pd.crosstab(adult_test_data['sex'], adult_test_data['Income']).plot(kind='bar')
plt.title('Income per personal sex')
plt.xlabel('Sex')
plt.ylabel('Income')
plt.show()<jupyter_output><empty_output><jupyter_text>Mutual Information<jupyter_code>sex = adult_train_data['sex'].copy()

for i in range(len(sex)):
    if sex[i] == 'Female':
        sex[i] = 1
    else:
        sex[i] = 0    
        
sex = np.asarray(sex, dtype='float64')
y_df = adult_train_data['Income'].copy()

for i in range(len(y_df)):
    if y_df[i] == '>50K':
        y_df[i] = 1
    else:
        y_df[i] = 0    
        
y = np.asarray(y_df, dtype='float64')
normalized_mutual_info_score(sex, y)
normalized_mutual_info_score(adult_test_data['sex'], adult_test_data['Income'])<jupyter_output><empty_output><jupyter_text>Splitting Data<jupyter_code>features = adult_train_data.drop(['fnlwgt', 'Income'], axis=1)
features = pd.get_dummies(features)
test_features = adult_test_data.drop(['fnlwgt', 'Income'], axis=1)
test_features = pd.get_dummies(test_features)
s = pd.get_dummies(adult_train_data['sex'])
test_s = pd.get_dummies(adult_test_data['sex'])
y_df = adult_train_data['Income'].copy()

for i in range(len(y_df)):
    if y_df[i] == '>50K':
        y_df[i] = 1
    else:
        y_df[i] = 0    
        
y = np.asarray(y_df, dtype='float64')
test_y_df = adult_test_data['Income'].copy()

for i in range(len(test_y_df)):
    if test_y_df[i] == '>50K':
        test_y_df[i] = 1
    else:
        test_y_df[i] = 0    
    
test_y = np.asarray(test_y_df, dtype='float64')
print('Training Data shapes:')
print('Features shape: {}'.format(features.shape))
print('Sensitive attribute shape: {}'.format(s.shape))
print('Output label shape: {}'.format(y.shape))
print('Test Data shapes:')
print('Features shape: {}'.format(test_features.shape))
print('Sensitive attribute shape: {}'.format(s.shape))
print('Output label shape: {}'.format(test_y.shape))<jupyter_output>Test Data shapes:
Features shape: (15060, 58)
Sensitive attribute shape: (30162, 2)
Output label shape: (15060,)<jupyter_text>Disparate Impact (P%-rule)<jupyter_code>def p_rule(files):
    df = pd.DataFrame()
    
    for f in files:
        df = pd.concat([df, f], axis=1)
        
    prot_pos = 0
    unprot_pos = 0

    if 'sex' in df.columns:
        for row in df.iterrows():
            if row[1][9] == 'Female' and row[1][-1] == '>50K':
                prot_pos += 1
            elif row[1][9] == 'Male' and row[1][-1] == '>50K': 
                unprot_pos += 1  
                
        total_females = df['sex'].value_counts()[1]
        total_males = df['sex'].value_counts()[0]
        
    if 'Female' in df.columns:
        for row in df.iterrows():
            if row[1][0] == 1 and row[1][-1] == 1:
                prot_pos += 1
            elif row[1][1] == 1 and row[1][-1] == 1: 
                unprot_pos += 1
                
        total_females = df['Female'].value_counts()[1]
        total_males = df['Female'].value_counts()[0]              
    
    prot_per = prot_pos / total_females * 100
    unprot_per = unprot_pos / total_males * 100
    
    train_p_rule = prot_per / unprot_per
    
    print('Number of Females in the positive class (>50K): {}'.format(prot_pos))
    print('Percentage of Females in the positive class (>50K): {:.2f}%'.format(prot_per))
    print('\nNumber of Males in the positive class (>50K): {}'.format(unprot_pos))
    print('Percentage of Males in the positive class (>50K): {:.2f}%'.format(unprot_per))
    print('\nTraining set p-rule: {:.2f}%'.format(train_p_rule * 100))<jupyter_output><empty_output><jupyter_text>P%-rule on training data<jupyter_code>p_rule([adult_train_data])<jupyter_output>Number of Females in the positive class (>50K): 1112
Percentage of Females in the positive class (>50K): 11.37%

Number of Males in the positive class (>50K): 6396
Percentage of Males in the positive class (>50K): 31.38%

Training set p-rule: 36.22%<jupyter_text>P%-rule on test data<jupyter_code>p_rule([adult_test_data])<jupyter_output>Number of Females in the positive class (>50K): 557
Percentage of Females in the positive class (>50K): 11.34%

Number of Males in the positive class (>50K): 3143
Percentage of Males in the positive class (>50K): 30.97%

Training set p-rule: 36.60%<jupyter_text>Disparate Mistreatment<jupyter_code>def print_disparate_mistreatment_stats(s, y, pred_y):

    prot_FP = 0
    unprot_FP = 0
    
    prot_FN = 0
    unprot_FN = 0
    
    prot_TN = 0
    unprot_TN = 0
    
    prot_TP = 0
    unprot_TP = 0

    for i in range(0, len(s)):
        if s['sex_Female'][i] == 1:
            if y[i] == 1:
                if pred_y[i] != y[i]:
                    prot_FN += 1
                else:
                    prot_TP += 1
            else:
                if pred_y[i] != y[i]:
                    prot_FP += 1
                else:
                    prot_TN += 1
        else:
            if y[i] == 1:
                if pred_y[i] != y[i]:
                    unprot_FN += 1
                else:
                    unprot_TP += 1
            else:
                if pred_y[i] != y[i]:
                    unprot_FP += 1
                else:
                    unprot_TN += 1

    prot_FPR = prot_FP / (prot_FP + prot_TN)
    prot_FNR = prot_FN / (prot_FN + prot_TP)
    if prot_FN == 0 and prot_TN == 0:
        prot_FOR = 0
    else:
        prot_FOR = prot_FP / (prot_FN + prot_TN)
    prot_FDR = prot_FN / (prot_FP + prot_TP)
    
    unprot_FPR = unprot_FP / (unprot_FP + unprot_TN)
    unprot_FNR = unprot_FN / (unprot_FN + unprot_TP)
    if unprot_FN == 0 and unprot_TN == 0:
        unprot_FOR = 0
    else:
        unprot_FOR = unprot_FP / (unprot_FN + unprot_TN)
    unprot_FDR = unprot_FN / (unprot_FP + unprot_TP)

    print()
    print("--- Disparate Mistreatment ---")
    print("# protected False Positive Rate: %.2f" % (prot_FPR))
    print("# unprotected False Positive Rate: %.2f" % (unprot_FPR))
    print()
    print("# protected False Negative Rate: %.2f" % (prot_FNR))
    print("# unprotected False Negative Rate: %.2f" % (unprot_FNR))
    print()
    print("# protected False Omission Rate: %.2f" % (prot_FOR))
    print("# unprotected False Omission Rate: %.2f" % (unprot_FOR))
    print()
    print("# protected False Discovery Rate: %.2f" % (prot_FDR))
    print("# unprotected False Discovery Rate: %.2f" % (unprot_FDR))
    print()

#     return prot_FPR, unprot_FPR, prot_FNR, unprot_FNR, prot_FDR, unprot_FDR, prot_FOR, unprot_FOR<jupyter_output><empty_output><jupyter_text>Multinomial Naive Bayes<jupyter_code>multi_nb = MultinomialNB()
multi_nb.fit(X=features, y=y)
'Class log-priors: {}'.format(multi_nb.class_log_prior_)
test_pred = multi_nb.predict(X=test_features)
score = accuracy_score(test_y, test_pred)
print('Accuracy score of multinomial Naive Bayes classifier: {}'.format(score))
cm_norm = confusion_matrix(test_y, test_pred)/confusion_matrix(test_y, test_pred).sum(axis=1)[:, np.newaxis]
sns.heatmap(cm_norm, xticklabels=['>50K', '<=50K'], yticklabels=['>50K', '<=50K'], vmin=0., vmax=1., annot=True)
plt.title('Confusion Matrix')
plt.ylabel('True label')
plt.xlabel('Predicted label')
#plt.figure()
plt.show()
sort_idx = np.argsort(multi_nb.feature_log_prob_[0])
log_probs = multi_nb.feature_log_prob_[:, sort_idx]
ticklabs = features.columns[sort_idx].tolist()

N = len(ticklabs)
ind = np.arange(N)  # the x locations for the groups
width = 0.35        # the width of the bars
fig, ax = plt.subplots(figsize=(16,6))
pos = ax.bar(ind, log_probs[0], width)
neg = ax.bar(ind + width, log_probs[1], width)
ax.set_ylabel('Log Probability')
ax.set_xticks(ind + width)
ax.set_xticklabels(ticklabs, rotation='vertical')
ax.legend((pos[0], neg[0]), ('>50K', '<=50K'), loc='best')
plt.show()<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code># Creating the logistic regression classifier:
log_reg = LogisticRegression(solver='lbfgs', max_iter=5000)

# Fitting the training data:
fitted = log_reg.fit(features, y)

# Calculating the class predictions:
pred = fitted.predict(test_features)

print('Accuracy of Logistic Regressor: ', fitted.score(test_features, test_y))
# Creating the confusion matrix:
cm_norm = confusion_matrix(test_y, pred)/confusion_matrix(test_y, pred).sum(axis=1)[:, np.newaxis]

sns.heatmap(cm_norm, xticklabels=['>50K', '<=50K'], yticklabels=['>50K', '<=50K'], vmin=0., vmax=1., annot=True)
plt.title('Confusion Matrix')
plt.ylabel('True label')
plt.xlabel('Predicted label')
#plt.figure()
plt.show()<jupyter_output><empty_output><jupyter_text>P%-rule on predictions<jupyter_code>test_preds = test_s.copy()
test_preds['Income'] = pred
p_rule([test_preds])<jupyter_output>Number of Females in the positive class (>50K): 380
Percentage of Females in the positive class (>50K): 7.73%

Number of Males in the positive class (>50K): 2674
Percentage of Males in the positive class (>50K): 26.35%

Training set p-rule: 29.35%<jupyter_text>Disparate Mistreatment<jupyter_code>print_disparate_mistreatment_stats(test_features, test_y, pred)<jupyter_output>--- Disparate Mistreatment ---
# protected False Positive Rate: 0.02
# unprotected False Positive Rate: 0.10

# protected False Negative Rate: 0.49
# unprotected False Negative Rate: 0.38

# protected False Omission Rate: 0.02
# unprotected False Omission Rate: 0.10

# protected False Discovery Rate: 0.72
# unprotected False Discovery Rate: 0.45<jupyter_text>Standardized<jupyter_code># Standardizing the training and testing feature variables to achieve zero mean and unit variance.
stand_X_train = scale(features)
stand_X_test = scale(test_features)
# Creating the logistic regression classifier:
log_reg = LogisticRegression(solver='lbfgs', max_iter=5000)

# Fitting the training data:
fitted = log_reg.fit(stand_X_train, y)

# Calculating the class predictions:
stand_pred = fitted.predict(stand_X_test)

print('Accuracy of Logistic Regressor: ', fitted.score(stand_X_test, test_y))
# Creating the confusion matrix:
cm_norm = confusion_matrix(test_y, stand_pred)/confusion_matrix(test_y, stand_pred).sum(axis=1)[:, np.newaxis]

sns.heatmap(cm_norm, xticklabels=['>50K', '<=50K'], yticklabels=['>50K', '<=50K'], vmin=0., vmax=1., annot=True)
plt.title('Confusion Matrix')
plt.ylabel('True label')
plt.xlabel('Predicted label')
#plt.figure()
plt.show()<jupyter_output><empty_output><jupyter_text>P%-rule on predictions<jupyter_code>test_preds = test_s.copy()
test_preds['Income'] = stand_pred
p_rule([test_preds])<jupyter_output>Number of Females in the positive class (>50K): 385
Percentage of Females in the positive class (>50K): 7.84%

Number of Males in the positive class (>50K): 2650
Percentage of Males in the positive class (>50K): 26.12%

Training set p-rule: 30.01%<jupyter_text>Disaprate Mistreatment<jupyter_code>print_disparate_mistreatment_stats(test_features, test_y, stand_pred)<jupyter_output>--- Disparate Mistreatment ---
# protected False Positive Rate: 0.02
# unprotected False Positive Rate: 0.10

# protected False Negative Rate: 0.48
# unprotected False Negative Rate: 0.39

# protected False Omission Rate: 0.02
# unprotected False Omission Rate: 0.10

# protected False Discovery Rate: 0.70
# unprotected False Discovery Rate: 0.46<jupyter_text>Decision Tree Classifier<jupyter_code>clf = DecisionTreeClassifier(random_state=0)
cross_val_score(clf, features, adult_train_data['Income'], cv=10)
np.mean(cross_val_score(clf, features, adult_train_data['Income'], cv=10))
clf = DecisionTreeClassifier(random_state=0)
cross_val_score(clf, features, adult_train_data['Income'], cv=20)
np.mean(cross_val_score(clf, features, adult_train_data['Income'], cv=20))<jupyter_output><empty_output><jupyter_text>Random Forest Classifier<jupyter_code># Creating the Random Forest Classifier:
rfc = RandomForestClassifier(n_estimators=500, random_state=42, criterion='entropy', max_depth=20)

# Fitting the training data:
rfc_fitted = rfc.fit(features, y)

pred = rfc_fitted.predict(test_features)

# Extracting the feature importances:
feature_importances = rfc_fitted.feature_importances_

# Displaying the training and test set accuracies:
print('Accuracy of random forest classifier on the training set: ', rfc_fitted.score(features, y))
print('Accuracy of random forest classifier on the testing set: ', rfc_fitted.score(test_features, test_y))
test_preds = test_s.copy()
test_preds['Income'] = pred
p_rule([test_preds])
print_disparate_mistreatment_stats(test_features, test_y, pred)<jupyter_output>--- Disparate Mistreatment ---
# protected False Positive Rate: 0.03
# unprotected False Positive Rate: 0.08

# protected False Negative Rate: 0.45
# unprotected False Negative Rate: 0.38

# protected False Omission Rate: 0.02
# unprotected False Omission Rate: 0.08

# protected False Discovery Rate: 0.60
# unprotected False Discovery Rate: 0.48<jupyter_text>Feature importance<jupyter_code># Creating and displaying a dataframe for the top 10 most important features:
feature_importances_df = pd.DataFrame(data = np.array(feature_importances).T, index = features.columns[np.newaxis].T, columns = ['Importance'])
feature_importances_df = feature_importances_df.sort_values(['Importance'], ascending = False)
print('Top 10 features according to the Random Forest Classifier:')
display(feature_importances_df.iloc[0:10])<jupyter_output>Top 10 features according to the Random Forest Classifier:<jupyter_text>Neural Networks (Multi-Layer Perceptron)<jupyter_code>clf = MLPClassifier()
clf.get_params()
clf.fit(features, y)
pred = clf.predict(test_features)
clf.score(test_features, test_y)
test_preds = test_s.copy()
test_preds['Income'] = pred
p_rule([test_preds])
print_disparate_mistreatment_stats(test_features, test_y, pred)<jupyter_output>--- Disparate Mistreatment ---
# protected False Positive Rate: 0.04
# unprotected False Positive Rate: 0.10

# protected False Negative Rate: 0.51
# unprotected False Negative Rate: 0.46

# protected False Omission Rate: 0.03
# unprotected False Omission Rate: 0.09

# protected False Discovery Rate: 0.66
# unprotected False Discovery Rate: 0.60
{'fraction_non_alphanumeric': 0.13513513513513514, 'fraction_numerical': 0.02674072377462208, 'mean_word_length': 4.525150268902246, 'pattern_counts': {'":': 0, '<': 76, '<?xml version=': 0, '>': 101, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '206035', 'license': 'MIT', 'n_tokens_mistral': 6651, 'n_tokens_neox': 6106, 'n_words': 1400, 'path': 'ML_and_Fairness_Experiments/.ipynb_checkpoints/ML and Fairness Tests - Original Dataset-checkpoint.ipynb', 'repo_name': 'kyriacosar/FairTGAN', 'size': '17464'}