<jupyter_start><jupyter_text>This notebook demonstrates how to remove the bias during "Pre-processing stage" using AI 360 fairness toolkit by adjusting the weights  Pre-processing algorithmA bias mitigation algorithm that is applied to training data.  Insert your credentials as credentials in the below cellClick on dropdown from Pipeline_LabelEncoder-0.1.zip under Data tab and select 'Credentials'<jupyter_code># @hidden_cell
# The following code contains the credentials for a file in your IBM Cloud Object Storage.
# You might want to remove those credentials before you share your notebook.
credentials = {

}

from ibm_botocore.client import Config
import ibm_boto3

cos = ibm_boto3.client(service_name='s3',
    ibm_api_key_id=credentials['IBM_API_KEY_ID'],
    ibm_service_instance_id=credentials['IAM_SERVICE_ID'],
    ibm_auth_endpoint=credentials['IBM_AUTH_ENDPOINT'],
    config=Config(signature_version='oauth'),
    endpoint_url=credentials['ENDPOINT'])
import os
os.getcwd()
cos.download_file(Bucket=credentials['BUCKET'],Key='Pipeline_LabelEncoder-0.1.zip',Filename='/home/wsuser/work/Pipeline_LabelEncoder-0.1.zip')
!ls
!pip install Pipeline_LabelEncoder-0.1.zip
!pip install aif360
import tensorflow as tf
tf.__version__
!pip install 'tensorflow>=1.13.1,< 2'
import pandas as pd
from aif360.metrics import BinaryLabelDatasetMetric
from aif360.datasets import BinaryLabelDataset
from IPython.display import Markdown, display
from aif360.algorithms.preprocessing.reweighing import Reweighing
import warnings
warnings.filterwarnings("ignore")<jupyter_output>WARNING:root:No module named 'numba.decorators': LFR will be unavailable. To install, run:
pip install 'aif360[LFR]'<jupyter_text>Insert the data as Pandas Dataframe and change the name from df_data_ to df<jupyter_code>import types
import pandas as pd
from botocore.client import Config
import ibm_boto3

def __iter__(self): return 0

# @hidden_cell
# The following code accesses a file in your IBM Cloud Object Storage. It includes your credentials.
# You might want to remove those credentials before you share the notebook.
client_943f0d0348cb4b5fb50f9c338e8d8cc1 = ibm_boto3.client(service_name='s3',
    ibm_api_key_id='<KEY>',
    ibm_auth_endpoint="https://iam.cloud.ibm.com/oidc/token",
    config=Config(signature_version='oauth'),
    endpoint_url='https://s3-api.us-geo.objectstorage.service.networklayer.com')

body = client_943f0d0348cb4b5fb50f9c338e8d8cc1.get_object(Bucket='ai360series-donotdelete-pr-1zay1c1uizeatc',Key='fraud_data.csv')['Body']
# add missing __iter__ method, so pandas accepts body as file-like object
if not hasattr(body, "__iter__"): body.__iter__ = types.MethodType( __iter__, body )

df = pd.read_csv(body)
df.head()

df.describe(include = 'all')
privileged_groups = [{'Gender': 1}]
unprivileged_groups = [{'Gender': 0}]
favorable_label = 1 
unfavorable_label = 0
from sklearn import preprocessing
categorical_column = ['Gender', 'Married', 'Fraud_risk']

data_encoded = df.copy(deep=True)
#Use Scikit-learn label encoding to encode character data
lab_enc = preprocessing.LabelEncoder()
for col in categorical_column:
        data_encoded[col] = lab_enc.fit_transform(df[col])
        le_name_mapping = dict(zip(lab_enc.classes_, lab_enc.transform(lab_enc.classes_)))
        print('Feature', col)
        print('mapping', le_name_mapping)
        

data_encoded.head()
from Pipeline_LabelEncoder.sklearn_label_encoder import PipelineLabelEncoder
preprocessed_data = PipelineLabelEncoder(columns = ['Gender','Married', 'Fraud_risk']).fit_transform(data_encoded)
print('-------------------------')
#print('validation data encoding')
#validation_enc_data = PipelineLabelEncoder(columns = ['Gender','Married', 'Fraud_risk']).transform(validation_input_data)
#Create binary label dataset that can be used by bias mitigation algorithms
fraud_dataset = BinaryLabelDataset(favorable_label=favorable_label,
                                unfavorable_label=unfavorable_label,
                                df=preprocessed_data,
                                label_names=['Fraud_risk'],
                                protected_attribute_names=['Gender', 'Married'],
                                unprivileged_protected_attributes=unprivileged_groups)
display(Markdown("#### Training Data Details"))
print("shape of the training dataset", fraud_dataset.features.shape)
print("Training data favorable label", fraud_dataset.favorable_label)
print("Training data unfavorable label", fraud_dataset.unfavorable_label)
print("Training data protected attribute", fraud_dataset.protected_attribute_names)
print("Training data privileged protected attribute (1:Male and 0:Female)", 
      fraud_dataset.privileged_protected_attributes)
print("Training data unprivileged protected attribute (1:Male and 0:Female)",
      fraud_dataset.unprivileged_protected_attributes)
metric_orig_train = BinaryLabelDatasetMetric(fraud_dataset, 
                                             unprivileged_groups=unprivileged_groups,
                                             privileged_groups=privileged_groups)
print("Difference in mean outcomes between unprivileged and privileged groups = %f" % 
      metric_orig_train.mean_difference())
RW = Reweighing(unprivileged_groups=unprivileged_groups,
               privileged_groups=privileged_groups)
RW.fit(fraud_dataset)
train_tf_dataset = RW.transform(fraud_dataset)
train_tf_dataset.labels
metric_orig_train = BinaryLabelDatasetMetric(train_tf_dataset, 
                                             unprivileged_groups=unprivileged_groups,
                                             privileged_groups=privileged_groups)

print("Difference in mean outcomes between unprivileged and privileged groups = %f"
      % metric_orig_train.mean_difference())<jupyter_output>Difference in mean outcomes between unprivileged and privileged groups = 0.000000
{'fraction_non_alphanumeric': 0.08830022075055188, 'fraction_numerical': 0.01664119544914247, 'mean_word_length': 4.674373795761079, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '190976', 'license': 'Apache-2.0', 'n_tokens_mistral': 1808, 'n_tokens_neox': 1696, 'n_words': 407, 'path': 'notebooks/Pre-processing.ipynb', 'repo_name': 'Anam-Mahmood/bias-mitigation-of-machine-learning-models-using-aif360', 'size': '5928'}