<jupyter_start><jupyter_text>Credit Risk ModellingBuilding a risk assessment model that eliminates social biases like age, sex and race. Improve the fairness, transparency and inclusitivity of access to credit  Importing toolsWe tested out a few different algorithms and XGBoost Classifier was so far the most accurate (75% prediction rate)<jupyter_code># Defining tools 

import xgboost
import pandas as pd
from sklearn.model_selection import train_test_split
from xgboost import XGBClassifier
from sklearn.metrics import accuracy_score<jupyter_output><empty_output><jupyter_text>Loading dataWe uses a public dataset given by Lending Club, with more than 2 million different accounts and 44 billion dollars in lending.<jupyter_code># Loading data

original_data_path = "accepted_2007_to_2018Q3.csv"

original_data = pd.read_csv(original_data_path, nrows=100000, engine="python")

features = ["loan_amnt", "int_rate", "emp_length", "home_ownership", "annual_inc",
           "verification_status", "loan_status", "title", "delinq_2yrs", "fico_range_low", "fico_range_high",
            "pub_rec", "revol_bal", "revol_util", "application_type", "tot_cur_bal", "chargeoff_within_12_mths", "pub_rec_bankruptcies",
           "total_il_high_credit_limit"]

# Features that are not numbers:
# One Hot encoding: home_ownership, verification_status, title ==== done
# Direct change:
# - emp_length: a years (2 <= a <= 10, 10+ years, <1 year)  ===== done
# - loan_status: only takes Charged Off and Fully Paid  ===== done
# - application_type: deals only with individual   ====== done

# Direct change first:

chosen_data = original_data[features]
chosen_data.drop(chosen_data[(chosen_data.loan_status != "Fully Paid") & (chosen_data.loan_status != "Charged Off")].index, inplace = True)

# Choose only individuals
chosen_data.drop(chosen_data[(chosen_data.application_type != "Individual")].index, inplace = True)
chosen_data = chosen_data.drop("application_type", axis = 1)

# Reformatting emp_length:
chosen_data["emp_length"].fillna(0, inplace = True)
emp_length_di = {"< 1 year": 0.5, "1 year": 1, "2 years": 2, "3 years": 3, "4 years": 4, "5 years": 5, "6 years": 6, "7 years": 7,
                "8 years": 8, "9 years": 9, "10+ years": 10}
chosen_data = chosen_data.replace({"emp_length": emp_length_di})

# Fill known empty columns
chosen_data['title'].fillna("Other", inplace = True)
util_median = chosen_data['revol_util'].median()
chosen_data['revol_util'].fillna(util_median, inplace = True)

# Double checking for empty cells
cols_with_missing = [col for col in chosen_data.columns if chosen_data[col].isnull().any()]
chosen_data = chosen_data.replace({"loan_status": {"Charged Off": 0, "Fully Paid": 1}})

chosen_data.to_csv("clean_data.csv")<jupyter_output><empty_output><jupyter_text>Training dataWe get the clean set of data in and use it to train our data.<jupyter_code># Loading clean data

clean_data_path = "clean_data.csv"
clean_data = pd.read_csv(clean_data_path, engine="python")

X = clean_data.drop(["loan_status", "int_rate"], axis = 1)

hot_encoded_X = pd.get_dummies(X)
y = clean_data.loan_status


# split data into train and test sets
X_train, X_test, y_train, y_test = train_test_split(hot_encoded_X, y, test_size=0.3, random_state=1)

# fit model no training data
model = XGBClassifier(n_estimators = 150, learning_rate = 0.05)
model.fit(X_train, y_train)

# make predictions for test data
y_pred = model.predict(X_test)

# evaluate accuracy of predictions
accuracy = accuracy_score(y_test, y_pred)
print("Accuracy: %.2f%%" % (accuracy * 100.0))


# evaluate increase in profit and interest rates
default_risk = model.predict_proba(hot_encoded_X)[:, 0] * 100
clean_data['default_risk'] = default_risk

# if default_risk > 25% we are not lending
mask = (clean_data['default_risk'] <= 25)
clean_data_not_default = clean_data[mask]

clean_data['new_loan_amnt'] = 0
clean_data.loc[mask, 'new_loan_amnt'] = clean_data_not_default['loan_amnt']

# total lost
# we have profit = interest_revenue - default_lost

clean_data['default_lost_predicted'] = clean_data.default_risk * clean_data.new_loan_amnt / 100
total_lost_predicted = clean_data['default_lost_predicted'].sum()

clean_data['default_lost_actual'] = clean_data.loan_amnt * abs(clean_data.loan_status - 1)
total_lost_actual = clean_data['default_lost_actual'].sum()
clean_data['interest_collected_actual'] = clean_data.loan_amnt * clean_data.int_rate / 100
total_interest_actual = clean_data['interest_collected_actual'].sum()
total_profit_actual = total_interest_actual - total_lost_actual

print(total_lost_predicted)
print(total_lost_actual)
print(total_interest_actual)
print(total_profit_actual)


# interest

clean_data<jupyter_output>61282074.150980055
174754225.0
99877677.35499999
-74876547.64500001
{'fraction_non_alphanumeric': 0.11366485582950271, 'fraction_numerical': 0.03092352695361471, 'mean_word_length': 6.503134796238244, 'pattern_counts': {'":': 15, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '250469', 'license': 'MIT', 'n_tokens_mistral': 1764, 'n_tokens_neox': 1605, 'n_words': 400, 'path': 'Untitled.ipynb', 'repo_name': 'nt7nguye/AI-credit-modelling', 'size': '4786'}