<jupyter_start><jupyter_text>Data cleanup**Checking null values**<jupyter_code>df.isnull().sum()
df = df.fillna(round(df.median()))<jupyter_output><empty_output><jupyter_text>One Hot Encoding<jupyter_code># df = pd.get_dummies(df)
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 500 entries, 0 to 499
Data columns (total 20 columns):
 #   Column                             Non-Null Count  Dtype  
---  ------                             --------------  -----  
 0   Credit-Application-Result          500 non-null    int64  
 1   Account-Balance                    500 non-null    int64  
 2   Duration-of-Credit-Month           500 non-null    int64  
 3   Payment-Status-of-Previous-Credit  500 non-null    int64  
 4   Purpose                            500 non-null    int64  
 5   Credit-Amount                      500 non-null    int64  
 6   Value-Savings-Stocks               500 non-null    int64  
 7   Length-of-current-employment       500 non-null    int64  
 8   Instalment-per-cent                500 non-null    int64  
 9   Guarantors                         500 non-null    int64  
 10  Duration-in-Current-address        500 non-null    float64
 11  Most-valuable-available-asset      500 non-null    int64  [...]<jupyter_text>Features Selection<jupyter_code>plt.figure(figsize=(12,12)) 
sns.heatmap(df.corr(),annot=True,cmap='cubehelix_r')
plt.show()
corr = df.corr()
#Correlation with output variable
cor_target = abs(corr["Credit-Application-Result"])
#Selecting highly correlated features
relevant_features = cor_target[cor_target>=0.09]
df_corr = pd.DataFrame(relevant_features)
df_corr = df_corr.reset_index()
df_corr = df_corr.rename(columns = {'Credit-Application-Result' : 'Corr', 'index': 'features'})
df_corr = df_corr.sort_values(['Corr'], ascending=False)
df_corr
X = df.drop(columns=['Credit-Application-Result'])
y = df['Credit-Application-Result']

print(X.shape)
print(y.shape)
X = X.dropna(axis=1, how='any')<jupyter_output><empty_output><jupyter_text>Extra Trees Classifier<jupyter_code>model = ExtraTreesClassifier()
model.fit(X, y)

df_ext=pd.DataFrame(model.feature_importances_,X.columns, columns=["extratrees"])
df_ext = df_ext.sort_values(['extratrees'], ascending=False)
df_ext =  df_ext[df_ext > 0.05]
df_ext = df_ext.dropna(how='all')
df_ext = df_ext.reset_index()
df_ext = df_ext.rename(columns = {'index': 'features', 'extratrees' : 'EXT'})
df_ext
df_features = reduce(lambda x,y: pd.merge(x,y, on='features', how='outer'), [df_corr, df_ext])
df_features = df_features.dropna()
df_features
fs_list = df_features.iloc[:,0].tolist()
b = ["Credit-Application-Result"]
fs_list += b 
fs_list
df.drop(df.columns.difference(fs_list), 1, inplace=True)
df
X1 = df.drop(columns=['Credit-Application-Result'])
y1 = df['Credit-Application-Result']

print(X1.shape)
print(y1.shape)

print(df.info())

X = np.array(X1)
y = np.array(y1)<jupyter_output><empty_output><jupyter_text>**Check whether the data is balanced or not**<jupyter_code>positive = df[df["Credit-Application-Result"]== 1]
negative = df[df["Credit-Application-Result"]== 0]

ratio = float(len(negative) / len(positive))
print(ratio)


print("positive:{}".format(len(positive)))
print("negative:{}".format(len(negative)))
random_state = 1
X_train, X_val, y_train,y_val = train_test_split(X,y, test_size = 0.2, random_state=random_state)
model = sm.OLS(y, X)
fit = model.fit()
print(fit.summary())<jupyter_output>OLS Regression Results                                
=======================================================================================
Dep. Variable:                      y   R-squared (uncentered):                   0.716
Model:                            OLS   Adj. R-squared (uncentered):              0.713
Method:                 Least Squares   F-statistic:                              207.9
Date:                Fri, 18 Jun 2021   Prob (F-statistic):                   1.10e-131
Time:                        12:57:00   Log-Likelihood:                         -310.95
No. Observations:                 500   AIC:                                      633.9
Df Residuals:                     494   BIC:                                      659.2
Df Model:                           6                                                  
Covariance Type:            nonrobust                                                  
================================[...]<jupyter_text>Provided test dataset<jupyter_code>df_test = pd.read_csv("customers-to-score.csv")
df_test
df_test['Account-Balance'].unique()
df_test['Account-Balance'].replace({'Some Balance': 1, 'No Account': 0}, inplace=True)
df_test['Payment-Status-of-Previous-Credit'].unique()
df_test['Payment-Status-of-Previous-Credit'].replace({'Paid Up': 2, 'No Problems (in this bank)': 1, 'Some Problems': 0}, inplace=True)
df_test["Purpose"].unique()
df_test['Purpose'].replace({
                    'New car': 3, 
                    'Used car': 2,
                    'Home Related': 1,
                    'Other': 0}, inplace=True)
df_test['Value-Savings-Stocks'].unique()
df_test['Value-Savings-Stocks'].replace({
                    '< £100': 2, 
                    '£100-£1000': 1,
                    'None': 0}, inplace=True)
df_test['Length-of-current-employment'].unique()
df_test['Length-of-current-employment'].replace({
                    '4-7 yrs': 2, 
                    '1-4 yrs': 1,
                    '< 1yr': 0}, inplace=True)
df_test['Concurrent-Credits'].replace({'Other Banks/Depts': 1}, inplace=True)
df_test['No-of-Credits-at-this-Bank'].replace({'More than 1': 1, '1': 0}, inplace=True)
df_test['Guarantors'].replace({'Yes': 1, 'None': 0}, inplace=True)
df_test.info()
df_test.drop(df_test.columns.difference(fs_list), 1, inplace=True)
df_test.info()
X_test = np.array(df_test)<jupyter_output><empty_output><jupyter_text>Machine Learning Training Logistic Regression<jupyter_code>clf = LogisticRegression().fit(X_train, y_train)
y_val_pred = clf.predict(X_val)
y_val_score = clf.decision_function(X_val)
cf = confusion_matrix(y_val, y_val_pred)
sns.heatmap(cf, annot=True)
print(classification_report(y_val, y_val_pred, target_names=['Non-Creditworthy', 'Creditworthy']))
plot_roc_curve(clf, X_val, y_val)  
y_test_pred = clf.predict(X_test)
y_test_score = clf.decision_function(X_test)
cf = confusion_matrix(y, y_test_pred)
sns.heatmap(cf, annot=True, fmt='d')
print(classification_report(y, y_test_pred, target_names=['Non-Creditworthy', 'Creditworthy']))<jupyter_output>precision    recall  f1-score   support

Non-Creditworthy       0.41      0.17      0.24       142
    Creditworthy       0.73      0.90      0.81       358

        accuracy                           0.69       500
       macro avg       0.57      0.54      0.52       500
    weighted avg       0.64      0.69      0.65       500<jupyter_text>Decision Tree Classifier<jupyter_code>clf = DecisionTreeClassifier().fit(X_train, y_train)
y_val_pred = clf.predict(X_val)
y_val_score = clf.predict_proba(X_val)
cf = confusion_matrix(y_val, y_val_pred)
sns.heatmap(cf, annot=True, fmt='d')
print(classification_report(y_val, y_val_pred, target_names=['Non-Creditworthy', 'Creditworthy']))
plot_roc_curve(clf, X_val, y_val) 
y_test_pred = clf.predict(X_test)
y_test_score = clf.predict_proba(X_test)
cf = confusion_matrix(y, y_test_pred)
sns.heatmap(cf, annot=True, fmt='d')
print(classification_report(y, y_test_pred, target_names=['Non-Creditworthy', 'Creditworthy']))<jupyter_output>precision    recall  f1-score   support

Non-Creditworthy       0.30      0.27      0.28       142
    Creditworthy       0.72      0.76      0.74       358

        accuracy                           0.62       500
       macro avg       0.51      0.51      0.51       500
    weighted avg       0.60      0.62      0.61       500<jupyter_text>Random Forest Classifier<jupyter_code>clf = RandomForestClassifier(n_estimators=100).fit(X_train, y_train)
y_val_pred = clf.predict(X_val)
y_val_score = clf.predict_proba(X_val)
cf = confusion_matrix(y_val, y_val_pred)
sns.heatmap(cf, annot=True, fmt='d')
print(classification_report(y_val, y_val_pred, target_names=['Non-Creditworthy', 'Creditworthy']))
plot_roc_curve(clf, X_val, y_val) 
y_test_pred = clf.predict(X_test)
y_test_score = clf.predict_proba(X_test)
cf = confusion_matrix(y, y_test_pred)
sns.heatmap(cf, annot=True, fmt='d')
print(classification_report(y, y_test_pred, target_names=['Non-Creditworthy', 'Creditworthy']))<jupyter_output>precision    recall  f1-score   support

Non-Creditworthy       0.36      0.26      0.30       142
    Creditworthy       0.73      0.81      0.77       358

        accuracy                           0.66       500
       macro avg       0.55      0.54      0.54       500
    weighted avg       0.63      0.66      0.64       500<jupyter_text>AdaBoost Classifier<jupyter_code>clf = AdaBoostClassifier().fit(X_train, y_train)
y_val_pred = clf.predict(X_val)
y_val_score = clf.predict_proba(X_val)
cf = confusion_matrix(y_val, y_val_pred)
sns.heatmap(cf, annot=True, fmt='d')
print(classification_report(y_val, y_val_pred, target_names=['Non-Creditworthy', 'Creditworthy']))
plot_roc_curve(clf, X_val, y_val)
y_test_pred = clf.predict(X_test)
y_test_score = clf.predict_proba(X_test)
cf = confusion_matrix(y, y_test_pred)
sns.heatmap(cf, annot=True, fmt='d')
print(classification_report(y, y_test_pred, target_names=['Non-Creditworthy', 'Creditworthy']))<jupyter_output>precision    recall  f1-score   support

Non-Creditworthy       0.31      0.23      0.26       142
    Creditworthy       0.72      0.80      0.76       358

        accuracy                           0.64       500
       macro avg       0.52      0.51      0.51       500
    weighted avg       0.61      0.64      0.62       500<jupyter_text>XGBoost<jupyter_code>clf = XGBClassifier(learning_rate = 0.001, n_estimators=200, random_state=random_state).fit(X_train, y_train)
y_val_pred = clf.predict(X_val)
y_val_score = clf.predict_proba(X_val)
cf = confusion_matrix(y_val, y_val_pred)
sns.heatmap(cf, annot=True, fmt='d')
print(classification_report(y_val, y_val_pred, target_names=['Non-Creditworthy', 'Creditworthy']))
plot_roc_curve(clf, X_val, y_val)
y_test_pred = clf.predict(X_test)
y_test_score = clf.predict_proba(X_test)
cf = confusion_matrix(y, y_test_pred)
sns.heatmap(cf, annot=True, fmt='d')
print(classification_report(y, y_test_pred, target_names=['Non-Creditworthy', 'Creditworthy']))<jupyter_output>precision    recall  f1-score   support

Non-Creditworthy       0.38      0.18      0.25       142
    Creditworthy       0.73      0.88      0.80       358

        accuracy                           0.68       500
       macro avg       0.55      0.53      0.52       500
    weighted avg       0.63      0.68      0.64       500
{'fraction_non_alphanumeric': 0.13519015452947164, 'fraction_numerical': 0.04441565651892292, 'mean_word_length': 2.7671662600209133, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '106236', 'license': 'MIT', 'n_tokens_mistral': 4104, 'n_tokens_neox': 3687, 'n_words': 665, 'path': 'project-4/.ipynb_checkpoints/Predicting-Default-Risk-checkpoint.ipynb', 'repo_name': 'FairozaAmira/Predictive-Analytics-for-Business-ND-Udacity', 'size': '10807'}