<jupyter_start><jupyter_text>Data cleanup**Checking null values**<jupyter_code>df.isnull().sum()
df = df.fillna(round(df.median()))
df.info()
df.describe()<jupyter_output><empty_output><jupyter_text>Features Selection<jupyter_code>plt.figure(figsize=(12,12)) 
sns.heatmap(df.corr(),annot=True,cmap='cubehelix_r')
plt.show()
corr = df.corr()
#Correlation with output variable
cor_target = abs(corr["Credit-Application-Result"])
#Selecting highly correlated features
relevant_features = cor_target[cor_target>=0]
df_corr = pd.DataFrame(relevant_features)
df_corr = df_corr.reset_index()
df_corr = df_corr.rename(columns = {'Credit-Application-Result' : 'Corr', 'index': 'features'})
df_corr = df_corr.sort_values(['Corr'], ascending=False)
df_corr
df.nunique()
df.skew()
df['No-of-dependents'].value_counts().sort_index().plot.bar()
df['No-of-Credits-at-this-Bank'].value_counts().sort_index().plot.bar()
df['Type-of-apartment'].value_counts().sort_index().plot.bar()
df['Guarantors'].value_counts().sort_index().plot.bar()
df['Telephone'].value_counts().sort_index().plot.bar()<jupyter_output><empty_output><jupyter_text>Data to be excluded:1. Concurrent-credits and Occupation (Because only have 1 values)2. Foreign-Worker, Guarantors and No-of-dependents (Because of skewed to 1 data only)3. Duration in current address is removed because has 344 empty records<jupyter_code>df = df.drop(columns=["Duration-in-Current-address",
                      'Concurrent-Credits',
                      'Guarantors',
                      'Foreign-Worker', 
                      'Occupation', 
                      'Telephone', 
                      'No-of-dependents',
                     'Duration-of-Credit-Month',
                     "Instalment-per-cent"])
X1 = df.drop(columns=['Credit-Application-Result'])
y1 = df['Credit-Application-Result']

print(X1.shape)
print(y1.shape)

print(df.info())

X = np.array(X1)
y = np.array(y1)<jupyter_output><empty_output><jupyter_text>**Check whether the data is balanced or not**<jupyter_code>positive = df[df["Credit-Application-Result"]== 1]
negative = df[df["Credit-Application-Result"]== 0]

ratio = float(len(negative) / len(positive))
print(ratio)


print("positive:{}".format(len(positive)))
print("negative:{}".format(len(negative)))
random_state = 1
X_train, X_val, y_train,y_val = train_test_split(X,y, test_size = 0.3, stratify=y, random_state=random_state)
model = sm.OLS(y1, X1)
fit = model.fit()
print(fit.summary())<jupyter_output>OLS Regression Results                                    
==============================================================================================
Dep. Variable:     Credit-Application-Result   R-squared (uncentered):                   0.759
Model:                                   OLS   Adj. R-squared (uncentered):              0.754
Method:                        Least Squares   F-statistic:                              154.1
Date:                       Sun, 08 Aug 2021   Prob (F-statistic):                   2.68e-144
Time:                               06:45:48   Log-Likelihood:                         -270.49
No. Observations:                        500   AIC:                                      561.0
Df Residuals:                            490   BIC:                                      603.1
Df Model:                                 10                                                  
Covariance Type:                   nonrobust      [...]<jupyter_text>Provided test dataset<jupyter_code>df_test = pd.read_csv("../content/drive/My Drive/Colab Notebooks/Udacity/customers-to-score.csv")
df_test
df_test['Account-Balance'].unique()
df_test['Account-Balance'].replace({'Some Balance': 1, 'No Account': 0}, inplace=True)
df_test['Payment-Status-of-Previous-Credit'].unique()
df_test['Payment-Status-of-Previous-Credit'].replace({'Paid Up': 2, 'No Problems (in this bank)': 1, 'Some Problems': 0}, inplace=True)
df_test["Purpose"].unique()
df_test['Purpose'].replace({
                    'New car': 3, 
                    'Used car': 2,
                    'Home Related': 1,
                    'Other': 0}, inplace=True)
df_test['Value-Savings-Stocks'].unique()
df_test['Value-Savings-Stocks'].replace({
                    '< £100': 2, 
                    '£100-£1000': 1,
                    'None': 0}, inplace=True)
df_test['Length-of-current-employment'].unique()
df_test['Length-of-current-employment'].replace({
                    '4-7 yrs': 2, 
                    '1-4 yrs': 1,
                    '< 1yr': 0}, inplace=True)
df_test['Concurrent-Credits'].replace({'Other Banks/Depts': 1}, inplace=True)
df_test['No-of-Credits-at-this-Bank'].replace({'More than 1': 1, '1': 0}, inplace=True)
df_test['Guarantors'].replace({'Yes': 1, 'None': 0}, inplace=True)
df_test.info()
df_test = df_test.drop(columns=["Duration-in-Current-address",
                      'Concurrent-Credits',
                      'Guarantors',
                      'Foreign-Worker', 
                      'Occupation', 
                      'Telephone', 
                      'No-of-dependents',
                     'Duration-of-Credit-Month',
                     "Instalment-per-cent"])
df_test.info()
X_test = np.array(df_test)<jupyter_output><empty_output><jupyter_text>Machine Learning Training<jupyter_code># Instantiate the classfiers and make a list
classifiers = [LogisticRegression(random_state=random_state), 
               DecisionTreeClassifier(random_state=random_state),
               RandomForestClassifier(n_estimators=100, random_state=random_state),
               AdaBoostClassifier(random_state=random_state)]

# Define a result table as a DataFrame
result_table = pd.DataFrame(columns=['classifiers', 'fpr','tpr','auc'])

# Train the models and record the results
for cls in classifiers:
    model = cls.fit(X_train, y_train)
    yproba = model.predict_proba(X_val)[::,1]
    
    fpr, tpr, _ = roc_curve(y_val,  yproba)
    auc = roc_auc_score(y_val, yproba)
    
    result_table = result_table.append({'classifiers':cls.__class__.__name__,
                                        'fpr':fpr, 
                                        'tpr':tpr, 
                                        'auc':auc}, ignore_index=True)

# Set name of the classifiers as index labels
result_table.set_index('classifiers', inplace=True)
fig = plt.figure(figsize=(8,6))

for i in result_table.index:
    plt.plot(result_table.loc[i]['fpr'], 
             result_table.loc[i]['tpr'], 
             label="{}, AUC={:.3f}".format(i, result_table.loc[i]['auc']))
    
plt.plot([0,1], [0,1], color='orange', linestyle='--')

plt.xticks(np.arange(0.0, 1.1, step=0.1))
plt.xlabel("Flase Positive Rate", fontsize=15)

plt.yticks(np.arange(0.0, 1.1, step=0.1))
plt.ylabel("True Positive Rate", fontsize=15)

plt.title('ROC Curve Analysis', fontweight='bold', fontsize=15)
plt.legend(prop={'size':13}, loc='lower right')

plt.show()
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

def make_confusion_matrix(cf,
                          group_names=None,
                          categories='auto',
                          count=True,
                          percent=True,
                          cbar=True,
                          xyticks=True,
                          xyplotlabels=True,
                          sum_stats=True,
                          figsize=None,
                          cmap='Blues',
                          title=None):
    '''
    This function will make a pretty plot of an sklearn Confusion Matrix cm using a Seaborn heatmap visualization.
    Arguments
    ---------
    cf:            confusion matrix to be passed in
    group_names:   List of strings that represent the labels row by row to be shown in each square.
    categories:    List of strings containing the categories to be displayed on the x,y axis. Default is 'auto'
    count:         If True, show the raw number in the confusion matrix. Default is True.
    normalize:     If True, show the proportions for each category. Default is True.
    cbar:          If True, show the color bar. The cbar values are based off the values in the confusion matrix.
                   Default is True.
    xyticks:       If True, show x and y ticks. Default is True.
    xyplotlabels:  If True, show 'True Label' and 'Predicted Label' on the figure. Default is True.
    sum_stats:     If True, display summary statistics below the figure. Default is True.
    figsize:       Tuple representing the figure size. Default will be the matplotlib rcParams value.
    cmap:          Colormap of the values displayed from matplotlib.pyplot.cm. Default is 'Blues'
                   See http://matplotlib.org/examples/color/colormaps_reference.html
                   
    title:         Title for the heatmap. Default is None.
    '''


    # CODE TO GENERATE TEXT INSIDE EACH SQUARE
    blanks = ['' for i in range(cf.size)]

    if group_names and len(group_names)==cf.size:
        group_labels = ["{}\n".format(value) for value in group_names]
    else:
        group_labels = blanks

    if count:
        group_counts = ["{0:0.0f}\n".format(value) for value in cf.flatten()]
    else:
        group_counts = blanks

    if percent:
        group_percentages = ["{0:.2%}".format(value) for value in cf.flatten()/np.sum(cf)]
    else:
        group_percentages = blanks

    box_labels = [f"{v1}{v2}{v3}".strip() for v1, v2, v3 in zip(group_labels,group_counts,group_percentages)]
    box_labels = np.asarray(box_labels).reshape(cf.shape[0],cf.shape[1])


    # CODE TO GENERATE SUMMARY STATISTICS & TEXT FOR SUMMARY STATS
    if sum_stats:
        #Accuracy is sum of diagonal divided by total observations
        accuracy  = np.trace(cf) / float(np.sum(cf))

        #if it is a binary confusion matrix, show some more stats
        if len(cf)==2:
            #Metrics for Binary Confusion Matrices
            precision = cf[1,1] / sum(cf[:,1])
            recall    = cf[1,1] / sum(cf[1,:])
            f1_score  = 2*precision*recall / (precision + recall)
            stats_text = "\n\nAccuracy={:0.3f}\nPrecision={:0.3f}\nRecall={:0.3f}\nF1 Score={:0.3f}".format(
                accuracy,precision,recall,f1_score)
        else:
            stats_text = "\n\nAccuracy={:0.3f}".format(accuracy)
    else:
        stats_text = ""


    # SET FIGURE PARAMETERS ACCORDING TO OTHER ARGUMENTS
    if figsize==None:
        #Get default figure size if not set
        figsize = plt.rcParams.get('figure.figsize')

    if xyticks==False:
        #Do not show categories if xyticks is False
        categories=False


    # MAKE THE HEATMAP VISUALIZATION
    plt.figure(figsize=figsize)
    sns.heatmap(cf,annot=box_labels,fmt="",cmap=cmap,cbar=cbar,xticklabels=categories,yticklabels=categories)

    if xyplotlabels:
        plt.ylabel('True label')
        plt.xlabel('Predicted label' + stats_text)
    else:
        plt.xlabel(stats_text)
    
    if title:
        plt.title(title)<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code>clf = LogisticRegression(solver='liblinear').fit(X_train, y_train)
y_val_pred = clf.predict(X_val)
y_val_score = clf.decision_function(X_val)
def plot_feature_importance(importance,names,model_type):
  
    #Create arrays from feature importance and feature names
    feature_importance = np.array(importance)
    feature_names = np.array(names)

    #Create a DataFrame using a Dictionary
    data={'feature_names':feature_names,'feature_importance':feature_importance}
    fi_df = pd.DataFrame(data)

    #Sort the DataFrame in order decreasing feature importance
    fi_df.sort_values(by=['feature_importance'], ascending=False,inplace=True)

    #Define size of bar plot
    plt.figure(figsize=(10,8))
    #Plot Searborn bar chart
    sns.barplot(x=fi_df['feature_importance'], y=fi_df['feature_names'])
    #Add chart labels
    plt.title(model_type + 'FEATURE IMPORTANCE')
    plt.xlabel('FEATURE IMPORTANCE')
    plt.ylabel('FEATURE NAMES')
cf = confusion_matrix(y_val, y_val_pred)
labels = ['True Neg','False Pos','False Neg','True Pos']
categories = ['Non-creditworthy', 'creditworthy']
make_confusion_matrix(cf, 
                      group_names=labels,
                      categories=categories)
print(classification_report(y_val, y_val_pred, target_names=['Non-Creditworthy', 'Creditworthy']))
y_test_pred = clf.predict(X_test)
y_test_score = clf.decision_function(X_test)
cf = confusion_matrix(y, y_test_pred)
labels = ['True Neg','False Pos','False Neg','True Pos']
categories = ['Non-creditworthy', 'creditworthy']
make_confusion_matrix(cf, 
                      group_names=labels,
                      categories=categories)
print(classification_report(y, y_test_pred, target_names=['Non-Creditworthy', 'Creditworthy']))<jupyter_output>precision    recall  f1-score   support

Non-Creditworthy       0.35      0.15      0.22       142
    Creditworthy       0.73      0.89      0.80       358

        accuracy                           0.68       500
       macro avg       0.54      0.52      0.51       500
    weighted avg       0.62      0.68      0.63       500<jupyter_text>Decision Tree Classifier<jupyter_code>clf = DecisionTreeClassifier().fit(X_train, y_train)
plot_feature_importance(clf.feature_importances_,X1.columns,'DECISION TREE ')
y_val_pred = clf.predict(X_val)
y_val_score = clf.predict_proba(X_val)
cf = confusion_matrix(y_val, y_val_pred)
labels = ['True Neg','False Pos','False Neg','True Pos']
categories = ['Non-creditworthy', 'creditworthy']
make_confusion_matrix(cf, 
                      group_names=labels,
                      categories=categories)
print(classification_report(y_val, y_val_pred, target_names=['Non-Creditworthy', 'Creditworthy']))
y_test_pred = clf.predict(X_test)
y_test_score = clf.predict_proba(X_test)
cf = confusion_matrix(y, y_test_pred)
labels = ['True Neg','False Pos','False Neg','True Pos']
categories = ['Non-creditworthy', 'creditworthy']
make_confusion_matrix(cf, 
                      group_names=labels,
                      categories=categories)
print(classification_report(y, y_test_pred, target_names=['Non-Creditworthy', 'Creditworthy']))<jupyter_output>precision    recall  f1-score   support

Non-Creditworthy       0.30      0.35      0.32       142
    Creditworthy       0.72      0.67      0.70       358

        accuracy                           0.58       500
       macro avg       0.51      0.51      0.51       500
    weighted avg       0.60      0.58      0.59       500<jupyter_text>Random Forest Classifier<jupyter_code>clf = RandomForestClassifier(n_estimators=100).fit(X_train, y_train)
plot_feature_importance(clf.feature_importances_,X1.columns,'RANDOM FOREST ')
y_val_pred = clf.predict(X_val)
y_val_score = clf.predict_proba(X_val)
cf = confusion_matrix(y_val, y_val_pred)
labels = ['True Neg','False Pos','False Neg','True Pos']
categories = ['Non-creditworthy', 'creditworthy']
make_confusion_matrix(cf, 
                      group_names=labels,
                      categories=categories)
print(classification_report(y_val, y_val_pred, target_names=['Non-Creditworthy', 'Creditworthy']))
y_test_pred = clf.predict(X_test)
y_test_score = clf.predict_proba(X_test)
cf = confusion_matrix(y, y_test_pred)
labels = ['True Neg','False Pos','False Neg','True Pos']
categories = ['Non-creditworthy', 'creditworthy']
make_confusion_matrix(cf, 
                      group_names=labels,
                      categories=categories)
print(classification_report(y, y_test_pred, target_names=['Non-Creditworthy', 'Creditworthy']))<jupyter_output>precision    recall  f1-score   support

Non-Creditworthy       0.29      0.20      0.24       142
    Creditworthy       0.72      0.80      0.76       358

        accuracy                           0.63       500
       macro avg       0.51      0.50      0.50       500
    weighted avg       0.60      0.63      0.61       500<jupyter_text>AdaBoost Classifier<jupyter_code>clf = AdaBoostClassifier().fit(X_train, y_train)
plot_feature_importance(clf.feature_importances_,X1.columns,'ADABOOST ')
y_val_pred = clf.predict(X_val)
y_val_score = clf.predict_proba(X_val)
cf = confusion_matrix(y_val, y_val_pred)
labels = ['True Neg','False Pos','False Neg','True Pos']
categories = ['Non-creditworthy', 'creditworthy']
make_confusion_matrix(cf, 
                      group_names=labels,
                      categories=categories)
print(classification_report(y_val, y_val_pred, target_names=['Non-Creditworthy', 'Creditworthy']))
y_test_pred = clf.predict(X_test)
y_test_score = clf.predict_proba(X_test)
cf = confusion_matrix(y, y_test_pred)
labels = ['True Neg','False Pos','False Neg','True Pos']
categories = ['Non-creditworthy', 'creditworthy']
make_confusion_matrix(cf, 
                      group_names=labels,
                      categories=categories)
print(classification_report(y, y_test_pred, target_names=['Non-Creditworthy', 'Creditworthy']))<jupyter_output>precision    recall  f1-score   support

Non-Creditworthy       0.31      0.20      0.25       142
    Creditworthy       0.72      0.82      0.77       358

        accuracy                           0.64       500
       macro avg       0.52      0.51      0.51       500
    weighted avg       0.60      0.64      0.62       500<jupyter_text>XGBoost<jupyter_code>clf = XGBClassifier(learning_rate = 0.001, n_estimators=200, random_state=random_state).fit(X_train, y_train)
plot_feature_importance(clf.feature_importances_,X1.columns,'XGBOOST ')
y_val_pred = clf.predict(X_val)
y_val_score = clf.predict_proba(X_val)
cf = confusion_matrix(y_val, y_val_pred)
labels = ['True Neg','False Pos','False Neg','True Pos']
categories = ['Non-creditworthy', 'creditworthy']
make_confusion_matrix(cf, 
                      group_names=labels,
                      categories=categories)
print(classification_report(y_val, y_val_pred, target_names=['Non-Creditworthy', 'Creditworthy']))
y_test_pred = clf.predict(X_test)
y_test_score = clf.predict_proba(X_test)
cf = confusion_matrix(y, y_test_pred)
labels = ['True Neg','False Pos','False Neg','True Pos']
categories = ['Non-creditworthy', 'creditworthy']
make_confusion_matrix(cf, 
                      group_names=labels,
                      categories=categories)
print(classification_report(y, y_test_pred, target_names=['Non-Creditworthy', 'Creditworthy']))<jupyter_output>precision    recall  f1-score   support

Non-Creditworthy       0.32      0.11      0.16       142
    Creditworthy       0.72      0.91      0.80       358

        accuracy                           0.68       500
       macro avg       0.52      0.51      0.48       500
    weighted avg       0.61      0.68      0.62       500
{'fraction_non_alphanumeric': 0.1224098439247856, 'fraction_numerical': 0.025089223885367284, 'mean_word_length': 2.6124687319607465, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '636401', 'license': 'MIT', 'n_tokens_mistral': 6288, 'n_tokens_neox': 5800, 'n_words': 1194, 'path': 'project-4/Predicting_Default_Risk.ipynb', 'repo_name': 'FairozaAmira/Predictive-Analytics-for-Business-ND-Udacity', 'size': '18773'}