<jupyter_start><jupyter_text>Breast Cancer Wisconsin Diagnostic Prediction**Benign or Malignant Cancer Tumors** Classification Models after Principal component analysis **Developed by** [**<NAME>** ](https://github.com/officialpm)  🖖        ---  Downloading dataset from UCI---Dataset Link - https://archive.ics.uci.edu/ml/datasets/Breast+Cancer+Wisconsin+%28Diagnostic%29<jupyter_code>!wget https://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/wdbc.data
import pandas as pd
headers = ["id","diagnosis","radius_mean","texture_mean","perimeter_mean","area_mean","smoothness_mean","compactness_mean","concavity_mean","concave points_mean","symmetry_mean","fractal_dimension_mean","radius_se","texture_se","perimeter_se","area_se","smoothness_se","compactness_se","concavity_se","concave points_se","symmetry_se","fractal_dimension_se","radius_worst","texture_worst","perimeter_worst","area_worst","smoothness_worst","compactness_worst","concavity_worst","concave points_worst","symmetry_worst","fractal_dimension_worst"]
data = pd.read_csv('wdbc.data', names=headers)<jupyter_output><empty_output><jupyter_text>Classification Models  Importing Packages<jupyter_code>import numpy as np
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import seaborn as sns # data visualization library  
import matplotlib.pyplot as plt
# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory
# !pip install tqdm
import time
from tqdm import tqdm_notebook as tqdm
from sklearn.metrics import confusion_matrix
from sklearn import metrics
from sklearn.linear_model import LogisticRegression
import warnings
warnings.simplefilter("ignore")<jupyter_output><empty_output><jupyter_text>Preparing Data<jupyter_code># feature names as a list
col = data.columns  # .columns gives columns names in data 
data.columns
# Y includes our labels and x includes our features
Y = data.diagnosis                          # M or B 
list = ['id','diagnosis']
X = data.drop(list,axis = 1 )
X.head()
# Function is used to convert text class to neumerical class
def Y_class(Y):
  if Y == 'B':
    return 0
  elif Y == 'M':
    return 1

#changing 
actual_value = Y
category = actual_value.map(Y_class) 
Y_class = category
drop_list = ['perimeter_mean','area_mean','radius_worst','perimeter_worst','radius_se']
x = X.drop(drop_list,axis = 1 )
from sklearn.preprocessing import StandardScaler
X_std_model = StandardScaler()
X_std = X_std_model.fit(X)
X_std = X_std_model.transform(X)

import pickle
pickle_out = open("X-STANDARD-MODEL.pickle","wb")
pickle.dump(X_std_model, pickle_out)
pickle_out.close()

mean_vec = np.mean(X_std, axis=0)
cov_mat = (X_std - mean_vec).T.dot((X_std - mean_vec)) / (X_std.shape[0]-1)
from numpy.linalg import eig
values, vectors = eig(cov_mat)
eigen_pair = sorted([(np.abs(values[i]), vectors[:,i]) for i in range(len(values))], key = lambda x: x[0], reverse = True)

var_exp = [(i / sum(values))*100 for i in sorted(values, reverse=True)]
cum_var_exp = np.cumsum(var_exp)
matrix = []
for values in eigen_pair[:15]:
  matrix.append(values[1])
X_pca = np.array(matrix).transpose()

pickle_out = open('X-PCA.pickle','wb')
pickle.dump(X_pca, pickle_out)
pickle_out.close()

X_re = X_std.dot(X_pca)

from sklearn.model_selection import train_test_split
X_t_re, X_test_re, Y_t_re, Y_test_re = train_test_split(X_re, Y_class, test_size=0.05, random_state=42)
X_train_re, X_val_re, Y_train_re, Y_val_re = train_test_split(X_t_re, Y_t_re, test_size=0.05, random_state=42)<jupyter_output><empty_output><jupyter_text>Applying Models  Logistic Regression<jupyter_code>C_list = []
i = 0.0001
for count in range(25):
    C_list.append(i)
    i = round(i+0.001,4)
def AUC_curve(X_train, Y_train, X_cv, Y_cv):
    auc_train = []
    auc_cv = []
    for C in C_list:
    #     Applying Logistic Regression with l1 regularization on trainng data for each C
        clf = LogisticRegression(C=C,penalty="l2")
        clf.fit(X_train, Y_train)
                # predict the response
        probs_train = clf.predict_proba(X_train)
        preds_train = probs_train[:,1]
        fpr_train, tpr_train, threshold_train = metrics.roc_curve(Y_train, preds_train)
        auc_train.append(metrics.auc(fpr_train, tpr_train))
    #     predicting probabilistic value from the cross validation data.
        probs_cv = clf.predict_proba(X_cv)
        preds_cv = probs_cv[:,1]
        fpr_cv, tpr_cv, threshold_cv = metrics.roc_curve(Y_cv, preds_cv)
        auc_cv.append(metrics.auc(fpr_cv, tpr_cv)) 
    
    df = pd.DataFrame({'Train_Accuracy': auc_train, 'CV_Accuracy': auc_cv},  index = C_list) 

    #   Ploting AUC curve for each value of C to find best which is neihter overfit nor underfit much.
    plt.plot(C_list,auc_train,label="Train Data")
    # for xy in zip(alpha, np.round(auc_train,2)):
    #         plt.annotate('(%s, %s)' % xy, xy=xy, textcoords='data')
    plt.plot(C_list,auc_cv,label="CV Data")
    for xy in zip(C_list, np.round(auc_cv,2)):
            plt.annotate('(%s, %s)' % xy, xy=xy, textcoords='data')

    plt.xlabel('C Value')
    plt.ylabel('AUC Data')
    plt.legend()
    plt.show()
    return df<jupyter_output><empty_output><jupyter_text>**Learning Curve**<jupyter_code># Applying logistic regression on each value of C to find best C.
AUC_curve(X_train_re, Y_train_re, X_val_re, Y_val_re)
def final_Model(C,X_train, Y_train, X_test, Y_test):
    clf = LogisticRegression(C=C,penalty="l2")
    clf.fit(X_train, Y_train)
    # getting probabilistic value from my model on Train data
    probs_train = clf.predict_proba(X_train)
    preds_train= probs_train[:,1]
    y_train_pred = clf.predict(X_train)
    fpr_train, tpr_train, threshold_train = metrics.roc_curve(Y_train, preds_train)
    auc_train = metrics.auc(fpr_train, tpr_train)

    # getting probabilistic and predicted value from my model on Train data
    probs_test = clf.predict_proba(X_test)
    y_pred = clf.predict(X_test)
    preds_test = probs_test[:,1]
    fpr_test, tpr_test, threshold_test = metrics.roc_curve(Y_test, preds_test)
    auc_test = metrics.auc(fpr_test, tpr_test)

    # Graph to show Accuracy percentage on train and test set data.
    plt.plot(fpr_train,tpr_train,label="AUC of Train Data"+str(np.round(auc_train,2)), color = 'green')
    plt.plot(fpr_test,tpr_test,label="AUC of Test Data"+str(np.round(auc_test,2)), color = 'red')
    plt.plot([0, 1], [0, 1], color='black', lw=1, linestyle='--')
    plt.xlabel('Number of Neighbors K')
    plt.ylabel('AUC Data')
    plt.legend()
    plt.show()
    
    return clf, y_pred, y_train_pred
# For the value of C at 0.01 data is not much overfit and underfit, so we are taking that as a C value for our LR model
clf, y_pred, y_train_pred = final_Model(0.0241,X_train_re, Y_train_re, X_test_re, Y_test_re)
import itertools
# Plotting Confusion matrix to know How much data is correctly and incorrectly predicted by our model.

def plot_confusion_matrix(cm, classes=['Benign', 'Malignant'],
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.tight_layout()
# Compute confusion matrix
cnf_matrix = confusion_matrix(Y_train_re, y_train_pred)
np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, 
                      title='Confusion matrix on train data, without normalization')
# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix,  normalize=True,
                      title='Normalized confusion matrix')

plt.show()
# Compute confusion matrix
cnf_matrix = confusion_matrix(Y_test_re, y_pred)
np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, 
                      title='Confusion matrix on train data, without normalization')
# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix,  normalize=True,
                      title='Normalized confusion matrix')

plt.show()
# This function will give top 10 positive class feature which has hightest probability from BOW data with l1 regularizer
def most_informative_feature_for_class(data, classifier):
    topn = sorted(zip(classifier.coef_[0], [i for i in range(data.shape[1])] ), reverse=True)
    column_names = ["word", "coef"]
    df = pd.DataFrame(columns = column_names)
    row = 0
    for coe, name in topn:
        df.loc[row] = [name, coe]
        row+=1
    return df
X_re.shape
print("Most Important Feature for class Logistic Regression : ")
df = most_informative_feature_for_class(X_re,clf)
ax = df.plot.barh(x='word', y='coef')
clf.coef_<jupyter_output><empty_output><jupyter_text>**From the confusion Matrix we can see that AUC performance for Maliganant Cancer is decreased by 2, but it is not effecting our test result.**But at the end we need to select best model, so we will not consider this model for predicting the result.  SVMSupport vector machine is used to solve Supervised Machine Learning Problem(Classification). In support Vector Machine the algorithm will try to predict the best plane or line on the basis of support vectors of different classes.This algorithm uses hinge loss as a loss function and l2 as regularization parameter.<jupyter_code>from sklearn.linear_model import SGDClassifier
from sklearn.calibration import CalibratedClassifierCV
def AUC_svm_curve(X_train, Y_train, X_cv, Y_cv):
    auc_train = []
    auc_cv = []
    for alpha in C_list:
    #     Applying Logistic Regression with l1 regularization on trainng data for each C
        clf = SGDClassifier(alpha=alpha,penalty="l2", loss="hinge")
        clf.fit(X_train, Y_train)
                # predict the response
        preds_train = clf.predict(X_train)
#       SGDClassifier using sigmoid calibration
        clf_sigmoid = CalibratedClassifierCV(clf,cv = 4, method = "isotonic")
        clf_sigmoid.fit(X_train, Y_train)
        
#       predicting probabilistic value from the Train data.
        prob_train_sigmoid = clf_sigmoid.predict_proba(X_train)[:,1]
        fpr_train, tpr_train, threshold_train = metrics.roc_curve(Y_train, prob_train_sigmoid)
        auc_train.append(metrics.auc(fpr_train, tpr_train))
        
        
#       predicting probabilistic value from the cross validation data.
        preds_cv = clf.predict(X_cv)
        prob_cv_sigmoid = clf_sigmoid.predict_proba(X_cv)[:,1]
        fpr_cv, tpr_cv, threshold_cv = metrics.roc_curve(Y_cv, prob_cv_sigmoid)
        auc_cv.append(metrics.auc(fpr_cv, tpr_cv)) 

    #   Ploting AUC curve for each value of C to find best which is neihter overfit nor underfit much.
    # alpha_log = [round(log10(x),2) for x in r]
    df = pd.DataFrame({'Train_Accuracy': auc_train, 'CV_Accuracy': auc_cv},  index = C_list) 

    # Plotting the graph between alpha and its AUC value to find best alpha for our model.    
    plt.plot(C_list,auc_train,label="Train Data")
    # for xy in zip(alpha, np.round(auc_train,2)):
    #         plt.annotate('(%s, %s)' % xy, xy=xy, textcoords='data')
    plt.plot(C_list,auc_cv,label="CV Data")
    for xy in zip(C_list, np.round(auc_cv,2)):
            plt.annotate('(%s, %s)' % xy, xy=xy, textcoords='data')

    plt.xlabel('Value of Alpha')
    plt.ylabel('AUC Data')
    plt.legend()
    plt.show()
    return df<jupyter_output><empty_output><jupyter_text>**Learning Curve**<jupyter_code># Applying logistic regression on each value of C to find best C.
AUC_svm_curve(X_train_re, Y_train_re, X_val_re, Y_val_re)
def final_Model(C,X_train, Y_train, X_test, Y_test):
    clf = SGDClassifier(alpha=C,penalty="l2", loss="hinge")
    clf.fit(X_train, Y_train)
            # predict the response
    preds_train = clf.predict(X_train)
#       SGDClassifier using sigmoid calibration
    clf_sigmoid = CalibratedClassifierCV(clf,cv = 4, method = "isotonic")
    clf_sigmoid.fit(X_train, Y_train)

#       predicting probabilistic value from the Train data.
    prob_train_sigmoid = clf_sigmoid.predict_proba(X_train)[:,1]
    fpr_train, tpr_train, threshold_train = metrics.roc_curve(Y_train, prob_train_sigmoid)
    auc_train = metrics.auc(fpr_train, tpr_train)


#       predicting probabilistic value from the cross validation data.
    preds_test = clf.predict(X_test)
    prob_test_sigmoid = clf_sigmoid.predict_proba(X_test)[:,1]
    fpr_test, tpr_test, threshold_test = metrics.roc_curve(Y_test, prob_test_sigmoid)
    auc_test = metrics.auc(fpr_test, tpr_test)

    # Graph to show Accuracy percentage on train and test set data.
    plt.plot(fpr_train,tpr_train,label="AUC of Train Data"+str(np.round(auc_train,2)), color = 'green')
    plt.plot(fpr_test,tpr_test,label="AUC of Test Data"+str(np.round(auc_test,2)), color = 'red')
    plt.plot([0, 1], [0, 1], color='black', lw=1, linestyle='--')
    plt.xlabel('Number of Neighbors K')
    plt.ylabel('AUC Data')
    plt.legend()
    plt.show()
    
    return clf, preds_test, preds_train
# For the value of C at 0.01 data is not much overfit and underfit, so we are taking that as a C value for our LR model
clf, y_pred, y_train_pred = final_Model(0.0121,X_train_re, Y_train_re, X_test_re, Y_test_re)
# Compute confusion matrix
cnf_matrix = confusion_matrix(Y_train_re, y_train_pred)
np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, 
                      title='Confusion matrix on train data, without normalization')
# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix,  normalize=True,
                      title='Normalized confusion matrix')

plt.show()

# Compute confusion matrix
cnf_matrix = confusion_matrix(Y_test_re, y_pred)
np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, 
                      title='Confusion matrix on test data, without normalization')
# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix,  normalize=True,
                      title='Normalized confusion matrix')

plt.show()
print("Most Important Feature for class Logistic Regression : ")
df = most_informative_feature_for_class(X_re,clf)
ax = df.plot.barh(x='word', y='coef')
clf.coef_<jupyter_output><empty_output><jupyter_text>From the confusion matrix of Training Data, we can see that SVM model has classified 322 record correct for Benign out of 323 and 185 record correct for Malignant out of 190.At Test Time SVM Model has classified 12 record correct for Benign out of 18 and 10 record correct for Malignant out of 11.  RandomForestClassifier<jupyter_code>max_depth = [1,5,10,50,500,1000]
n_estimators = [5,10,100,500]
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
def AUC_curve_rf(X_train,Y_train,X_val, Y_val,max_depth,n_estimators):
    AUC_Train = np.zeros((len(max_depth), len(n_estimators)))
    AUC_CV = np.zeros((len(max_depth), len(n_estimators)))
    for depth in range(len(max_depth)):
        for estimators in range(len(n_estimators)):
    #     Applying Logistic Regression with l1 regularization on trainng data for each C
            clf = RandomForestClassifier(max_depth=max_depth[depth], n_estimators=n_estimators[estimators])
            clf.fit(X_train, Y_train)
                # predict the response
            probs_train = clf.predict_proba(X_train)
            preds_train = probs_train[:,1]
            fpr_train, tpr_train, threshold_train = metrics.roc_curve(Y_train, preds_train)
#             auc_train.append(metrics.auc(fpr_train, tpr_train))
    #     predicting probabilistic value from the cross validation data.
            probs_cv = clf.predict_proba(X_val)
            preds_cv = probs_cv[:,1]
            fpr_cv, tpr_cv, threshold_cv = metrics.roc_curve(Y_val, preds_cv)
#             auc_cv.append(metrics.auc(fpr_cv, tpr_cv)) 
#             print("Depth ",max_depth[depth],"Split ",min_samples_split[split], "AUC_train ",metrics.auc(fpr_train, tpr_train),
#                  "AUC_cv ",metrics.auc(fpr_cv, tpr_cv))
            
            AUC_Train[depth][estimators] = metrics.auc(fpr_train, tpr_train)
            AUC_CV[depth][estimators] = metrics.auc(fpr_cv, tpr_cv)
    return AUC_Train, AUC_CV
def plot_graph(AUC_train, AUC_cv):
    df_train = pd.DataFrame(AUC_train, columns = n_estimators,index = max_depth)
    df_cv = pd.DataFrame(AUC_cv, columns = n_estimators,index = max_depth)
    # plt.subplot(2, 1, 1)
    sns.heatmap(df_train, annot=True, fmt="g", cmap='viridis')
    plt.ylabel("Depth")
    plt.xlabel("n_estimators")
    plt.title("AUC values on Train data")
    plt.show()
    # plt.subplot(2, 1, 2)
    sns.heatmap(df_cv, annot=True, fmt="g", cmap='viridis')
    plt.ylabel("Depth")
    plt.xlabel("n_estimators")
    plt.title("AUC values on CV data")
    plt.show()
    # print(df)

def final_Model_rf(X_train,Y_train,X_test,Y_test,max_depth,n_estimators):
    clf = RandomForestClassifier(max_depth=max_depth, n_estimators=n_estimators)
    clf.fit(X_train, Y_train)
    # getting probabilistic value from my model on Train data
    probs_train = clf.predict_proba(X_train)
    preds_train= probs_train[:,1]
    y_train_pred = clf.predict(X_train)
    fpr_train, tpr_train, threshold_train = metrics.roc_curve(Y_train, preds_train)
    auc_train = metrics.auc(fpr_train, tpr_train)

    # getting probabilistic and predicted value from my model on Train data
    probs_test = clf.predict_proba(X_test)
    y_pred = clf.predict(X_test)
    preds_test = probs_test[:,1]
    fpr_test, tpr_test, threshold_test = metrics.roc_curve(Y_test, preds_test)
    auc_test = metrics.auc(fpr_test, tpr_test)

    # Graph to show Accuracy percentage on train and test set data.
    plt.plot(fpr_train,tpr_train,label="AUC of Train Data"+str(np.round(auc_train,2)), color = 'green')
    plt.plot(fpr_test,tpr_test,label="AUC of Test Data"+str(np.round(auc_test,2)), color = 'red')
    plt.plot([0, 1], [0, 1], color='black', lw=1, linestyle='--')
    plt.xlabel('Number of Neighbors K')
    plt.ylabel('AUC Data')
    plt.legend()
    plt.show()
    
    return clf, y_pred, y_train_pred
AUC_train, AUC_cv = AUC_curve_rf(X_train_re,Y_train_re,X_val_re, Y_val_re,max_depth,n_estimators)<jupyter_output><empty_output><jupyter_text>**Learning Curve**<jupyter_code>plot_graph(AUC_train, AUC_cv)
clf, y_pred, y_train_pred = final_Model_rf(X_train_re, Y_train_re, X_test_re, Y_test_re,50,10)
# Compute confusion matrix on train Data
# Compute confusion matrix
cnf_matrix = confusion_matrix(Y_train_re, y_train_pred)
np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, 
                      title='Confusion matrix on train data, without normalization')
# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix,  normalize=True,
                      title='Normalized confusion matrix')

plt.show()
# Compute confusion matrix on train Data
# Compute confusion matrix
cnf_matrix = confusion_matrix(Y_test_re, y_pred)
np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, 
                      title='Confusion matrix on test data, without normalization')
# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix,  normalize=True,
                      title='Normalized confusion matrix')

plt.show()
pickle_out = open('RandomForestClassifier_model.pickle','wb')
pickle.dump(clf, pickle_out)
pickle_out.close()
# most important positive class feature
def most_important_feature_rf(data, classifier):
    weight = classifier.feature_importances_
    topn = sorted(zip(weight, [i for i in range(data.shape[1])]), key = lambda x: x[0])
    column_names = ["word", "coef"]
    df = pd.DataFrame(columns = column_names)
    row = 0
    for coe, name in topn:
        df.loc[row] = [name, coe]
        row+=1
    return df
clf.feature_importances_
print("Most Important Feature for class Logistic Regression : ")
df = most_important_feature_rf(X_re,clf)
ax = df.plot.barh(x='word', y='coef')
import pydotplus
from PIL import Image
from sklearn.tree import export_graphviz
def decision_trees_save(clf, iteration, feature_names, target_names):
    file_name = "cancer_"+str(iteration)+"_pca.png"
    dot_data = export_graphviz(clf, out_file = None,feature_names=feature_names, class_names = target_names)
    # print(dot_data)
    graph = pydotplus.graph_from_dot_data(dot_data)
    # print(graph)
    graph.write_png(file_name)
for i in range(len(clf.estimators_)):
    decision_trees_save(clf.estimators_[i], i, [i for i in range(X_re.shape[1])], ['Benign', 'Malignant'])
image = Image.open("cancer_9_pca.png")
image<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10916411218373977, 'fraction_numerical': 0.008464107608546462, 'mean_word_length': 4.26825741142444, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 39, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '564696', 'license': 'MIT', 'n_tokens_mistral': 7858, 'n_tokens_neox': 7488, 'n_words': 1754, 'path': 'notebooks/Classification_Models_after_PCA_Breast_Cancer_Wisconsin_Diagnostic.ipynb', 'repo_name': 'officialpm/Breast-Cancer-Wisconsin-Diagnostic', 'size': '21863'}