<jupyter_start><jupyter_text>Applying Models after Feature Selection<jupyter_code># Importing Libraries
import pandas as pd
import numpy as np
from sklearn.metrics import precision_score, recall_score, f1_score, roc_auc_score, accuracy_score, classification_report
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.impute import KNNImputer

from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.neural_network import MLPClassifier

from sklearn.metrics import *
import zipfile
from zipfile import ZipFile
import pickle
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
# Loading Datasets

df_knn_imputed_uniform_3 = pd.read_csv(r'data\df_knn_imputed_uniform_3_FS.csv')
df_knn_imputed_uniform_5 = pd.read_csv(r'data\df_knn_imputed_uniform_5_FS.csv')
df_knn_imputed_uniform_7 = pd.read_csv(r'data\df_knn_imputed_uniform_7_FS.csv')
df_knn_imputed_uniform_9 = pd.read_csv(r'data\df_knn_imputed_uniform_9_FS.csv')
df_knn_imputed_uniform_11 = pd.read_csv(r'data\df_knn_imputed_uniform_11_FS.csv')
df_mean_imputed = pd.read_csv(r'data\df_mean_imputed_FS.csv')
df_knn_imputed_uniform_3.columns<jupyter_output><empty_output><jupyter_text>Applying all the models(LOG, RF, KNN, SVC, NB and combined model of LOG & RF).<jupyter_code>dfs = [df_knn_imputed_uniform_3,df_knn_imputed_uniform_5,
        df_knn_imputed_uniform_7,df_knn_imputed_uniform_9,
        df_knn_imputed_uniform_11,df_mean_imputed]
    
dfs_name = ["knn_imputed_3", "knn_imputed_5",
                "knn_imputed_7","knn_imputed_9",
                "knn_imputed_11","mean_imputed"]<jupyter_output><empty_output><jupyter_text>Logistic Regression model<jupyter_code>lr_classifier = LogisticRegression()
def log():
    logistic = {}

    for i, df in enumerate(dfs):
        x = df.drop(['class'], axis=1)
        y = df['class']
        x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.25, shuffle = True, random_state=0)
        
        ss = StandardScaler()
        x_train = ss.fit_transform(x_train)
        x_test = ss.transform(x_test)
        lr_classifier.fit(x_train,y_train)
        
        test_pred = lr_classifier.predict(x_test)
        train_pred = lr_classifier.predict(x_train).round(7)
       
        # print(test_pred)
        train_acc = accuracy_score(y_train, train_pred).round(7)
        test_acc = accuracy_score(y_test, test_pred).round(7)
        
        tn, fp, fn, tp = confusion_matrix(y_test, test_pred).ravel()
#         print(classification_report(y_test, test_pred))
        logistic[dfs_name[i]] = [train_acc, test_acc, tn, fp, fn, tp]
        
    return logistic
pd.DataFrame.from_dict(data = log(),orient='index', columns = ['train_acc', 'test_acc', 'true_neg', 'false_pos', 'false_neg', 'true_pos'])<jupyter_output><empty_output><jupyter_text>It is observed that for above Logistic Regression FP are between 2 or 3 and FN is zero for KNN imputed datasets. This is good.  But for mean imputed dataset there are FN which is not desirable in CKD prediction use case  Random Forest Model<jupyter_code>rf_classifier = RandomForestClassifier(n_estimators = 500)
def rf():
    rand_forest = {}

    for i, df in enumerate(dfs):
        x = df.drop('class',axis=1)
        y = df['class']
        x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.25, shuffle = True, random_state=42)
        
        ss = StandardScaler()
        x_train = ss.fit_transform(x_train)
        x_test = ss.transform(x_test)
        rf_classifier.fit(x_train,y_train)
        
        test_pred = rf_classifier.predict(x_test)
        train_pred = rf_classifier.predict(x_train)
       
        # print(test_pred)
        train_acc = accuracy_score(y_train, train_pred)
        test_acc = accuracy_score(y_test, test_pred)
        
        tn, fp, fn, tp = confusion_matrix(y_test, test_pred).ravel()
#         print(classification_report(y_test, test_pred))
        rand_forest[dfs_name[i]] = [train_acc, test_acc, tn, fp, fn, tp]
        
    return rand_forest
pd.DataFrame.from_dict(data = rf(),orient='index', columns = ['train_acc', 'test_acc', 'true_neg', 'false_pos', 'false_neg', 'true_pos'])<jupyter_output><empty_output><jupyter_text>In case of Random Forest, same performance is observed for all datasets. Mean imputed dataset is having perfect score.  KNN, SVC and Naive Bayes models<jupyter_code>def models(x):
    
    if(x==0):
        classifier = KNeighborsClassifier()
    elif(x==1):
        classifier = SVC(kernel='rbf')
    else:
        classifier = GaussianNB()
        
        
    res = {}

    for i, df in enumerate(dfs):
        x = df.drop('class',axis=1)
        y = df['class']
        x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.25, shuffle = True, random_state=0)
        
        ss = StandardScaler()
        x_train = ss.fit_transform(x_train)
        x_test = ss.transform(x_test)
        classifier.fit(x_train,y_train)
        
        test_pred = classifier.predict(x_test)
        train_pred = classifier.predict(x_train)
       
        # print(test_pred)
        train_acc = accuracy_score(y_train, train_pred)
        test_acc = accuracy_score(y_test, test_pred)
        
        tn, fp, fn, tp = confusion_matrix(y_test, test_pred).ravel()
#         print(classification_report(y_test, test_pred))
        res[dfs_name[i]] = [train_acc, test_acc, tn, fp, fn, tp]
        
    return res
for i in range(3):
    result = models(i)
    res = pd.DataFrame.from_dict(data = result, orient='index', columns = ['train_acc', 'test_acc', 'true_neg', 'false_pos', 'false_neg', 'true_pos'])
    if(i==0):
        print("\033[1m",i+1,".PERFORMANCE INDICATORS for KNN \n")
    elif(i==1):
        print(i+1,"\033[1m"+".PERFORMANCE INDICATORS for SVC \n")
    
    else:
       print(i+1,"\033[1m"+".PERFORMANCE INDICATORS for Naive Bayes \n")
#     result = models(i)
    print(res,'\n')<jupyter_output>[1m 1 .PERFORMANCE INDICATORS for KNN 

                train_acc  test_acc  true_neg  false_pos  false_neg  true_pos
knn_imputed_3    0.976667      0.97        37          1          2        60
knn_imputed_5    0.993333      0.95        37          1          4        58
knn_imputed_7    0.983333      0.98        38          0          2        60
knn_imputed_9    0.986667      0.99        38          0          1        61
knn_imputed_11   0.996667      0.98        38          0          2        60
mean_imputed     0.986667      0.99        38          0          1        61 

2 [1m.PERFORMANCE INDICATORS for SVC 

                train_acc  test_acc  true_neg  false_pos  false_neg  true_pos
knn_imputed_3    0.993333      0.98        36          2          0        62
knn_imputed_5    0.996667      0.98        36          2          0        62
knn_imputed_7    1.000000      0.98        36          2          0        62
knn_imputed_9    0.996667      0.97        35          3   [...]<jupyter_text>Combined Model of Logistic Regression & Random Forest<jupyter_code>from sklearn.ensemble import VotingClassifier
def combined():

    estimators=[ ('lr', lr_classifier), ('rf', rf_classifier)]

    majority_voting = VotingClassifier(estimators, voting='hard')

    sets = {}

        #dfs: array of imputed df
    dfs = [df_knn_imputed_uniform_3,df_knn_imputed_uniform_5,
            df_knn_imputed_uniform_7,df_knn_imputed_uniform_9,
            df_knn_imputed_uniform_11,df_mean_imputed]

    dfs_name = ["knn_imputed_3", "knn_imputed_5",
                    "knn_imputed_7","knn_imputed_9",
                    "knn_imputed_11","mean_imputed"]

    for i, df in enumerate(dfs):
        x = df.drop('class',axis=1)
        y = df['class']

        x_train, x_test, y_train, y_test = train_test_split(x, y, test_size = 0.25,random_state = 42)

        ss = StandardScaler()
        x_train = ss.fit_transform(x_train)
        x_test = ss.transform(x_test)
        majority_voting.fit(x_train,y_train)

        test_pred = majority_voting.predict(x_test)
        train_pred = majority_voting.predict(x_train)

        # print(test_pred)
        train_acc = accuracy_score(y_train, train_pred)*100
        test_acc = accuracy_score(y_test, test_pred)*100

        tn, fp, fn, tp = confusion_matrix(y_test, test_pred).ravel()
#         print(classification_report(y_test, test_pred))
        sets[dfs_name[i]] = [train_acc, test_acc, tn, fp, fn, tp]
    return sets
pd.DataFrame.from_dict(data = combined(), orient='index', columns = ['train_acc', 'test_acc', 'true_neg', 'false_pos', 'false_neg', 'true_pos'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09891721182043763, 'fraction_numerical': 0.02988946537333634, 'mean_word_length': 3.0122171945701357, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '246803', 'license': 'MIT', 'n_tokens_mistral': 3342, 'n_tokens_neox': 3134, 'n_words': 618, 'path': '5.After_Feature_Selection.ipynb', 'repo_name': 'sreekanthmutthoju/ML_Medical_Domain', 'size': '8866'}