<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Mounted at /content/drive<jupyter_text>Dataset Exploration1.   Head of dataset2.   Check for null values (absent)3.   Check for class imbalance (present - remove it by upsampling)<jupyter_code># Read the csv file
import pandas as pd
df=pd.read_csv('/content/drive/My Drive/Diabetes/diabetes.csv')
df.head()
# Print the null values of every column
df.isna().sum()

# Print class count to check for imbalance
df['Outcome'].value_counts()
from sklearn.utils import resample
df_majority = df[df.Outcome==0]
df_minority = df[df.Outcome==1]
 
# Upsample minority class
df_minority_upsampled = resample(df_minority, 
                                 replace=True,     # sample with replacement
                                 n_samples=500,    # to match majority class
                                 random_state=42) # reproducible results
 
# Combine majority class with upsampled minority class
df = pd.concat([df_majority, df_minority_upsampled])

print(df['Outcome'].value_counts())<jupyter_output>1    500
0    500
Name: Outcome, dtype: int64<jupyter_text>Stratified K Folds Cross Validation<jupyter_code># Add a "kfolds" column which will indicate the validation set number
df['kfolds']=-1

# Shuffle all the rows and then reset the index
df=df.sample(frac=1,random_state=42).reset_index(drop=True)
df.head()
from sklearn import model_selection

# Create 5 sets of training,validation sets
strat_kf=model_selection.StratifiedKFold(n_splits=5)

# .split() returns a list of 5 lists (corresponding to the n_splits value)
# Each of these 5 lists consists of 2 lists. 1st one contains training set indices and 2nd one contains validation set indices
# In a dataset of 10 data points, data 1 and 2 will be the validation in 1st fold, data 3 and 4 in the second fold and so on
# 1st iteration of the for loop : trn_ = 3,4,5,6,7,8,9,10 and val_ = 1,2 and fold : 0
# Assign 1st and 2nd row's kfolds value as 0 representing that they will be the validation points for 1st (0th) fold

for fold,(trn_,val_) in enumerate(strat_kf.split(X=df,y=df['Outcome'])):
  df.loc[val_,'kfolds']=fold
df.head()<jupyter_output><empty_output><jupyter_text>Scale the features using StandardScaler<jupyter_code>from sklearn.preprocessing import StandardScaler
scaler=StandardScaler()
df_2=pd.DataFrame(scaler.fit_transform(df),index=df.index,columns=df.columns)

# Target column and kfolds column dont need to be scaled
df_2['Outcome']=df['Outcome']
df_2['kfolds']=df['kfolds']
df_2.head()<jupyter_output><empty_output><jupyter_text>Feature Selection1.   KNN2.   Naive Bayes<jupyter_code>from sklearn import metrics
import matplotlib.pyplot as plt
def run(fold,df,models,print_details=False):
  
  # Training and validation sets
  df_train=df[df['kfolds']!=fold].reset_index(drop=True)
  df_valid=df[df['kfolds']==fold].reset_index(drop=True)


  # x and y of training dataset
  x_train=df_train.drop('Outcome',axis=1).values
  y_train=df_train.Outcome.values

  # x and y of validation dataset
  x_valid=df_valid.drop('Outcome',axis=1).values
  y_valid=df_valid.Outcome.values

  # accuracy => will store accuracies of the models  (same for confusion_matrices)
  accuracy=[]
  confusion_matrices=[]
  classification_report=[]

  for model_name,model_constructor in list(models.items()):
    clf=model_constructor
    clf.fit(x_train,y_train)

    # preds_train, preds_valid => predictions when training and validation x are fed into the trained model
    preds_train=clf.predict(x_train)
    preds_valid=clf.predict(x_valid)

    acc_train=metrics.accuracy_score(y_train,preds_train)
    acc_valid=metrics.accuracy_score(y_valid,preds_valid)
    conf_matrix=metrics.confusion_matrix(y_valid,preds_valid)
    class_report=metrics.classification_report(y_valid,preds_valid)

    accuracy.append(acc_valid)
    confusion_matrices.append(conf_matrix)
    classification_report.append(class_report)

    if(print_details==True):
      print(f'Model => {model_name} => Fold = {fold} => Training Accuracy = {acc_train} => Validation Accuracy = {acc_valid}')

  if(print_details==True):
    print('\n--------------------------------------------------------------------------------------------\n')
    
  return accuracy,confusion_matrices,classification_report<jupyter_output><empty_output><jupyter_text>Greedy Feature Selection<jupyter_code>def greedy_feature_selection(fold,df,models,target_name):

  # target_index => stores the index of the target variable in the dataset
  # kfolds_index => stores the index of kfolds column in the dataset
  target_index=df.columns.get_loc(target_name)
  kfolds_index=df.columns.get_loc('kfolds')

  # good_features => stores the indices of all the optimal features
  # best_scores => keeps track of the best scores 
  good_features=[]
  best_scores=[]

  # df has X and y and a kfolds column. 
  # no of features (no of columns in X) => total columns in df - 1 (there's 1 y) - 1 (there's 1 kfolds)
  num_features=df.shape[1]-2


  while True:

    # this_feature => the feature added to the already selected features to measure the effect of the former on the model
    # best_score => keeps track of the best score achieved while selecting features 1 at a time and checking its effect on the model
    this_feature=None
    best_score=0


    for feature in range(num_features):

      # if the feature is already in the good_features list, ignore and move ahead
      if feature in good_features:
        continue
      
      # add the currently selected feature to the already discovered good features
      selected_features=good_features+[feature]

      # all the selected features + target and kfolds column
      df_train=df.iloc[:, selected_features + [target_index,kfolds_index]]

      # fit the selected dataset to a model 
      accuracy,confusion_matrices,classification_report=run(fold,df_train,models)

      # if any improvement is observed over the previous set of features
      if(accuracy[0]>best_score):
        this_feature=feature
        best_score=accuracy[0]
      
    if(this_feature!=None):
      good_features.append(this_feature)
      best_scores.append(best_score)
    
    if(len(best_scores)>2):
      if(best_scores[-1]<best_scores[-2]):
        break
    
  return best_scores[:-1] , df.iloc[:, good_features[:-1] + [target_index,kfolds_index]]<jupyter_output><empty_output><jupyter_text>Recursive Feature Selection<jupyter_code>from sklearn.feature_selection import RFE

def recursive_feature_selection(df,models,n_features_to_select,target_name):
  X=df.drop(labels=[target_name,'kfolds'],axis=1).values
  y=df[target_name]
  kfolds=df.kfolds.values

  model_name,model_constructor=list(models.items())[0]

  rfe=RFE(
      estimator=model_constructor,
      n_features_to_select=n_features_to_select
  )

  try:
    rfe.fit(X,y)
  except RuntimeError:
    print(f"{model_name} does not support feature importance... Returning original dataframe\n")
    return df
  else:
    X_transformed = rfe.transform(X)
    df_optimal=pd.DataFrame(data=[X,y,kfolds])
    return df_optimal
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.naive_bayes import MultinomialNB
print('Greedy Feature Selection : ')
print('\n')
models={'KNN': KNeighborsClassifier()}
best_scores,df_optimal_KNN=greedy_feature_selection(fold=4,df=df_2,models=models,target_name='Outcome')
print(df_optimal_KNN.head())

print('\n')
print("Recursive Feature Selection : ")
print('\n')
df_recursive_optimal_KNN=recursive_feature_selection(df=df_2,models=models,n_features_to_select=5,target_name='Outcome')
print(df_recursive_optimal_KNN.head())
models={'Naive Bayes' : GaussianNB()}
best_scores,df_optimal_NB=greedy_feature_selection(fold=4,df=df_2,models=models,target_name='Outcome')
print(df_optimal_NB.head())

print('\n')
df_recursive_optimal_NB=recursive_feature_selection(df=df_2,models=models,n_features_to_select=5,target_name='Outcome')
print(df_recursive_optimal_NB.head())<jupyter_output>Glucose  Pregnancies       BMI  Outcome  kfolds
0 -0.230248    -0.053578 -0.363725        1       0
1  2.011697     0.787960  0.691889        1       0
2 -0.614581     1.068473  1.430819        1       0
3 -0.518498     1.629498 -0.007455        1       0
4  0.250169    -1.175629 -0.007455        0       0


Naive Bayes does not support feature importance... Returning original dataframe

   Pregnancies   Glucose  BloodPressure  ...       Age  Outcome  kfolds
0    -0.053578 -0.230248      -0.423269  ... -0.367793        1       0
1     0.787960  2.011697      -0.111124  ...  0.579018        1       0
2     1.068473 -0.614581       1.553653  ...  0.923314        1       0
3     1.629498 -0.518498      -0.215172  ...  0.665092        1       0
4    -1.175629  0.250169       0.409119  ... -1.142458        0       0

[5 rows x 10 columns]<jupyter_text>Hyperparameter Tuning in KNN  Manually finding the optimal value of n_neighbors parameter<jupyter_code># Find the optimal value of the n_neighbors parameter

models={f'KNN_{i}':KNeighborsClassifier(n_neighbors=i) for i in range(2,31)}
# run the model only for fold number 4 ie the 5th fold

accuracy,confusion_matrices,classification_report=run(fold=4,df=df_optimal_KNN,models=models,print_details=True)
x=[i for i in range(2,31)]
y=accuracy
plt.plot(x,y)
plt.xlabel('Number of Nearest Neighbors')
plt.ylabel('Accuracy Score')
plt.title("Optimal n_neighbors value")
plt.show()<jupyter_output><empty_output><jupyter_text>Using Grid Search to find optimal values of n_neighbors and p<jupyter_code>from sklearn import model_selection
from sklearn import metrics

def hyperparameter_tune_and_run(df,num_folds,models,target_name,param_grid,evaluation_metric,print_details=False):
  X=df.drop(labels=[target_name,'kfolds'],axis=1).values
  y=df[target_name]

  model_name,model_constructor=list(models.items())[0]

  model = model_selection.GridSearchCV(
      estimator = model_constructor,
      param_grid = param_grid,
      scoring = evaluation_metric,
      verbose = 10,
      cv = num_folds
  )

  model.fit(X,y)

  if(print_details==True):
    print(f"Best score : {model.best_score_}")

    print("Best parameters : ")
    best_parameters=model.best_estimator_.get_params()
    for param_name in sorted(param_grid.keys()):
      print(f"\t{param_name}: {best_parameters[param_name]}")
  
  return model

models={'KNN': KNeighborsClassifier()}
param_grid = {
    "n_neighbors" : [i for i in range(2,31)],
    "p" : [2,3]
}
model = hyperparameter_tune_and_run(df=df_optimal_KNN,num_folds=5,models=models,target_name='Outcome',param_grid=param_grid,evaluation_metric="accuracy",print_details=True)<jupyter_output>Fitting 5 folds for each of 58 candidates, totalling 290 fits
[CV] n_neighbors=2, p=2 ..............................................
[CV] .................. n_neighbors=2, p=2, score=0.770, total=   0.0s
[CV] n_neighbors=2, p=2 ..............................................
[CV] .................. n_neighbors=2, p=2, score=0.730, total=   0.0s
[CV] n_neighbors=2, p=2 ..............................................
[CV] .................. n_neighbors=2, p=2, score=0.785, total=   0.0s
[CV] n_neighbors=2, p=2 ..............................................
[CV] .................. n_neighbors=2, p=2, score=0.755, total=   0.0s
[CV] n_neighbors=2, p=2 ..............................................
[CV] .................. n_neighbors=2, p=2, score=0.740, total=   0.0s
[CV] n_neighbors=2, p=3 ..............................................
[CV] .................. n_neighbors=2, p=3, score=0.775, total=   0.0s
[CV] n_neighbors=2, p=3 ..............................................
[CV] ..........[...]<jupyter_text>Comparison between KNN and NB1.   Dataset when KNN was considered for feature selection2.   Dataset when NB was considered for feature selection<jupyter_code># Compare between KNN and Naive Bayes

models={
        'KNN': KNeighborsClassifier(n_neighbors=12,p=3),
        'Gaussian Naive Bayes': GaussianNB(),
        }
# accuracies => list of 5 lists. Each list will contain 3 values ie KNN accuracy, Gaussian Naive Bayes

accuracies,confusion_matrices,classification_reports=[],[],[]
for f in range(5):
  accuracy,confusion_matrix,classification_report=run(f,df_optimal_KNN,models=models,print_details=True)
  accuracies.append(accuracy)
  confusion_matrices.append(confusion_matrix)
  classification_reports.append(classification_report)
print(accuracies)
x_axis_labels=['Predicted Normal','Predicted Diabetic']
y_axis_labels=['True Normal','True Diabetic']
import seaborn as sns

# Heatmap of confusion matrix of 5th fold of KNN

sns.heatmap(confusion_matrices[4][0],xticklabels=x_axis_labels,yticklabels=y_axis_labels,annot=True)

# Heatmap of confusion matrix of 5th fold of Naive Bayes

sns.heatmap(confusion_matrices[4][1],xticklabels=x_axis_labels,yticklabels=y_axis_labels,annot=True)

# Classification report of 5th fold of KNN

print("KNN")
print(classification_reports[4][0])
# Classification report of 5th fold of Naive Bayes

print("Naive Bayes")
print(classification_reports[4][1])



accuracies,confusion_matrices,classification_reports=[],[],[]
for f in range(5):
  accuracy,confusion_matrix,classification_report=run(f,df_optimal_NB,models=models,print_details=True)
  accuracies.append(accuracy)
  confusion_matrices.append(confusion_matrix)
  classification_reports.append(classification_report)
import seaborn as sns

# Heatmap of confusion matrix of 5th fold of KNN

sns.heatmap(confusion_matrices[4][0],xticklabels=x_axis_labels,yticklabels=y_axis_labels,annot=True)
# Heatmap of confusion matrix of 5th fold of Naive Bayes

sns.heatmap(confusion_matrices[4][1],xticklabels=x_axis_labels,yticklabels=y_axis_labels,annot=True)
# Classification report of 5th fold of KNN

print("KNN")
print(classification_reports[4][0])
# Classification report of 5th fold of Naive Bayes

print("Naive Bayes")
print(classification_reports[4][1])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1372786115655452, 'fraction_numerical': 0.0346397325556583, 'mean_word_length': 3.9437412095639943, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '668329', 'license': 'MIT', 'n_tokens_mistral': 5022, 'n_tokens_neox': 4662, 'n_words': 1059, 'path': 'Diabetes.ipynb', 'repo_name': 'AryanMethil/Diabetes-KNN-vs-Naive-Bayes', 'size': '14059'}