<jupyter_start><jupyter_text>DATA MODELING ML PIPELINE  Import Libraries<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Read & Store Data<jupyter_code>df_clean = pd.read_csv('DiabetesCleaned_2018.csv', index_col=0) # Read csv file<jupyter_output><empty_output><jupyter_text>Copy Data<jupyter_code>df_model = df_clean.copy()<jupyter_output><empty_output><jupyter_text>Sample Data<jupyter_code>all_ones  = df_clean.loc[df_model['DIABETES'] == 1]
all_zeros = df_clean.loc[df_model['DIABETES'] == 0]

print("{} is the length of all_ones".format(len(all_ones)))
print("{} is the length of all_zeros".format(len(all_zeros)))
sampled_ones = all_ones.sample(n=30000)
samples_zeros = all_zeros.sample(n=30000)

print("{} is the length of sampled_ones".format(len(sampled_ones)))
print("{} is the length of samples_zeros".format(len(samples_zeros)))
df_model = pd.concat((sampled_ones, samples_zeros), axis=0)
df_model = df_model.sample(frac=1)
df_model.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 60000 entries, 283235 to 133961
Data columns (total 19 columns):
EXERCISE                60000 non-null float64
HEARTATTACK             60000 non-null float64
CORONARYHEARTDISEASE    60000 non-null float64
STROKE                  60000 non-null float64
ASTHMA                  60000 non-null float64
SKINCANCER              60000 non-null float64
OTHERCANCER             60000 non-null float64
CHRONICBRONCHITIS       60000 non-null float64
ARTHRITIS               60000 non-null float64
DEPRESSIVEDISORDER      60000 non-null float64
KIDNEYDISEASE           60000 non-null float64
DIABETES                60000 non-null float64
SLEEPTIME_GROUP         60000 non-null object
SEX_GROUP               60000 non-null object
WEIGHT_KILOGRAM         60000 non-null float64
HEIGHT_METER            60000 non-null float64
BMI_GROUP               60000 non-null object
RACE_GROUP              60000 non-null object
AGE_GROUP               60000 non-null obje[...]<jupyter_text>Split Data (Independent Variables & Depedent Varaibles)<jupyter_code>dependentVar = 'DIABETES'

X = df_model.loc[:, df_model.columns != dependentVar]
y = df_model[dependentVar].values

print("Number of observations and dimensions in 'X':", X.shape)
print("Number of observations in 'y':", y.shape)
print(X, '\n')
print(y)<jupyter_output>EXERCISE  HEARTATTACK  CORONARYHEARTDISEASE  STROKE  ASTHMA  \
283235       0.0          0.0                   0.0     0.0     0.0   
265051       0.0          1.0                   0.0     0.0     1.0   
374463       0.0          0.0                   0.0     0.0     0.0   
156849       1.0          0.0                   0.0     0.0     0.0   
32830        1.0          0.0                   1.0     0.0     0.0   
...          ...          ...                   ...     ...     ...   
143113       1.0          0.0                   0.0     0.0     0.0   
399750       1.0          0.0                   0.0     0.0     0.0   
250087       1.0          0.0                   0.0     0.0     0.0   
414675       1.0          0.0                   0.0     0.0     0.0   
133961       1.0          1.0                   1.0     0.0     0.0   

        SKINCANCER  OTHERCANCER  CHRONICBRONCHITIS  ARTHRITIS  \
283235         0.0          1.0                0.0        0.0   
265051         0.[...]<jupyter_text>Split Data (Train & Test)<jupyter_code>from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.25, random_state = 0)

print("Number of observations and dimensions in training set:", X_train.shape)
print("Number of observations and dimensions in test set:", X_test.shape)
print("Number of observations in training set:", y_train.shape)
print("Number of observations in test set:", y_test.shape)<jupyter_output>Number of observations and dimensions in training set: (45000, 18)
Number of observations and dimensions in test set: (15000, 18)
Number of observations in training set: (45000,)
Number of observations in test set: (15000,)<jupyter_text>Transform Data<jupyter_code>from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.decomposition import PCA
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.model_selection import GridSearchCV
categorical_features = ['SLEEPTIME_GROUP', 'SEX_GROUP', 'BMI_GROUP', 'RACE_GROUP', 'AGE_GROUP']
numerical_features   = [col for col in X.columns if col not in categorical_features]
preprocessor         = ColumnTransformer(
                        transformers=[('onehot-encoder', OneHotEncoder(), categorical_features),
                                     ('scalar', StandardScaler(), numerical_features)])<jupyter_output><empty_output><jupyter_text>Pipeline Creation<jupyter_code>from sklearn.linear_model import LogisticRegression
pipeline_lr = Pipeline([('preprocessor', preprocessor),
                        ('pca', PCA(n_components=2)),
                        ('lr_classifier', LogisticRegression(class_weight='balanced', random_state=1234))])
from sklearn.neighbors import KNeighborsClassifier
pipeline_knn = Pipeline([('preprocessor', preprocessor),
                        ('pca', PCA(n_components=2)),
                        ('knn_classifier', KNeighborsClassifier())])
from sklearn.svm import SVC
pipeline_svc = Pipeline([('preprocessor', preprocessor),
                        ('pca', PCA(n_components=2)),
                        ('svc_classifier', SVC(class_weight='balanced', random_state=1234))])
from sklearn.naive_bayes import GaussianNB
pipeline_nb = Pipeline([('preprocessor', preprocessor),
                        ('pca', PCA(n_components=2)),
                        ('nb_classifier', GaussianNB())])
from sklearn.tree import DecisionTreeClassifier
pipeline_dt = Pipeline([('preprocessor', preprocessor),
                     ('pca', PCA(n_components=2)),
                     ('dt_classifier', DecisionTreeClassifier(class_weight='balanced', random_state=1234))])
from sklearn.ensemble import RandomForestClassifier
pipeline_rf = Pipeline([('preprocessor', preprocessor),
                     ('pca', PCA(n_components=2)),
                     ('rf_classifier', RandomForestClassifier(class_weight='balanced', random_state=1234))])
from xgboost import XGBClassifier
pipeline_xgb = Pipeline([('preprocessor', preprocessor),
                     ('pca', PCA(n_components=2)),
                     ('xgb_classifier', XGBClassifier(class_weight='balanced', random_state=1234))])
from sklearn.ensemble import AdaBoostClassifier
pipeline_adb = Pipeline([('preprocessor', preprocessor),
                     ('pca', PCA(n_components=2)),
                     ('adb_classifier', AdaBoostClassifier(random_state=1234))])<jupyter_output><empty_output><jupyter_text>Build Pipeline on Training Data<jupyter_code>pipelines = [pipeline_lr, pipeline_knn, pipeline_svc, pipeline_nb, pipeline_dt, pipeline_rf, pipeline_xgb, 
             pipeline_adb]
pipe_dict = {0: 'Logistic Regression', 1: 'K-Nearest Neighbors', 2: 'Kernal Support Vector Machine',  
             3: 'Naive Bayes', 4: 'Decision Tree', 5: 'Random Forest', 6: 'XGBoost', 7: 'AdaBoost'}
for pipe in pipelines:
	pipe.fit(X_train, y_train)<jupyter_output>/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.
  FutureWarning)
/opt/anaconda3/lib/python3.7/site-packages/sklearn/svm/base.py:193: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.
  "avoid this warning.", FutureWarning)
/opt/anaconda3/lib/python3.7/site-packages/sklearn/ensemble/forest.py:245: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.
  "10 in version 0.20 to 100 in 0.22.", FutureWarning)<jupyter_text>Predict Pipeline on Testing Data<jupyter_code>for i,model in enumerate(pipelines):
    print("{0:s} Test Accuracy: {1:.3f}".format(pipe_dict[i], model.score(X_test,y_test)))
best_accuracy   = 0
best_classifier = 0
best_pipeline   = ""

for i,model in enumerate(pipelines):
    if model.score(X_test,y_test) > best_accuracy:
        best_accuracy = model.score(X_test,y_test)
        best_pipeline = model
        best_classifier = i
print('Classifier with best accuracy: {}'.format(pipe_dict[best_classifier]))<jupyter_output>Classifier with best accuracy: XGBoost<jupyter_text>Hyperparameter Tuning<jupyter_code>param_grid_lr = {
    'lr_classifier__max_iter': [100, 200, 500, 1000],                              # Number of iterations
    'lr_classifier__solver': ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga'],   # Algorithm to use for optimization
    'lr_classifier__class_weight': ['balanced']                                    # Troubleshoot unbalanced data sampling
}

gdLR = GridSearchCV(estimator=pipeline_lr, param_grid=param_grid_lr, cv=3, n_jobs=-1, verbose=True)
param_grid_knn = {
    'knn_classifier__n_neighbors': [5, 10, 15],                                      # Number of K
    #'knn_classifier__algorithm': ['ball_tree', 'kd_tree', 'brute', 'auto'],         # Algorithm to compute nearest neighbors
    'knn_classifier__metric': ['minkowski', 'euclidean', 'manhattan', 'chebyshev']   # Algorithm to find the distance
}

gdKNN = GridSearchCV(estimator=pipeline_knn, param_grid=param_grid_knn, cv=3, n_jobs=-1, verbose=True)
param_grid_svc = {
    'svc_classifier__C': [0.1, 1, 10, 100],                           # Regularization parameter
    'svc_classifier__gamma': [1, 0.1, 0.01, 0.001],                   # Kernel coef for ‘rbf’, ‘poly’ and ‘sigmoid’
    #'svc_classifier__kernel': ['linear', 'rbf', 'poly', 'sigmoid'],  # Kernel type to be used in the algorithm
    #'svc_classifier__degree': [1, 2, 3, 4, 5, 6],                    # Degree for ‘poly’
    'svc_classifier__class_weight': ['balanced']                      # Troubleshoot unbalanced data sampling
}

gdSVC = GridSearchCV(estimator=pipeline_svc, param_grid=param_grid_svc, cv=2, n_jobs=-1, verbose=True)
param_grid_dt = {
    'dt_classifier__criterion': ['gini', 'entropy'],   # Measure quality of split
    'dt_classifier__splitter': ['best', 'random'],     # Strategy used to choose split at each node
    'dt_classifier__max_depth': [2, 4, 6, 8, 10],      # Maximum depth of the tree
    'dt_classifier__class_weight': ['balanced']        # Troubleshoot unbalanced data sampling
}

gdDT = GridSearchCV(estimator=pipeline_dt, param_grid=param_grid_dt, cv=3, n_jobs=-1, verbose=True)
param_grid_rf = {
    'rf_classifier__criterion': ['gini', 'entropy'],        # Measure quality of split
    'rf_classifier__max_depth': [2, 3, 5, 9],               # Maximum depth of the tree
    'rf_classifier__n_estimators': [100, 200, 300, 1000],   # Number of trees in the forest
    'rf_classifier__class_weight': ['balanced']             # Troubleshoot unbalanced data sampling
}

gdRF = GridSearchCV(estimator=pipeline_rf, param_grid=param_grid_rf, cv=3, n_jobs=-1, verbose=True)
param_grid_xgb = {
    'xgb_classifier__max_depth': [2, 3, 5, 9],                   # Maximum depth of a tree
    'xgb_classifier__n_estimators': [50, 100, 200, 300, 1000],   # Maximum number of estimators at which boosting is terminated
    'xgb_classifier__learning_rate': [1, 0.1, 0.01, 0.001],      # Step size shrinkage used in update to prevents overfitting range: [0,1]
    'xgb_classifier__class_weight': ['balanced']                 # Troubleshoot unbalanced data sampling
}

gdXGB = GridSearchCV(estimator=pipeline_xgb, param_grid=param_grid_xgb, cv=3, n_jobs=-1, verbose=True)
param_grid_adb = {
    'adb_classifier__n_estimators': [50, 100, 200, 300, 1000],   # Maximum number of estimators at which boosting is terminated
    'adb_classifier__learning_rate': [1, 0.1, 0.01, 0.001]       # Step size shrinkage used in update to prevents overfitting range: [0,1]   
}

gdADB = GridSearchCV(estimator=pipeline_adb, param_grid=param_grid_adb, cv=3, n_jobs=-1, verbose=True)<jupyter_output><empty_output><jupyter_text>Build Pipeline Grid Search on Training Data<jupyter_code>pipelines_grid = [gdLR, gdKNN, gdSVC, gdDT, gdRF, gdXGB, gdADB]
pipe_grid_dict = {0: 'Logistic Regression', 1: 'K-Nearest Neighbors', 2: 'Kernal Support Vector Machine', 
                  3: 'Decision Tree', 4: 'Random Forest', 5: 'XGBoost', 6: 'AdaBoost'}
for pipe in pipelines_grid:
	pipe.fit(X_train, y_train)<jupyter_output>Fitting 3 folds for each of 20 candidates, totalling 60 fits<jupyter_text>Predict Pipeline Grid Search on Testing Data<jupyter_code>for i,model in enumerate(pipelines_grid):
    print("{0:s} Test Accuracy: {1:.3f}".format(pipe_grid_dict[i], model.score(X_test,y_test)))
best_accuracy   = 0
best_classifier = 0
best_pipeline   = ""

for i,model in enumerate(pipelines):
    if model.score(X_test,y_test) > best_accuracy:
        best_accuracy = model.score(X_test,y_test)
        best_pipeline = model
        best_classifier = i
print('Classifier with best accuracy: {}'.format(pipe_dict[best_classifier]))<jupyter_output>Classifier with best accuracy: XGBoost
{'fraction_non_alphanumeric': 0.10769345906301452, 'fraction_numerical': 0.04610088310133214, 'mean_word_length': 3.269329073482428, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '310460', 'license': 'MIT', 'n_tokens_mistral': 4819, 'n_tokens_neox': 4331, 'n_words': 1037, 'path': 'notebooks/DATA MODELING ML PIPELINE [5].ipynb', 'repo_name': 'kopaljain95/CDC-Diabetes-DataModeling', 'size': '13362'}