<jupyter_start><jupyter_text>Predict the onset of diabetes based on diagnostic measures* [Dataset](https://www.kaggle.com/uciml/pima-indians-diabetes-database)<jupyter_code>from IPython.core.interactiveshell import InteractiveShell

InteractiveShell.ast_node_interactivity = "all"
import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import pyplot as plt
df = pd.read_csv("dataset/diabetes.csv")
df.columns
df.head(5)
df.info()
df.describe()<jupyter_output><empty_output><jupyter_text>Data Exploration<jupyter_code>df.corr()["Outcome"]
sns.lmplot(x="Glucose", y="DiabetesPedigreeFunction", data=df, hue="Outcome")
sns.lmplot(x="Glucose", y="Age", data=df, hue="Outcome")<jupyter_output><empty_output><jupyter_text>Handle missing values<jupyter_code>cols_with_missing_values = ["Glucose", "BloodPressure", "SkinThickness", "Insulin", "BMI"]
df[cols_with_missing_values] = df.loc[:, cols_with_missing_values].replace(0, np.NAN)
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 768 entries, 0 to 767
Data columns (total 9 columns):
 #   Column                    Non-Null Count  Dtype  
---  ------                    --------------  -----  
 0   Pregnancies               768 non-null    int64  
 1   Glucose                   763 non-null    float64
 2   BloodPressure             733 non-null    float64
 3   SkinThickness             541 non-null    float64
 4   Insulin                   394 non-null    float64
 5   BMI                       757 non-null    float64
 6   DiabetesPedigreeFunction  768 non-null    float64
 7   Age                       768 non-null    int64  
 8   Outcome                   768 non-null    int64  
dtypes: float64(6), int64(3)
memory usage: 54.1 KB<jupyter_text>Split training and testing data<jupyter_code>from sklearn.model_selection import train_test_split

X = df.iloc[:, :-1].values
y = df.iloc[:, -1].values

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, stratify=y, random_state=42
)<jupyter_output><empty_output><jupyter_text>Find the best imputation strategy<jupyter_code># from sklearn.model_selection import cross_val_score, RepeatedStratifiedKFold

# def evaluate_best_param_for_imputer(imputer_cls, params):
#     params_scores = {}
#     for strategy in strategies:
#     cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=3, random_state=42)
#     scores = cross_val_score(estimator, X_train, y_train, scoring="accuracy", n_jobs=-1)
#     print(f"Parameter value {strategy} score:  {scores.mean()}")
#     strategy_scores[strategy] = scores.mean()
from sklearn.ensemble import RandomForestClassifier

# using simple imputation strategy
from sklearn.impute import SimpleImputer
from sklearn.model_selection import RepeatedStratifiedKFold, cross_val_score
from sklearn.pipeline import Pipeline

# lets find the best strategy to use for imputing
strategies = ["mean", "median", "most_frequent", "constant"]

strategy_scores = {}
for strategy in strategies:
    pipeline = Pipeline(
        [
            ("imputer", SimpleImputer(strategy=strategy, fill_value=0)),
            ("classifier", RandomForestClassifier(n_estimators=500, random_state=42)),
        ]
    )
    cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=3, random_state=42)
    scores = cross_val_score(pipeline, X_train, y_train, cv=cv, scoring="accuracy", n_jobs=-1)
    print(f"Strategy and score: {strategy}, {scores.mean()}")
    strategy_scores[strategy] = scores.mean()

max_score = max(strategy_scores.values())
for k, v in strategy_scores.items():
    if v == max_score:
        print(f"Parameter with best score: {k}")
        break
from sklearn.impute import KNNImputer

# Highest accuracy with n=9
for neighbour in range(3, 20):
    pipeline = Pipeline(
        [
            ("imputer", KNNImputer(n_neighbors=neighbour)),
            ("classifier", RandomForestClassifier(n_estimators=500, random_state=42)),
        ]
    )
    cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=3, random_state=42)
    scores = cross_val_score(pipeline, X_train, y_train, cv=cv, scoring="accuracy", n_jobs=-1)
    print(f"Neighbours and score: {neighbour}, {scores.mean()}")
# uniform performs better
for weight in ["uniform", "distance"]:
    pipeline = Pipeline(
        [
            ("imputer", KNNImputer(n_neighbors=9, weights=weight)),
            ("classifier", RandomForestClassifier(n_estimators=500, random_state=42)),
        ]
    )
    cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=3, random_state=42)
    scores = cross_val_score(pipeline, X_train, y_train, cv=cv, scoring="accuracy", n_jobs=-1)
    print(f"Weight and score: {weight}, {scores.mean()}")<jupyter_output>Weight and score: uniform, 0.766049135901195
Weight and score: distance, 0.765507130481141<jupyter_text>KNNImputer with 10 neighbours and uniform strategy seems to be performing the best.  Evaluate models<jupyter_code>from lightgbm import LGBMClassifier
from sklearn.ensemble import (
    AdaBoostClassifier,
    ExtraTreesClassifier,
    GradientBoostingClassifier,
    RandomForestClassifier,
)
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from xgboost import XGBClassifier


# NOTE: I didnot do feature scaling for SVC, KNN
def get_models():
    return [
        ("Logistic", LogisticRegression(random_state=42)),
        ("SVC", SVC(random_state=42)),
        ("KNN", KNeighborsClassifier(n_neighbors=10)),
        ("RandomForest", RandomForestClassifier(n_estimators=500, random_state=42)),
        ("ExtraTrees", ExtraTreesClassifier(n_estimators=500, random_state=42)),
        (
            "AdaBoost",
            AdaBoostClassifier(
                base_estimator=DecisionTreeClassifier(max_depth=2),
                n_estimators=500,
                random_state=42,
            ),
        ),
        ("GradientBoost", GradientBoostingClassifier(n_estimators=500, random_state=42)),
        ("LightGradientBoost", LGBMClassifier(random_state=42)),
        ("XGBoost", XGBClassifier(n_estimators=100, use_label_encoder=False, random_state=42)),
    ]
for model in get_models():
    pipeline = Pipeline([("imputer", KNNImputer(n_neighbors=9, weights="uniform")), model])
    cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=3, random_state=42)
    scores = cross_val_score(pipeline, X_train, y_train, scoring="accuracy", cv=cv, n_jobs=-1)
    print(f"Model and score: {model[0]}, {scores.mean()}")<jupyter_output>Model and score: Logistic, 0.7812475009996002
Model and score: SVC, 0.7654760318094984
Model and score: KNN, 0.7448531698431737
Model and score: RandomForest, 0.766049135901195
Model and score: ExtraTrees, 0.7638766715536008
Model and score: AdaBoost, 0.7182327069172332
Model and score: GradientBoost, 0.7394686569816518
Model and score: LightGradientBoost, 0.7448487271757962
Model and score: XGBoost, 0.741045803900662<jupyter_text>Feature scaling<jupyter_code>from sklearn.preprocessing import MinMaxScaler, StandardScaler

scalers = [("standardizer", StandardScaler()), ("normalizer", MinMaxScaler())]

# we can see the increase in accuracy after feature scaling
for scaler in scalers:
    estimator_pipeline = Pipeline(
        [
            ("imputer", KNNImputer(n_neighbors=9, weights="uniform")),
            scaler,
            ("KNN", KNeighborsClassifier(n_neighbors=10)),
        ]
    )
    cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=3, random_state=42)
    scores = cross_val_score(
        estimator_pipeline, X_train, y_train, scoring="accuracy", cv=cv, n_jobs=-1
    )
    print(f"Scaler and score: {scaler[0]}, {scores.mean()}")
# NOTE: normalizer is performing well with SVC
for scaler in scalers:
    estimator_pipeline = Pipeline(
        [
            ("imputer", KNNImputer(n_neighbors=9, weights="uniform")),
            scaler,
            ("SVC", SVC(random_state=42)),
        ]
    )
    cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=3, random_state=42)
    scores = cross_val_score(
        estimator_pipeline, X_train, y_train, scoring="accuracy", cv=cv, n_jobs=-1
    )
    print(f"Scaler and score: {scaler[0]}, {scores.mean()}")
# Does logistic regression accuracy increases after feature scaling
# using saga solver with feature scaling the accuracy is increased slightly(0.5%)
for scaler in scalers:
    estimator_pipeline = Pipeline(
        [
            ("imputer", KNNImputer(n_neighbors=9, weights="uniform")),
            scaler,
            (
                "Logistic",
                LogisticRegression(
                    penalty="elasticnet", solver="saga", l1_ratio=0.4, random_state=42
                ),
            ),
        ]
    )
    cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=3, random_state=42)
    scores = cross_val_score(
        estimator_pipeline, X_train, y_train, scoring="accuracy", cv=cv, n_jobs=-1
    )
    print(f"Scaler and score: {scaler[0]}, {scores.mean()}")<jupyter_output>Scaler and score: standardizer, 0.7893775823004131
Scaler and score: normalizer, 0.7855791016926563<jupyter_text>Hyperparameter tuning with KNN<jupyter_code>data_prep_pipeline = Pipeline(
    [("imputer", KNNImputer(n_neighbors=9, weights="uniform")), ("standardizer", StandardScaler())]
)

X_train_trans = data_prep_pipeline.fit_transform(X_train, y_train)
from sklearn.model_selection import GridSearchCV

params = {"n_neighbors": range(5, 50, 5), "weights": ["uniform", "distance"]}

clf = KNeighborsClassifier()
cv = RepeatedStratifiedKFold(n_splits=5, n_repeats=3, random_state=42)
grid_search = GridSearchCV(
    clf, params, cv=cv, n_jobs=-1, scoring="accuracy", return_train_score=True
)
grid_search.fit(X_train_trans, y_train)
# With hyperparameter tuning we are able to get to 77.5%
grid_search.best_params_
grid_search.best_score_
# Get the train and test score of the best params
best_params_index = grid_search.cv_results_["params"].index(grid_search.best_params_)
grid_search.cv_results_["mean_train_score"][best_params_index], grid_search.cv_results_[
    "std_train_score"
][best_params_index]
grid_search.cv_results_["mean_test_score"][best_params_index], grid_search.cv_results_[
    "std_test_score"
][best_params_index]
# Look at the results of all the models with params
for param, train_mean, train_std, test_mean, test_std in zip(
    grid_search.cv_results_["params"],
    grid_search.cv_results_["mean_train_score"],
    grid_search.cv_results_["std_train_score"],
    grid_search.cv_results_["mean_test_score"],
    grid_search.cv_results_["std_test_score"],
):
    print(f"{param}, {train_mean}, {train_std}, {test_mean}, {test_std}")<jupyter_output>{'n_neighbors': 5, 'weights': 'uniform'}, 0.8317031775205737, 0.01269102882134063, 0.7502465680394508, 0.043624284983511535
{'n_neighbors': 5, 'weights': 'distance'}, 1.0, 0.0, 0.7502465680394508, 0.04382584206657892
{'n_neighbors': 10, 'weights': 'uniform'}, 0.7968246871878086, 0.01731856916819127, 0.7502776667110934, 0.041994408266736824
{'n_neighbors': 10, 'weights': 'distance'}, 1.0, 0.0, 0.7578657425918521, 0.045410897096262656
{'n_neighbors': 15, 'weights': 'uniform'}, 0.8080917214467461, 0.010654681055283898, 0.7730996490292772, 0.03499170331308661
{'n_neighbors': 15, 'weights': 'distance'}, 1.0, 0.0, 0.7752721133768714, 0.03278201299945618
{'n_neighbors': 20, 'weights': 'uniform'}, 0.7973636569359583, 0.010137936828824981, 0.7627482340397175, 0.034222633006098076
{'n_neighbors': 20, 'weights': 'distance'}, 1.0, 0.0, 0.7660180372295528, 0.0363507167725729
{'n_neighbors': 25, 'weights': 'uniform'}, 0.7897653701587932, 0.011130061720658637, 0.7589586387667154, 0.03771018767206458
[...]<jupyter_text>Testing the model<jupyter_code>model_pipeline = Pipeline(
    [
        ("imputer", KNNImputer(n_neighbors=9, weights="uniform")),
        scaler,
        (
            "Logistic",
            LogisticRegression(penalty="elasticnet", solver="saga", l1_ratio=0.4, random_state=42),
        ),
    ]
)

model_pipeline.fit(X_train, y_train)
y_hat = model_pipeline.predict(X_test)
from sklearn.metrics import (
    accuracy_score,
    confusion_matrix,
    f1_score,
    precision_score,
    recall_score,
    roc_auc_score,
)

confusion_matrix(y_test, y_hat)
accuracy_score(y_test, y_hat)
precision_score(y_test, y_hat)
recall_score(y_test, y_hat)
f1_score(y_test, y_hat)
roc_auc_score(y_test, y_hat)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11744912674250921, 'fraction_numerical': 0.0769908668482615, 'mean_word_length': 4.387570133793699, 'pattern_counts': {'":': 2, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '386959', 'license': 'MIT', 'n_tokens_mistral': 5036, 'n_tokens_neox': 4412, 'n_words': 855, 'path': 'pima_diabetic/predict_diabetes.ipynb', 'repo_name': 'girip11/ml_problems', 'size': '12482'}