<jupyter_start><jupyter_text>Predicting Diabetes using Gradient boosting classifier<jupyter_code>from IPython.core.interactiveshell import InteractiveShell

InteractiveShell.ast_node_interactivity = "all"
import numpy as np
import pandas as pd
import seaborn as sns
df = pd.read_csv("dataset/diabetes.csv")
df.head(5)
df.describe()
from sklearn.model_selection import train_test_split

X = df.iloc[:, :-1]
y = df.iloc[:, -1]
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, stratify=y, random_state=42
)
X_train.shape, X_test.shape
def replace_with_nan(X: pd.DataFrame, *, columns):
    X.loc[:, columns] = X.loc[:, columns].replace(0, np.nan)
    return X
def remove_columns(X: pd.DataFrame, *, columns):
    return X.drop(columns=columns)
from typing import List, Mapping

from sklearn.base import BaseEstimator, TransformerMixin


class FeatureInteractions(BaseEstimator, TransformerMixin):
    def __init__(self, feature_interactions: Mapping[str, List[str]]):
        self.feature_interactions = feature_interactions
        self.added_columns_ = []

    def fit(self, X, y, **fitparams):
        return self

    def transform(self, X):
        for feat1 in self.feature_interactions:
            for feat2 in self.feature_interactions[feat1]:
                self.added_columns_.append(f"{feat1}_{feat2}")
                X.loc[:, f"{feat1}_{feat2}"] = X[feat1] * X[feat2]
        return X
from sklearn.impute import KNNImputer


class KNNDataFrameImputer(KNNImputer):
    def transform(self, X):
        return pd.DataFrame(super().transform(X), columns=X.columns)
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import FunctionTransformer, StandardScaler

missing_value_cols = ["Glucose", "BloodPressure", "SkinThickness", "Insulin", "BMI"]

feature_interactions = {
    "Glucose": [
        "Age",
        "Pregnancies",
        "Insulin",
        "BMI",
        "BloodPressure",
        "DiabetesPedigreeFunction",
    ],
    "Insulin": ["Age", "BMI", "DiabetesPedigreeFunction"],
}

features_to_drop = ["SkinThickness", "Pregnancies", "BloodPressure"]

preprocessing_pipeline = Pipeline(
    steps=[
        (
            "nan_marker",
            FunctionTransformer(replace_with_nan, kw_args={"columns": missing_value_cols}),
        ),
        ("nan_imputer", KNNDataFrameImputer(n_neighbors=10)),
        ("feature_interactions", FeatureInteractions(feature_interactions)),
        (
            "features_remover",
            FunctionTransformer(remove_columns, kw_args={"columns": features_to_drop}),
        ),
        ("scaler", StandardScaler()),
    ]
)
from sklearn.ensemble import GradientBoostingClassifier

model_pipeline = Pipeline(
    steps=[
        ("data_preprocessing", preprocessing_pipeline),
        ("model", GradientBoostingClassifier(random_state=42)),
    ]
)
import warnings

# grid search
from sklearn.model_selection import RandomizedSearchCV

warnings.simplefilter("ignore")

gb_clf_params = {
    "model__n_estimators": range(100, 1000, 50),
    "model__learning_rate": [0.01, 0.1, 0.5, 1.0],
    "model__subsample": [0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
    "model__max_features": [5, 7, 9, 11],
    "model__max_depth": range(2, 5),
    "model__min_samples_split": [2, 5, 10],
}

# random_cv = RandomizedSearchCV(model_pipeline, gb_clf_params, n_iter = 100, cv=10, random_state=42, n_jobs=-1, verbose=2, scoring="roc_auc")
# random_cv.fit(X_train, y_train)
# random_cv.best_params_
best_performing_params = {
    "model__subsample": 0.8,
    "model__n_estimators": 750,
    "model__min_samples_split": 5,
    "model__max_features": 5,
    "model__max_depth": 2,
    "model__learning_rate": 0.01,
}
model_pipeline.set_params(**best_performing_params)
from sklearn.model_selection import RepeatedStratifiedKFold, cross_val_score

cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=42)
scores = cross_val_score(model_pipeline, X_train, y_train, cv=cv, scoring="roc_auc", n_jobs=-1)
scores.mean(), scores.std()
# train tuned model
model_pipeline.fit(X_train, y_train)
from sklearn.metrics import accuracy_score, confusion_matrix, f1_score, roc_auc_score

y_test_pred = model_pipeline.predict(X_test)
print(f"Accuracy: {accuracy_score(y_test, y_test_pred)}")
print(f"ROC AUC score: {roc_auc_score(y_test, y_test_pred)}")
print(f"F1 score: {f1_score(y_test, y_test_pred)}")
print(f"Confusion Matrix: {confusion_matrix(y_test, y_test_pred)}")<jupyter_output>Accuracy: 0.7207792207792207
ROC AUC score: 0.6785185185185186
F1 score: 0.5742574257425743
Confusion Matrix: [[82 18]
 [25 29]]
{'fraction_non_alphanumeric': 0.12459087933667903, 'fraction_numerical': 0.03294785075278202, 'mean_word_length': 4.645320197044335, 'pattern_counts': {'":': 16, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '199291', 'license': 'MIT', 'n_tokens_mistral': 1718, 'n_tokens_neox': 1604, 'n_words': 310, 'path': 'pima_diabetic/diabetes_gradient_boosting.ipynb', 'repo_name': 'girip11/ml_problems', 'size': '4583'}