<jupyter_start><jupyter_text>Split training and test set<jupyter_code>from sklearn.model_selection import train_test_split

X = df.iloc[:, :-1].values
y = df.iloc[:, -1].values
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.1, stratify=y, random_state=42
)<jupyter_output><empty_output><jupyter_text>Impute missing valuesImpute after splitting training and test set to avoid data leakage.<jupyter_code>from sklearn.impute import KNNImputer

imputer = KNNImputer(n_neighbors=9)
X_train = imputer.fit_transform(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Model training<jupyter_code>def get_dataframe(X, y, columns):
    return pd.DataFrame(np.c_[X, y], columns=columns)
from sklearn.linear_model import LogisticRegression

# from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from sklearn.metrics import accuracy_score, confusion_matrix, roc_auc_score

# Perform cross validation
from sklearn.model_selection import cross_val_predict
from sklearn.preprocessing import StandardScaler


def perform_cross_validation(X, y):
    std_scaler = StandardScaler()
    X_train_scaled = std_scaler.fit_transform(X, y)
    clf = LogisticRegression(random_state=42)
    return cross_val_predict(clf, X_train_scaled, y, cv=5, n_jobs=-1)


def print_metrics(y_test, y_hat):
    print(f"Accuracy: {accuracy_score(y_test, y_hat)}")
    print(f"ROC AUC score: {roc_auc_score(y_test, y_hat)}")
    print(f"Confusion Matrix: {confusion_matrix(y_test, y_hat)}")<jupyter_output><empty_output><jupyter_text>No data preprocessing<jupyter_code>df_train = get_dataframe(X_train, y_train, df.columns)
df_train.info()
df_train.corr()["Outcome"]
y_pred = perform_cross_validation(
    df_train.drop(columns=["Outcome"]).values, df_train.loc[:, "Outcome"].values
)

print_metrics(y_train, y_pred)<jupyter_output>Accuracy: 0.76410998552822
ROC AUC score: 0.7148224988473951
Confusion Matrix: [[395  55]
 [108 133]]<jupyter_text>Perform log transformation<jupyter_code>df_train = get_dataframe(X_train, y_train, df.columns)
df_train.info()
df_train.corr()["Outcome"]
df_train.hist(figsize=(20, 10))
# lets perform log transformation on age, pregnancies, diabeticpredigreefunction, Insulin
# Glucose, bloodpressure and BMI already look like normally distributed
cols_to_normal_dist = ["Pregnancies", "Insulin", "DiabetesPedigreeFunction", "Age"]
df_train.loc[:, cols_to_normal_dist] = pd.DataFrame(
    np.log(1 + df_train.loc[:, cols_to_normal_dist].values), columns=cols_to_normal_dist
)

df_train.loc[:, cols_to_normal_dist].hist(figsize=(20, 10))
df_train.corr()["Outcome"]
y_pred = perform_cross_validation(
    df_train.drop(columns=["Outcome"]).values, df_train.loc[:, "Outcome"].values
)

print_metrics(y_train, y_pred)<jupyter_output>Accuracy: 0.768451519536903
ROC AUC score: 0.7200829875518672
Confusion Matrix: [[396  54]
 [106 135]]<jupyter_text>Accuracy increased from 0.764 to 0.768  BoxCox transformation<jupyter_code>df_train = get_dataframe(X_train, y_train, df.columns)
df_train.info()
from scipy import stats as spstats

l, opt_lambda = spstats.boxcox(1 + df_train["Pregnancies"])
opt_lambda
pregnancies_transformed = spstats.boxcox(1 + df_train["Pregnancies"], lmbda=opt_lambda)
df_train["Pregnancies"] = pregnancies_transformed
sns.histplot(data=pregnancies_transformed)
df_train.corr()["Outcome"]
y_pred = perform_cross_validation(
    df_train.drop(columns=["Outcome"]).values, df_train.loc[:, "Outcome"].values
)

print_metrics(y_train, y_pred)<jupyter_output>Accuracy: 0.7670043415340086
ROC AUC score: 0.7189718764407561
Confusion Matrix: [[395  55]
 [106 135]]<jupyter_text>Binning Glucose and BMI<jupyter_code>df_train = get_dataframe(X_train, y_train, df.columns)
df_train.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 691 entries, 0 to 690
Data columns (total 9 columns):
 #   Column                    Non-Null Count  Dtype  
---  ------                    --------------  -----  
 0   Pregnancies               691 non-null    float64
 1   Glucose                   691 non-null    float64
 2   BloodPressure             691 non-null    float64
 3   SkinThickness             691 non-null    float64
 4   Insulin                   691 non-null    float64
 5   BMI                       691 non-null    float64
 6   DiabetesPedigreeFunction  691 non-null    float64
 7   Age                       691 non-null    float64
 8   Outcome                   691 non-null    float64
dtypes: float64(9)
memory usage: 48.7 KB<jupyter_text>> **Oral glucose tolerance test**-For this test, you fast overnight, and the fasting blood sugar level is measured. Then you drink a sugary liquid, and blood sugar levels are tested periodically for the next two hours.A blood sugar level less than 140 mg/dL (7.8 mmol/L) is normal. A reading of more than 200 mg/dL (11.1 mmol/L) after two hours indicates diabetes. A reading between 140 and 199 mg/dL (7.8 mmol/L and 11.0 mmol/L) indicates prediabetes.<jupyter_code>glucose = df_train["Glucose"]
glucose.min(), glucose.max(), glucose.mean()
# introduce three categories normal, prediabetes and diabetes
glucose_bins = [0, 100, 140, 200]
glucose_bin_labels = ["low", "normal", "prediabetic"]
df_train["GlucoseLevel"] = pd.cut(
    df_train.loc[:, "Glucose"], bins=glucose_bins, right=False, labels=glucose_bin_labels
)
df_train.dtypes
df_train.loc[:, ["Glucose", "GlucoseLevel", "Outcome"]].groupby(["GlucoseLevel", "Outcome"]).count()
df_train.loc[:, ["Glucose", "Outcome"]].groupby("Outcome", as_index=False).mean()
bmi_bins = [0, 19, 25, 30, df_train["BMI"].max() + 1]
bmi_bin_categories = ["underweight", "normal", "overweight", "obese"]
df_train["BMILevel"] = pd.cut(
    df_train.loc[:, "BMI"], bins=bmi_bins, right=False, labels=bmi_bin_categories
)
df_train.dtypes
df_train.loc[:, ["BMI", "BMILevel", "Outcome"]].groupby(["BMILevel", "Outcome"]).count()
df_train.corr()["Outcome"]
# perform cross validation using logistic regression classifier
from sklearn.preprocessing import OrdinalEncoder

ordinal_encoder = OrdinalEncoder()
df_train.loc[:, ["GlucoseLevelEnc", "BMILevelEnc"]] = ordinal_encoder.fit_transform(
    df_train.loc[:, ["GlucoseLevel", "BMILevel"]], df_train["Outcome"]
)
df_train.head()
df_train.loc[:, ["GlucoseLevelEnc", "BMILevelEnc", "Outcome"]].corr()["Outcome"]
# predicting with only the ordinal encoded features
y_pred = perform_cross_validation(
    df_train.drop(columns=["Glucose", "GlucoseLevel", "BMI", "BMILevel", "Outcome"]).values,
    df_train.loc[:, "Outcome"].values,
)

print_metrics(y_train, y_pred)<jupyter_output>Accuracy: 0.7496382054992764
ROC AUC score: 0.703711387736284
Confusion Matrix: [[385  65]
 [108 133]]<jupyter_text>Accuracy reduced from 0.764 to 0.749  Using feature interactions<jupyter_code>df_train = get_dataframe(X_train, y_train, df.columns)
df_train.info()
from sklearn.preprocessing import PolynomialFeatures

interaction_feat_enc = PolynomialFeatures(degree=2, interaction_only=True, include_bias=False)

features = ["Pregnancies", "Glucose", "BloodPressure", "Insulin", "BMI", "Age"]
interac_feat_names = [
    "Pregnancies_Glucose",
    "Pregnancies_BloodPressure",
    "Pregnancies_Insulin",
    "Pregnancies_BMI",
    "Pregnancies_Age",
    "Glucose_BloodPressure",
    "Glucose_Insulin",
    "Glucose_BMI",
    "Glucose_Age",
    "BloodPressure_Insulin",
    "BloodPressure_BMI",
    "BloodPressure_Age",
    "Insulin_BMI",
    "Insulin_Age",
    "BMI_Age",
]

interac_feat_values = interaction_feat_enc.fit_transform(df_train.loc[:, features])

interac_feat_values.shape
interaction_feat_enc.powers_
interac_feat_df = pd.DataFrame(interac_feat_values[:, len(features) :], columns=interac_feat_names)
interac_feat_df.head()
df_train = pd.concat([df_train, interac_feat_df], axis=1)
df_train.head()
df_train.corr()["Outcome"]
# predicting with all features
y_pred = perform_cross_validation(
    df_train.drop(columns=["Outcome"]).values,
    df_train.loc[:, "Outcome"].values,
)

print_metrics(y_train, y_pred)
# predicting with only the interactive features
y_pred = perform_cross_validation(
    df_train.drop(columns=features + ["Outcome"]).values,
    df_train.loc[:, "Outcome"].values,
)

print_metrics(y_train, y_pred)<jupyter_output>Accuracy: 0.768451519536903
ROC AUC score: 0.7181558321807284
Confusion Matrix: [[398  52]
 [108 133]]
{'fraction_non_alphanumeric': 0.12450569203115638, 'fraction_numerical': 0.04409826243259437, 'mean_word_length': 5.5458823529411765, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '261733', 'license': 'MIT', 'n_tokens_mistral': 3285, 'n_tokens_neox': 2920, 'n_words': 605, 'path': 'pima_diabetic/feature_engineering.ipynb', 'repo_name': 'girip11/ml_problems', 'size': '8345'}