<jupyter_start><jupyter_text>POC work on statsmodels logistic regressionPOC on statsmodels Logistic Regression including1. Training the model2. Pulling predictions3. Basic evaluation metrics4. Wrapping Logistic Regression statsmodel into an sklearn Pipeline<jupyter_code>import statsmodels.api as sm
import pandas as pd
import numpy as np
import seaborn as sns
import scipy

from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.base import BaseEstimator, RegressorMixin
from sklearn.pipeline import Pipeline

from sklearn.metrics import (
    roc_auc_score, 
    recall_score, 
    accuracy_score, 
    precision_score,
    confusion_matrix,
    roc_curve
)

# To help when Jupyter Notebook autocomplete is really slow
%config Completer.use_jedi = False
obs = 100000
population_df = (
    # Make columns of noise
    pd.DataFrame(np.random.rand(obs, 3), columns=[f"f{i}" for i in range(3)])
    .assign(target_class=np.random.rand(obs, 1) > 0.5)
    # Make columns that can help predict the target
    .assign(f3=lambda p_df: p_df.target_class.apply(lambda tc: tc + np.random.standard_normal()))
)
population_df.head()
X_train, X_test, y_train, y_test = train_test_split(
    population_df.drop(columns=['target_class']), 
    population_df.target_class, 
    test_size=0.2, random_state=42)
# By default there is no intercept but can be added into the X_train as a column of 1's
logit_mod = sm.Logit(y_train, X_train)
logit_mod_fit = logit_mod.fit()
logit_mod_fit.summary()
# Be default these are the probability of target_class == True
# For not just use the standard cutoff of 0.5
predictions = logit_mod_fit.predict(X_test) > 0.5
print(f"recall: {recall_score(y_test, predictions)}")
print(f"accuracy: {accuracy_score(y_test, predictions)}")
print(f"precision: {precision_score(y_test, predictions)}")
print(f"ROC: {roc_auc_score(y_test, predictions)}")
display(pd.DataFrame(confusion_matrix(y_test, predictions), index=[False, True], columns=[False, True]))
fpr, tpr, _ = roc_curve(y_test, predictions)

ax = sns.lineplot(x=fpr, y=tpr)
ax.set_title('ROC Curve for Logistic Regression model')<jupyter_output><empty_output><jupyter_text>Wrap the Logit predictor in an sklearn Pipeline<jupyter_code>class SMWrapper(BaseEstimator, RegressorMixin):
    """ A universal sklearn-style wrapper for statsmodels regressors """
    def __init__(self, model_class, fit_intercept=True):
        self.model_class = model_class
        self.fit_intercept = fit_intercept
    def fit(self, X, y):
        if isinstance(X, scipy.sparse.csr.csr_matrix):
            X = X.todense()  # statsmodels has trouble with csr_matrix
        if self.fit_intercept:
            X = sm.add_constant(X)
        self.model_ = self.model_class(y, X)
        self.results_ = self.model_.fit()
    def predict(self, X):
        if self.fit_intercept:
            X = sm.add_constant(X)
        return self.results_.predict(X)
pipeline_logit_model = Pipeline(
    steps=[
        ('classifier', SMWrapper(sm.Logit, fit_intercept=False))
    ])

pipeline_logit_model.fit(X_train, y_train)
pl_predictions = pipeline_logit_model.predict(X_test) > 0.5
assert all(pl_predictions == predictions), 'Base predictions without the pipeline should match these predictions'<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09597335755373902, 'fraction_numerical': 0.008174386920980926, 'mean_word_length': 4.766143106457243, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '263663', 'license': 'MIT', 'n_tokens_mistral': 1121, 'n_tokens_neox': 1090, 'n_words': 278, 'path': 'statsmodels_poc/statsmodels_logistic_reg.ipynb', 'repo_name': 'pettinato/poc_work', 'size': '3303'}