<jupyter_start><jupyter_text>Let's see how Logistic Regression acts with 5 techniques:1. Standardization of Numerical Variables2. Encoding of Categorical Variables3. Data Imbalance4. Colinearity5. Missing Values<jupyter_code>import numpy as np
import pandas as pd
from sklearn.impute import SimpleImputer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score, average_precision_score
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OrdinalEncoder, StandardScaler, OneHotEncoder
from sklearn_pandas import DataFrameMapper
from snape.make_dataset import make_dataset<jupyter_output><empty_output><jupyter_text>Check out snape [here](https://github.com/mbernico/snape)<jupyter_code>def get_data(categorical_features=True,
             balanced=True, 
             correlated_features=False, 
             missing_values=False,
             dataset_size=12000):
    
    if categorical_features:
        label_list = []
        N_CATEGORICAL = 4
        for i in range(N_CATEGORICAL):
            num_classes = np.random.randint(2, 10)
            labels = list(np.arange(num_classes))
            labels = [f'str_{i}' for i in labels]
            label_list.append(labels)
            
    if correlated_features:
        N_REDUNDANT = 1
        N_REPEATED = 1
        N_INFORMATIVE = 8 - N_REDUNDANT - N_REPEATED
    
    conf = {
        "type": "classification",
        "n_classes": 2,
        "n_samples": dataset_size,
        "n_features": 8,
        "out_path": "./",
        "output": "my_dataset",
        "n_informative": N_INFORMATIVE if correlated_features else 8,
        "n_repeated": N_REPEATED if correlated_features else 0,
        "n_redundant": N_REDUNDANT if correlated_features else 0,
        "n_clusters": 2,
        "weights": [0.5, 0.5] if balanced else [0.9, 0.1],
        "pct_missing": 0.70 if missing_values else 0.00,
        "n_categorical": N_CATEGORICAL if categorical_features else 0,
        "random_seed":42,
        "label_list":label_list if categorical_features else []
    }

    make_dataset(config=conf)
    df = pd.read_csv('my_dataset_train.csv')
    
    label = 'y'
    categorical_features = [col for col in df.columns if (df[col].dtype==object) & (col != label)]
    numerical_features = [col for col in df.columns if (col not in categorical_features) & (col != label)]
    
    return df, label, categorical_features, numerical_features

def evaluation(pipeline, X, y):
    y_predict_proba = pipeline.predict_proba(X)[:, 1]
    return{
        'auc': roc_auc_score(y, y_predict_proba),
        'pr-auc': average_precision_score(y, y_predict_proba)
    }<jupyter_output><empty_output><jupyter_text>Logistic Regression  1.1 Standardiazation  Without Standardization<jupyter_code>df, label, categorical_features, numerical_features = get_data(categorical_features=False, dataset_size=120000)
df[numerical_features].describe()
train_df, test_df = train_test_split(df, test_size=0.1, shuffle=False)
X_train, y_train = train_df[categorical_features + numerical_features], train_df[label]
X_test, y_test = test_df[categorical_features + numerical_features], test_df[label]
clf = LogisticRegression()
pipeline = Pipeline([
    ('clf', clf)
], verbose=True)

pipeline.fit(X_train[numerical_features], y_train)
evaluation(pipeline, X_test[numerical_features], y_test)<jupyter_output>[Pipeline] ............... (step 1 of 1) Processing clf, total=   0.3s<jupyter_text>With Standardization<jupyter_code>num = [([n], [StandardScaler()]) for n in numerical_features]
mapper = DataFrameMapper(num, df_out=True)

clf = LogisticRegression()
pipeline = Pipeline([
    ('preprocess', mapper),
    ('clf', clf)
], verbose=True)

pipeline.fit(X_train[numerical_features], y_train)
evaluation(pipeline, X_test[numerical_features], y_test)<jupyter_output>[Pipeline] ........ (step 1 of 2) Processing preprocess, total=   0.1s
[Pipeline] ............... (step 2 of 2) Processing clf, total=   0.1s<jupyter_text>**Result**- No need to scale for logistic regression accuracy. But convergence is faster. [More info here](https://stats.stackexchange.com/questions/48360/is-standardization-needed-before-fitting-logistic-regression:~:text=3%20Answers&text=Standardization%20isn't%20required%20for,the%20technique%20used%20for%20optimization.&text=Otherwise%2C%20you%20can%20run%20your,standardization%20treatment%20on%20the%20features)  1.2 Encoding We need numeric encoding for logistic regression.<jupyter_code>df, label, categorical_features, numerical_features = get_data()<jupyter_output>--------------------------------------------------------------------------------
Warning: n_repeated not in configuration, defaulting to 0
Warning: n_clusters_per_class not in configuration, defaulting to 2
Warning: effective_rank not in configuration, defaulting to None
Warning: tail_strength not in configuration, defaulting to 0.5
Warning: noise not in configuration, defaulting to 0.0
Warning: shuffle not in configuration, defaulting to True
Creating Classification Dataset...
Creating Categorical Features...
Warning: insert_dollar not in configuration, defaulting to 'No'
Warning: insert_percent not in configuration, defaulting to 'No'
Warning: star_schema not in configuration, defaulting to 'No'
Writing Train/Test Datasets<jupyter_text>One Hot Encoding<jupyter_code>train_df, test_df = train_test_split(df, test_size=0.1, shuffle=False)
X_train, y_train = train_df[categorical_features + numerical_features], train_df[label]
X_test, y_test = test_df[categorical_features + numerical_features], test_df[label]

num = [([n], [SimpleImputer()]) for n in numerical_features]
cat = [([c], [OneHotEncoder()]) for c in categorical_features]
mapper = DataFrameMapper(cat + num, df_out=True)

clf = LogisticRegression(max_iter=1000)
pipeline = Pipeline([
    ('preprocess', mapper),
    ('clf', clf)
], verbose=True)

pipeline.fit(X_train, y_train)
evaluation(pipeline, X_test, y_test)
preprocessed_X_test = mapper.transform(X_test)
preprocessed_X_test.head().T<jupyter_output><empty_output><jupyter_text>Ordinal Encoding<jupyter_code>num = [([n], [SimpleImputer()]) for n in numerical_features]
cat = [([c], [OrdinalEncoder()]) for c in categorical_features]
mapper = DataFrameMapper(cat + num, df_out=True)

clf = LogisticRegression()
pipeline = Pipeline([
    ('preprocess', mapper),
    ('clf', clf)
], verbose=True)

pipeline.fit(X_train, y_train)
evaluation(pipeline, X_test, y_test)
preprocessed_X_test = mapper.transform(X_test)
preprocessed_X_test.head().T<jupyter_output><empty_output><jupyter_text>**Result**: - `OrdinalEncoding` works when relationships exist between categorical variables (size, weather). Otherwise, prefer `OneHotEncoding`- `OneHotEncoding` takes up space. Hence more training time  1.3 Data Imbalance What happens if the training data isn't balanced?  Unbalanced<jupyter_code>df, label, categorical_features, numerical_features = get_data(balanced=False)

train_df, test_df = train_test_split(df, test_size=0.1, shuffle=False)
X_train, y_train = train_df[categorical_features + numerical_features], train_df[label]
X_test, y_test = test_df[categorical_features + numerical_features], test_df[label]
df[label].value_counts()
num = [([n], [SimpleImputer()]) for n in numerical_features]
cat = [([c], [OrdinalEncoder()]) for c in categorical_features]
mapper = DataFrameMapper(cat + num, df_out=True)

clf = LogisticRegression()
pipeline = Pipeline([
    ('preprocess', mapper),
    ('clf', clf)
])

pipeline.fit(X_train, y_train)
evaluation(pipeline, X_test, y_test)
y_predict_proba = pipeline.predict_proba(X_test)[:, 1]
y_predict_proba.mean()<jupyter_output><empty_output><jupyter_text>Balanced<jupyter_code>df, label, categorical_features, numerical_features = get_data(balanced=True)

train_df, test_df = train_test_split(df, test_size=0.1, shuffle=False)
X_train, y_train = train_df[categorical_features + numerical_features], train_df[label]
X_test, y_test = test_df[categorical_features + numerical_features], test_df[label]

num = [([n], [SimpleImputer()]) for n in numerical_features]
cat = [([c], [OrdinalEncoder()]) for c in categorical_features]
mapper = DataFrameMapper(cat + num, df_out=True)

clf = LogisticRegression()
pipeline = Pipeline([
    ('preprocess', mapper),
    ('clf', clf)
])

pipeline.fit(X_train, y_train)
evaluation(pipeline, X_test, y_test)
y_predict_proba = pipeline.predict_proba(X_test)[:, 1]
y_predict_proba.mean()<jupyter_output><empty_output><jupyter_text>Dealing with unbalanced data by over weighting<jupyter_code>df, label, categorical_features, numerical_features = get_data(balanced=False)

train_df, test_df = train_test_split(df, test_size=0.1, shuffle=False)
X_train, y_train = train_df[categorical_features + numerical_features], train_df[label]
X_test, y_test = test_df[categorical_features + numerical_features], test_df[label]

num = [([n], [SimpleImputer()]) for n in numerical_features]
cat = [([c], [OrdinalEncoder()]) for c in categorical_features]
mapper = DataFrameMapper(cat + num, df_out=True)

clf = LogisticRegression(class_weight='balanced')
pipeline = Pipeline([
    ('preprocess', mapper),
    ('clf', clf)
])

pipeline.fit(X_train, y_train)
evaluation(pipeline, X_test, y_test)<jupyter_output>--------------------------------------------------------------------------------
Warning: n_clusters_per_class not in configuration, defaulting to 2
Warning: effective_rank not in configuration, defaulting to None
Warning: tail_strength not in configuration, defaulting to 0.5
Warning: noise not in configuration, defaulting to 0.0
Warning: shuffle not in configuration, defaulting to True
Creating Classification Dataset...
Creating Categorical Features...
Warning: insert_dollar not in configuration, defaulting to 'No'
Warning: insert_percent not in configuration, defaulting to 'No'
Warning: star_schema not in configuration, defaulting to 'No'
Writing Train/Test Datasets<jupyter_text>**Result**:- Having an unbalanced dataset doesn't harm accuracy, but harms precision-recall metrics of the positive class.- This is mostly due to lower predicted probability values.   1.4 Correlated Features<jupyter_code>df, label, categorical_features, numerical_features = get_data(categorical_features=False, correlated_features=True)
train_df, test_df = train_test_split(df, test_size=0.1, shuffle=False)
X_train, y_train = train_df[categorical_features + numerical_features], train_df[label]
X_test, y_test = test_df[categorical_features + numerical_features], test_df[label]

num = [([n], [SimpleImputer()]) for n in numerical_features]
mapper = DataFrameMapper(num, df_out=True)

clf = LogisticRegression()
pipeline = Pipeline([
    ('preprocess', mapper),
    ('clf', clf)
])

pipeline.fit(X_train, y_train)
evaluation(pipeline, X_test, y_test)
import statsmodels.api as sm
preprocessed_X_train = mapper.transform(X_train)
preprocessed_X_train = sm.add_constant(preprocessed_X_train)
results = sm.OLS(y_train, preprocessed_X_train).fit()
results.summary()
from statsmodels.stats.outliers_influence import variance_inflation_factor
for column in numerical_features:
    print(f"""{column}, {variance_inflation_factor(
                                preprocessed_X_train.values, 
                                list(preprocessed_X_train.columns).index(column))}""")
df.corr()<jupyter_output><empty_output><jupyter_text>Start with removing perfectly multicolinearity<jupyter_code>numerical_features = ['x0', 'x1', 'x2', 'x4', 'x5', 'x6', 'x7'] # remove x3

train_df, test_df = train_test_split(df, test_size=0.1, shuffle=False)
X_train, y_train = train_df[categorical_features + numerical_features], train_df[label]
X_test, y_test = test_df[categorical_features + numerical_features], test_df[label]

num = [([n], [SimpleImputer()]) for n in numerical_features]
mapper = DataFrameMapper(num, df_out=True)

clf = LogisticRegression()
pipeline = Pipeline([
    ('preprocess', mapper),
    ('clf', clf)
])

pipeline.fit(X_train, y_train)
evaluation(pipeline, X_test, y_test)
preprocessed_X_train = mapper.transform(X_train)
preprocessed_X_train = sm.add_constant(preprocessed_X_train)
results = sm.OLS(y_train, preprocessed_X_train).fit()
results.summary()
for column in numerical_features:
    print(f"""{column}, {variance_inflation_factor(
                                preprocessed_X_train.values, 
                                list(preprocessed_X_train.columns).index(column))}""")<jupyter_output>x0, inf
x1, inf
x2, inf
x4, inf
x5, inf
x6, inf
x7, inf<jupyter_text>Removing feature with perfect multicolinearity:- Improves interpretability of the coefficients (like `x0` here)- Logistic Regression doesn't lose performance.   Remove multicolinearity<jupyter_code>numerical_features = ['x0', 'x1', 'x2', 'x4', 'x5', 'x7'] # remove x3, x6

train_df, test_df = train_test_split(df, test_size=0.1, shuffle=False)
X_train, y_train = train_df[categorical_features + numerical_features], train_df[label]
X_test, y_test = test_df[categorical_features + numerical_features], test_df[label]

num = [([n], [SimpleImputer()]) for n in numerical_features]
mapper = DataFrameMapper(num, df_out=True)

clf = LogisticRegression()
pipeline = Pipeline([
    ('preprocess', mapper),
    ('clf', clf)
])

pipeline.fit(X_train, y_train)
evaluation(pipeline, X_test, y_test)
preprocessed_X_train = mapper.transform(X_train)
preprocessed_X_train = sm.add_constant(preprocessed_X_train)
results = sm.OLS(y_train, preprocessed_X_train).fit()
results.summary()
for column in numerical_features:
    print(f"""{column}, {variance_inflation_factor(
                                preprocessed_X_train.values, 
                                list(preprocessed_X_train.columns).index(column))}""")<jupyter_output>x0, 4.358204798860465
x1, 1.8526871839909662
x2, 5.622338237184614
x4, 4.123960180952725
x5, 2.6095687697415917
x7, 10.922197872534808<jupyter_text>Removing `x6`, we didn't lose explainability nor performance   Remove x7 with high VAR<jupyter_code>numerical_features = ['x0', 'x1', 'x2', 'x4', 'x5'] # remove x3, x6, x7

train_df, test_df = train_test_split(df, test_size=0.1, shuffle=False)
X_train, y_train = train_df[categorical_features + numerical_features], train_df[label]
X_test, y_test = test_df[categorical_features + numerical_features], test_df[label]

num = [([n], [SimpleImputer()]) for n in numerical_features]
mapper = DataFrameMapper(num, df_out=True)

clf = LogisticRegression()
pipeline = Pipeline([
    ('preprocess', mapper),
    ('clf', clf)
])

pipeline.fit(X_train, y_train)
evaluation(pipeline, X_test, y_test)
preprocessed_X_train = mapper.transform(X_train)
preprocessed_X_train = sm.add_constant(preprocessed_X_train)
results = sm.OLS(y_train, preprocessed_X_train).fit()
results.summary()
for column in numerical_features:
    print(f"""{column}, {variance_inflation_factor(
                                preprocessed_X_train.values, 
                                list(preprocessed_X_train.columns).index(column))}""")<jupyter_output>x0, 1.0434492528061576
x1, 1.2487373171729157
x2, 1.089610333638892
x4, 1.1174255753042328
x5, 1.2630916367080673<jupyter_text>Removing `x7`:- Helped explainability - Negatively impacted performance Remedy: Add polynomial terms, Try other models that capture more complex interactions.   1.5 Missing Values<jupyter_code>df, label, categorical_features, numerical_features = get_data(missing_values=True)
train_df, test_df = train_test_split(df, test_size=0.1, shuffle=False)
X_train, y_train = train_df[categorical_features + numerical_features], train_df[label]
X_test, y_test = test_df[categorical_features + numerical_features], test_df[label]

num = [([n], [SimpleImputer()]) for n in numerical_features]
mapper = DataFrameMapper(num, df_out=True)

clf = LogisticRegression()
pipeline = Pipeline([
    #('preprocess', mapper),
    ('clf', clf)
])

pipeline.fit(X_train[numerical_features], y_train)
evaluation(pipeline, X_test[numerical_features], y_test)
train_df, test_df = train_test_split(df, test_size=0.1, shuffle=False)
X_train, y_train = train_df[categorical_features + numerical_features], train_df[label]
X_test, y_test = test_df[categorical_features + numerical_features], test_df[label]

num = [([n], [SimpleImputer()]) for n in numerical_features] # Impute values
mapper = DataFrameMapper(num, df_out=True)
                                                                                                                                                           
clf = LogisticRegression()
pipeline = Pipeline([
    ('preprocess', mapper),
    ('clf', clf)
])

pipeline.fit(X_train[numerical_features], y_train)
evaluation(pipeline, X_test[numerical_features], y_test)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12077179609337779, 'fraction_numerical': 0.022451167222486898, 'mean_word_length': 5.117668488160292, 'pattern_counts': {'":': 15, '<': 54, '<?xml version=': 0, '>': 54, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '522261', 'license': 'MIT', 'n_tokens_mistral': 5801, 'n_tokens_neox': 5526, 'n_words': 1216, 'path': 'Preprocessing Cheat - Logistic Regression.ipynb', 'repo_name': 'ajhalthor/preprocessing-cheat-sheet', 'size': '16792'}