<jupyter_start><jupyter_text>Optimize Logistic RegressionThis Notebook contains the optimized Logistic Regression with the 51 selected features. A logistic regression was choosen, because it is the model _Die Zeit_ uses and we tried to beat. Therefore we wanted to see if we could allready beat the model with our engineered and selecte features, as well as tuning. Furthermore, when working with our data that accumulates to a binary separation, we want to classify our observations as the customer “will churn” or “won’t churn”. A logistic Regression determines the probability of belonging in one or the other group. The model predicts relationships between the target feature churn and the remaining features to apply probabilistic calculations for determining which class the customer should belong to. Threfore we see a Logistic regression a good fit to predict customer churn.<jupyter_code># data analysis and wrangling
import pandas as pd
import numpy as np
import math

# own modules
import eda_methods as eda

# visualization
import seaborn as sns
sns.set(style="white")  
import matplotlib
import matplotlib.pyplot as plt
%matplotlib inline
plt.style.use('ggplot')
from pandas.plotting import scatter_matrix

# warnings handler
import warnings
warnings.filterwarnings("ignore")

# Machine Learning Libraries
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import fbeta_score, accuracy_score, f1_score, recall_score, precision_score
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from xgboost import XGBClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import make_scorer 
from sklearn.model_selection import KFold
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score

#Pipeline
from sklearn.model_selection import train_test_split, cross_val_predict
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer
from imblearn.under_sampling import RandomUnderSampler
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import RandomizedSearchCV

from sklearn.metrics import fbeta_score, make_scorer

random_state=1
from sklearn.metrics import fbeta_score, accuracy_score

def train_predict(modelname, y_train, y_test, predictions_train, predictions_test):
    '''
    inputs:
       - learner: the learning algorithm to be trained and predicted on
       -
       - y_train: income training set
       -
       - y_test: income testing set
    '''
    
    results = {}
    # model name
    results['model'] = modelname
    # accuracy
    results['acc_train'] = accuracy_score(y_train,predictions_train)
    results['acc_test'] = accuracy_score(y_test,predictions_test)
    # F-score
    #results[‘f_train’] = fbeta_score(y_train,predictions_train,0.5)
    #results[‘f_test’] = fbeta_score(y_test,predictions_test,0.5)
    # F1-score
    results['f1_train'] = f1_score(y_train,predictions_train)
    results['f1_test'] = f1_score(y_test,predictions_test)
    # Recall
    results['recall_train'] = recall_score(y_train,predictions_train)
    results['recall_test'] = recall_score(y_test,predictions_test)
    # Precision
    results['precision_train'] = precision_score(y_train,predictions_train)
    results['precision_test'] = precision_score(y_test,predictions_test)
    #fbets
    results['fbeta_train'] = fbeta_score(y_train, predictions_train, beta = .5, average = 'weighted').round(2)
    results['fbeta_test'] =fbeta_score(y_test, predictions_test, beta = .5, average = 'weighted').round(2)
    # Return the results
    return results<jupyter_output><empty_output><jupyter_text>Data<jupyter_code># new feature dataframe
df = pd.read_csv('data/df_clean_engineered_all.csv')
y = df['churn']
# Drop some obvious unnecessary features
df = df.drop(['churn','plz_3','abo_registrierung_min','nl_registrierung_min','ort'], axis = 1)
# Get dummies for the categorial features
df = pd.get_dummies(df, columns = ['kanal', 'objekt_name', 'aboform_name', 'zahlung_rhythmus_name',
                                   'zahlung_weg_name', 'plz_1', 'plz_2', 'land_iso_code', 
                                   'anrede','titel'], drop_first = True)
df.shape
# 51 selected features 
X = df[['zahlung_weg_name_Rechnung',
 'zahlung_rhythmus_name_halbjährlich',
 'rechnungsmonat',
 'received_anzahl_6m',
 'openedanzahl_6m',
 'objekt_name_ZEIT Digital',
 'nl_zeitbrief',
 'nl_aktivitaet',
 'liefer_beginn_evt',
 'cnt_umwandlungsstatus2_dkey',
 'clickrate_3m',
 'anrede_Frau',
 'aboform_name_Geschenkabo',
 'unsubscribed_anzahl_1m',
 'studentenabo',
 'received_anzahl_bestandskunden_6m',
 'openrate_produktnews_3m',
 'opened_anzahl_bestandskunden_6m',
 'objekt_name_DIE ZEIT - CHRIST & WELT',
 'nl_zeitshop',
 'nl_opt_in_sum',
 'nl_opened_1m',
 'kanal_andere',
 'kanal_B2B',
 'clicked_anzahl_6m',
 'che_reg',
 'MONTH_DELTA_nl_min',
 'zon_zp_red',
 'zahlung_rhythmus_name_vierteljährlich',
 'unsubscribed_anzahl_hamburg_1m',
 'unsubscribed_anzahl_6m',
 'sum_zon',
 'sum_reg',
 'shop_kauf',
 'plz_2_10',
 'plz_1_7',
 'plz_1_1',
 'openrate_zeitbrief_3m',
 'openrate_produktnews_1m',
 'openrate_3m',
 'openrate_1m',
 'nl_unsubscribed_6m',
 'nl_fdz_organisch',
 'metropole',
 'cnt_abo_magazin',
 'cnt_abo_diezeit_digital',
 'cnt_abo',
 'clicked_anzahl_bestandskunden_3m',
 'aboform_name_Probeabo',
 'aboform_name_Negative Option',
 'MONTH_DELTA_abo_min']]
        
X.shape<jupyter_output><empty_output><jupyter_text>RandomSearch CV<jupyter_code>def pipeline_optimization(X,y,balance=None):
    
    # devide features
    categoric_features = list(X.columns[X.dtypes==object])

    numeric_features = list(X.columns[X.dtypes != object])

    # split train and test set
    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=random_state,stratify=y)
    
    if balance == 'over':
        # define oversampling strategy
        print('Oversampling')
        oversample = RandomOverSampler(sampling_strategy='minority')
        X_train, y_train = oversample.fit_resample(X_train, y_train)

    if balance == 'under':
        print('Undersampling')
        # define undersample strategy
        undersample = RandomUnderSampler(sampling_strategy='majority')
        X_train, y_train = undersample.fit_resample(X_train, y_train)
        
    # Hyperparameter grid
    param_logreg = {'logreg__penalty':['l1','l2'],
                'logreg__C': (range(0, 100, 10)),
                'logreg__solver': ['liblinear', 'saga']
               }

    
    models={
        'logreg': LogisticRegression(random_state=random_state,n_jobs=-1) 
        }  
    
    # create preprocessors
    numeric_transformer = Pipeline(steps=[
        ('imputer_num', SimpleImputer(strategy='median')),
        ('scaler', MinMaxScaler())
        #('scaler', StandardScaler())
    ])

    categorical_transformer = Pipeline(steps=[
            ('imputer_cat', SimpleImputer(strategy='constant', fill_value='missing')),
            ('onehot', OneHotEncoder(handle_unknown='ignore'))
    ])

    preprocessor = ColumnTransformer(
        transformers=[
            ('num', numeric_transformer, numeric_features),
            ('cat', categorical_transformer, categoric_features)
        ])

    model_results = pd.DataFrame(columns=['model','acc_train','acc_test','f1_train','f1_test','recall_train','recall_test','precision_train','precision_test'])
    
    
    # process pipeline for every model
    for model in models.items():
        
        print(model[0])
        pipe = Pipeline(steps=[('preprocessor', preprocessor), 
                               (model[0], model[1])
                               ])

        grid_logreg = RandomizedSearchCV(pipe, param_logreg, cv=3, scoring='recall', 
                           verbose=5, n_jobs=-1, n_iter = 100)#, refit='f1')
        # fit model
        grid_logreg.fit(X_train, y_train)
        
        # Show best parameters
        print('Best score:\n{:.2f}'.format(grid_logreg.best_score_))
        print("Best parameters:\n{}".format(grid_logreg.best_params_))
        
        # Save best model as best_model
        best_model = grid_logreg.best_estimator_
        
        #predict results
        y_train_pred = grid_logreg.predict(X_train)
       

        y_test_pred = grid_logreg.predict(X_test)
        
        results = train_predict(model[0],y_train, y_test, y_train_pred, y_test_pred)
        
        model_results = pd.concat([model_results, pd.DataFrame(results,index=[0])])
        # print results
        
        print("\nConfusion matrix on test")
        print(confusion_matrix(y_test, y_test_pred))
        print("\n")
        
    return model_results
RandomizedSearch_logreg = pipeline_optimization(X,y,balance='under')
RandomizedSearch_logreg<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09696771378708552, 'fraction_numerical': 0.007962478184991274, 'mean_word_length': 4.233447488584475, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '628998', 'license': 'MIT', 'n_tokens_mistral': 3010, 'n_tokens_neox': 2918, 'n_words': 624, 'path': 'ml-modelle/LogReg_Tuning.ipynb', 'repo_name': 'jb-ds2020/Capstone_Churn', 'size': '9168'}