<jupyter_start><jupyter_text>1. define functions2. Define paths and capture data3. evaluate categorical features4. missing data5. anomaly detection6. Visualize pairwise relations7. Check if data is imbalanced<jupyter_code>import os
import re
import sys
import warnings

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import pickle as pkl
import seaborn as sns
from impyute.imputation.cs import fast_knn
from sklearn.ensemble import IsolationForest
from sklearn.preprocessing import StandardScaler
from scipy import stats

# %matplotlib inline
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>set sandbox_mode boolean for image building* if sandbox_mode = True: faster to run, but images won't be generated<jupyter_code>sandbox_mode = False<jupyter_output><empty_output><jupyter_text>define functions<jupyter_code>def get_numerical_mask(df):
    type_mask = []
    for i in df.dtypes:
        if str(i).startswith('float') or str(i).startswith('int'): # or str(i).startswith('bool')
            type_mask.append(True)
        else: type_mask.append(False)
    num_cols = list(np.array(df.columns)[type_mask])
    other_cols = list(np.array(df.columns)[[not elem for elem in type_mask]])
    
    return num_cols, other_cols<jupyter_output><empty_output><jupyter_text>functions for visualizations<jupyter_code># build boxplots
def build_boxplots(df, x, file):
    
    plt.close()
    
    cols = df.columns
    fig, axes = plt.subplots(1,len(cols),figsize=(15,5))

    for ax,col in zip(axes, cols):
        sns.boxplot(x = x, y = col, data = df, ax = ax, width=.1)
    plt.tight_layout()
    plt.plot()
    
    # save file
    file = '01boxplot_'+file+'.jpg'
    plt.savefig(os.path.join(reports,file), bbox_inches = "tight")
    
# plot distributions
def plot_dist(feature, target, name, xlabel):
    fig, axes = plt.subplots(1,2,figsize=(15,5))
    ax1, ax2 = axes[0], axes[1]

    ax1 = sns.distplot( feature, hist=True, kde=True, bins=int(180/5), ax=ax1 )
    ax1.set_ylabel('frequency and KDE')
    ax1.set_xlabel(xlabel)
    ax1.set_title('{} histogram'.format(xlabel));

    ax2 = sns.kdeplot(feature[(target == 0) ],
                    color="Red", shade = True, ax = ax2)
    ax2 = sns.kdeplot(feature[(target == 1) ],
                    color="Blue", shade= True, ax = ax2)
    ax2.legend(["Not Churn", "Churn"], loc='upper right')
    ax2.set_ylabel('Density')
    ax2.set_xlabel('{}'.format(xlabel))
    ax2.set_title('Density function of {}'.format(xlabel));

    # save file
    file = '03hist_kde_'+name+'.jpg'
    plt.savefig(os.path.join(reports,file), bbox_inches = "tight")<jupyter_output><empty_output><jupyter_text>functions related to missing data<jupyter_code>def get_missing(df):
    total = df.isnull().sum()
    percent = (df.isnull().sum()/df.isnull().count())
    missing_data = pd.concat([total, percent], axis=1, keys=['total', 'percent_missing'])
    missing_data['percent_missing'] = missing_data['percent_missing']
    missing_data['percent_missing'] = missing_data['percent_missing'].apply(lambda x: round(x,2))
    
    return missing_data

def drop_missing_from_threshold(df, row_threshold, col_threshold):
    row_count, col_count = df.shape
    # drop columns according to threshold of missing; use mask of columns which have less missing than threshold
    df = df.iloc[:, (df_missing['percent_missing'] < col_threshold).to_list()]
    
    # drop row according to threshold of missing
    n_cols = df.shape[1]
    df['ratio_mis'] = df.apply(lambda x: (n_cols - x.count())/n_cols, axis=1)
    df = df[df['ratio_mis']<row_threshold]
    df.drop(['ratio_mis'], axis=1, inplace=True)
    
    # count number of removals
    row_count_new, col_count_new = df.shape
    row_count_removal = row_count - row_count_new
    col_count_removal = col_count - col_count_new
    print('{} rows and {} columns were removed from database'.format(row_count_removal, col_count_removal))
    
    return df

def apply_imputation(df, method = 'knn', k=30, manual_val=-1):
    try:
        assert method in ['knn', 'mode', 'median', -1, 'manual']
    except AssertionError:
        raise ValueError('error: select a correct method for imputation: [knn, mode, median, -1, manual]')
        
    if method == 'knn':
        sys.setrecursionlimit(100000) #Increase the recursion limit of the OS
        numerical_cols, other_cols = get_numerical_mask(df)
        
#         df =  StandardScaler().fit_transform(df) # scale for knn to work properly (it's distance based)

        # start the KNN training
        imputed_training = fast_knn(df[numerical_cols], k=30)

        # retrieve column names
        imp_cols = imputed_training.columns.to_list()
        imputed_training.rename({imp_cols[i]: numerical_cols[i] for i in range(len(imp_cols))}, axis = 1, inplace=True)
        df.reset_index(inplace=True)
        other_cols.append('id')
        df = df[other_cols].merge(imputed_training, left_index=True, right_index=True)
        df.set_index('id', inplace=True)
        
    elif method == 'mode':
        df.fillna(data.mode().iloc[0], inplace=True)
        
    elif method == 'median':
        df.fillna(df.median(), inplace=True)

    elif method == -1:
        df.fillna(-1, inplace=True)
    
    elif method == 'manual':
        df.fillna(manual_val, inplace=True)
        
    try:
        assert df[df.isna().any(axis=1)].shape[0] == 0
    except AssertionError:
        raise ValueError('there are still missing values')
    
    return df<jupyter_output><empty_output><jupyter_text>functions related to outliers<jupyter_code>def apply_standard_deviation_tol(df, tol=3):
    #scale data for operation
    df = pd.DataFrame(StandardScaler().fit_transform(df[numerical_cols]))
    
    z = np.abs(stats.zscore(df))
    z = pd.DataFrame(z, columns = df.columns, index=df.index)
    z.fillna(0, inplace=True)
    for col in z.columns[2:]:
        z = z[z[col]<tol]
    print("{0:.2%} of data was removed after dealing with outliers".format((df.shape[0]-z.shape[0])/df.shape[0]))
    df = df.loc[z.index, :]
    
    return df

def apply_isolation_forest(df, contamination=0.05):
    clf = IsolationForest(max_samples='auto', contamination=contamination, random_state=42) # contamination='auto' or 0.05
    clf.fit(df)

    outlier_pred = clf.predict(df)
    print('number of outliers:', np.count_nonzero(outlier_pred == -1), 'from a total of {}'.format(len(outlier_pred)))
    print('percentage of outliers: {0:.0%}'.format(np.count_nonzero(outlier_pred == -1)/np.count_nonzero(outlier_pred == 1)))
    
    return outlier_pred

def get_outliers(df, label, cols, method = 'isolation_forest', if_contamination = 0.05, z_tol = 3):
    
    # separate numerical data for treatment
    df_filtered = df[cols]
    
    if method == 'isolation_forest':
        outliers = apply_isolation_forest(df_filtered, if_contamination)
    elif method == 'standard_deviation_tol':
        df = apply_standard_deviation_tol(df_filtered, z_tol)
    
    return outliers, label<jupyter_output><empty_output><jupyter_text>outlier analysis<jupyter_code>import operator

def get_outlier_dict(df):
    df = df.copy()
    mult = 1.5
    n_outliers = {}
    for var in df.columns:
        quart1 = data[var].quantile(0.25)
        quart3 = data[var].quantile(0.75)
        iqr = quart3 - quart1
        lower_bound = quart1 - mult*iqr
        upper_bound = quart3 + mult*iqr
        df.loc[(df[var] < lower_bound) | (df[var] > upper_bound), [var]] = True
        df.loc[(df[var] != True), [var]] = False
        out_count = df[df[var]==True].shape[0]
        n_outliers[var] = out_count

    tup_results = sorted(n_outliers.items(), key=operator.itemgetter(1), reverse=True)
    ordered_cols = list(list(zip(*tup_results))[0])

    out_check = []
    n_outliers = {}
    for var in ordered_cols:
        for check in out_check:
            df.loc[(df[check] == True), [var]] = False
        out_check.append(var)
        out_count = df[df[var]==True].shape[0]
        if out_count != 0: n_outliers[var] = out_count
    return n_outliers
            
def plot_outliers(results, strings):
    file = '{}.jpg'.format(strings['filename'])
    tup_results = sorted(results.items(), key=operator.itemgetter(1))

    N = len(results)
    ind = np.arange(N)  # the x locations for the groups
    width = 0.40       # the width of the bars

    fig = plt.figure(figsize=(16,2))
    ax = fig.add_subplot(111)
    rects = ax.bar(ind+0.5, list(zip(*tup_results))[1], width,)
    for rect in rects:
        height = rect.get_height()
        ax.text(rect.get_x()+rect.get_width()/2., 
                1.005*height, 
                height, 
                ha='center', 
                va='bottom',)

    ax.set_xlabel('{}'.format(strings['xlabel']))
    ax.set_ylabel('{}'.format(strings['ylabel']))
    ax.set_title('{}'.format(strings['title']))
    
    # set ylim
    ylim = max(results.values())*1.2
    ax.set_ylim([0, ylim])
        
    ax.set_xticks(ind + width/2.)
    ax.set_xticklabels(list(zip(*tup_results))[0], rotation=22)
#     print("Order of models:", list(zip(*tup_results))[0])
    
    plt.plot()
    plt.savefig(os.path.join(reports,file), bbox_inches = "tight")<jupyter_output><empty_output><jupyter_text>Define paths and capture data<jupyter_code>inputs = os.path.join('..', 'data', '02_intermediate')
outputs = os.path.join('..', 'data', '02_intermediate')
reports = os.path.join('..', 'data', '06_reporting')
data               = pd.read_csv(os.path.join(inputs, 'X_train.csv'), index_col='id')
data_test          = pd.read_csv(os.path.join(inputs, 'X_test.csv'), index_col='id')
y_train            = pd.read_csv(os.path.join(inputs, 'y_train.csv'), index_col='id')
y_test             = pd.read_csv(os.path.join(inputs, 'y_test.csv'), index_col='id')
print('Dataset dimensions:', data.shape)
data.head()<jupyter_output>Dataset dimensions: (4930, 19)<jupyter_text>Checking  for possible anomalies in the database<jupyter_code>data.describe()<jupyter_output><empty_output><jupyter_text>get types of columns<jupyter_code>numerical_cols, other_cols = get_numerical_mask(data)
boolean_vars = ['partner', 'dependents', 'phoneservice', 'onlinesecurity', 'onlinebackup', 'deviceprotection', 
            'techsupport', 'streamingmovies', 'paperlessbilling']
boolean_vars.extend(['seniorcitizen', 'gender_male'])
for col in boolean_vars:
    numerical_cols.remove(col)<jupyter_output><empty_output><jupyter_text>boxplotsThis is useful to check for outliers, which will be handled later.<jupyter_code>partition = int(len(numerical_cols)/1)
if not sandbox_mode:
    build_boxplots(data[numerical_cols[:partition*1]], y_train['y'], '01')<jupyter_output><empty_output><jupyter_text>evaluate categorical features<jupyter_code>categories_dict = {}
for cat in data.columns:
    if data[cat].dtypes == 'object':
        categories_dict[cat] = list(data[cat].unique())
        n_categories = data[cat].unique().size
        print("number of categories for variable '{}': {} ".format(cat, n_categories))<jupyter_output>number of categories for variable 'multiplelines': 3 
number of categories for variable 'internetservice': 3 
number of categories for variable 'streamingtv': 3 
number of categories for variable 'contract': 3 
number of categories for variable 'paymentmethod': 4<jupyter_text>distribution of categorical varsBy assessing the proportion of positive cases over the categorical variables, we can observe some interesting factors:* Except for the variable "multiplelines", the proportion might vary a lot according to the specific category.* For internet service, the fiber optic users have a higher proclivity to churn.* month-to-month users have a high chance to churn (40%). 2 years contracts have extremely low churn (just 7%).* for payment method, payment checks users also might churn more frequently.<jupyter_code>def get_cat_vis(df, cat, y):
    df = df.copy()
    df['y'] = y
    print('selected feature:', feature)
    print('proportion of positive cases')
    multiplers = (df[df['y']==1][cat].value_counts() / df[cat].value_counts()).sort_values(ascending=False).head()
    print(multiplers)
    sns.catplot(x=cat, kind="count", hue="y", data=df, height=4);
print('there are {} categorical variables'.format(len(categories_dict.keys())))
print('from a total of {}'.format(data.shape[1]))
# cat = cat_list[0]
# plt.xticks(rotation=45)
# data[cat].hist(bins=16, alpha = .5, figsize=(10,3));
cat_list = list(categories_dict.keys())
feature = cat_list[0]
get_cat_vis(data, feature, y_train['y'])
feature = cat_list[1]
get_cat_vis(data, feature, y_train['y'])
feature = cat_list[2]
get_cat_vis(data, feature, y_train['y'])
feature = cat_list[3]
get_cat_vis(data, feature, y_train['y'])
feature = cat_list[4]
get_cat_vis(data, feature, y_train['y'])<jupyter_output>selected feature: paymentmethod
proportion of positive cases
electronic check             0.454711
mailed check                 0.186941
bank transfer (automatic)    0.164534
credit card (automatic)      0.141659
Name: paymentmethod, dtype: float64<jupyter_text>distribution of boolean vars<jupyter_code>feature = boolean_vars[0]
get_cat_vis(data, feature, y_train['y'])
feature = boolean_vars[1]
get_cat_vis(data, feature, y_train['y'])
feature = boolean_vars[2]
get_cat_vis(data, feature, y_train['y'])
feature = boolean_vars[3]
get_cat_vis(data, feature, y_train['y'])
feature = boolean_vars[4]
get_cat_vis(data, feature, y_train['y'])
feature = boolean_vars[5]
get_cat_vis(data, feature, y_train['y'])
feature = boolean_vars[6]
get_cat_vis(data, feature, y_train['y'])
feature = boolean_vars[7]
get_cat_vis(data, feature, y_train['y'])
feature = boolean_vars[8]
get_cat_vis(data, feature, y_train['y'])<jupyter_output>selected feature: paperlessbilling
proportion of positive cases
1.0    0.330462
0.0    0.162722
Name: paperlessbilling, dtype: float64<jupyter_text>missing datausually, missing data is solved by filling it with some metric such as median. If the number of missing data in some entry is too high, we must evaluate for the removal of those entries.For categorical with missing data, if you want to encode missing values, first change its type to a string:```pythona[pd.isnull(a)]  = 'NaN'```Some refs:* https://stackoverflow.com/questions/36808434/label-encoder-encoding-missing-valuesAbout the missing values, we can't assume beforehand if those are Missing at Random (MAR) or Missing not at Random (MNAR). Further investigation would be necessary to properly decide over how to handle it.For now, I am assuming they are Missing at Random. So I will remove some of them through a threshold, and apply imputation for the rest. By applying a proper imputation I observed a slight improvement over the score.The catch is that applying imputation over euclidean distances can be extremely imprecise.  drop columns and rows for threshold of missing<jupyter_code>print('SORTED LIST OF MISSING VALUES')
df_missing = get_missing(data)
df_missing_vis = df_missing[df_missing['total'] > 0]
df_missing_vis['percent_missing'] = df_missing_vis['percent_missing'].apply(lambda x: round(x, 2))
# df_missing_vis.sort_values(by='percent_missing', ascending=False).head(20)
df_missing_vis.sort_values(by='percent_missing', ascending=False)
col_mis_threshold = 0.8
row_mis_threshold = 0.5
data = drop_missing_from_threshold(data, row_mis_threshold, col_mis_threshold)<jupyter_output>0 rows and 0 columns were removed from database<jupyter_text>visualize rows with missingwe already know that the critical columns are related to geo_location<jupyter_code># sandbox_mode = True
if sandbox_mode:
    print('number of missing:', data[data.isna().any(axis=1)].shape[0])
    data[data.isna().any(axis=1)].tail()<jupyter_output><empty_output><jupyter_text>imputation of missing valuesFor continuous values, I would prefer knn over median, but it depends on normalized dataset. Nevertheless, we don't have missing on continuous datasets, even though we could encode categorical data. But the encoding step wasn't organized to precede this notebook, so I will stick to 'mode', which imputes the most frequent value.Some refs:* https://jamesrledoux.com/code/imputation:~:text=One%20approach%20to%20imputing%20categorical,given%20in%20Pandas'%20value_counts%20function. the apply_imputation function accepts the following methods: knn, median, mode, or -1 (impute as category -1 [for categorical vars])<jupyter_code># imputation over numerical variables
data[numerical_cols] = data[numerical_cols].astype(float)
if data[numerical_cols].isnull().values.any():
    data[numerical_cols] = apply_imputation(data[numerical_cols], method = 'knn', k = 30)

# imputation over categorical variables
# if data[other_cols].isnull().values.any():
data[other_cols] = apply_imputation(data[other_cols], method = 'mode', k = 30)

# manual imputation on lag and forecast variables
# manual_cols = ['lag_1', 'lag_2', 'lag_3']
# if data[manual_cols].isnull().values.any():
#     data[manual_cols] = apply_imputation(data[manual_cols], method = 'manual', manual_val = False)
# data_test['y'].fillna(value=data_test['y'], inplace=True)
# imputation over numerical variables
data_test[numerical_cols] = data_test[numerical_cols].astype(float)
if data_test[numerical_cols].isnull().values.any():
    data_test[numerical_cols] = apply_imputation(data_test[numerical_cols], method = 'knn', k = 30)

# imputation over categorical variables
if data_test[other_cols].isnull().values.any():
    data_test[other_cols] = apply_imputation(data_test[other_cols], method = 'mode', k = 30)<jupyter_output><empty_output><jupyter_text>anomaly detectionremove outliers from choosing one of the following methods: isolation_forest, standard_deviation_tol (using z_score on standardized version)other parameters are:* if_contamination: isolation forest level of contamination* z_tol: tolerance for standard deviation (if using zscore)It is not advisable to remove outliers without proper consideration.Some interesting refs:* https://towardsdatascience.com/ways-to-detect-and-remove-the-outliers-404d16608dba* https://towardsdatascience.com/anomaly-detection-with-isolation-forest-visualization-23cd75c281e2* https://towardsdatascience.com/outlier-detection-with-isolation-forest-3d190448d45e<jupyter_code># temporarily add train to test data (no leakage here)
test_start = data_test.index[0]
data_test = data.append(data_test)

# get outliers on train data
data['if_anomaly'], _ = get_outliers(data[numerical_cols], y_train, numerical_cols, 
                                                  method = 'isolation_forest', if_contamination = 0.05)

# get outliers on test data (needs trainset)

data_test['if_anomaly'], _ = get_outliers(data_test[numerical_cols], y_train, numerical_cols, 
                                                  method = 'isolation_forest', if_contamination = 0.05)

# removes trainset again
data_test = data_test.iloc[len(data):,:]<jupyter_output>number of outliers: 247 from a total of 4930
percentage of outliers: 5%
number of outliers: 353 from a total of 7043
percentage of outliers: 5%<jupyter_text>outlier removal isn't justified for now<jupyter_code>remove_outliers = False
if remove_outliers:
    data = data[data['if_anomaly'] == 1]
    y_train = y_train[y_train.index.isin(data.index.to_list())]<jupyter_output><empty_output><jupyter_text>data_temp, outliers_data, y_train_temp = remove_outliers(data, y_train, numerical_cols,                                                   method = 'isolation_forest', if_contamination = 0.01)<jupyter_code>ploting_dict = {'filename': 'outlier_analysis', 'xlabel': 'feature', 'ylabel': 'outlier count', 
                'title': 'number of outliers per feature'}

plot_outliers(get_outlier_dict(data[numerical_cols]), ploting_dict)<jupyter_output><empty_output><jupyter_text>build new variables  Visualize pairwise relationsWhen datasets have just a few variables (10–15), pairplots allow for a quick visual inspection of those relations, as well as bariable distributions.<jupyter_code>numerical_cols, other_cols = get_numerical_mask(data)
boolean_vars = ['partner', 'dependents', 'phoneservice', 'onlinesecurity', 'onlinebackup', 'deviceprotection', 
            'techsupport', 'streamingmovies', 'paperlessbilling']
boolean_vars.extend(['seniorcitizen', 'gender_male'])
boolean_vars.append('if_anomaly')
for col in boolean_vars:
    numerical_cols.remove(col)
separator = int(len(numerical_cols)/1)<jupyter_output><empty_output><jupyter_text>group 1<jupyter_code>if not sandbox_mode:
    data_vis = data[numerical_cols[:separator]]
    data_vis['y'] = y_train['y']
    sns.pairplot(data_vis, plot_kws={'alpha': 0.1});
    plt.savefig(os.path.join(reports,'01_pairplots_1.jpg'), bbox_inches = "tight")<jupyter_output><empty_output><jupyter_text>check selected distributions<jupyter_code>y = y_train['y']
var_name = 'tenure'
plot_dist(data[var_name], y, var_name, xlabel = 'tenure (months)')
var_name = 'monthlycharges'
plot_dist(data[var_name], y, var_name, xlabel = 'monthly charges')
var_name = 'totalcharges'
plot_dist(data[var_name], y, var_name, xlabel = 'total charges')<jupyter_output><empty_output><jupyter_text>Check if data is imbalancedWhen data is imbalanced, we must evaluate for solutions such as oversampling or undersamplig, which might be done with techniques such as SMOTE (Synthetic Minority Oversampling Technique).<jupyter_code>y = y_train.append(y_test)
color = sns.color_palette()
classif = y_train['y'].value_counts()
plt.figure(figsize=(8,4))
sns.barplot(classif.index, classif.values, alpha=0.8, color=color[0])
plt.ylabel('Number of occurrences', fontsize=12)
plt.xlabel('Failed', fontsize=12)
plt.plot()
plt.savefig(os.path.join(reports,'imbalance.jpg'), bbox_inches = "tight")
counts = y['y'].value_counts()
counts
print('ratio between classes:',round(counts[0]/counts[1],2))
print('\r\nCheck proportions below:')
y['y'].value_counts(normalize=True)<jupyter_output><empty_output><jupyter_text>Description (2nd round)<jupyter_code>if sandbox_mode:
    print(data.shape[0])
    print(y_train.shape[0])
    print(data_test.shape[0])
    print(y_test.shape[0])
    data.describe()<jupyter_output><empty_output><jupyter_text>save intermediate data<jupyter_code>data.to_csv(os.path.join(outputs, 'X_train.csv'))
data_test.to_csv(os.path.join(outputs, 'X_test.csv'))

y_train.to_csv(os.path.join(outputs, 'y_train.csv'))
y_test.to_csv(os.path.join(outputs, 'y_test.csv'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12687263836408091, 'fraction_numerical': 0.01644809957768393, 'mean_word_length': 4.7860082304526745, 'pattern_counts': {'":': 0, '<': 103, '<?xml version=': 0, '>': 101, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '26', 'dir': 'jupyter-structured-clean-dedup', 'id': '230751', 'license': 'MIT', 'n_tokens_mistral': 7818, 'n_tokens_neox': 7381, 'n_words': 1699, 'path': 'notebooks/03_eda_cleansing.ipynb', 'repo_name': 'Marcelobbr/churn', 'size': '22495'}