<jupyter_start><jupyter_text>Telecom Customer Churn Prediction Feature Engineering This tutorial is based on [this](https://www.kaggle.com/pavanraj159/telecom-customer-churn-prediction/comments6.-Model-Performances) Kaggle notebook and [this](https://github.com/gojek/feast/tree/master/examples/feast-xgboost-churn-prediction-tutorial) Feast notebook  1. Churn Feature Engineering<jupyter_code>from hops import pandas_helper as pandas
from hops import featurestore
import pandas as pd
import numpy as np
import os
import itertools
import warnings
warnings.filterwarnings("ignore")
import sklearn # Tested with 0.22.1
from slugify import slugify
import s3fs<jupyter_output>Starting Spark application<jupyter_text>1.1 Data<jupyter_code>#telcom = pd.read_csv('http://feast-examples.storage.googleapis.com/telco-churn-xgboost/telco_customer_churn.csv')
telcom = pandas.read_csv('Resources/telco_customer_churn.csv')
#telcom = pd.read_csv('s3://sagemakerjim/telco_customer_churn.csv')
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 2000)
telcom.head(5)<jupyter_output>customer_id  gender  senior_citizen partner dependents  tenure phone_service    multiple_lines internet_service online_security online_backup device_protection tech_support streaming_tv streaming_movies        contract paperless_billing             payment_method  monthly_charges total_charges churn
0  7590-VHVEG  Female               0     Yes         No       1            No  No phone service              DSL              No           Yes                No           No           No               No  Month-to-month               Yes           Electronic check            29.85         29.85    No
1  5575-GNVDE    Male               0      No         No      34           Yes                No              DSL             Yes            No               Yes           No           No               No        One year                No               Mailed check            56.95        1889.5    No
2  3668-QPYBK    Male               0      No         No       2           Yes             [...]<jupyter_text>1.2 Data Manipulation<jupyter_code># Replacing spaces with null values in total charges column
telcom['total_charges'] = telcom["total_charges"].replace(" ",np.nan)

# Dropping null values from total charges column which contain .15% missing data 
telcom = telcom[telcom["total_charges"].notnull()]
telcom = telcom.reset_index()[telcom.columns]

# Convert to float type
telcom["total_charges"] = telcom["total_charges"].astype(float)

# Replace 'No internet service' to No for the following columns
replace_cols = [ 'online_security', 'online_backup', 'device_protection',
                'tech_support','streaming_tv', 'streaming_movies']
for i in replace_cols : 
    telcom[i]  = telcom[i].replace({'internet_service' : 'No'})
    
# Replace binary values with strings
telcom["senior_citizen"] = telcom["senior_citizen"].replace({1:"Yes",0:"No"})

# Tenure to categorical column
def tenure_lab(telcom) :
    
    if telcom["tenure"] <= 12 :
        return "Tenure_0-12"
    elif (telcom["tenure"] > 12) & (telcom["tenure"] <= 24 ):
        return "Tenure_12-24"
    elif (telcom["tenure"] > 24) & (telcom["tenure"] <= 48) :
        return "Tenure_24-48"
    elif (telcom["tenure"] > 48) & (telcom["tenure"] <= 60) :
        return "Tenure_48-60"
    elif telcom["tenure"] > 60 :
        return "Tenure_gt_60"
telcom["tenure_group"] = telcom.apply(lambda telcom:tenure_lab(telcom),
                                      axis = 1)

# Separating churn and non churn customers
churn     = telcom[telcom["churn"] == "Yes"]
not_churn = telcom[telcom["churn"] == "No"]

# Separating catagorical and numerical columns
Id_col     = ['customer_id']
target_col = ["churn"]
cat_cols   = telcom.nunique()[telcom.nunique() < 6].keys().tolist()
cat_cols   = [x for x in cat_cols if x not in target_col]
num_cols   = [x for x in telcom.columns if x not in cat_cols + target_col + Id_col]

telcom.head()<jupyter_output>customer_id  gender senior_citizen partner dependents  tenure phone_service    multiple_lines internet_service online_security online_backup device_protection tech_support streaming_tv streaming_movies        contract paperless_billing             payment_method  monthly_charges  total_charges churn  tenure_group
0  7590-VHVEG  Female             No     Yes         No       1            No  No phone service              DSL              No           Yes                No           No           No               No  Month-to-month               Yes           Electronic check            29.85          29.85    No   Tenure_0-12
1  5575-GNVDE    Male             No      No         No      34           Yes                No              DSL             Yes            No               Yes           No           No               No        One year                No               Mailed check            56.95        1889.50    No  Tenure_24-48
2  3668-QPYBK    Male             No      No     [...]<jupyter_text>1.3 Data Processing<jupyter_code>from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler

# Customer id col
Id_col     = ['customer_id']

# Target columns
target_col = ["churn"]

# Categorical columns
cat_cols   = telcom.nunique()[telcom.nunique() < 6].keys().tolist()
cat_cols   = [x for x in cat_cols if x not in target_col]

# Numerical columns
num_cols   = [x for x in telcom.columns if x not in cat_cols + target_col + Id_col]

# Binary columns with 2 values
bin_cols   = telcom.nunique()[telcom.nunique() == 2].keys().tolist()

# Columns more than 2 values
multi_cols = [i for i in cat_cols if i not in bin_cols]

# Label encoding Binary columns
le = LabelEncoder()
for i in bin_cols :
    telcom[i] = le.fit_transform(telcom[i])
    
# Duplicating columns for multi value columns
telcom = pd.get_dummies(data = telcom,columns = multi_cols )

# Scaling Numerical columns
std = StandardScaler()
scaled = std.fit_transform(telcom[num_cols])
scaled = pd.DataFrame(scaled,columns=num_cols)

# Dropping original values merging scaled values for numerical columns
df_telcom_og = telcom.copy()
telcom = telcom.drop(columns = num_cols,axis = 1)
telcom = telcom.merge(scaled,left_index=True,right_index=True,how = "left")

# Clean up column names
telcom.columns = [slugify(col, lowercase=True, separator='_') for col in telcom.columns]
telcom.columns = map(str.lower, telcom.columns)
telcom.head()<jupyter_output>customer_id  gender  senior_citizen  partner  dependents  phone_service  paperless_billing  churn  multiple_lines_no  multiple_lines_no_phone_service  multiple_lines_yes  internet_service_dsl  internet_service_fiber_optic  internet_service_no  online_security_no  online_security_no_internet_service  online_security_yes  online_backup_no  online_backup_no_internet_service  online_backup_yes  device_protection_no  device_protection_no_internet_service  device_protection_yes  tech_support_no  tech_support_no_internet_service  tech_support_yes  streaming_tv_no  streaming_tv_no_internet_service  streaming_tv_yes  streaming_movies_no  streaming_movies_no_internet_service  streaming_movies_yes  contract_month_to_month  contract_one_year  contract_two_year  payment_method_bank_transfer_automatic  payment_method_credit_card_automatic  payment_method_electronic_check  payment_method_mailed_check  tenure_group_tenure_0_12  tenure_group_tenure_12_24  tenure_group_tenure_24_48  tenure_group_tenur[...]<jupyter_text>1.4 Create FeatureGroups<jupyter_code>from hops import featurestore
featurestore.create_featuregroup(telcom, "telcom_featuregroup", description="Telecom churn prediction feature group", online=True)
featurestore.set_featuregroup_tag("telcom_featuregroup", "privacy_level", "pii")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.078559528386518, 'fraction_numerical': 0.02101755734973728, 'mean_word_length': 3.0477178423236513, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '181905', 'license': 'Apache-2.0', 'n_tokens_mistral': 2548, 'n_tokens_neox': 2375, 'n_words': 579, 'path': 'churn_create_featuregroups.ipynb', 'repo_name': 'jimdowling/churn', 'size': '7803'}