<jupyter_start><jupyter_text>Telecom Churn Case StudyPG Diploma in Machine Learning and AI March 2018Group Members1. <NAME>2. <NAME>3. <NAME>4. <NAME>The analysis is divided into following parts1. Filter high-value customers2. Handlling The Missing Values3. Derive new features4. Splitting the Test and Train Data5. PCA6. Logistic Regression7. SVM8. XGBoost  Data understanding, preparation, and feature engineering<jupyter_code>import pandas as pd
import seaborn as sns
import numpy as np
data = pd.read_csv('telecom_churn_data.csv')
#data.describe()
#data.columns
print(data.shape)
#pd.set_option('display.max_columns', None)
#pd.set_option('display.max_rows', None)
# data.isnull().sum().sort_values()/data.shape[0]*100 > 73
data.head(5)
data.filter(like='total_rech_data').head()
#data.mobile_number.unique().size
#sample = data.head()
#months_data = sample.filter(like='rech')
#months_data<jupyter_output><empty_output><jupyter_text>1. Filter high-value customersfiltering the high value custmer in data frame max_revenue_customers_data<jupyter_code>data['average_recharge']=(data.total_rech_amt_6+data.total_rech_amt_7)/2
data.average_recharge.describe(percentiles=[0.7,0.9,0.99,0.999,0.999999])
#70% is at 368.5 so filtering based on that
max_revenue_customers_data = data[data.average_recharge >368.5]<jupyter_output><empty_output><jupyter_text>2. Handlling The Missing Values Finding how much missing value we have for each coloum and handle them<jupyter_code>#missing value before any change
round(data.isnull().sum()/len(data.index), 2)*100
#first find high missing value (above 60%) columns
missing_columns = data.columns[100*(data.isnull().sum()/len(data.index)) > 60]
# missing value (above 60%) columns
missing_columns
# divide data for each month as all month have same coloum, to easy to undestand
churn_data_jun = data.filter(regex='(_6$|^jun_)')
churn_data_jul = data.filter(regex='(_7$|^jul_)')
churn_data_aug = data.filter(regex='(_8$|^aug_)')
churn_data_sep = data.filter(regex='(_9$|^sep_)') 
# each mounth have around 55 column
churn_data_jun.columns.shape, churn_data_jul.columns.shape, churn_data_aug.columns.shape, churn_data_sep.columns.shape 
# each mounth have around 55 column with following name
churn_data_jun.columns
# see data for jun mounth
churn_data_jun.head(10)
# treat the lower missing values by replacing 0 
data['total_rech_data_6'] = data['total_rech_data_6'].replace(np.nan, 0)
data['total_rech_data_7'] = data['total_rech_data_7'].replace(np.nan, 0)
data['total_rech_data_8'] = data['total_rech_data_8'].replace(np.nan, 0)
data['total_rech_data_9'] = data['total_rech_data_9'].replace(np.nan, 0)
data['av_rech_amt_data_6'] = data['av_rech_amt_data_6'].replace(np.nan, 0)
data['av_rech_amt_data_7'] = data['av_rech_amt_data_7'].replace(np.nan, 0)
data['av_rech_amt_data_8'] = data['av_rech_amt_data_8'].replace(np.nan, 0)
data['av_rech_amt_data_9'] = data['av_rech_amt_data_9'].replace(np.nan, 0)
# treat the missing values based on the values in other coloum
data['max_rech_data_6'] = data['date_of_last_rech_data_6'].apply(lambda x: 0 if x == np.nan else 1)
data['max_rech_data_7'] = data['date_of_last_rech_data_7'].apply(lambda x: 0 if x == np.nan else 1)
data['max_rech_data_8'] = data['date_of_last_rech_data_8'].apply(lambda x: 0 if x == np.nan else 1)
data['max_rech_data_9'] = data['date_of_last_rech_data_9'].apply(lambda x: 0 if x == np.nan else 1)
#  treat the higher missing values by replacing -1
data['night_pck_user_6'] = data['night_pck_user_6'].replace(np.nan, -1)
data['night_pck_user_7'] = data['night_pck_user_7'].replace(np.nan, -1)
data['night_pck_user_8'] = data['night_pck_user_8'].replace(np.nan, -1)
data['night_pck_user_9'] = data['night_pck_user_9'].replace(np.nan, -1)
data['fb_user_6'] = data['fb_user_6'].replace(np.nan, -1)
data['fb_user_7'] = data['fb_user_7'].replace(np.nan, -1)
data['fb_user_8'] = data['fb_user_8'].replace(np.nan, -1)
data['fb_user_9'] = data['fb_user_9'].replace(np.nan, -1)
# treat the missing values based on the values in other coloum
data['count_rech_2g_6'] = data['date_of_last_rech_data_6'].apply(lambda x: 0 if x == np.nan else 1)
data['count_rech_3g_6'] = data['date_of_last_rech_data_6'].apply(lambda x: 0 if x == np.nan else 1)
data['count_rech_2g_7'] = data['date_of_last_rech_data_7'].apply(lambda x: 0 if x == np.nan else 1)
data['count_rech_3g_7'] = data['date_of_last_rech_data_7'].apply(lambda x: 0 if x == np.nan else 1)
data['count_rech_2g_8'] = data['date_of_last_rech_data_8'].apply(lambda x: 0 if x == np.nan else 1)
data['count_rech_3g_8'] = data['date_of_last_rech_data_8'].apply(lambda x: 0 if x == np.nan else 1)
data['count_rech_2g_9'] = data['date_of_last_rech_data_9'].apply(lambda x: 0 if x == np.nan else 1)
data['count_rech_3g_9'] = data['date_of_last_rech_data_9'].apply(lambda x: 0 if x == np.nan else 1)
#missing value after above changes
round(data.isnull().sum()/len(data.index), 2)*100
# now no coloum left with large number of missing value 
missing_row_percentage = 100 * data.T.isnull().sum().sort_values(ascending=False) / data.shape[1]
missing_row_percentage[missing_row_percentage > 50].shape
missing_columns = data.columns[100*(data.isnull().sum()/len(data.index)) > 60]
data = data.drop(missing_columns,axis=1)
#missing value after above changes
round(data.isnull().sum().sort_values(ascending=False)/len(data.index), 2)*100
# drop the coloum mobile_number (not a feature) and circle id (with no variance)
data.drop(['mobile_number','circle_id'],axis=1,inplace=True)
data = data[data.columns.drop(list(data.filter(regex='date')))]
data.shape
# fill rest of the coloum with meadian values
data = data.fillna(data.median()).dropna(axis=1, how='all')
data.info()
# data after handlling all the missing values
round(data.isnull().sum().sort_values(ascending=False)/len(data.index), 2)*100<jupyter_output><empty_output><jupyter_text>3. Derive new featurescreate new coloum/feature from existing<jupyter_code>#calculting total Operator to Operator talk time uages
data['t2t_total_6']=data['loc_og_t2t_mou_6']+data['std_og_t2t_mou_6']+data['loc_ic_t2t_mou_6']+data['std_ic_t2t_mou_6']
data['t2t_total_7']=data['loc_og_t2t_mou_7']+data['std_og_t2t_mou_7']+data['loc_ic_t2t_mou_7']+data['std_ic_t2t_mou_7']
data['t2t_total_8']=data['loc_og_t2t_mou_8']+data['std_og_t2t_mou_8']+data['loc_ic_t2t_mou_8']+data['std_ic_t2t_mou_8']
data['t2t_total_9']=data['loc_og_t2t_mou_9']+data['std_og_t2t_mou_9']+data['loc_ic_t2t_mou_9']+data['std_ic_t2t_mou_9']

#calculting total Operator to Other talk time uages
data['t2m_total_6']=data['loc_og_t2m_mou_6']+data['std_og_t2m_mou_6']+data['loc_ic_t2m_mou_6']+data['std_ic_t2m_mou_6']
data['t2m_total_7']=data['loc_og_t2m_mou_7']+data['std_og_t2m_mou_7']+data['loc_ic_t2m_mou_7']+data['std_ic_t2m_mou_7']
data['t2m_total_8']=data['loc_og_t2m_mou_8']+data['std_og_t2m_mou_8']+data['loc_ic_t2m_mou_8']+data['std_ic_t2m_mou_8']
data['t2m_total_9']=data['loc_og_t2m_mou_9']+data['std_og_t2m_mou_9']+data['loc_ic_t2m_mou_9']+data['std_ic_t2m_mou_9']

#calculting total Operator to Fixed line talk time uages
data['t2f_total_6']=data['loc_og_t2f_mou_6']+data['std_og_t2f_mou_6']+data['loc_ic_t2f_mou_6']+data['std_ic_t2f_mou_6']
data['t2f_total_7']=data['loc_og_t2f_mou_7']+data['std_og_t2f_mou_7']+data['loc_ic_t2f_mou_7']+data['std_ic_t2f_mou_7']
data['t2f_total_8']=data['loc_og_t2f_mou_8']+data['std_og_t2f_mou_8']+data['loc_ic_t2f_mou_8']+data['std_ic_t2f_mou_8']
data['t2f_total_9']=data['loc_og_t2f_mou_9']+data['std_og_t2f_mou_9']+data['loc_ic_t2f_mou_9']+data['std_ic_t2f_mou_9']

#calculting total Operator to own call center talk time uages
# this may help to find the customer calling more to call center usually have issues in services
data['t2c_total_6']=data['loc_og_t2c_mou_6']+data['std_og_t2c_mou_6']
data['t2c_total_7']=data['loc_og_t2c_mou_7']+data['std_og_t2c_mou_7']
data['t2c_total_8']=data['loc_og_t2c_mou_8']+data['std_og_t2c_mou_8']
data['t2c_total_9']=data['loc_og_t2c_mou_9']+data['std_og_t2c_mou_9']

'''
data['cust_care_calls_6']=data['t2c_total_6']/data['total_og_mou_6'].apply(lambda x: 0 if x == np.nan else x)
data['cust_care_calls_7']=data['t2c_total_7']/data['total_og_mou_7'].apply(lambda x: 0 if x == np.nan else x)
data['cust_care_calls_8']=data['t2c_total_8']/data['total_og_mou_8'].apply(lambda x: 0 if x == np.nan else x)
data['cust_care_calls_9']=data['t2c_total_9']/data['total_og_mou_9'].apply(lambda x: 0 if x == np.nan else x)
'''
data['total_vol_6']=data['vol_2g_mb_6']+data['vol_3g_mb_6']
data['total_vol_7']=data['vol_2g_mb_7']+data['vol_3g_mb_7']
data['total_vol_8']=data['vol_2g_mb_8']+data['vol_3g_mb_8']
data['total_vol_9']=data['vol_2g_mb_9']+data['vol_3g_mb_9']
# calculting Change in the usage for a customer from month 6 to month 7 and month 7 to month 8
# calculting Change in outgoing minutes of usage
data['total_og_mou_diff_7_6'] = data.total_og_mou_7 - data.total_og_mou_6
data['total_og_mou_diff_8_7'] = data.total_og_mou_8 - data.total_og_mou_7

# calculting Change in incoming minutes of usage
data['total_ic_mou_diff_7_6'] = data.total_ic_mou_7 - data.total_ic_mou_6
data['total_ic_mou_diff_8_7'] = data.total_ic_mou_8 - data.total_ic_mou_7

# calculting Change in number of rechages
data['total_rech_num_diff_7_6'] = data.total_rech_num_7 - data.total_rech_num_6
data['total_rech_num_diff_8_7'] = data.total_rech_num_8 - data.total_rech_num_7

# calculting Change in amount of rechages
data['total_rech_amt_diff_7_6'] = data.total_rech_amt_7 - data.total_rech_amt_6
data['total_rech_amt_diff_8_7'] = data.total_rech_amt_8 - data.total_rech_amt_7
#TODO Duplicate operation
data.average_recharge.describe(percentiles=[0.7,0.9,0.99,0.999,0.999999])
#TODO Duplicate operation
data = data[data.average_recharge > 368.5]
#Mark label/tag Churned customers (churn=1, else 0) based on the Month 9
data['churn'] = np.where((data['total_ic_mou_9'] + data['total_og_mou_9']+data['total_vol_9']) == 0, 1, 0)
#Now drop all the Month 9 coloums
data = data[data.columns.drop(list(data.filter(like='_9')))]
#check for any missing value at this level
round(data.isnull().sum().sort_values(ascending=False)/len(data.index), 2)*100
data.shape
#check the Coloum with No varaince (not changing) value i.e. a column with same value all across
#TODO Remove coloum with nunique = 1
pd.set_option('display.max_rows', None)
data.nunique(axis=0).sort_values(ascending=True)
'''
fb_user_8                      3
fb_user_7                      3
fb_user_6                      3
night_pck_user_6               3
night_pck_user_7               3
night_pck_user_8               3
monthly_2g_6                   5
monthly_2g_8                   6
monthly_2g_7                   6
monthly_3g_8                  12
monthly_3g_6                  12
monthly_3g_7                  15
sachet_3g_6                   25
sachet_3g_7                   27
sachet_3g_8                   29
sachet_2g_6                   30
sachet_2g_8                   34
sachet_2g_7                   34
'''
data[data.churn == 1].shape
#TODO: Finding the coloum with catagorical values 
data[data.churn == 1].groupby('sachet_3g_6').size()
#data[(data.sachet_3g_8 == 0) & (data.churn == 1)].shape
# TODO Finding the coloum with catagorical values  
categorical_variables = ['fb_user_8','fb_user_7','fb_user_6','night_pck_user_6','night_pck_user_7','night_pck_user_8','monthly_2g_6','monthly_2g_8','monthly_2g_7','monthly_3g_8','monthly_3g_6','monthly_3g_7','sachet_3g_6','sachet_3g_7','sachet_3g_8','sachet_2g_6','sachet_2g_8','sachet_2g_7']
# TODO: the coloum with catagorical values 
data_cleaned = data.copy()
data_cleaned.to_csv('imputed_clean_churn_data.csv')
predicted_val = 'churn'
for cat_var in categorical_variables:
    churn_rate_replace_dict = data_cleaned[[cat_var, predicted_val]].groupby(cat_var).mean().to_dict()[predicted_val]
    data[cat_var] =  data[cat_var].replace(churn_rate_replace_dict)
continuous_variables = list(data.columns)
decision_tree_data = data

continuous_variables.remove('churn')
continuous_data = data[continuous_variables]
churn_frame = data['churn']
data = data.drop(['churn'],axis =1)
cols = list(continuous_data)
nunique = continuous_data.apply(pd.Series.nunique)
cols_to_drop = nunique[nunique == 1].index
continuous_data = continuous_data.drop(cols_to_drop, axis=1)
normalized_df=(continuous_data-continuous_data.mean())/continuous_data.std()
data = data.drop(continuous_variables, 1)
data = pd.concat([data,normalized_df,churn_frame],axis=1)
data.head()
type(data)
from sklearn.utils import resample
data_majority = data[data.churn==0]
data_minority = data[data.churn==1]

df_minority_upsampled = resample(data_minority, replace=True, n_samples=data[data.churn == 0].shape[0], random_state=123)
data = pd.concat([df_minority_upsampled, data_majority])
data = data.sample(frac=1).reset_index(drop=True)<jupyter_output><empty_output><jupyter_text>4. Splitting the Test and Train Data<jupyter_code>from sklearn.model_selection import train_test_split

# Putting feature variable to X
X = data.drop(['churn'],axis=1)

# Putting response variable to y
y = data['churn']
# Splitting the data into train and test
X_train, X_test, y_train, y_test = train_test_split(X,y, train_size=0.7,test_size=0.3,random_state=100)
X_train.shape,X_test.shape<jupyter_output><empty_output><jupyter_text>5. PCA<jupyter_code>#Improting the PCA module
from sklearn.decomposition import PCA
pca = PCA(svd_solver='randomized', random_state=42)
null_columns=data.columns[data.isnull().any()]
data[null_columns].isnull().sum()
#Doing the PCA on the train data
pca.fit(X_train)
pca.components_
colnames = list(X_train.columns)
pcs_df = pd.DataFrame({'PC1':pca.components_[0],'PC2':pca.components_[1], 'Feature':colnames})
pcs_df.head()
import matplotlib.pyplot as plt
%matplotlib inline
fig = plt.figure(figsize = (8,8))
plt.scatter(pcs_df.PC1, pcs_df.PC2)
plt.xlabel('Principal Component 1')
plt.ylabel('Principal Component 2')
for i, txt in enumerate(pcs_df.Feature):
    plt.annotate(txt, (pcs_df.PC1[i],pcs_df.PC2[i]))
plt.tight_layout()
plt.show()
#Making the screeplot - plotting the cumulative variance against the number of components
%matplotlib inline
fig = plt.figure(figsize = (12,8))
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.xlabel('number of components')
plt.ylabel('cumulative explained variance')
plt.show()
#Using incremental PCA for efficiency - saves a lot of time on larger datasets
from sklearn.decomposition import IncrementalPCA
pca_final = IncrementalPCA(n_components=50)
df_train_pca = pca_final.fit_transform(X_train)
df_train_pca.shape
#creating correlation matrix for the principal components
corrmat = np.corrcoef(df_train_pca.transpose())
#plotting the correlation matrix
%matplotlib inline
plt.figure(figsize = (20,10))
sns.heatmap(corrmat,annot = True)
# 1s -> 0s in diagonals
corrmat_nodiag = corrmat - np.diagflat(corrmat.diagonal())
print("max corr:",corrmat_nodiag.max(), ", min corr: ", corrmat_nodiag.min(),)
# we see that correlations are indeed very close to 0
#Applying selected components to the test data - 16 components
df_test_pca = pca_final.transform(X_test)
df_test_pca.shape
#Training the model on the train data
from sklearn.linear_model import LogisticRegression
from sklearn import metrics

learner_pca_logistic = LogisticRegression()
model_pca_logistic = learner_pca_logistic.fit(df_train_pca,y_train)
#Making prediction on the test data
pred_probs_test = model_pca_logistic.predict_proba(df_test_pca)[:,1]
"{:2.2}".format(metrics.roc_auc_score(y_test, pred_probs_test))<jupyter_output><empty_output><jupyter_text>6. Logistic Regression<jupyter_code>pca_again = PCA(0.90)
df_train_pca2 = pca_again.fit_transform(X_train)
df_train_pca2.shape
#training the regression model
learner_pca2_logistic = LogisticRegression()
model_pca2_logistic = learner_pca2_logistic.fit(df_train_pca2,y_train)
df_test_pca2 = pca_again.transform(X_test)
df_test_pca2.shape
#Making prediction on the test data
pred_probs_test2 = model_pca2_logistic.predict_proba(df_test_pca2)[:,1]
"{:2.2f}".format(metrics.roc_auc_score(y_test, pred_probs_test2))<jupyter_output><empty_output><jupyter_text>7. SVM<jupyter_code>from sklearn.svm import SVC
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import validation_curve
from sklearn.model_selection import KFold
svm_model = SVC(kernel='linear', class_weight='balanced', C=1.0, random_state=123) 
svm_pca_model = svm_model.fit(df_train_pca2,y_train)
df_test_pca2 = pca_again.transform(X_test)
df_test_pca2.shape
pred_probs_test2 = svm_pca_model.predict(df_test_pca2)
"{:2.2f}".format(metrics.roc_auc_score(y_test, pred_probs_test2))
svm_model = SVC(kernel='linear', class_weight='balanced', C=1.0, random_state=123) 
svm_original_model = svm_model.fit(X_train,y_train)
pred_probs_test_original = svm_model.predict(X_test)
"{:2.2f}".format(metrics.roc_auc_score(y_test, pred_probs_test_original))

# Putting feature variable to X
X = decision_tree_data.drop(['churn'],axis=1)

# Putting response variable to y
y = decision_tree_data['churn']
# Splitting the data into train and test
X_train, X_test, y_train, y_test = train_test_split(X,y, train_size=0.7,test_size=0.3,random_state=100)
# Importing decision tree classifier from sklearn library
from sklearn.tree import DecisionTreeClassifier

# Fitting the decision tree with default hyperparameters, apart from
# max_depth which is 5 so that we can plot and read the tree.
dt_default = DecisionTreeClassifier(max_depth=5)
dt_default.fit(X_train, y_train)
# Let's check the evaluation metrics of our default model

# Importing classification report and confusion matrix from sklearn metrics
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score

# Making predictions
y_pred_default = dt_default.predict(X_test)

# Printing classification report
print(classification_report(y_test, y_pred_default))
# Printing confusion matrix and accuracy
print(confusion_matrix(y_test,y_pred_default))
print(accuracy_score(y_test,y_pred_default))
# Importing required packages for visualization
from IPython.display import Image  
from sklearn.externals.six import StringIO  
from sklearn.tree import export_graphviz
import pydot, graphviz

# Putting features
features = list(decision_tree_data.columns[:-1])
features
# plotting tree with max_depth=3
dot_data = StringIO()  
export_graphviz(dt_default, out_file=dot_data,
                feature_names=features, filled=True,rounded=True)

graph = pydot.graph_from_dot_data(dot_data.getvalue())  
Image(graph[0].create_png())
# GridSearchCV to find optimal max_depth
from sklearn.model_selection import KFold
from sklearn.model_selection import GridSearchCV


# specify number of folds for k-fold CV
n_folds = 5

# parameters to build the model on
parameters = {'max_depth': range(1, 40)}

# instantiate the model
dtree = DecisionTreeClassifier(criterion = "gini", 
                               random_state = 100)

# fit tree on training data
tree = GridSearchCV(dtree, parameters, 
                    cv=n_folds, 
                   scoring="accuracy")
tree.fit(X_train, y_train)
# scores of GridSearch CV
scores = tree.cv_results_
pd.DataFrame(scores).head()
# plotting accuracies with max_depth
plt.figure()
plt.plot(scores["param_max_depth"], 
         scores["mean_train_score"], 
         label="training accuracy")
plt.plot(scores["param_max_depth"], 
         scores["mean_test_score"], 
         label="test accuracy")
plt.xlabel("max_depth")
plt.ylabel("Accuracy")
plt.legend()
plt.show()

# GridSearchCV to find optimal max_depth
from sklearn.model_selection import KFold
from sklearn.model_selection import GridSearchCV


# specify number of folds for k-fold CV
n_folds = 5

# parameters to build the model on
parameters = {'min_samples_leaf': range(5, 200, 20)}

# instantiate the model
dtree = DecisionTreeClassifier(criterion = "gini", 
                               random_state = 100)

# fit tree on training data
tree = GridSearchCV(dtree, parameters, 
                    cv=n_folds, 
                   scoring="accuracy")
tree.fit(X_train, y_train)
# scores of GridSearch CV
scores = tree.cv_results_
pd.DataFrame(scores).head()
# plotting accuracies with min_samples_leaf
plt.figure()
plt.plot(scores["param_min_samples_leaf"], 
         scores["mean_train_score"], 
         label="training accuracy")
plt.plot(scores["param_min_samples_leaf"], 
         scores["mean_test_score"], 
         label="test accuracy")
plt.xlabel("min_samples_leaf")
plt.ylabel("Accuracy")
plt.legend()
plt.show()

# GridSearchCV to find optimal min_samples_split
from sklearn.model_selection import KFold
from sklearn.model_selection import GridSearchCV


# specify number of folds for k-fold CV
n_folds = 5

# parameters to build the model on
parameters = {'min_samples_split': range(5, 200, 20)}

# instantiate the model
dtree = DecisionTreeClassifier(criterion = "gini", 
                               random_state = 100)

# fit tree on training data
tree = GridSearchCV(dtree, parameters, 
                    cv=n_folds, 
                   scoring="accuracy")
tree.fit(X_train, y_train)
# scores of GridSearch CV
scores = tree.cv_results_
pd.DataFrame(scores).head()
# plotting accuracies with min_samples_leaf
plt.figure()
plt.plot(scores["param_min_samples_split"], 
         scores["mean_train_score"], 
         label="training accuracy")
plt.plot(scores["param_min_samples_split"], 
         scores["mean_test_score"], 
         label="test accuracy")
plt.xlabel("min_samples_split")
plt.ylabel("Accuracy")
plt.legend()
plt.show()
# Create the parameter grid 
param_grid = {
    'max_depth': range(5, 15, 5),
    'min_samples_leaf': range(50, 150, 50),
    'min_samples_split': range(50, 150, 50),
    'criterion': ["entropy", "gini"]
}

n_folds = 5

# Instantiate the grid search model
dtree = DecisionTreeClassifier()
grid_search = GridSearchCV(estimator = dtree, param_grid = param_grid, 
                          cv = n_folds, verbose = 1)

# Fit the grid search to the data
grid_search.fit(X_train,y_train)
# cv results
cv_results = pd.DataFrame(grid_search.cv_results_)
cv_results.head()
# printing the optimal accuracy score and hyperparameters
print("best accuracy", grid_search.best_score_)
print(grid_search.best_estimator_)
# model with optimal hyperparameters
clf_gini = DecisionTreeClassifier(criterion = "gini", 
                                  random_state = 100,
                                  max_depth=10, 
                                  min_samples_leaf=50,
                                  min_samples_split=50)
clf_gini.fit(X_train, y_train)
# accuracy score
clf_gini.score(X_test,y_test)
# plotting the tree
dot_data = StringIO()  
export_graphviz(clf_gini, out_file=dot_data,feature_names=features,filled=True,rounded=True)

graph = pydot.graph_from_dot_data(dot_data.getvalue())  
Image(graph[0].create_png())
# tree with max_depth = 3
clf_gini = DecisionTreeClassifier(criterion = "gini", 
                                  random_state = 100,
                                  max_depth=3, 
                                  min_samples_leaf=50,
                                  min_samples_split=50)
clf_gini.fit(X_train, y_train)

# score
print(clf_gini.score(X_test,y_test))
# plotting tree with max_depth=3
dot_data = StringIO()  
export_graphviz(clf_gini, out_file=dot_data,feature_names=features,filled=True,rounded=True)

graph = pydot.graph_from_dot_data(dot_data.getvalue())  
Image(graph[0].create_png())
# classification metrics
from sklearn.metrics import classification_report,confusion_matrix
y_pred = clf_gini.predict(X_test)
print(classification_report(y_test, y_pred))
# confusion matrix
print(confusion_matrix(y_test,y_pred))<jupyter_output><empty_output><jupyter_text>8. XGBoost<jupyter_code>from xgboost import XGBClassifier
from xgboost import plot_tree
churn_data = pd.read_csv('imputed_clean_churn_data.csv')
# Putting feature variable to X
X = churn_data.drop(['churn'],axis=1)

# Putting response variable to y
y = churn_data['churn']
# Splitting the data into train and test
X_train, X_test, y_train, y_test = train_test_split(X,y, train_size=0.7,test_size=0.3,random_state=100)
model = XGBClassifier(scale_pos_weight=0.5 )
model.fit(X_train, y_train)
# plot single tree

%matplotlib inline
plt.figure(figsize=(50,50))
plot_tree(model)
plt.show()
y_pred = model.predict_proba(X_test)
y_pred[:10]
roc = metrics.roc_auc_score(y_test, y_pred[:, 1])
print("AUC: %.2f%%" % (roc * 100.0))
from sklearn.model_selection import GridSearchCV


# specify number of folds for k-fold CV
n_folds = 5

# parameters to build the model on
parameters = {'max_depth': range(1, 40),
             'learning_rate': np.arange(0.0, 1.0, 0.01),
             'gamma': np.arange(0.0, 3.0, 0.1)}

# instantiate the model
xgboost = XGBClassifier(scale_pos_weight=0.5, random_state=40)

# fit tree on training data
xgboost = GridSearchCV(xgboost, parameters, 
                    cv=n_folds, 
                   scoring="accuracy")
xgboost.fit(X_train, y_train)
scores = xgboost.cv_results_
pd.DataFrame(scores).head()
# plotting accuracies with max_depth
plt.figure()
plt.plot(scores["param_max_depth"], 
         scores["mean_train_score"], 
         label="training accuracy")
plt.plot(scores["param_max_depth"], 
         scores["mean_test_score"], 
         label="test accuracy")
plt.xlabel("max_depth")
plt.ylabel("Accuracy")
plt.legend()
plt.show()
plt.figure()
plt.plot(scores["param_learning_rate"], 
         scores["mean_train_score"], 
         label="training accuracy")
plt.plot(scores["param_learning_rate"], 
         scores["mean_test_score"], 
         label="test accuracy")
plt.xlabel("Learning Rate")
plt.ylabel("Accuracy")
plt.legend()
plt.show()
plt.figure()
plt.plot(scores["param_gamma"], 
         scores["mean_train_score"], 
         label="training accuracy")
plt.plot(scores["param_gamma"], 
         scores["mean_test_score"], 
         label="test accuracy")
plt.xlabel("Gamma")
plt.ylabel("Accuracy")
plt.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1234454168585905, 'fraction_numerical': 0.030132043605097497, 'mean_word_length': 5.735522233712513, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '324652', 'license': 'MIT', 'n_tokens_mistral': 10235, 'n_tokens_neox': 9725, 'n_words': 1788, 'path': 'Telecom_churn_case_study.ipynb', 'repo_name': 'gettingtoknowdata/predict-churn', 'size': '26101'}