<jupyter_start><jupyter_text>Data Preprocessing<jupyter_code># pandas-profiling packages
from pandas_profiling import ProfileReport
profile = ProfileReport(df_train,minimal=True)
profile.to_widgets()
# first find the numerical and categorical features
df_train_num = df_train.select_dtypes(exclude='object')
df_train_cat = df_train.select_dtypes(include='object')
num_features = df_train_num.columns.tolist()
cat_features = df_train_cat.columns.tolist()
# remove the target variable "Churn" and the unuseful features "customerID","Unnamed: 0"
# cat_features.remove('Churn')
# cat_features.remove('customerID')
cat_features = [e for e in cat_features if e not in ('Churn','customerID')]
# classified some numerical variables into categorical variables
num_to_cat = []

df_train['SeniorCitizen'].value_counts()
num_to_cat.append('SeniorCitizen')
# change this features to categorical
num_features = [element for element in num_features if element not in num_to_cat]
cat_features += num_to_cat<jupyter_output><empty_output><jupyter_text>Outliers<jupyter_code># build the IQR outlier remover
# note that you have to remove the rows for both X and y 

def find_outlier_iqr(X):
    # calculate IQR
    q1 = X.quantile(0.25)
    q3 = X.quantile(0.75)
    iqr = q3 - q1
    
    outlier_index = ((X < (q1 - 1.5 * iqr)) | (X > (q3 + 1.5 * iqr))).any(axis=1)
    
    return outlier_index
# this shows how many outliers will be removed from the dataset
outlier_index = find_outlier_iqr(df_train[num_features])
# before remove outliers
len_df_train = len(df_train)
# remove the outliers from the dataset
df_train_iqr = df_train[~outlier_index]
# after remove outliers
len_df_train_after = len(df_train_iqr)
print(f'we will remove {(len_df_train-len_df_train_after)/len_df_train:.2%} outliers')<jupyter_output>we will remove 0.28% outliers<jupyter_text>Building the Pipeline<jupyter_code># prepare the data
X = df_train_iqr.drop(['Churn'],axis=1)
y = df_train_iqr['Churn']
df_train_iqr.columns
# Split the data into a training set and a test set. 
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

print(X_train.shape)
print(X_test.shape)
# building the pipeline
# Create the preprocessing pipeline for numerical and categorical features
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler, OneHotEncoder

num_pipeline = Pipeline(
    steps=[
        ('num_imputer', SimpleImputer(strategy='mean')),
        ('scaler', StandardScaler()),
        ]
)

cat_pipeline = Pipeline(
    steps=[
        ('cat_imputer', SimpleImputer(strategy='most_frequent')),
        ('onehot', OneHotEncoder()),
    ]
)
# Combine two pipelines to form the preprocessor
from sklearn.compose import ColumnTransformer

preprocessor = ColumnTransformer(
    transformers=[
        ('num_pipeline', num_pipeline, num_features),
        ('cat_pipeline', cat_pipeline, cat_features),
    ]
)
# Make a full pipeline by combining preprocessor and the model
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC


tree_clf_model = Pipeline(
    steps=[
        ('preprocessor', preprocessor),
        ('tree_clf', DecisionTreeClassifier()),
    ]
)

log_reg_pipeline = Pipeline(
    steps=[
        ('preprocessor', preprocessor),
        ('log_reg', LogisticRegression()),
    ]
)

svm_clf_model = Pipeline(
    steps=[
        ('preprocessor', preprocessor),
        ('svm_clf',SVC()),
    ]
)<jupyter_output><empty_output><jupyter_text>Decision Tree<jupyter_code># use grid search to find the best decision tree model
from sklearn.model_selection import GridSearchCV


# here you must use the step names as the prefix followed by two under_scores to sepecify the parameter names
# you also need to specify the "full path" of the steps
# as you can see we can even grid search parameters for preprocessing pipeline step
param_grid = [
    {
        'preprocessor__num_pipeline__num_imputer__strategy': ['mean', 'median'],
        'tree_clf__criterion': ['gini', 'entropy'], 
        'tree_clf__max_depth': [2, 3, 4, 5, 6, 7],
    }
]

# set up the grid search 
grid_search = GridSearchCV(tree_clf_model, param_grid, cv=5,
                          scoring='accuracy',
                          return_train_score=True)
# train the model using the full pipeline
grid_search.fit(X_train, y_train)
# check the best performing parameter combination
grid_search.best_params_
# get the best trained model
tree_clf_best = grid_search.best_estimator_

# prediction for the test set
y_pred = tree_clf_best.predict(X_test)
# make predictions for kaggle submission
churn_test = pd.read_csv('udel-churn-test.csv')

y_pred_churn = tree_clf_best.predict(churn_test)

# combine id and prediction for kaggle submission
churn_submit = pd.DataFrame({
    'customerID': churn_test['customerID'],
    'Churn': y_pred_churn
})

# generate the csv
churn_submit.to_csv('churn-submit_decisiontree.csv', index=False)
print('csv saved! please submit the prediction csv to Kaggle.com')<jupyter_output>csv saved! please submit the prediction csv to Kaggle.com<jupyter_text>Logistic Regression<jupyter_code># conduct cross validation with the full pipeline
from sklearn.model_selection import cross_val_score

log_reg_scores_accu = cross_val_score(log_reg_pipeline, X_train, y_train, scoring="accuracy", cv=10)
print(f'cross validation average accuracy score: {log_reg_scores_accu.mean()}')
# we train the final model using ALL training data
log_reg_pipeline.fit(X_train, y_train)

# # use named_steps[] to access any step in your pipeline
print(log_reg_pipeline.named_steps['log_reg'].intercept_)
print(log_reg_pipeline.named_steps['log_reg'].coef_)

# get the prediction results from the testing set
y_pred = log_reg_pipeline.predict(X_test)

# calculate accuracy
# we have similiar score for the testing set as the cross validation score - good
from sklearn.metrics import accuracy_score

print(f'Testing set accuracy score : {accuracy_score(y_test,y_pred)}')
# make prediction using linear regression model
y_pred_logistic = log_reg_pipeline.predict(churn_test)

# combine id and prediction for kaggle submission
churn_submit = pd.DataFrame({
    'customerID': churn_test['customerID'],
    'Churn': y_pred_logistic
})

# generate the csv
churn_submit.to_csv('churn-submit_log.csv', index=False)
print('csv saved! please submit the prediction csv to Kaggle.com')<jupyter_output>csv saved! please submit the prediction csv to Kaggle.com<jupyter_text>SVM<jupyter_code># use grid search to find the best decision tree model
from sklearn.model_selection import GridSearchCV


# here you must use the step names as the prefix followed by two under_scores to sepecify the parameter names
# you also need to specify the "full path" of the steps
# as you can see we can even grid search parameters for preprocessing pipeline step
param_grid = [
    {
        'svm_clf__kernel': ['linear','poly','rbf'],
        'svm_clf__degree': [1,2,3], 
        'svm_clf__C': [1,3,5],
    }
]

# set up the grid search 
grid_search = GridSearchCV(svm_clf_model, param_grid, cv=5,
                          scoring='accuracy',
                          return_train_score=True)
grid_search.fit(X_train, y_train)
grid_search.best_params_
SVM_reg_best = grid_search.best_estimator_
SVM_reg_best
# make predictions for kaggle submission
y_pred_churn = SVM_reg_best.predict(churn_test)

# combine id and prediction for kaggle submission
churn_submit = pd.DataFrame({
    'customerID': churn_test['customerID'],
    'Churn': y_pred_churn
})

# generate the csv
churn_submit.to_csv('churn-submit_SVM.csv', index=False)
print('csv saved! please submit the prediction csv to Kaggle.com')<jupyter_output>csv saved! please submit the prediction csv to Kaggle.com
{'fraction_non_alphanumeric': 0.09105855571013026, 'fraction_numerical': 0.005438219299355002, 'mean_word_length': 5.027439024390244, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '336967', 'license': 'MIT', 'n_tokens_mistral': 2615, 'n_tokens_neox': 2537, 'n_words': 701, 'path': 'Assignment11/Assignment11-Logistic-Regression-finished-XigangHuang.ipynb', 'repo_name': 'hxg10636/misy467-assignments', 'size': '7907'}