<jupyter_start><jupyter_text>Model selection*Note:* this notebook has been runned on Microsoft Azure.<jupyter_code># Set up X and y dataframe, train and test split
y=df['Satisfaction']
X=df.iloc[:, :-1]
X_train, X_test, y_train, y_test = train_test_split(X, y, 
                                                    test_size =0.30, 
                                                    stratify=y,  
                                                    random_state= 42)
print(X_train.shape[0])
print(X_test.shape[0])
# Useful function
def hyperp_search(classifier, parameters):
    gs = GridSearchCV(classifier, parameters, cv=3, scoring = 'f1', verbose=0, n_jobs=-1)
    gs = gs.fit(X_train, y_train)
    print("f1_train: %f using %s" % (gs.best_score_, gs.best_params_))

    best_model = gs.best_estimator_
    y_pred = best_model.predict(X_test)
    y_pred_train = best_model.predict(X_train)

    print("f1=====\ntrain: %.3f\ntest: %.3f\n=====" % (f1_score(y_train, y_pred_train), f1_score(y_test, y_pred) )) 
    print("\nPrecision=====\ntrain: %.3f\ntest: %.3f\n=====" % (precision_score(y_train, y_pred_train), precision_score(y_test, y_pred) )) 
    print("")
    sns.heatmap(confusion_matrix(y_test, y_pred), annot=True, fmt = '2d', cmap='Reds')
    plt.show()
    print(classification_report(y_test, y_pred))

def roc(model,X_train,y_train,X_test,y_test):
    model.fit(X_train, y_train)
    y_pred = model.predict(X_test)

    y_probs = model.predict_proba(X_test) #predict_proba gives the probabilities for the target (0 and 1 in your case) 

    fpr, tpr, thresholds1=metrics.roc_curve(y_test,  y_probs[:,1])

    import matplotlib.pyplot as plt
    plt.plot(fpr, tpr, label='ROC')
    plt.plot([0, 1], [0, 1], color='darkblue', linestyle='--')
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver Operating Characteristic (ROC) Curve')
    plt.legend()
    plt.show()

    auc = metrics.roc_auc_score(y_test, y_probs[:,1])
    print('AUC: %.2f' % auc)
    return (fpr, tpr)<jupyter_output><empty_output><jupyter_text>KNN model<jupyter_code>from sklearn.neighbors import KNeighborsClassifier
classifier = KNeighborsClassifier()
parameters = {'n_neighbors':np.arange(10,500,20)}
hyperp_search(classifier,parameters)
parameters = {'n_neighbors':np.arange(8,15,1)}
hyperp_search(classifier, parameters)
plt.savefig('KNN_confusionMatrix.png', dpi = 200)
model_knn = KNeighborsClassifier(n_neighbors=13)
fpr1,tpr1=roc(model_knn,X_train,y_train,X_test,y_test)<jupyter_output><empty_output><jupyter_text>Tree classification<jupyter_code>from sklearn.tree import DecisionTreeClassifier
classifier = DecisionTreeClassifier()
parameters = {'criterion': ['entropy','gini'], 
              'max_depth': np.arange(8,15,1),
              'min_samples_split': np.arange(8,15,1),
              'min_samples_leaf': np.arange(8,15,1)}
hyperp_search(classifier,parameters)
parameters = {'criterion': ['entropy','gini'], 
              'max_depth': np.arange(5,12,1),
              'min_samples_split': np.arange(5,12,1),
              'min_samples_leaf': np.arange(5,15,1)}

hyperp_search(classifier,parameters)
model_tree = DecisionTreeClassifier(criterion='gini', max_depth=11, min_samples_leaf=7, min_samples_split=9)
fpr2,tpr2=roc(model_tree,X_train,y_train,X_test,y_test)<jupyter_output><empty_output><jupyter_text>Naive Bayes - Gaussian<jupyter_code># Naive Bayes
from sklearn.naive_bayes import GaussianNB
model = GaussianNB()

model.fit(X_train, y_train)
y_pred=model.predict(X_test)

from sklearn.metrics import f1_score
print("f1_score: ", f1_score(y_test, y_pred))

print("f1_test: ", f1_score(y_test, y_pred))


print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))
fpr3,tpr3=roc(model,X_train,y_train,X_test,y_test)<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code># Logistic

from sklearn.linear_model import LogisticRegression

classifier = LogisticRegression()
parameters = {"C":[1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1, 10, 100, 1000], "max_iter":[1000] }
hyperp_search(classifier,parameters)
parameters = {"C":np.arange(1, 10,1), "max_iter": [1000]}
hyperp_search(classifier,parameters)
model = LogisticRegression(C=10, max_iter=1000)

fpr4,tpr4=roc(model,X_train,y_train,X_test,y_test)
model.fit(X_train,y_train)

coeff=pd.DataFrame()
coeff["feature"]=X_train.columns
coeff["w"]=model.coef_[0]

coeff.sort_values(by=['w'], inplace=True)
fig = plt.figure(figsize=(15,15))
sns.barplot(data=coeff, y="feature", x="w", palette="Blues_d", orient="h")
sns.set(rc={'figure.figsize':(6,4)})<jupyter_output><empty_output><jupyter_text>SVM<jupyter_code>from sklearn.svm import SVC

classifier = SVC()
parameters = {"kernel":['linear','rbf'], "C":[0.1,100]}
hyperp_search(classifier,parameters)
parameters = {"kernel":['rbf'], "C":[10,20]}
hyperp_search(classifier,parameters)
parameters = {"kernel":['rbf'], "C":[5, 6, 7, 8, 9]}
hyperp_search(classifier,parameters)
model = SVC(C=6, kernel='rbf',probability=True)

fpr5,tpr5=roc(model,X_train,y_train,X_test,y_test)<jupyter_output><empty_output><jupyter_text>Multi-layer perceptron<jupyter_code>from sklearn.neural_network import MLPClassifier

classifier = MLPClassifier()
parameters = {"hidden_layer_sizes":[(10, 5),(100,20,5)],  "max_iter": [2000], "alpha": [0.001,0.1]}
hyperp_search(classifier,parameters)
parameters = {"hidden_layer_sizes":[(100,20,5), (3,2,3), (100), (3), (150, 35, 7)],  "max_iter": [2000], "alpha": [0.1, 0.7]}
hyperp_search(classifier,parameters)
parameters = {"hidden_layer_sizes":[(150, 35, 7), (180, 40, 10)],  "max_iter": [2000], "alpha": [0.7, 0.8, 0.9]}
hyperp_search(classifier,parameters)
model_MLP=MLPClassifier(hidden_layer_sizes=(180, 40, 10), alpha=0.7, max_iter=2000)

fpr6,tpr6=roc(model_MLP,X_train,y_train,X_test,y_test)
plt.plot(fpr1, tpr1, label= "KNN")
plt.plot(fpr2, tpr2, label= "Tree")
plt.plot(fpr3, tpr3, label= "NB")
plt.plot(fpr4, tpr4, label= "Logistic")    
plt.plot(fpr5, tpr5, label= "SVM")
plt.plot(fpr6, tpr6, label= "NeuralNet")
plt.plot([0, 1], [0, 1], color='darkblue', linestyle='--')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver Operating Characteristic (ROC) Curve')
plt.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16382741601655787, 'fraction_numerical': 0.040120999840789685, 'mean_word_length': 5.806067172264355, 'pattern_counts': {'":': 19, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '44418', 'license': 'MIT', 'n_tokens_mistral': 2632, 'n_tokens_neox': 2443, 'n_words': 388, 'path': 'notebooks/3.0-fc-HyperSearch.ipynb', 'repo_name': 'FedericoCantarelli/Customer-Satisfaction-Yojo', 'size': '6281'}