<jupyter_start><jupyter_text>Split in train and validation validation condiviso con le varie tecniche per il confronto, fatto con lo stratified per tenere tutto bilanciato con le classi.<jupyter_code>attributes = [col for col in df.columns if col != 'IsBadBuy']
X = df[attributes].values
y = df['IsBadBuy']

X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.3, stratify=y)<jupyter_output><empty_output><jupyter_text>Sampling Method Abbiamo scelto di utilizzare l'undersampling visto che con il Decision Tree era quello con il quale si ottenevano risultati migliori.<jupyter_code>rus = RandomUnderSampler(random_state=42)
print('Resampled dataset shape %s' % Counter(y_train))
X_train_res, y_train_res = rus.fit_resample(X_train, y_train)
print('Resampled dataset shape %s' % Counter(y_train_res))<jupyter_output>Resampled dataset shape Counter({0: 4670, 1: 4670})<jupyter_text>Naive Bayes<jupyter_code>gnb = GaussianNB()
%%timeit -n 1
gnb.fit(X_train_res, y_train_res)
gnb.fit(X_train_res, y_train_res)
%%timeit -n 1
gnb.predict(X_val)
y_pred = gnb.predict(X_val)
y_train_pred = gnb.predict(X_train_res)
#y_pred = gnb.fit(X_train_res, y_train_res).predict(X_val)
print("Number of mislabeled points out of a total %d points : %d" % (X_val.shape[0], (y_val != y_pred).sum()))<jupyter_output>Number of mislabeled points out of a total 16482 points : 5877<jupyter_text>35,7% di misclassified sul validation  Analyze the results<jupyter_code>roc_auc_models_u = []
for i in range(0,len(cnfs)):
    fpr, tpr, _ = roc_curve(y_train_res, y_pred_trains_u[i])
    roc_auc = auc(fpr, tpr)
    roc_auc = roc_auc_score(y_train_res, y_pred_trains_u[i], average=None)
    print("model {} - roc_auc: {}".format(i, roc_auc))
    roc_auc_models_u.append(roc_auc)
fpr, tpr, _ = roc_curve(y_train_res, y_train_pred)
roc_auc = auc(fpr, tpr)
roc_auc = roc_auc_score(y_train_res, y_train_pred, average=None)
print("model {} - roc_auc: {}".format(0, roc_auc))
print('Train Accuracy %s' % accuracy_score(y_train_res, y_train_pred))
print('Train F1-score %s' % f1_score(y_train_res, y_train_pred, average=None))
fpr, tpr, _ = roc_curve(y_val, y_pred)
roc_auc = auc(fpr, tpr)
roc_auc = roc_auc_score(y_val, y_pred, average=None)
print("model {} - roc_auc: {}".format(0, roc_auc))
print('Val Accuracy %s' % accuracy_score(y_val, y_pred))
print('Val F1-score %s' % f1_score(y_val,y_pred, average=None))
%matplotlib inline
plt.figure(figsize=(8, 5))
plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % (roc_auc))
    
plt.plot([0, 1], [0, 1], 'k--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate') 
plt.tick_params(axis='both', which='major')
plt.legend(loc="lower right", fontsize=14, frameon=False)
plt.show()<jupyter_output><empty_output><jupyter_text>Naive Bayes with SMOTE<jupyter_code>sm = SMOTE(random_state=42)
print('Resampled dataset shape %s' % Counter(y_train))
X_train_res_smote, y_train_res_smote = sm.fit_resample(X_train, y_train)
print('Resampled dataset shape %s' % Counter(y_train_res_smote))
gnb2 = GaussianNB()
gnb2.fit(X_train_res_smote, y_train_res_smote)
%%timeit -n 1
gnb2.predict(X_val)
#y_pred_over della cella precedente non viene salvato
y_pred_smote = gnb2.predict(X_val)
print("Number of mislabeled points out of a total %d points : %d" % (X_val.shape[0], (y_val != y_pred_smote).sum()))<jupyter_output>Number of mislabeled points out of a total 16482 points : 7631<jupyter_text>46,3% di misclassified  Naive Bayes with oversampling<jupyter_code>ros = RandomOverSampler(random_state=42)
print('Resampled dataset shape %s' % Counter(y_train))
X_train_res_over, y_train_res_over = ros.fit_resample(X_train, y_train)
print('Resampled dataset shape %s' % Counter(y_train_res_over))
%%timeit -n 1
gnb2.predict(X_val)
y_pred_over = gnb2.predict(X_val)
print("Number of mislabeled points out of a total %d points : %d" % (X_val.shape[0], (y_val != y_pred_over).sum()))<jupyter_output>Number of mislabeled points out of a total 16482 points : 7631<jupyter_text>46,3% di misclassified  Random Forest Ho visto che Random Forest non è richiesta.  Gridsearch<jupyter_code>param_list = {'n_estimators': list(np.arange(2, 100)),
              'criterion': ['gini', 'entropy'],
              'max_depth': [None] + list(np.arange(2, 100)),
              'min_samples_split': list(np.arange(2, 100)),
              'min_samples_leaf': list(np.arange(1, 100)),
             }
new_params = {'randomforestclassifier__' + key: param_list[key] for key in param_list}

skf = StratifiedKFold(n_splits=3)
clf = RandomForestClassifier(n_estimators=2, criterion='gini', max_depth=None, min_samples_split=2, min_samples_leaf=1)
imba_pipeline = make_pipeline(RandomUnderSampler(), clf)
scoring = ['accuracy', 'f1', 'roc_auc' ]

random_search = RandomizedSearchCV(imba_pipeline, param_distributions=new_params, n_iter=1000, cv=skf, scoring=scoring, refit = 'roc_auc', n_jobs = 4, verbose = 1, return_train_score=True)
random_search.fit(X_train, y_train)
cnfs = report_multiple(random_search.cv_results_, n_top=3, scoring = 'roc_auc')<jupyter_output>Fitting 3 folds for each of 1000 candidates, totalling 3000 fits<jupyter_text>Perform Classification<jupyter_code>models_u = []
y_pred_vals_u = []
y_pred_trains_u = []
hyper_ps = random_search.cv_results_
for cnf in cnfs.values():
    n_estimators = cnf['randomforestclassifier__n_estimators']
    criterion = cnf['randomforestclassifier__criterion']
    max_depth = cnf['randomforestclassifier__max_depth']
    min_samples_split = cnf['randomforestclassifier__min_samples_split']
    min_samples_leaf = cnf['randomforestclassifier__min_samples_leaf']
    clf = RandomForestClassifier(n_estimators=n_estimators, criterion=criterion, max_depth=max_depth, min_samples_split=min_samples_split, min_samples_leaf=min_samples_leaf)
    clf = clf.fit(X_train_res, y_train_res)
    models_u.append(clf)
    y_pred = clf.predict(X_val)
    y_pred_tr = clf.predict(X_train_res)
    y_pred_vals_u.append(y_pred)
    y_pred_trains_u.append(y_pred_tr)<jupyter_output><empty_output><jupyter_text>Analyze the classification results<jupyter_code>for i in range(0,len(cnfs)):
    print("model {}".format(i))
    print('Train Accuracy %s' % accuracy_score(y_train_res, y_pred_trains_u[i]))
    print('Train F1-score %s' % f1_score(y_train_res, y_pred_trains_u[i], average=None))
    print()

    print('Test Accuracy %s' % accuracy_score(y_val, y_pred_vals_u[i]))
    print('Test F1-score %s' % f1_score(y_val, y_pred_vals_u[i], average=None))
    print(classification_report(y_val, y_pred_vals_u[i]))
    print(confusion_matrix(y_val, y_pred_vals_u[i]))
roc_auc_models_u = []
for i in range(0,len(cnfs)):
    fpr, tpr, _ = roc_curve(y_train_res, y_pred_trains_u[i])
    roc_auc = auc(fpr, tpr)
    roc_auc = roc_auc_score(y_train_res, y_pred_trains_u[i], average=None)
    print("model {} - roc_auc: {}".format(i, roc_auc))
    roc_auc_models_u.append(roc_auc)
roc_auc_models_u = []
for i in range(0,len(cnfs)):
    fpr, tpr, _ = roc_curve(y_val, y_pred_vals_u[i])
    roc_auc = auc(fpr, tpr)
    roc_auc = roc_auc_score(y_val, y_pred_vals_u[i], average=None)
    print("model {} - roc_auc: {}".format(i, roc_auc))
    roc_auc_models_u.append(roc_auc)<jupyter_output>model 0 - roc_auc: 0.6357027295908513
model 1 - roc_auc: 0.6439090605527069
model 2 - roc_auc: 0.6403717829132193<jupyter_text>Choose the best model Come miglior modello scelgo il model 2, essendo quello con la ROC AUC migliore.{'randomforestclassifier__n_estimators': 87, 'randomforestclassifier__min_samples_split': 23, 'randomforestclassifier__min_samples_leaf': 3, 'randomforestclassifier__max_depth': 59, 'randomforestclassifier__criterion': 'gini'}<jupyter_code>clf = RandomForestClassifier(n_estimators=87, criterion='gini', max_depth=59, min_samples_split=23, min_samples_leaf=3)
%%timeit -n 1
clf.fit(X_train_res, y_train_res)
%%timeit -n 1
clf.predict(X_val)
y_pred = clf.predict(X_val)
fpr, tpr, _ = roc_curve(y_val, y_pred)
roc_auc = auc(fpr, tpr)
roc_auc = roc_auc_score(y_val, y_pred, average=None)
print("model {} - roc_auc: {}".format(0, roc_auc))
%matplotlib inline
plt.figure(figsize=(8, 5))
plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % (roc_auc))
    
plt.plot([0, 1], [0, 1], 'k--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate') 
plt.tick_params(axis='both', which='major')
plt.legend(loc="lower right", fontsize=14, frameon=False)
plt.show()<jupyter_output><empty_output><jupyter_text>Features importance<jupyter_code>for col, imp in zip(attributes, clf.feature_importances_):
    print(col, imp)
importances = clf.feature_importances_
# Sort feature importances in descending order
indices = np.argsort(importances)[::-1]

df1 = df
del df1['IsBadBuy']

# Rearrange feature names so they match the sorted feature importances
names = [df1.columns[i] for i in indices]

# Create plot
plt.figure(figsize=(15, 5))

# Create plot title
plt.title("Feature Importance")

# Add bars
plt.bar(range(X.shape[1]), importances[indices])

# Add feature names as x-axis labels
plt.xticks(range(X.shape[1]), names, rotation=90)

# Show plot
plt.show()<jupyter_output><empty_output><jupyter_text>K-NN  Gridsearch<jupyter_code>param_list = {'n_neighbors': list(np.arange(2, 200)),
              'weights': ['uniform', 'distance'],
              'algorithm': ['auto'],
              'leaf_size': list(np.arange(2, 200)),
             }
new_params = {'kneighborsclassifier__' + key: param_list[key] for key in param_list}

skf = StratifiedKFold(n_splits=3)
clf = KNeighborsClassifier(n_neighbors=2, weights='uniform', algorithm='auto', leaf_size=2)
imba_pipeline = make_pipeline(RandomUnderSampler(), clf)
scoring = ['accuracy', 'f1', 'roc_auc' ]

random_search = RandomizedSearchCV(imba_pipeline, param_distributions=new_params, n_iter=1000, cv=skf, scoring=scoring, refit = 'roc_auc', n_jobs = 4, verbose = 1, return_train_score=True)
random_search.fit(X_train, y_train)
cnfs = report_multiple(random_search.cv_results_, n_top=3, scoring = 'roc_auc')<jupyter_output>Fitting 3 folds for each of 1000 candidates, totalling 3000 fits<jupyter_text>Non capisco se è andato in overfitting o no. Lo provo sul validation esterno e dopo provo un'altra grid.  Perform Classification<jupyter_code>models_u = []
y_pred_vals_u = []
y_pred_trains_u = []
hyper_ps = random_search.cv_results_
for cnf in cnfs.values():
    n_neighbors = cnf['kneighborsclassifier__n_neighbors']
    weights = cnf['kneighborsclassifier__weights']
    algorithm = cnf['kneighborsclassifier__algorithm']
    leaf_size = cnf['kneighborsclassifier__leaf_size']
    clf = KNeighborsClassifier(n_neighbors=n_neighbors, weights=weights, algorithm=algorithm, leaf_size=leaf_size)
    clf = clf.fit(X_train_res, y_train_res)
    models_u.append(clf)
    y_pred = clf.predict(X_val)
    y_pred_tr = clf.predict(X_train_res)
    y_pred_vals_u.append(y_pred)
    y_pred_trains_u.append(y_pred_tr)<jupyter_output><empty_output><jupyter_text>Analyze the classification results<jupyter_code>for i in range(0,len(cnfs)):
    print("model {}".format(i))
    print('Train Accuracy %s' % accuracy_score(y_train_res, y_pred_trains_u[i]))
    print('Train F1-score %s' % f1_score(y_train_res, y_pred_trains_u[i], average=None))
    print()

    print('Test Accuracy %s' % accuracy_score(y_val, y_pred_vals_u[i]))
    print('Test F1-score %s' % f1_score(y_val, y_pred_vals_u[i], average=None))
    print(classification_report(y_val, y_pred_vals_u[i]))
    print(confusion_matrix(y_val, y_pred_vals_u[i]))
roc_auc_models_u = []
for i in range(0,len(cnfs)):
    fpr, tpr, _ = roc_curve(y_train_res, y_pred_trains_u[i])
    roc_auc = auc(fpr, tpr)
    roc_auc = roc_auc_score(y_train_res, y_pred_trains_u[i], average=None)
    print("model {} - roc_auc: {}".format(i, roc_auc))
    roc_auc_models_u.append(roc_auc)
roc_auc_models_u = []
for i in range(0,len(cnfs)):
    fpr, tpr, _ = roc_curve(y_val, y_pred_vals_u[i])
    roc_auc = auc(fpr, tpr)
    roc_auc = roc_auc_score(y_val, y_pred_vals_u[i], average=None)
    print("model {} - roc_auc: {}".format(i, roc_auc))
    roc_auc_models_u.append(roc_auc)<jupyter_output>model 0 - roc_auc: 0.6102710825086516
model 1 - roc_auc: 0.6096736481750294
model 2 - roc_auc: 0.6075214495449303<jupyter_text>Choose the best model Il modello migliore sembrerebbe il model 0, anche se probabilmente è in overfitting.{'kneighborsclassifier__weights': 'distance', 'kneighborsclassifier__n_neighbors': 135, 'kneighborsclassifier__leaf_size': 142, 'kneighborsclassifier__algorithm': 'auto'}<jupyter_code>neigh = KNeighborsClassifier(n_neighbors=135, weights='distance', algorithm='auto', leaf_size=142)
%%timeit -n 1
neigh.fit(X_train_res, y_train_res)
neigh.fit(X_train_res, y_train_res)
%%timeit -n 1
neigh.predict(X_val)
y_pred = neigh.predict(X_val)
fpr, tpr, _ = roc_curve(y_val, y_pred)
roc_auc = auc(fpr, tpr)
roc_auc = roc_auc_score(y_val, y_pred, average=None)
print("model {} - roc_auc: {}".format(0, roc_auc))
%matplotlib inline
plt.figure(figsize=(8, 5))
plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % (roc_auc))
    
plt.plot([0, 1], [0, 1], 'k--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate') 
plt.tick_params(axis='both', which='major')
plt.legend(loc="lower right", fontsize=14, frameon=False)
plt.show()<jupyter_output><empty_output><jupyter_text>Another gridsearch<jupyter_code>param_list = {'n_neighbors': [200, 300, 400, 500, 600, 700, 800, 1000],
              'weights': ['distance'],
              'algorithm': ['auto'],
              'leaf_size': [100, 130, 160, 190, 220],
             }
new_params = {'kneighborsclassifier__' + key: param_list[key] for key in param_list}

skf = StratifiedKFold(n_splits=3)
clf = KNeighborsClassifier(n_neighbors=200, weights='distance', algorithm='auto', leaf_size=100)
imba_pipeline = make_pipeline(RandomUnderSampler(), clf)
scoring = ['accuracy', 'f1', 'roc_auc' ]

random_search = RandomizedSearchCV(imba_pipeline, param_distributions=new_params, n_iter=1000, cv=skf, scoring=scoring, refit = 'roc_auc', n_jobs = 4, verbose = 1, return_train_score=True)
random_search.fit(X_train, y_train)
cnfs = report_multiple(random_search.cv_results_, n_top=3, scoring = 'roc_auc')<jupyter_output>C:\Users\Giulia\Anaconda3\lib\site-packages\sklearn\model_selection\_search.py:281: UserWarning: The total space of parameters 40 is smaller than n_iter=1000. Running 40 iterations. For exhaustive searches, use GridSearchCV.
  % (grid_size, self.n_iter, grid_size), UserWarning)
[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.<jupyter_text>Other tests<jupyter_code>neigh = KNeighborsClassifier(n_neighbors=9000, weights='distance', algorithm='auto', leaf_size=150)
neigh.fit(X_train_res, y_train_res)
y_pred = neigh.predict(X_val)
y_pred_train = neigh.predict(X_train_res)

fpr, tpr, _ = roc_curve(y_train_res, y_pred_train)
roc_auc = auc(fpr, tpr)
roc_auc = roc_auc_score(y_train_res, y_pred_train, average=None)
print("model {} - roc_auc: {}".format(0, roc_auc))

fpr, tpr, _ = roc_curve(y_val, y_pred)
roc_auc = auc(fpr, tpr)
roc_auc = roc_auc_score(y_val, y_pred, average=None)
print("model {} - roc_auc: {}".format(0, roc_auc))<jupyter_output>model 0 - roc_auc: 1.0
model 0 - roc_auc: 0.5910739812673513<jupyter_text>Ho aumentato gradualmente il numero di vicini per cercare di migliorare la roc sul validation e di peggiorare quella sul training, ma niente.  Choose the final best model Prendo quello selezionato dopo la prima gridsearch<jupyter_code>neigh = KNeighborsClassifier(n_neighbors=135, weights='distance', algorithm='auto', leaf_size=142)
%%timeit -n 1
neigh.fit(X_train_res, y_train_res)
neigh.fit(X_train_res, y_train_res)
%%timeit -n 1
neigh.predict(X_val)
y_pred = neigh.predict(X_val)
fpr, tpr, _ = roc_curve(y_val, y_pred)
roc_auc = auc(fpr, tpr)
roc_auc = roc_auc_score(y_val, y_pred, average=None)
print("model {} - roc_auc: {}".format(0, roc_auc))
%matplotlib inline
plt.figure(figsize=(8, 5))
plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % (roc_auc))
    
plt.plot([0, 1], [0, 1], 'k--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate') 
plt.tick_params(axis='both', which='major')
plt.legend(loc="lower right", fontsize=14, frameon=False)
plt.show()<jupyter_output><empty_output><jupyter_text>Decision Tree<jupyter_code>dtc = DecisionTreeClassifier(criterion='gini', max_depth=5, min_samples_split=26, min_samples_leaf=25)
%%timeit -n 1
dtc.fit(X_train_res, y_train_res)
dtc.fit(X_train_res, y_train_res)
%%timeit -n 1
dtc.predict(X_val)
y_pred = dtc.predict(X_val)
fpr, tpr, _ = roc_curve(y_val, y_pred)
roc_auc = auc(fpr, tpr)
roc_auc = roc_auc_score(y_val, y_pred, average=None)
print("model {} - roc_auc: {}".format(0, roc_auc))
%matplotlib inline
plt.figure(figsize=(8, 5))
plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % (roc_auc))
    
plt.plot([0, 1], [0, 1], 'k--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate') 
plt.tick_params(axis='both', which='major')
plt.legend(loc="lower right", fontsize=14, frameon=False)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1373263888888889, 'fraction_numerical': 0.02991898148148148, 'mean_word_length': 6.461571675302245, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '442483', 'license': 'MIT', 'n_tokens_mistral': 7384, 'n_tokens_neox': 6877, 'n_words': 1307, 'path': 'notebook/4_classification/9_CFU/c_9_CFU.ipynb', 'repo_name': 'alessandrocuda/carvana', 'size': '17280'}