<jupyter_start><jupyter_text>Libraries<jupyter_code>from numpy import mean, std
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import auc
from sklearn.metrics import make_scorer
from sklearn.dummy import DummyClassifier
from sklearn.pipeline import Pipeline
from sklearn.model_selection import RepeatedStratifiedKFold, StratifiedKFold
from sklearn.tree import DecisionTreeClassifier
from sklearn.preprocessing import StandardScaler
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from xgboost import XGBClassifier
from sklearn.model_selection import cross_val_score, cross_validate
from sklearn.utils import resample
from imblearn.over_sampling import SMOTE
from sklearn import tree
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score, f1_score
from sklearn.inspection import permutation_importance
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import MinMaxScaler,PowerTransformer
import os<jupyter_output><empty_output><jupyter_text>Load data and resample<jupyter_code># load the dataset
def load_dataset(file_path):
    # load the dataset as a dataframe
    data_df = pd.read_pickle(file_path)
    # correct data types
    fix_cols = data_df.columns[data_df.dtypes == 'object']
    data_df.loc[:, fix_cols] = data_df.loc[:, fix_cols].astype('bool')
    # remove redundant features
    data_df.drop(['total_review_count', 'halal_review_count', 'halal_negation_count', 'halal_truck_count'], inplace=True, axis=1)
    # split into features and target
    X, y = data_df.drop('halal', axis=1), data_df.halal
    return X, y

# define resampling method
def split_and_resample(X, y, test_size=0, resampling=None):
    if test_size > 0:
        # setting up testing and training sets
        x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=test_size)
    else:
        x_train, x_test, y_train, y_test = X, pd.DataFrame(columns=X.columns, dtype=X.dtypes.values),\
                                            y, pd.Series(name=y.name, dtype=y.dtypes)
    # concatenate our training data back together
    X = pd.concat([x_train, y_train], axis=1)
    # separate minority and majority classes
    min_class = X[X.halal==X.halal.value_counts().sort_values().index[0]]
    max_class = X[X.halal==X.halal.value_counts().sort_values().index[1]]
    if not resampling:
        return x_train, y_train, x_test, y_test
    # oversampling minority
    if resampling == 'oversample':
        new_min_class = resample(min_class, replace=True, n_samples=(len(max_class)))
        upsampled = pd.concat([max_class, new_min_class])
        return upsampled.drop('halal', axis=1), upsampled.halal, x_test, y_test
    elif resampling == 'undersample':
        new_max_class = resample(max_class, replace=True, n_samples=(len(min_class)))
        downsampled = pd.concat([new_max_class, min_class])
        return downsampled.drop('halal', axis=1), downsampled.halal, x_test, y_test
    elif resampling == 'SMOTE':
        x_train, y_train = SMOTE().fit_sample(x_train, y_train)
        return x_train, y_train, x_test, y_test

# define models to test
def get_models():
    models = list()
    # BASE
    models.append(DummyClassifier(strategy='constant', constant=1))
    # CART
    models.append(DecisionTreeClassifier()) # max depth set to 15 to avoid overfitting
    # KNN
    steps = [('s',StandardScaler()),('m',KNeighborsClassifier())]
    models.append(Pipeline(steps=steps))
    # RF
    models.append(RandomForestClassifier(n_estimators=10))
    # XGBoost
    models.append(XGBClassifier(learning_rate = 0.05, n_estimators=300))
    return models

# calculate precision-recall area under curve
def pr_auc(y_true, probas_pred):
   # calculate precision-recall curve
   p, r, _ = precision_recall_curve(y_true, probas_pred)
   # calculate area under curve
   return auc(r, p)

# evaluate a model
def evaluate_model(X, y, model):
    # define evaluation procedure
    cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3)
    # define the model evaluation the metric
    metric = make_scorer(pr_auc, needs_proba=True)
    # evaluate model
    scores = cross_val_score(model, X, y, scoring=metric, cv=cv, error_score='raise')
    return scores<jupyter_output><empty_output><jupyter_text>Compare models performance after modifying halal_truck to include 'halal cart(s)'<jupyter_code># define the location of the dataset
file_path_v1 = '/Users/wesamazaizeh/Desktop/Projects/halal_o_meter/src/features/feature_engineering/restaurant_cat_and_num.pkl'
file_path_v2 = '/Users/wesamazaizeh/Desktop/Projects/halal_o_meter/src/features/feature_engineering/restaurant_cat_and_num_v2.pkl'

def run_model_comparison(file_path):
    # load the dataset
    X, y = load_dataset(file_path)
    # encode boolean data as integers
    y = y*1
    X.loc[:, X.dtypes == 'bool'] = X.loc[:, X.dtypes == 'bool']*1
    # split to train and test
    X_train, y_train, X_test, y_test = split_and_resample(X, y, test_size=0.2, resampling='oversample')
    # define models
    models = get_models()
    # evaluate each model
    print('From: ', file_path.split('/')[-1].split('.')[0])
    results = []
    for i, model in enumerate(models):
        # evaluate the model and store results
        scores = evaluate_model(X_train, y_train, model)
        results.append(scores)
        # summarize performance
        print('>%s %.3f (%.3f)' % (model.__class__.__name__, mean(scores), std(scores)))
    print('\n')
    return results

res_list = []
for file_path in [file_path_v1, file_path_v2]:
    res_list.extend(run_model_comparison(file_path=file_path))<jupyter_output>From:  restaurant_cat_and_num
>DummyClassifier 0.750 (0.002)
>DecisionTreeClassifier 0.935 (0.020)
>Pipeline 0.884 (0.027)
>RandomForestClassifier 0.935 (0.023)
>XGBClassifier 0.934 (0.019)


From:  restaurant_cat_and_num_v2
>DummyClassifier 0.750 (0.002)
>DecisionTreeClassifier 0.928 (0.029)
>Pipeline 0.899 (0.031)
>RandomForestClassifier 0.943 (0.021)
>XGBClassifier 0.936 (0.027)<jupyter_text>Comapre resampling methods with new dataset<jupyter_code># define the location of the dataset
file_path_v2 = '/Users/wesamazaizeh/Desktop/Projects/halal_o_meter/src/features/feature_engineering/restaurant_cat_and_num_v2.pkl'
# load the dataset
X, y = load_dataset(file_path_v2)
# encode boolean data as integers
y = y*1
X.loc[:, X.dtypes == 'bool'] = X.loc[:, X.dtypes == 'bool']*1
# define models and resampling methods
models = get_models()
resampling = [None, 'oversample', 'undersample', 'SMOTE']
# evaluate each model
results = []
for resample_method in resampling:
    # split and oversample
    X_train, y_train, X_test, y_test = split_and_resample(X, y, test_size=0.2, resampling=resample_method)
    print('>>%s: ' % str(resample_method).upper())
    for model in models:
        # evaluate the model and store results
        scores = evaluate_model(X_train, y_train, model)
        results.append(scores)
        # summarize performance
        print('>%s %.3f (%.3f)' % (model.__class__.__name__, scores.mean(), scores.std()))
        # evaluate against test set
        model.fit(X_train, y_train)
        y_pred = model.predict(X_test)
        print(' >accuracy: %.3f'  % accuracy_score(y_test, y_pred))
        print(' >recall: %.3f' % recall_score(y_test, y_pred))
    print()<jupyter_output>>>NONE: 
>DummyClassifier 0.865 (0.002)
 >accuracy: 0.738
 >recall: 1.000
>DecisionTreeClassifier 0.891 (0.019)
 >accuracy: 0.762
 >recall: 0.855
>Pipeline 0.928 (0.020)
 >accuracy: 0.768
 >recall: 0.879
>RandomForestClassifier 0.924 (0.032)
 >accuracy: 0.780
 >recall: 0.847
>XGBClassifier 0.940 (0.022)
 >accuracy: 0.833
 >recall: 0.935

>>OVERSAMPLE: 
>DummyClassifier 0.750 (0.003)
 >accuracy: 0.696
 >recall: 1.000
>DecisionTreeClassifier 0.943 (0.017)
 >accuracy: 0.738
 >recall: 0.821
>Pipeline 0.915 (0.020)
 >accuracy: 0.786
 >recall: 0.778
>RandomForestClassifier 0.961 (0.015)
 >accuracy: 0.750
 >recall: 0.838
>XGBClassifier 0.953 (0.024)
 >accuracy: 0.786
 >recall: 0.872

>>UNDERSAMPLE: 
>DummyClassifier 0.750 (0.006)
 >accuracy: 0.708
 >recall: 1.000
>DecisionTreeClassifier 0.810 (0.045)
 >accuracy: 0.702
 >recall: 0.706
>Pipeline 0.806 (0.063)
 >accuracy: 0.690
 >recall: 0.782
>RandomForestClassifier 0.825 (0.084)
 >accuracy: 0.732
 >recall: 0.739
>XGBClassifier 0.848 (0.063)
 >[...]<jupyter_text>Evaluate against test set<jupyter_code># plot the results
plot_df = pd.DataFrame(columns=['version', 'model', 'results', 'mean', 'sd'])
plot_df['version'] = ['v1'] * 5 + ['v2'] * 5
models=get_models()
plot_df['model'] = [model.__class__.__name__ for model in models] * 2
plot_df['results'] = [result for result in res_list]
plot_df['mean'] = plot_df['results'].apply(mean)
plot_df['sd'] = plot_df['results'].apply(std)

def grouped_barplot(df, cat,subcat, val , err):
    u = df[cat].unique()
    x = np.arange(len(u))
    subx = df[subcat].unique()
    offsets = (np.arange(len(subx))-np.arange(len(subx)).mean())/(len(subx)+1.)
    width= np.diff(offsets).mean()
    sns.set_style('whitegrid')
    plt.figure(figsize=(14,10))
    for i,gr in enumerate(subx):
        dfg = df[df[subcat] == gr]
        plt.bar(x+offsets[i], dfg[val].values, width=width, 
                label="{} {}".format(subcat, gr), yerr=dfg[err].values)
    plt.xlabel(cat, size=14)
    plt.ylabel(val, size=14)
    plt.xticks(x, u, size=14)
    plt.ylim(0.7,1)
    plt.legend(loc='upper left', fontsize=12)
    plt.show()

cat = 'version'
subcat = 'model'
val = 'mean'
err = 'sd'
grouped_barplot(plot_df, cat, subcat, val, err )<jupyter_output><empty_output><jupyter_text>Model metrics for Random Forest with oversampling to compare to Baseline-3<jupyter_code># define the location of the dataset
file_path_v2 = '/Users/wesamazaizeh/Desktop/Projects/halal_o_meter/src/features/feature_engineering/restaurant_cat_and_num_v2.pkl'
# load the dataset
X, y = load_dataset(file_path_v2)
# encode boolean data as integers
y = y*1
X.loc[:, X.dtypes == 'bool'] = X.loc[:, X.dtypes == 'bool']*1
# split to train and test
X_train, y_train, X_test, y_test = split_and_resample(X, y, test_size=0.2, resampling='oversample')

# initialize RF with different depths
depths = range(1, X.shape[1])
cv = 5
plot_df = pd.DataFrame(columns=['depth', 'accuracy', 'precision', 'recall', 'f1'], dtype=float)
depths_list = [[depth]*cv for depth in depths]
plot_df['depth'] = [item for items in depths_list for item in items]
scoring = {'acc': 'accuracy', 'prec': 'precision', 'rec': 'recall', 'f1': 'f1'}

for i in range(0, plot_df.shape[0],5): 
    model = RandomForestClassifier(max_depth=plot_df.loc[i, 'depth'])
    # train model and collect metrics
    cv_scores = cross_validate(model, X_train, y_train, scoring = scoring, cv = cv, n_jobs = -1)
    plot_df.loc[i:i+cv-1, 'accuracy'] = cv_scores['test_acc']
    plot_df.loc[i:i+cv-1, 'precision'] = cv_scores['test_prec']
    plot_df.loc[i:i+cv-1, 'recall'] = cv_scores['test_rec']
    plot_df.loc[i:i+cv-1, 'f1'] = cv_scores['test_f1']
# visualize decision tree metrics vs max_depth
plt.figure(figsize=(10,8))
sns.lineplot(x='depth', y='value', hue='variable', data=pd.melt(plot_df, ['depth']))
plt.axvline(x=10, color='red')
plt.show()
# use max depth of 12
model = RandomForestClassifier(max_depth=10)
model.fit(X_train, y_train)

# model performance on test set
y_pred = model.predict(X_test)
print('ROC-AUC:   %.3f' % roc_auc_score(y_test, y_pred))
print('Accuracy:  %.3f'  % accuracy_score(y_test, y_pred))
print('Precision: %.3f' % precision_score(y_test, y_pred))
print('Recall:    %.3f' % recall_score(y_test, y_pred))
print('f1:        %.3f' % f1_score(y_test, y_pred))
print('Classification report: \n', classification_report(y_test, y_pred))
print('Confusion matrix: \n', confusion_matrix(y_test, y_pred))


# Feature importance from RF
plot_df = pd.DataFrame(data={'importances': model.feature_importances_, 'features': X.columns})

# plot
plt.figure(figsize=(10,8))
g = sns.barplot(x=plot_df['features'], y=plot_df['importances'],
                order=plot_df.sort_values('importances', ascending=False).features)
g_labels = g.set_xticklabels(g.get_xticklabels(), rotation=45, horizontalalignment='right')
g.set_title('Random Forest Feature Importance', size=14)
g.set_ylabel('Importance score (%)', size=14)
g.set_xlabel('Features', size=14)
g.set_xticklabels(g.get_xticklabels(), size=12)
plt.show()<jupyter_output>ROC-AUC:   0.741
Accuracy:  0.786
Precision: 0.840
Recall:    0.855
f1:        0.847
Classification report: 
               precision    recall  f1-score   support

           0       0.65      0.63      0.64        51
           1       0.84      0.85      0.85       117

    accuracy                           0.79       168
   macro avg       0.75      0.74      0.74       168
weighted avg       0.78      0.79      0.78       168

Confusion matrix: 
 [[ 32  19]
 [ 17 100]]<jupyter_text>Noramlize numeric values?  Try XGBoost with oversampling<jupyter_code># define the location of the dataset
file_path_v2 = '/Users/wesamazaizeh/Desktop/Projects/halal_o_meter/src/features/feature_engineering/restaurant_cat_and_num_v2.pkl'
# load the dataset
X, y = load_dataset(file_path_v2)
# encode boolean data as integers
y = y*1
X.loc[:, X.dtypes == 'bool'] = X.loc[:, X.dtypes == 'bool']*1
# split to train and test
X_train, y_train, X_test, y_test = split_and_resample(X, y, test_size=0.2, resampling='oversample')
# define model 
model = XGBClassifier(max_depth=10,
                        subsample=0.8,
                        objective='binary:logistic',
                        n_estimators=300,
                        learning_rate = 0.001)
eval_set = [(X_train, y_train), (X_test, y_test)]
model.fit(X_train, y_train.values.ravel(), early_stopping_rounds=15, eval_metric='aucpr', eval_set=eval_set)
# evaluate against test set
y_pred = model.predict(X_test)
print('Accuracy:  %.3f'  % accuracy_score(y_test, y_pred))
print('Precision: %.3f' % precision_score(y_test, y_pred))
print('Recall:    %.3f' % recall_score(y_test, y_pred))
print('f1:        %.3f' % f1_score(y_test, y_pred))
print('Confusion matrix: \n', confusion_matrix(y_test, y_pred))
print('Classification report:\n', classification_report(y_test, y_pred))<jupyter_output>[0]	validation_0-aucpr:0.95443	validation_1-aucpr:0.94669
Multiple eval metrics have been passed: 'validation_1-aucpr' will be used for early stopping.

Will train until validation_1-aucpr hasn't improved in 15 rounds.
[1]	validation_0-aucpr:0.96109	validation_1-aucpr:0.93938
[2]	validation_0-aucpr:0.96293	validation_1-aucpr:0.93980
[3]	validation_0-aucpr:0.96164	validation_1-aucpr:0.93639
[4]	validation_0-aucpr:0.96272	validation_1-aucpr:0.93751
[5]	validation_0-aucpr:0.96374	validation_1-aucpr:0.93815
[6]	validation_0-aucpr:0.96412	validation_1-aucpr:0.93755
[7]	validation_0-aucpr:0.96509	validation_1-aucpr:0.93647
[8]	validation_0-aucpr:0.96483	validation_1-aucpr:0.93678
[9]	validation_0-aucpr:0.96515	validation_1-aucpr:0.94161
[10]	validation_0-aucpr:0.96478	validation_1-aucpr:0.94244
[11]	validation_0-aucpr:0.96499	validation_1-aucpr:0.94188
[12]	validation_0-aucpr:0.96497	validation_1-aucpr:0.94201
[13]	validation_0-aucpr:0.96510	validation_1-aucpr:0.93982
[14]	validation_0-aucpr[...]<jupyter_text>Pickle and save False Positives for more feature engineering<jupyter_code># get indices of FP from XGBoost
FP_list = y_test.index[(y_test==False) & (y_pred == True)]
FP_list = pd.Series(FP_list, name='platform_id')

# get indices of TN from XGBoost
TN_list = y_test.index[(y_test==False) & (y_pred == False)]
TN_list = pd.Series(TN_list, name='platform_id')

# # pickle and save list of restaurant id's
# FP_pkl_name = '/Users/wesamazaizeh/Desktop/Projects/halal_o_meter/src/features/feature_engineering/FP_list_v2.pkl'
# TN_pkl_name = '/Users/wesamazaizeh/Desktop/Projects/halal_o_meter/src/features/feature_engineering/TN_list_v2.pkl'
# FP_list.to_pickle(FP_pkl_name)
# TN_list.to_pickle(TN_pkl_name)
FP<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12910110989145018, 'fraction_numerical': 0.04701792901573362, 'mean_word_length': 4.5703125, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 96, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '176501', 'license': 'FTL', 'n_tokens_mistral': 6486, 'n_tokens_neox': 6016, 'n_words': 1277, 'path': 'src/models/baseline/baseline_v4.ipynb', 'repo_name': 'WAzaizeh/Halal_o_Meter', 'size': '16398'}