<jupyter_start><jupyter_text>We see that for Bogo and Discount XGB provides the best result, while for Informational offer the Linear model is the most suiting one.<jupyter_code># Print the best model
pprint.pprint(best_model)
# Create Transformers
for prefix in ['bogo','discount', 'info']:
    print(prefix)

    # Create XGB Transformer
    transformer = best_model[prefix]['ll']['model'].transformer(\
                        instance_count = 1, instance_type = 'ml.m4.xlarge')

    # Batch transform
    transformer.transform(f's3://{bucket}/CapstoneProjectStarbucks/{prefix}/{prefix}_test.csv',
                   content_type='text/csv', split_type='Line', logs=False)
    
    transformer.wait(logs=False)
    
    # Download the data
    session.download_data(f'./data/{prefix}/{prefix}_ll_preds', bucket,
                  key_prefix='/'.join(transformer.output_path.split('/')[3:]))
    
    # Create Linear Transformer
    transformer = best_model[prefix]['xgb']['model'].transformer(\
                     instance_count = 1, instance_type = 'ml.m4.xlarge')

    # Batch transform
    transformer.transform(f's3://{bucket}/CapstoneProjectStarbucks/{prefix}/{prefix}_test.csv',
       content_type='text/csv', split_type='Line', wait=True, logs=False)

    # Download the data
    session.download_data(f'./data/{prefix}/{prefix}_xgb_preds', bucket,
                  key_prefix='/'.join(transformer.output_path.split('/')[3:]))
# Create a function to assess the results
def assess_model(y, y_prob, y_pred=None):
    '''
    This function provides performance measures for ML models, given
    the true labels and the predictions. In addition, it plots the 
    ROC Curve and the Confusion Matrix.
    
    Args:
    y (list/numpy.array) - the true labels
    y_prob (list/numpy.array) - model-returned probabilities
    y_pred (list/numpy.array) - Predictions returned by the model.
    
    Outputs:
    res (DataFrame) - dataset contains a list of the most used performance
    measures for the data provided.
    '''

    
    if y_pred is None:
        y_pred = pd.Series([1 if x >= 0.5 else 0 for x in y_prob])
        
    res = pd.DataFrame(
        {'value': [accuracy_score(y, y_pred),
                   balanced_accuracy_score(y, y_pred),
                   precision_score(y, y_pred), recall_score(y, y_pred),
                   f1_score(y, y_pred),
                   average_precision_score(y, y_prob),
                   roc_auc_score(y, y_prob)]
         },
        index=['accuracy', 'balanced_accuracy', 'precision', 'recall', 'f1',
               'average_precision', 'AUC']
    )

    fpr, tpr, _ = roc_curve(y, y_prob)
    plt.plot(fpr, tpr)
    plt.title('ROC Curve')
    plt.show()

    print('Confusion matrix')
    display(pd.crosstab(y, y_pred, normalize='index'))

    return res
# Plot the aggregated results
for prefix in ['bogo', 'discount', 'info']:
    print(prefix +' Offer')
    
    # Load True Labels
    y_test = pd.read_csv(f'./data/{prefix}/{prefix}_test_target.csv', 
                         squeeze=True, header=None)
    
    # XGBoost
    print('XGBoost Model')
    y_prob = pd.read_csv(f'./data/{prefix}/{prefix}_xgb_preds/{prefix}_test.csv.out', squeeze=True, header=None)
    display(assess_model(y_test, y_prob))

    # LinearLearner
    print('LinearLearner Model')
    with open(f'./data/{prefix}/{prefix}_ll_preds/{prefix}_test.csv.out', 
                                                                  'r') as f:
        jsons = [json.loads(x.replace('\n', '')) for x in f.readlines()]
    res = pd.DataFrame(jsons)
    display(assess_model(y_test, res['score'].values, y_pred=res[
                                                'predicted_label'].values))
    
    print('\n\n')<jupyter_output>bogo Offer
XGBoost Model
{'fraction_non_alphanumeric': 0.1296540797465012, 'fraction_numerical': 0.0036968576709796672, 'mean_word_length': 2.5939278937381403, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '440805', 'license': 'MIT', 'n_tokens_mistral': 1227, 'n_tokens_neox': 1176, 'n_words': 259, 'path': 'Model.ipynb', 'repo_name': 'ArtemShuv91/capstone_project_starbucks', 'size': '3787'}