<jupyter_start><jupyter_text><jupyter_code># Importing libraries
import numpy as np
import pandas as pd
import altair as alt
# Data Loading 
df_5a = pd.read_csv('/content/drive/MyDrive/PG_Diploma_AI_ML_2021_UOHYD/assignments/5_Performance_metrics/5_a.csv')

df_5b = pd.read_csv('/content/drive/MyDrive/PG_Diploma_AI_ML_2021_UOHYD/assignments/5_Performance_metrics/5_b.csv')

df_5c = pd.read_csv('/content/drive/MyDrive/PG_Diploma_AI_ML_2021_UOHYD/assignments/5_Performance_metrics/5_c.csv')

df_5d = pd.read_csv('/content/drive/MyDrive/PG_Diploma_AI_ML_2021_UOHYD/assignments/5_Performance_metrics/5_d.csv')<jupyter_output><empty_output><jupyter_text>Machine Learning Classification Model Metrics<jupyter_code># Y label predicted based on threshold at 0.5 
df_5a['y_pred_label'] = np.where(df_5a['proba']<0.5,False,True)
df_5b['y_pred_label'] = np.where(df_5b['proba']<0.5,False,True)
df_5c['y_pred_label'] = np.where(df_5c['prob']<0.5,False,True)
def my_confusion_matrix(y_actual,y_pred):
    """Custom Confusion matrix written by <NAME>"""
    TP,TN,FN,FP=0,0,0,0
    for item in zip(y_actual,y_pred):
        if bool(item[0])==True and item[1]==True:
            TP+=1
        elif bool(item[0])==False and item[1]==False:
            TN+=1
        elif bool(item[0])==True and item[1]==False:
            FN+=1
        else:
            FP+=1
    conf_matrix = np.array([TN,FP,FN,TP]).reshape(2,2)
    return conf_matrix
# Looking at confusion matrix
my_confusion_matrix(df_5c.y,df_5c.y_pred_label)
co_mat
def my_performance_metrics(conf_matrix):
    """Custom function to get classification model metrics written by <NAME>"""
    tn, fp, fn, tp = conf_matrix.ravel()
    accuracy = np.round((tp+tn)/(tp+tn+fn+fp),3)
    precision = np.round(tp/(tp+fp),3)
    recall = np.round(tp/(tp+fn),3)
    f2_score = np.round(2*precision*recall/(precision+recall),3)
    return f'Accuracy: {accuracy},Precision: {precision},Recall: {recall},F1-score {f2_score}'
my_performance_metrics(co_mat) 
def my_auc_roc_curve(y_true, y_prob,auc_score=True,roc_plot=False):

    """Custom function to calucate AUC score and plot ROC-AUC curve written by <NAME>"""

    fpr = []
    tpr = []
    
    y_prob = y_prob.sort_values(ascending=False)
    thresholds = np.percentile(y_prob,q=list(range(0,102,2)))[::-1]

    for tshold in thresholds:
        y_pred = np.where(y_prob >tshold, 1, 0)
        tn, fp, fn, tp = my_confusion_matrix(y_true,y_pred).ravel()
        tpr.append(tp / (tp + fn))
        fpr.append(fp / (fp + tn))
        
    df_X = pd.DataFrame({'tpr':tpr,'fpr':fpr,'threshold':thresholds})

    if auc_score is True:
        my_auc = np.trapz(df_X.tpr,df_X.fpr)
        print(f'AUC score:{my_auc}')

    if roc_plot is True:
        return alt.Chart(df_X).mark_area(fillOpacity = 0.8, fill = 'green').encode(
                                                alt.X('fpr', title="false positive rate"),
                                                alt.Y('tpr', title="true positive rate"))
my_auc_roc_curve(df_5a.y,df_5a.proba,roc_plot=True)
def my_best_threshold(y_true,y_prob):
    """Compute the best threshold of probability which gives lowest values of metric A for the given data"""
    y_prob = y_prob.sort_values(ascending=False)
    thresholds = np.percentile(y_prob,q=list(range(0,102,2)))[::-1]
    A_scores = dict()
    for tshold in thresholds:
        y_pred = np.where(y_prob >tshold, 1, 0)
        tn, fp, fn, tp = my_confusion_matrix(y_true,y_pred).ravel()
        A_scores[tshold]= (500*fn) + (100*fp)
    return f'Threshold and scores:{A_scores} \n and the one that minimizes the required:{min(A_scores,key=A_scores.get)}'
my_best_threshold(df_5c.y,df_5c.prob)<jupyter_output><empty_output><jupyter_text>Machine Learning Regression Model Metrics<jupyter_code>def my_r_squrd(y_actual,y_pred):
    """Regression model R Squared custom function written by <NAME>"""
    y_mean = np.mean(y_actual)
    err = np.array((y_actual - y_pred)**2)
    ssr = np.sum(err)
    sst = np.sum(np.array((y_actual-y_mean))**2)
    return np.round(1 - (ssr/sst),3)
my_r_squrd(df_5d.y,df_5d.pred)
def my_rmse(y_actual,y_pred):
    """Regression model RMSE custom function written by <NAME>"""
    N=len(y_actual)
    sum_err = np.sum(np.array((y_actual - y_pred)**2))
    rmse_cal = np.round(np.sqrt(sum_err/N),3)
    return rmse_cal
my_rmse(df_5d.y,df_5d.pred)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14259681093394078, 'fraction_numerical': 0.024145785876993165, 'mean_word_length': 4.029782359679267, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '567940', 'license': 'MIT', 'n_tokens_mistral': 1782, 'n_tokens_neox': 1704, 'n_words': 286, 'path': 'Machine_Learning_Models_Evaluation_metrics_Mallesham_Yamulla.ipynb', 'repo_name': 'myamullaciencia/pg_diploma_ai_ml_uohyd', 'size': '4445'}