<jupyter_start><jupyter_text>Exercise 1 Confusion Matrix<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

%matplotlib inline<jupyter_output><empty_output><jupyter_text>a) Load first dataset  - Scores<jupyter_code>datasetA = pd.read_csv('ex1-system-a.csv',names=['0','1','2','3','4','5','6','7','8','9','y_true','unused'], sep = ";")
datasetA = datasetA.drop('unused', axis = 1) # drop the last colomn without any information in it.
classes_name = ['0','1','2','3','4','5','6','7','8','9']
nb_classes = 10
datasetA.head()
y_true = datasetA['y_true'].values
y_scores_A = datasetA[classes_name].values   #isolate the matrix of scores
y_pred_A = np.argmax(y_scores_A,axis=1)      #elect winner class<jupyter_output><empty_output><jupyter_text>b) What is the overall error rate of the system ?<jupyter_code>### START YOUR CODE
overall_rate = 
### END YOUR CODE

print('overall rate of the system : ',overall_rate)<jupyter_output><empty_output><jupyter_text>c) Compute and report the confusion matrix of the system. We create a second one our selfs and ckeck if it is correct with the "correct" one from sklearn.<jupyter_code>def confusion_matrix2(y_true,y_pred,nb_classes):
    """
    Arguments:
    y_true -- groud truth labels
    y_pred -- predicted values
    n_classes -- number of classes
    
    Returns:
    the confusion matrix as 2d numpy array 
    """
    ### START YOUR CODE

    
    ### END YOUR CODE
    return counts
# TEST - we use the sklearn builit in functionality
from sklearn.metrics import confusion_matrix
verif_cm = confusion_matrix(y_true,y_pred_A)

cm_A = confusion_matrix2(y_true, y_pred_A,nb_classes)
# verification for our confusion matrix function
print((cm_A == verif_cm).sum() == nb_classes**2)<jupyter_output><empty_output><jupyter_text>Now we want to plot the confusion matrix.<jupyter_code>def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
plt.figure()
plot_confusion_matrix(cm_A, classes=classes_name,title='Confusion matrix for System A, without normalization')

plt.figure()
plot_confusion_matrix(cm_A, classes=classes_name, normalize=True,
                      title='Normalized confusion matrix for System B')

plt.show()<jupyter_output><empty_output><jupyter_text>d) What are the worst and best classes in terms of precision and sensitivity (recall) ?<jupyter_code>def precision_per_class(cm):
    """
    Aguments:
    cm -- confusion matrix
    Returns:
    Precision per class, i.e. a numpy array of length given by the number of classes
    """
    ### START YOUR CODE  
    
    ### END YOUR CODE
def recall_per_class(cm):
    """
    Aguments:
    cm -- confusion matrix
    Returns:
    Recall per class, i.e. a numpy array of length given by the number of classes
    """
    ### START YOUR CODE  

    ### END YOUR CODE
precisions = precision_per_class(cm_A)
# should return '1' as best (95.7%)
# and '5' as worst (81.4%) 

best_class = np.argmax(precisions)
worst_class = np.argmin(precisions)
print('best precision class  : ', best_class, '[', precisions[best_class],']')
print('worst precision class : ', worst_class, '[', precisions[worst_class],']')
recalls = recall_per_class(cm_A)
# should return '1' as best (97.97%)
# and '8' as worst (79.26%) 

best_class = np.argmax(recalls)
worst_class = np.argmin(recalls)
print('best recall class  : ', best_class, '[',recalls[best_class],']')
print('worst recall class : ', worst_class, '[',recalls[worst_class],']')<jupyter_output><empty_output><jupyter_text>e) Load second dataset  - ScoresFind the output of a second system B. It contains the same ground truth values - but different predictions.What is the best system between (a) and (b) in terms of error rate and F1.<jupyter_code>datasetB = pd.read_csv('ex1-system-b.csv',names=['0','1','2','3','4','5','6','7','8','9','y_true','unused'], sep = ";")
datasetB = datasetB.drop('unused', axis = 1) # drop the last colomn without any information in it.
datasetB.head()
y_true = datasetB['y_true'].values
y_scores_B = datasetB[classes_name].values   #isolate the matrix of scores
y_pred_B = np.argmax(y_scores_B,axis=1)      #elect winner class

cm_B = confusion_matrix2(y_true,y_pred_B,nb_classes)

plt.figure()
plot_confusion_matrix(cm_B, classes=classes_name,title='Confusion matrix for System B, without normalization')

plt.figure()
plot_confusion_matrix(cm_B, classes=classes_name, normalize=True,
                      title='Normalized confusion matrix for System B')

plt.show()
def system_precision(cm):
    ### START YOUR CODE

    ### END YOUR CODE

def system_recall(cm):
    ### START YOUR CODE

    ### END YOUR CODE
    
def system_accuracy(cm):
    ### START YOUR CODE

    ### END YOUR CODE
    
def system_f1_score(cm):
    ### START YOUR CODE

    ### END YOUR CODE
print('System A\n\trecall : ',system_recall(cm_A),'\n\tprecision : ',system_precision(cm_A),'\n\taccuracy : ',system_accuracy(cm_A),'\n\tf1-score : ',f1_score(cm_A))
print('System B\n\trecall : ',system_recall(cm_B),'\n\tprecision : ',system_precision(cm_B),'\n\taccuracy : ',system_accuracy(cm_B),'\n\tf1-score : ',f1_score(cm_B))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13298791018998274, 'fraction_numerical': 0.01226252158894646, 'mean_word_length': 4.397017707362535, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '527453', 'license': 'MIT', 'n_tokens_mistral': 2000, 'n_tokens_neox': 1926, 'n_words': 488, 'path': 'exercises/exercise4/Exercise 2/confusion_matrix_ex-stud.ipynb', 'repo_name': 'PatCH0816/TSM_DeLearn', 'size': '5790'}