<jupyter_start><jupyter_text>Import<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from itertools import product
from sklearn.metrics import confusion_matrix, classification_report

pd.set_option('expand_frame_repr', False)
pd.set_option('display.max_colwidth', 0)

%matplotlib inline
print("Complete")<jupyter_output>Complete<jupyter_text>Input<jupyter_code>filePathDataEval = r'C:\Users\nguyen.tt1\Downloads\eval.csv'
filePathDataTrain = r'C:\Users\nguyen.tt1\Documents\Nanibot_ZaloAIChallenge2019_VietnameseWikiQA\Dataset\train.json'
nTopQAPair = 10
evalData = pd.read_csv(filePathDataEval, encoding="utf8")
trainData = pd.read_json(filePathDataTrain, encoding="utf8")
evalData.insert(
    len(evalData.columns),
    'IsCorrect',
    ~(evalData['prediction'] ^ evalData['label'])
)<jupyter_output><empty_output><jupyter_text>Utilities<jupyter_code>def plotConfusionMatrix(cm, normalize=False):
    cmap = plt.get_cmap('Greens')

    plt.imshow(cm,cmap=cmap)
    plt.colorbar()
    plt.xticks([0,1],['True','False'])
    plt.yticks([0,1],['True','False'])
    
    if normalize:
        cm = (cm.astype('float') / cm.sum(axis=1)[:, np.newaxis])
    
    for i, j in product(range(cm.shape[0]), range(cm.shape[1])):
        if normalize:
            plt.text(j, i, "{:0.4f}%\n".format(cm[i, j]*100),
                    ha="center",
                    va="center",
                    size='large',
                    color="black",
                    weight="bold"
            )
        else:
            plt.text(j, i, "{:,}\n".format(cm[i, j]),
                    ha="center",
                    va="center",
                    size='large',
                    color="black",
                    weight="bold"
            )

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    
    if normalize:
        plt.title("Confusion matrix - Normalized\n\n")
    else:
        plt.title("Confusion matrix\n\n")
    plt.show()
def getTopQAPair(nQAPair, useCorrectPrediction=True,sortAsc=True):
    if (useCorrectPrediction):
        topQA = evalData[
            evalData['IsCorrect']
        ].sort_values(by=['probabilities'], ascending=sortAsc).iloc[:nQAPair][['guid','prediction','probabilities']]
    else:
        topQA = evalData[
            ~evalData['IsCorrect']
        ].sort_values(by=['probabilities'], ascending=sortAsc).iloc[:nQAPair][['guid','prediction','probabilities']]

    return pd.merge(
        right=trainData,
        left=topQA,
        right_on='id',
        left_on='guid'
    )[['id','question','title','text','label','prediction','probabilities']]<jupyter_output><empty_output><jupyter_text>Confusion Matrix<jupyter_code>plotConfusionMatrix(
    confusion_matrix(
            evalData['label'],
            evalData['prediction']
    )
)
plotConfusionMatrix(
    confusion_matrix(
            evalData['label'],
            evalData['prediction']
    ),
    normalize=True
)
print("Classification report\n")
print(classification_report(
    evalData['label'],
    evalData['prediction']
))<jupyter_output><empty_output><jupyter_text>Top x QA pairs with the most confidence & correct<jupyter_code>getTopQAPair(nTopQAPair, useCorrectPrediction=True, sortAsc=False)<jupyter_output><empty_output><jupyter_text>Top x QA pairs with the least confidence but correct<jupyter_code>getTopQAPair(nTopQAPair, useCorrectPrediction=True, sortAsc=True)<jupyter_output><empty_output><jupyter_text>Top x QA pairs with the most confidence but incorrect<jupyter_code>getTopQAPair(nTopQAPair, useCorrectPrediction=False, sortAsc=False)<jupyter_output><empty_output><jupyter_text>Top x QA pairs with the least confidence & incorrect<jupyter_code>getTopQAPair(nTopQAPair, useCorrectPrediction=False, sortAsc=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13785900783289817, 'fraction_numerical': 0.0060052219321148825, 'mean_word_length': 3.72962962962963, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '368035', 'license': 'MIT', 'n_tokens_mistral': 1319, 'n_tokens_neox': 1261, 'n_words': 180, 'path': 'Ultilities/AnalyzeModelResult.ipynb', 'repo_name': 'phateopera/Nanibot_ZaloAIChallenge2019_VietnameseWikiQA', 'size': '3830'}