<commit_before># -*- coding: utf-8 -*-

"""
    TestReport of a multi-class classifier

    Copyright Xerox(C) 2016 <NAME>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
    
    
    Developed  for the EU project READ. The READ project has received funding 
    from the European Union�s Horizon 2020 research and innovation programme 
    under grant agreement No 674943.
    
"""
from __future__ import absolute_import
from __future__ import  print_function
from __future__ import unicode_literals

import time
from functools import reduce

import numpy as np


from sklearn.metrics import confusion_matrix
from util.metrics import confusion_classification_report, confusion_list_classification_report, confusion_accuracy_score


class TestReport:
    """
    A class that encapsulates the result of a classification test
    """
    
    def __init__(self, name, l_Y_pred, l_Y, lsClassName, lsDocName=None):
        """
        takes a test name, a prediction and a groundtruth or 2 lists of that stuff
        optionally the list f class names
        optionally the list document names
        compute:
        - if lsClassName: the sub-list of seen class (in either prediction or GT)
        
        NOTE:   Baseline models may not know the number of classes, nor the docnames...
                But since these TestReport are attachd to th emain one, we have the info!
        """
        #Content of this object (computed below!!)
        self.t = time.time()
        self.name          = name
        self.lsDocName     = lsDocName
        self.lsClassName   = lsClassName
        self.nbClass       = len(self.lsClassName) if self.lsClassName else None
        self.lBaselineTestReport = []       #TestReport for baseline methods, if any
        
        if type(l_Y_pred) == list:
            self.l_Y_pred = l_Y_pred
            self.l_Y      = l_Y
        else:
            self.l_Y_pred = [l_Y_pred]
            self.l_Y      = [l_Y]

        #Computing the classes that are observed 
        aSeenCls = np.unique(self.l_Y_pred[0])
        for _Y in self.l_Y_pred[1:]: aSeenCls = np.unique( np.hstack([aSeenCls, np.unique(_Y)]) )
#         #... or that were expected to be observed
#         for _Y in self.l_Y:          aSeenCls = np.unique( np.hstack([aSeenCls, np.unique(_Y)]) )
        aSeenCls.sort()
        self.lSeenCls = aSeenCls.tolist()
        
        # --- 
        assert type(l_Y_pred) == type(l_Y), "Internal error: when creating a test report, both args must be of same type (list of np.array or np.array)"
        if l_Y is None:
            assert l_Y is None and l_Y_pred is None, "Internal error"
        else:
            assert len(l_Y) == len(l_Y_pred), "Internal error"
            if lsDocName: assert len(l_Y) == len(lsDocName), "Internal error"
        # --- 
        
    
    def attach(self, loTstRpt):
        """
        attach this testReport or list of TestReport to the current TestReport (typically the results of the baseline(s) )
        """
        if type(loTstRpt) == list:
            self.lBaselineTestReport.extend(loTstRpt)
        else:
            self.lBaselineTestReport.append(loTstRpt)
        return self.lBaselineTestReport
    
    # ------------------------------------------------------------------------------------------------------------------
    def getTestName(self):              return self.name
    def getNbClass(self):               return self.nbClass
    def getClassNameList(self):         return self.lsClassName
    def getDocNameList(self):           return self.lsDocName
    def getBaselineTestReports(self):   return self.lBaselineTestReport
    
    def getConfusionMatrix(self):
        """
        Return a confusion matrix covering all classes (not only those observed in this particular test)
        if you have the flat Y and Y_pred, pass them!
        """
        labels   = range(self.nbClass) if self.nbClass else None
        confumat = reduce(np.add, [confusion_matrix(_Y, _Y_pred, labels) for _Y, _Y_pred in zip(self.l_Y, self.l_Y_pred)])
        return confumat
    
    def getConfusionMatrixByDocument(self):
        """
        List of confusion matrices, one for each document in the test set
        """
        labels   = range(self.nbClass) if self.nbClass else None
        return [ confusion_matrix(_Y, _Y_pred, labels) for _Y, _Y_pred in zip(self.l_Y, self.l_Y_pred) ]
    
    def getClassificationReport(self, aConfuMat=None):
        """
        return the score and the (textual) classification report
        """
        if aConfuMat is None: aConfuMat = self.getConfusionMatrix()
        if self.lsClassName:
            sClassificationReport = confusion_classification_report(aConfuMat, target_names=self.lsClassName)
        else:
            sClassificationReport = confusion_classification_report(aConfuMat)
        
        fScore = confusion_accuracy_score(aConfuMat)    
        
        return fScore, sClassificationReport

    # ------------------------------------------------------------------------------------------------------------------
    def toString(self, bShowBaseline=True, bBaseline=False):
        """
        return a nicely indented test report
        if bShowBaseline, includes any attached report(s), with proper indentation
        if bBaseline: report for a baseline method
        """
        if bBaseline:
            sSpace  = " "*8
            sSepBeg = "\n" + sSpace + "~" * 30
            sTitle  = " BASELINE "
            sSepEnd = "~"*len(sSepBeg)
        else:
            sSepBeg = "--- " +  time.asctime(time.gmtime(self.t)) + "---------------------------------"
            sTitle  = "TEST REPORT FOR"
            sSpace  = ""
            sSepEnd = "-"*len(sSepBeg)<commit_msg>add detailled reporting (scor per documents)<commit_after># -*- coding: utf-8 -*-

"""
    TestReport of a multi-class classifier

    Copyright Xerox(C) 2016 <NAME>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
    
    
    Developed  for the EU project READ. The READ project has received funding 
    from the European Union�s Horizon 2020 research and innovation programme 
    under grant agreement No 674943.
    
"""
from __future__ import absolute_import
from __future__ import  print_function
from __future__ import unicode_literals

import time
import os
from functools import reduce

import numpy as np


from sklearn.metrics import confusion_matrix
from util.metrics import confusion_classification_report, confusion_list_classification_report, confusion_accuracy_score


class TestReport:
    """
    A class that encapsulates the result of a classification test
    """
    
    def __init__(self, name, l_Y_pred, l_Y, lsClassName, lsDocName=None):
        """
        takes a test name, a prediction and a groundtruth or 2 lists of that stuff
        optionally the list f class names
        optionally the list document names
        compute:
        - if lsClassName: the sub-list of seen class (in either prediction or GT)
        
        NOTE:   Baseline models may not know the number of classes, nor the docnames...
                But since these TestReport are attachd to th emain one, we have the info!
        """
        #Content of this object (computed below!!)
        self.t = time.time()
        self.name          = name
        self.lsDocName     = lsDocName
        self.lsClassName   = lsClassName
        self.nbClass       = len(self.lsClassName) if self.lsClassName else None
        self.lBaselineTestReport = []       #TestReport for baseline methods, if any
        
        if type(l_Y_pred) == list:
            self.l_Y_pred = l_Y_pred
            self.l_Y      = l_Y
        else:
            self.l_Y_pred = [l_Y_pred]
            self.l_Y      = [l_Y]

        #Computing the classes that are observed 
        aSeenCls = np.unique(self.l_Y_pred[0])
        for _Y in self.l_Y_pred[1:]: aSeenCls = np.unique( np.hstack([aSeenCls, np.unique(_Y)]) )
#         #... or that were expected to be observed
#         for _Y in self.l_Y:          aSeenCls = np.unique( np.hstack([aSeenCls, np.unique(_Y)]) )
        aSeenCls.sort()
        self.lSeenCls = aSeenCls.tolist()
        
        # --- 
        assert type(l_Y_pred) == type(l_Y), "Internal error: when creating a test report, both args must be of same type (list of np.array or np.array)"
        if l_Y is None:
            assert l_Y is None and l_Y_pred is None, "Internal error"
        else:
            assert len(l_Y) == len(l_Y_pred), "Internal error"
            if lsDocName: assert len(l_Y) == len(lsDocName), "Internal error"
        # --- 
        
    
    def attach(self, loTstRpt):
        """
        attach this testReport or list of TestReport to the current TestReport (typically the results of the baseline(s) )
        """
        if type(loTstRpt) == list:
            self.lBaselineTestReport.extend(loTstRpt)
        else:
            self.lBaselineTestReport.append(loTstRpt)
        return self.lBaselineTestReport
    
    # ------------------------------------------------------------------------------------------------------------------
    def getTestName(self):              return self.name
    def getNbClass(self):               return self.nbClass
    def getClassNameList(self):         return self.lsClassName
    def getDocNameList(self):           return self.lsDocName
    def getBaselineTestReports(self):   return self.lBaselineTestReport
    
    def getConfusionMatrix(self):
        """
        Return a confusion matrix covering all classes (not only those observed in this particular test)
        if you have the flat Y and Y_pred, pass them!
        """
        labels   = range(self.nbClass) if self.nbClass else None
        confumat = reduce(np.add, [confusion_matrix(_Y, _Y_pred, labels) for _Y, _Y_pred in zip(self.l_Y, self.l_Y_pred)])
        return confumat
    
    def getConfusionMatrixByDocument(self):
        """
        List of confusion matrices, one for each document in the test set
        """
        labels   = range(self.nbClass) if self.nbClass else None
        return [ confusion_matrix(_Y, _Y_pred, labels) for _Y, _Y_pred in zip(self.l_Y, self.l_Y_pred) ]
    
    def getClassificationReport(self, aConfuMat=None):
        """
        return the score and the (textual) classification report
        """
        if aConfuMat is None: aConfuMat = self.getConfusionMatrix()
        if self.lsClassName:
            sClassificationReport = confusion_classification_report(aConfuMat, target_names=self.lsClassName)
        else:
            sClassificationReport = confusion_classification_report(aConfuMat)
        
        fScore = confusion_accuracy_score(aConfuMat)    
        
        return fScore, sClassificationReport

    def getDetailledReport(self):
        """
            return Precision, Recall, F1 per label for each docucment
            0 5577_001.mpxml [ 0.882  0.922  0.976  1.     0.   ] [ 0.904  0.981  0.988  0.771  0.   ] [ 0.893  0.951  0.982  0.871  0.   ]
            1 5577_007.mpxml [ 0.969  0.952  0.992  1.     0.   ] [ 0.969  0.981  0.969  1.     0.   ] [ 0.969  0.967  0.981  1.     0.   ]
            
        """
        sReport = "\n Detailed Reporting \n" + "-"*20 +"\n"
        lConfMat = self.getConfusionMatrixByDocument()
        eps=1e-8
        # numpy bug!!
        np.set_printoptions(precision = 3, suppress = True)
        for i,conf in enumerate(lConfMat):
            p   = np.diag(conf)/(eps+conf.sum(axis=0))
            r   = np.diag(conf)/(eps+conf.sum(axis=1))
            f1  = 2*p*r/(eps+p+r)
            sReport += "%d\t%s\t%s %s %s\n"%(i,os.path.basename(self.getDocNameList()[i]),p,r,f1)
            
        return sReport
    # ------------------------------------------------------------------------------------------------------------------
    def toString(self, bShowBaseline=True, bBaseline=False):
        """
        return a nicely indented test report
        if bShowBaseline, includes any attached report(s), with proper indentation
        if bBaseline: report for a baseline method
        """
        if bBaseline:
            sSpace  = " "*8
            sSepBeg = "\n" + sSpace + "~" * 30
            sTitle  = " BASELINE "
            sSepEnd = "~"*len(sSepBeg)
        else:
            sSepBeg = "--- " +  time.asctime(time.gmtime(self.t)) + "---------------------------------"
            sTitle  = "TEST REPORT FOR"
            sSpace  = ""
            sSepEnd = "-"*len(sSepBeg)
{'fraction_non_alphanumeric': 0.11590394861688928, 'fraction_numerical': 0.011750967082694694, 'mean_word_length': 1.9670853183196189, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '101374', 'n_tokens_mistral': 4032, 'n_tokens_neox': 3847, 'n_words': 1241}