<jupyter_start><jupyter_text>Datasets:Datasets containing the experiments results: * Out of fold predictions* logits of each class* Original labels* Loss functions<jupyter_code># Out of Fold prediction Datasets
cleaned=pd.read_csv('../input/cassava-inference-analysis-cleaned/logits_cleaned.csv')
original= pd.read_csv('../input/cassava-inference-analysis-original/logits_original.csv')
bitemp=pd.read_csv('../input/cassava-resnext-384-16ep/oof_df.csv')
bitemp=bitemp.rename(columns={'preds':'pred'})
bitemp_sd=pd.read_csv('../input/trainer-cassava-resnext-bitemp-sd/oof_df.csv')
vit= pd.read_csv('../input/cassava-vit-bitemp-oof-analysis/vit_bitemp_sd.csv')
resnext= pd.read_csv('../input/trainer-cassava-resnext50-bitemp-sd/oof_df.csv')<jupyter_output><empty_output><jupyter_text>Presentation class:Class containing methods to print and plot the experiments results:* Confusion matrix* Cross validation report* Accuracy scores* Error scores* Comparative plot<jupyter_code>class presentation():
    """
    A Class to print analytic and visual reports of the cassava leaf disease competition results:
    
    Inputs:
    dataX: a Dataframe including the original labels and the models predictions/logits of each experiment
    ...
    
    Methods:
    cm: Plot confusion matrix
    report: print the accuracy score of each class + error scores
    accuracies: return a list of accuracy scores
    proba: return a list of error scores
    visualplot: plot error/accuracy across the 5 classes
    """
    
    def __init__(self, data, data2=None, data3=None, title1="",title2="",title3=""):
        self.data= data
        self.data2= data2
        self.data3= data3
        self.title1= title1
        self.title2= title2
        self.title3= title3
        
    #Plot confusion matrix
    def cm(self,first=True,second=False, third=False, color='Purples'):
        plt.figure(figsize=(8,5))
        if first:
            plt.title(f'Confusion matrix: {self.title1}', weight='bold', fontsize=18)
            cmatrix = confusion_matrix(self.data.label,self.data.pred)
        elif second:
            plt.title(f'Confusion matrix: {self.title2}', weight='bold', fontsize=18)
            cmatrix = confusion_matrix(self.data2.label,self.data2.pred)
        
        elif third:
            plt.title(f'Confusion matrix: {self.title3}', weight='bold', fontsize=18)
            cmatrix = confusion_matrix(self.data3.label,self.data3.pred)
        
        sns.heatmap(cmatrix,annot=True,fmt="d",cbar=True, annot_kws={'fontweight':'bold'}, cmap=color)
        plt.ylabel('Classes: Labels', weight='bold')
        plt.xlabel('Classes: Predictions', weight='bold')
        plt.show()
    
    #Print report: Accuracy
    def report(self, first=True, second=False, third=False):
        labels = ['Class 0', 'Class 1', 'Class 2', 'Class 3', 'Class 4']
        if first:
            print(f'--- {self.title1} DATASET ---')
            print(f'Average CV score: {round((self.data.label.values==self.data.pred.values).mean()*100,2)}%')
            cm=confusion_matrix(self.data.label, self.data.pred)

        elif second:
            print(f'--- {self.title2} DATASET ---')
            print(f'Average CV score: {round((self.data2.label.values==self.data2.pred.values).mean()*100,2)}%')
            cm=confusion_matrix(self.data2.label, self.data2.pred)
        
        elif third:
            print(f'--- {self.title3} DATASET ---')
            print(f'Average CV score: {round((self.data3.label.values==self.data3.pred.values).mean()*100,2)}%')
            cm=confusion_matrix(self.data3.label, self.data3.pred)
        
        print('------------------------------------')
        print('Accuracy score of each class:')
        accs=np.zeros((5,1))
        df=pd.DataFrame(labels, columns=['Class'])
        for i, val in enumerate(cm):
            print(f"Class {i}: {round(val[i]/sum(val)*100,1)}%")
            accs[i]+= round(val[i]/sum(val)*100,2)
        df1=df.copy()
        df1['Accuracy %']=accs
        #df1 = df1.style.background_gradient(cmap='Reds')
        #display(df1)
        print('------------------------------------')
        print('Probability of each class to mistake for healthy')
        prbs=np.zeros((4,1))
        for i, val in enumerate(cm[:-1]):
            print(f"Class {i}: : {round(val[4]/val[i]*100,1)}%")
            prbs[i]+= round(val[4]/val[i]*100,1)
        df2=df.head(4).copy()
        df2['Error %']=prbs
        #df2 = df2.style.background_gradient(cmap='Blues')
        #display(df2)
        del df, df1, df2

    #List of class accuracies
    def accuracies(self,df,first=False, *params):
        probs=[]
        cm=confusion_matrix(df.label, df.pred)
        for i, val in enumerate(cm):
            prob=round(val[i]/sum(val)*100,1)
            probs.append(prob)
        return probs

        #List of probability of disease classes predicted healthy    
    def proba(self,df, *params):
        probs=[]
        cm=confusion_matrix(df.label,df.pred)
        for i, val in enumerate(cm[:-1]):
            prob=round(val[4]/val[i]*100,1)
            probs.append(prob)
        return probs

    #Function to plot errors and accuracies
    def visualplot(self, first=False, second=False, *args, errors=True, color1='indigo', color2='gainsboro', color3='deeppink'):
        labels = ['Class 0:CBB', 'Class 1:CBSD', 'Class 2:CGM', 'Class 3:CMD']
        
        #Read error data
        if errors:
            probs1=self.proba(self.data)
            if self.data2 is not None and self.data3 is not None:
                probs2= self.proba(self.data2)
                probs3= self.proba(self.data3)
            if self.data2 is not None:
                probs2= self.proba(self.data2)
        
        #Read accuracy data
        else:
            labels.append('healthy')
            probs1=self.accuracies(self.data)
            if self.data2 is not None and self.data3 is not None:
                probs2= self.accuracies(self.data2)
                probs3= self.accuracies(self.data3)
            if self.data2 is not None:
                probs2= self.accuracies(self.data2)
        
          
        x = np.arange(len(labels))  # the label locations
        width = 0.25  # the width of the bars

        if self.data2 is not None and self.data3 is not None:
            fig, ax = plt.subplots(figsize=(9,4))
        else:
            fig, ax = plt.subplots(figsize=(8,4))
        rects1 = ax.bar(x - width/2, probs1, width, label=f'{self.title1}',edgecolor='black', color=color1)
        if self.data2 is not None and self.data3 is not None:
            if first:
                pass
            else:
                rects2 = ax.bar(x + width/2, probs2, width, label=f'{self.title2}',edgecolor='black', color=color2)
                if second:
                    pass
                else:
                    rects3 = ax.bar(x + width*1.54, probs3, width, label=f'{self.title3}',edgecolor='black', color=color3)
        
        if self.data2 is not None and self.data3 is None:
            if first:
                pass
            else:
                rects2 = ax.bar(x + width/2, probs2, width, label=f'{self.title2}',edgecolor='black', color=color2)
        

        #Add grid
        ax.grid(b=True, color='grey', linestyle='-.', linewidth=1, alpha=0.3)
        
        #Set Bars starting point values
        if not errors:
            ax.set(ylim=[50, 120])            

        # Add some text for labels, title and custom x-axis tick labels, etc.
        if errors:
            ax.set_ylabel('Error percentage', fontsize=13)
            ax.set_xlabel('Cassava classes', fontsize=13)
            ax.set_title('Probability of each class to mistake for healthy', weight='bold', fontsize=15)
        else:
            ax.set_ylabel('Accuracy', fontsize=13)
            ax.set_xlabel('Cassava classes', fontsize=13)
            ax.set_title('Accuracy of the 5 classes', weight='bold', fontsize=15)
        
        ax.set_xticks(x)
        plt.xticks(weight='bold')
        ax.set_xticklabels(labels)
        ax.legend()

        ax.bar_label(rects1, padding=0.8, weight='bold', color='grey', fontsize=9)
        if self.data2 is not None and self.data3 is not None:
            if first:
                pass
            else:
                ax.bar_label(rects2, padding=1, weight='bold', color='indigo', fontsize=9)
                if second:
                    pass
                else:
                    ax.bar_label(rects3, padding=1, weight='bold', color='deeppink', fontsize=9)
        if self.data2 is not None and self.data3 is None:
            if first:
                pass
            else:
                ax.bar_label(rects2, padding=1, weight='bold', color='indigo', fontsize=9)
                    
        
        fig.tight_layout()
        return plt.show()
    
def full_report(data1,data2=None,data3=None,title1='d',title2='f', title3='d', errors=False,first=True, second=False, third=False, color1="indigo", color2="gainsboro", color3="deeppink"):
    #Define class
    project=presentation(data1,data2,data3,title1=title1,title2=title2, title3=title3)
    #Print 1st dataset report
    if first:
        project.report()
        project.cm()
    #Print 2nd dataset report
    if second or third:
        project.report(first=first, second=second, third=third)
        project.cm(first=first, second=second, third=third)
    #Plot error difference
    project.visualplot(errors=errors, first=False, second=second, color1=color1, color2=color2, color3=color3)<jupyter_output><empty_output><jupyter_text>Experiment 1: BenchmarkResults of the first trained model to get a benchmark score<jupyter_code>full_report(data1=original,
            title1='Original EfficienNet B3',
           errors=True) #errors=False will plot the accuracy scores<jupyter_output>--- Original EfficienNet B3 DATASET ---
Average CV score: 87.43%
------------------------------------
Accuracy score of each class:
Class 0: 64.8%
Class 1: 83.5%
Class 2: 77.8%
Class 3: 94.9%
Class 4: 70.1%
------------------------------------
Probability of each class to mistake for healthy
Class 0: : 29.4%
Class 1: : 7.7%
Class 2: : 9.6%
Class 3: : 1.8%<jupyter_text>Experiment 2: Removing 300 mislabeled images<jupyter_code>full_report(data1=original, data2=cleaned,
            title1='benchmark', title2='Cleaned',
            first=False,second=True,
            color1='lightgreen', color2='lightpink',
           errors=True) #first=False to print only the 2nd datasets report<jupyter_output>--- Cleaned DATASET ---
Average CV score: 88.08%
------------------------------------
Accuracy score of each class:
Class 0: 69.3%
Class 1: 82.7%
Class 2: 78.5%
Class 3: 94.8%
Class 4: 72.1%
------------------------------------
Probability of each class to mistake for healthy
Class 0: : 20.1%
Class 1: : 8.4%
Class 2: : 9.7%
Class 3: : 1.8%<jupyter_text>Experiment 3: Bi-Tempered loss<jupyter_code>full_report(data1=original, data2=bitemp, 
            title1='Cross Entropy loss', title2='Bi-Tempered loss',
            first=False,second=True) #first=False to print only the 2nd datasets report
full_report(data1=cleaned, data2=bitemp, 
            title1='Cleaned dataset', title2='Bi-Tempered loss',
            color1='navajowhite', color2='lightskyblue', errors=True,
            first=False,second=True) #first=False to print only the 2nd datasets report<jupyter_output>--- Bi-Tempered loss DATASET ---
Average CV score: 88.51%
------------------------------------
Accuracy score of each class:
Class 0: 64.5%
Class 1: 79.3%
Class 2: 80.3%
Class 3: 96.4%
Class 4: 73.7%
------------------------------------
Probability of each class to mistake for healthy
Class 0: : 31.5%
Class 1: : 10.1%
Class 2: : 7.9%
Class 3: : 1.1%<jupyter_text>Experiment 4: Bi-Tempered loss + Self-Distillation<jupyter_code>full_report(data1=original, data2=bitemp, data3=bitemp_sd,
           title1='Benchmark', title2='Bi-Tempered', title3='Bi-Tempered + Self-Distillation',
           first=False,second=False, third=True) #first=False and second=False to print only the 3nd dataset's report<jupyter_output>--- Bi-Tempered + Self-Distillation DATASET ---
Average CV score: 88.77%
------------------------------------
Accuracy score of each class:
Class 0: 64.1%
Class 1: 77.8%
Class 2: 80.1%
Class 3: 96.7%
Class 4: 76.1%
------------------------------------
Probability of each class to mistake for healthy
Class 0: : 35.0%
Class 1: : 12.9%
Class 2: : 8.3%
Class 3: : 1.0%<jupyter_text>Experiment 5: Resnext50_32_4d model<jupyter_code>full_report(data1=bitemp_sd, data2=resnext,
            title1='EfficientNet B3', title2='Resnext50_32_4d',
            #errors=True, color1='crimson', color2='gainsboro',
            first=False,second=True) #first=False to print only the 2nd dataset's report<jupyter_output>--- Resnext50_32_4d DATASET ---
Average CV score: 89.21%
------------------------------------
Accuracy score of each class:
Class 0: 66.0%
Class 1: 78.9%
Class 2: 78.5%
Class 3: 97.6%
Class 4: 74.9%
------------------------------------
Probability of each class to mistake for healthy
Class 0: : 31.0%
Class 1: : 10.3%
Class 2: : 8.7%
Class 3: : 0.7%<jupyter_text>Experiment 6: Vision Transformer model<jupyter_code>full_report(data1=bitemp_sd, data2=resnext, data3=vit,
           title1='EfficientNet B3', title2='Resnext50_32_4d', title3='Vision Transformer',
           errors=False,color1='black', color2='gainsboro',color3='white',

           first=False,second=False, third=True) #first=False and second=False to print only the 3nd dataset's report<jupyter_output>--- Vision Transformer DATASET ---
Average CV score: 87.44%
------------------------------------
Accuracy score of each class:
Class 0: 64.5%
Class 1: 82.8%
Class 2: 82.0%
Class 3: 94.1%
Class 4: 70.2%
------------------------------------
Probability of each class to mistake for healthy
Class 0: : 29.9%
Class 1: : 7.9%
Class 2: : 7.8%
Class 3: : 2.0%<jupyter_text>Experiment 8: FINAL ENSEMBLE<jupyter_code>vit_logits=vit.iloc[:,1:6].to_numpy()
original_logits=original.iloc[:,1:6].to_numpy()
resnext_logits=resnext.iloc[:,6:11].to_numpy()
B3_logits=bitemp_sd.iloc[:,6:11].to_numpy()
ensemble= 0.2*vit_logits + 0.4*resnext_logits + 0.4*B3_logits
preds= np.argmax(ensemble, axis=1)
blend=pd.DataFrame(ensemble, columns=['logits0','logits1','logits2','logits3','logits4',])
blend['pred']=preds
blend['label']=resnext['label']
print('Ensemble weights: 0.2*ViT + 0.4*resnext + 0.4*B3')
print(f'Ensemble: Average CV score: {round((blend.label.values==blend.pred.values).mean()*100,2)}%')
def accuracies(df,first=False):
        probs=[]
        cm=confusion_matrix(df.label, df.pred)
        for i, val in enumerate(cm):
            prob=round(val[i]/sum(val)*100,1)
            probs.append(prob)
            
        return probs
#Function to plot errors and accuracies
def visualplot(data1,data2,data3,data4, title1, title2,title3,title4):
        labels = ['Class 0:CBB', 'Class 1:CBSD', 'Class 2:CGM', 'Class 3:CMD','healthy']
        
       
        #Read accuracy data
        probs1= accuracies(data1)
        probs2= accuracies(data2)
        probs3= accuracies(data3)
        probs4= accuracies(data4)
        
          
        x = np.arange(len(labels))  # the label locations
        width = 0.2  # the width of the bars

        fig, ax = plt.subplots(figsize=(12,4))
        rects1 = ax.bar(x - width/2, probs1, width, label=f'{title1}',edgecolor='black', color='navajowhite')
        rects2 = ax.bar(x + width/2, probs2, width, label=f'{title2}',edgecolor='black', color='lightskyblue')
        rects3 = ax.bar(x + width*1.54, probs3, width, label=f'{title3}',edgecolor='black', color='lightgreen')
        rects4 = ax.bar(x + width*2.57, probs4, width, label=f'{title4}',edgecolor='black', color='lightpink')
        

        #Add grid
        ax.grid(b=True, color='grey', linestyle='-.', linewidth=1, alpha=0.3)
        
        #Set Bars starting point values
        ax.set(ylim=[50, 120])            

        # Add some text for labels, title and custom x-axis tick labels, etc.
        ax.set_ylabel('Accuracy', fontsize=13)
        ax.set_xlabel('Cassava classes', fontsize=13)
        ax.set_title('Accuracy of the 5 classes', weight='bold', fontsize=15)
        
        ax.set_xticks(x)
        plt.xticks(weight='bold')
        ax.set_xticklabels(labels)
        ax.legend()

        ax.bar_label(rects1, padding=0.8, weight='bold', color='darkorange', fontsize=9)
        ax.bar_label(rects2, padding=1, weight='bold', color='blue', fontsize=9)
        ax.bar_label(rects3, padding=1, weight='bold', color='green', fontsize=9)
        ax.bar_label(rects4, padding=1, weight='bold', color='deeppink', fontsize=9)
                    
        
        fig.tight_layout()
        return plt.show()
visualplot(vit, bitemp_sd, resnext, blend, 'ViT', 'EfficientNetB3', 'Resnext50_32_4d', 'Ensemble')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16012725344644752, 'fraction_numerical': 0.03888299752562743, 'mean_word_length': 2.528372479733943, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '148727', 'license': 'Apache-2.0', 'n_tokens_mistral': 5965, 'n_tokens_neox': 5559, 'n_words': 1271, 'path': 'Experiments analysis/cassava-analysis-presentation.ipynb', 'repo_name': 'Nik404/Cassava-Disease-App', 'size': '16974'}