<jupyter_start><jupyter_text>Classification and Visualization https://en.wikipedia.org/wiki/Multi-label_classification<jupyter_code>from __future__ import print_function
import torch
import torch.nn as nn

import torchvision.datasets as dset
import torchvision.transforms as transforms
import torchvision.models

import os
import sys
import copy

import numpy as np
import pandas as pd

import sklearn
from sklearn import metrics

import matplotlib
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import seaborn as sns

from IPython.display import display, HTML
import ipywidgets as widgets

from textwrap import wrap
from itertools import compress

# Import MonaLIA library from the package in the subfolder of the notebook folder
module_path = os.path.abspath(os.path.join('../..'))
if module_path not in sys.path:
    sys.path.append(module_path)

import importlib 
import MonaLIA.util.visualization_helpers as vis
import MonaLIA.model.train as model
import MonaLIA.data.image_transforms
import MonaLIA.data.image_dataset

#importlib.reload(MonaLIA.util.visualization_helpers)
importlib.reload(MonaLIA.model.train)
#importlib.reload(MonaLIA.data.image_transforms)
#importlib.reload( MonaLIA.data.image_dataset)
    
from MonaLIA.data import image_transforms
from MonaLIA.data import image_dataset

from MonaLIA.data.image_dataset import JocondeDataset
from MonaLIA.data.binary_subset import BinarySubset
from MonaLIA.data.image_transforms import NormalizeMultiCrop, PadToSquare

from MonaLIA.util.visualization_helpers import target_to_text, prediction_to_text
from MonaLIA.util.visualization_helpers import show_confusion_matrix, show_random_images, image_tensor_to_np
from MonaLIA.util.visualization_helpers import visualize_classification_HTML as display_HTML
from MonaLIA.util.visualization_helpers import compare_classification_HTML as display_compare_HTML
print('PyTorch ver.' ,  torch.__version__ )
print('torchvision ver.' ,  torchvision.__version__ )
print('cuda ver.', torch.version.cuda)
print('sckit-learn ver.', sklearn.__version__)
print('matplotlib ver.', matplotlib.__version__)
print('pandas ver.', pd.__version__)<jupyter_output>PyTorch ver. 1.6.0
torchvision ver. 0.7.0
cuda ver. 10.2
sckit-learn ver. 0.23.2
matplotlib ver. 3.3.1
pandas ver. 1.1.1<jupyter_text>Load Data<jupyter_code>dataset = 'Joconde'

if dataset == 'Joconde':
    images_root = 'C:/Joconde/joconde'
    descr_path = 'C:/Datasets/Joconde/Forty classes'
    image_description_file = os.path.join(descr_path, 'dataset1.csv')
    exclude_labels = []# ['espèce animale+être humain' , 'none+none'] 

elif dataset == 'ImageNet': # refactor
    images_root = 'Z:\Documents\Python Scripts\data\ImageNet'

else:
    raise ValueError('unknown dataset')


multi_label = True
multi_crop = False
batch_size = 4

model_name = 'inception_v3'
model_checkpoint_file = '../../MonaLIA/output/inception_v3_Joconde_40_classes.test.1000.1.checkpoint.pth.tar'
model_metrics_file = '../../MonaLIA/output/inception_v3_Joconde_40_classes.test.1000.1.csv'

model_image_size = 299
if model_name == 'inception_v3':
    dataset_mean =  [0.5, 0.5, 0.5]
    dataset_std  =  [0.5, 0.5, 0.5]

elif model_name == 'vgg16_bn':
    if dataset == 'Joconde':
        dataset_mean =  image_transforms.joconde_mean_animals 
        dataset_std  =  image_transforms.joconde_std_animals 
    else:
        dataset_mean =  image_transforms.imagenet_mean 
        dataset_std  =  image_transforms.imagenet_std

else:
        dataset_mean =  image_transforms.imagenet_mean 
        dataset_std  =  image_transforms.imagenet_std   
    
if (multi_crop):
    test_trans = transforms.Compose([
                    transforms.Resize(max(256, model_image_size)),
                    transforms.FiveCrop(model_image_size),
                    transforms.Lambda(lambda crops: torch.stack([transforms.ToTensor()(crop) for crop in crops])), # returns a 4D tensor
                    NormalizeMultiCrop(mean = dataset_mean,
                                         std = dataset_std)
                    ])
else:

    test_trans = transforms.Compose([
        #PadToSquare(padding_mode='wrap'),
        #transforms.Resize((model_image_size, model_image_size)), 
        transforms.Resize(model_image_size),
        transforms.CenterCrop(model_image_size),
        transforms.ToTensor(),
        transforms.Normalize(mean = dataset_mean,
                             std = dataset_std),
    ])
    

if dataset == 'Joconde':
    test_set = JocondeDataset(image_description_file, 
                            images_root,
                            dataset_name = '40_classes',
                            exclude_labels=exclude_labels  ,
                            label_column='label',
                            multiple_labels = multi_label, 
                            filter_dict= {'usage': ['test']}, 
                            add_columns=['ref', 'repr'],
                            transform=test_trans)

elif dataset == 'ImageNet':
    test_set = dset.ImageFolder(root=os.path.join(images_root, 'test'),  transform=test_trans)


test_loader = torch.utils.data.DataLoader(dataset=test_set,
                                        batch_size=batch_size,
                                        shuffle=False,
                                        num_workers=2)

class_count = len(test_set.classes)
class_names = test_set.classes

print('Test', test_set)
print('    Labels:', test_set.labels_count)
print()<jupyter_output>Test Dataset JocondeDataset
    Number of datapoints: 4463
    Root location: C:/Joconde/joconde
    Description file: C:/Datasets/Joconde/Forty classes\dataset1.csv
    Number of classes: 40
    Number of uniqie labels: 449
    StandardTransform
Transform: Compose(
               Resize(size=299, interpolation=PIL.Image.BILINEAR)
               CenterCrop(size=(299, 299))
               ToTensor()
               Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
           )
    Labels: {'ange': 178, 'arbre': 451, 'armure': 85, 'bateau': 300, 'bateau à voiles': 102, 'casque': 63, 'cavalier': 137, 'chapeau': 153, 'cheval': 374, 'chien': 183, 'château': 155, 'couronne': 155, 'croix': 119, 'de face': 237, 'de profil': 317, 'drapeau': 58, 'draperie': 177, 'en buste': 360, 'feuille': 86, 'fleur': 247, 'lion': 64, 'livre': 171, 'main': 127, 'maison': 290, 'mer': 109, 'montagne': 121, 'mouton': 56, 'nu': 277, 'nuage': 52, 'nudité': 123, 'oiseau': 219, 'pont': 112, 'robe': 62, 'table': 82, [...]<jupyter_text>Adjust if Binary (balanced)<jupyter_code>class_count = len(checkpoint['classes'])

if(class_count == 2):

    test_set = BinarySubset(test_set, 'ange')
    
    test_loader = torch.utils.data.DataLoader(dataset=test_set,
                                        batch_size=batch_size,
                                        shuffle=False,
                                        num_workers=2)

    class_count = len(test_set.classes)
    class_names = test_set.classes

    print('Test', test_set)
    print('    Labels:', test_set.labels_count)
    print()<jupyter_output>Test Dataset BinarySubset
    Number of datapoints: 356
    Root location: C:/Joconde/joconde
    Number of classes: 2
    Number of positive labels: 178
    Number of negative labels: 178
    Labels: {'~ange': 178, 'ange': 178}<jupyter_text>Transform to Binary (unbalalnced)<jupyter_code>def transform_sample(sample, class_idx):
    sample_list = list(sample)
    sample_list[1] = [sample[1][class_idx] ^ 1 , sample[1][class_idx] ]
    return tuple(sample_list)

def transform_dataset(dataset, subset_class):
    class_idx =  dataset.class_to_idx[subset_class]
    
    dataset.samples = [ transform_sample(sample, class_idx)  for sample in dataset.samples ]
    dataset.targets = [sample[1] for sample in dataset.samples]
    dataset.imgs = dataset.samples

    dataset.classes = ['~' + subset_class, subset_class]
    dataset.class_to_idx = { '~' + subset_class: 0, subset_class: 1}

    labels_per_class = [sum([target[i] for target in dataset.targets]) for i in range(len(dataset.classes))]
    dataset.labels_count = dict(zip(dataset.classes, labels_per_class))
    
    return dataset


subset_class = checkpoint['classes'][1]
class_idx =  test_set.class_to_idx[subset_class]
print(subset_class, class_idx)

transform_dataset(test_set, subset_class)

print('Test', test_set)
print('    Labels:', test_set.labels_count)
print()<jupyter_output>Test Dataset JocondeDataset
    Number of datapoints: 4463
    Root location: C:/Joconde/joconde
    Description file: C:/Datasets/Joconde/Forty classes\dataset1.csv
    Number of classes: 2
    Number of uniqie labels: 2
    StandardTransform
Transform: Compose(
               Resize(size=299, interpolation=PIL.Image.BILINEAR)
               CenterCrop(size=(299, 299))
               ToTensor()
               Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
           )
    Labels: {'~ange': 4285, 'ange': 178}<jupyter_text>Read metrics from the cluster run<jupyter_code>print(model_checkpoint_file)
checkpoint = torch.load(model_checkpoint_file)
print(checkpoint.keys())
from livelossplot import PlotLosses

metrics_offset = 0
if os.path.exists(model_metrics_file): 
    metrics_df_old = pd.read_csv(model_metrics_file)
    metrics_df = metrics_df_old
    metrics_offset = 1
else:
    metrics_df = pd.DataFrame(checkpoint['metrics_history'])

liveloss_cluster = PlotLosses()#(max_epoch=metrics_df.shape[0], max_cols=2)

for i, row in metrics_df.iterrows():
    epoch_stats ={}
    for col in metrics_df.columns[metrics_offset:]:#.sort_values(ascending=False):
        epoch_stats[col] = row[col]

    liveloss_cluster.update(epoch_stats)
    
liveloss_cluster.draw()

print('Best model training accuracy: %.4f' % metrics_df.accuracy.max())
print('Best model validation accuracy: %.4f' % metrics_df.val_accuracy.max())
print('Best model mAP: %.4f' % metrics_df.val_mAP.max())
print('Best model epoch (mAP): %d' %  (metrics_df.val_mAP.idxmax() + 1))
print('Training time: %s' % checkpoint['elapsed_time'])

## read training set and validation set for the statistics
train_set = JocondeDataset(image_description_file, 
                            images_root,
                            exclude_labels=exclude_labels  ,
                            label_column='label',
                            multiple_labels = multi_label, 
                            filter_dict= {'usage': ['train']})
val_set = JocondeDataset(image_description_file, 
                            images_root,
                            exclude_labels=exclude_labels  ,
                            label_column='label',
                            multiple_labels = multi_label, 
                            filter_dict= {'usage': ['val']})

if(len(checkpoint['classes']) == 2):

    train_set = BinarySubset(train_set, checkpoint['classes'][1])
    
    val_set = BinarySubset(val_set, checkpoint['classes'][1])

    
print(train_set)
print(val_set)<jupyter_output>Dataset JocondeDataset
    Number of datapoints: 55990
    Root location: C:/Joconde/joconde
    Description file: C:/Datasets/Joconde/Forty classes\dataset1.csv
    Number of classes: 40
    Number of uniqie labels: 4893
Dataset JocondeDataset
    Number of datapoints: 4463
    Root location: C:/Joconde/joconde
    Description file: C:/Datasets/Joconde/Forty classes\dataset1.csv
    Number of classes: 40
    Number of uniqie labels: 449<jupyter_text>Load Model<jupyter_code>if os.path.exists(model_checkpoint_file.replace('checkpoint' , 'best')): 
    print(model_checkpoint_file.replace('checkpoint' , 'best'))
    checkpoint = torch.load(model_checkpoint_file.replace('checkpoint' , 'best'))
else:
    print(model_checkpoint_file)
    checkpoint = torch.load(model_checkpoint_file)
print(checkpoint.keys())
class_count = len(checkpoint['classes'])
use_cuda = torch.cuda.is_available()
device = torch.device('cuda:0' if use_cuda else 'cpu')
print('Using cuda? ', use_cuda)

net = model.load_net(model_name = model_name, class_count=class_count)
#net.load_state_dict(torch.load(model_param_file))
net.load_state_dict(checkpoint['state_dict'])
net = net.to(device)
#print(net.transform_input)<jupyter_output>Using cuda?  True<jupyter_text>Run Model<jupyter_code>th = checkpoint['threshold'].round(2)
print(th)
activation = torch.sigmoid
decision = model.decision_by_class_threshold
decision_param = th
scores = model.predict(net, test_loader, activation, decision, decision_param)<jupyter_output>images total: 1000, correct: 257, acc: 25.7% , mAP: 0.536
images total: 2000, correct: 514, acc: 25.7% , mAP: 0.530
images total: 3000, correct: 752, acc: 25.1% , mAP: 0.524
images total: 4000, correct: 997, acc: 24.9% , mAP: 0.527
images total: 4463, correct: 1112, acc: 24.9% , mAP: 0.528
Finished prediction<jupyter_text>Statistics https://www.datascienceblog.net/post/machine-learning/interpreting-roc-curves-auc/https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_curve.htmlsklearn.metrics.precision_recall_curvehttps://scikit-learn.org/stable/auto_examples/model_selection/plot_precision_recall.htmlsphx-glr-auto-examples-model-selection-plot-precision-recall-pyhttps://scikit-learn.org/stable/modules/generated/sklearn.metrics.average_precision_score.html<jupyter_code>scores = scores.cpu() 

decision_param = th#0.4999
print(decision_param)
y_true = np.array(test_set.targets , dtype = np.dtype('B')) 
y_pred = decision(scores, decision_param ).numpy().astype(np.dtype('B'))
#y_pred = model.decision_by_threshold(scores, decision_param).numpy().astype(np.dtype('B'))
y_scores = scores.numpy()
prfs = metrics.precision_recall_fscore_support(y_true= y_true,
                                               y_pred= y_pred)

acc = metrics.accuracy_score(y_true= y_true,
                            y_pred= y_pred)


sim = metrics.jaccard_score(y_true= y_true,
                            y_pred= y_pred,
                            average='samples')

mAP = 0.0
mAP =  metrics.average_precision_score(y_true= y_true,
                                       y_score=y_scores,
                                       average='macro')
report = metrics.classification_report(y_true= y_true,
                                 y_pred= y_pred, 
                                 #labels = test_set.classes,
                                 target_names = test_set.classes,
                                 output_dict = True)
ap = 0.0
ap =  metrics.average_precision_score(y_true= y_true,
                                      y_score=y_scores,
                                      average=None)

report_output = pd.concat((pd.Series([x for x in report] ),
                           pd.Series(list(train_set.labels_count.values()) + 4 * [sum(train_set.labels_count.values())]),
                           pd.Series(list(val_set.labels_count.values()) + 4 * [sum(val_set.labels_count.values())]),
                           pd.Series([x['support'] for x in report.values()]),
                           pd.Series([x['recall'] for x in report.values()]).round(3),
                           pd.Series([x['precision'] for x in report.values()]).round(3),
                           pd.Series([x['f1-score'] for x in report.values()]).round(3),
                           pd.Series(ap).round(3),
                           pd.Series(report['macro avg']['f1-score']).round(3),
                           pd.Series(acc).round(3),
                           pd.Series(mAP).round(3),
                           pd.Series(sim).round(3),
                           pd.Series(decision_param)
                           ), axis=1)
report_output.columns = ['class', 'train', 'val', 'test', 'recall', 'precision', 'f1-score', 'AP', 'f1 macroavg','acc', 'mAP', 'jaccard similarity score' , 'param']

pd.options.display.float_format = '{:,.2f}'.format
report_output<jupyter_output>C:\Users\abobashe\Anaconda3\lib\site-packages\sklearn\metrics\_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))<jupyter_text>Visualize<jupyter_code>cat_dd = widgets.Dropdown(options= test_set.classes,
                         description='Categories:')

query_options = [ 'NO EXACT MATCH',
                  'NO INTERSECTION MATCH',
                  '<50% INTERSECTION',
                  '>=50% INTERSECTION',
                  'INTERSECTION MATCH',
                  'EXACT MATCH', 
                  'ALL CONTAINING CLASS',
                  'FALSE POSITIVES'
                ]

opt_ss = widgets.RadioButtons(options = query_options,
                             description = 'Query options',
                             value = '>=50% INTERSECTION'  ,
                                layout=widgets.Layout(width='50%'))
display (cat_dd, opt_ss)
query = opt_ss.index
item_count = 5
is_random = False
show_class =  test_set.class_to_idx[cat_dd.value] 
if query == 0:
    #strict accuracy = exact match = subset accuracy
    classified_idx = np.nonzero((y_true == y_pred).sum(axis=1) < class_count)[0]
elif query == 5:
    classified_idx = np.nonzero((y_true == y_pred).sum(axis=1) == class_count)[0]

elif query == 1:    
    #no intersection
    classified_idx = np.nonzero((y_true & y_pred).sum(axis=1) == 0)[0]
elif query == 4:   
    #at least one intersection 
    classified_idx = np.nonzero((y_true & y_pred).sum(axis=1) > 0)[0]

elif query == 2:   
    #intersection "less than" accuracy
    classified_idx = np.nonzero( (y_true & y_pred).sum(axis=1) / y_true.sum(axis=1) < 0.5 )[0]
elif query == 3:   
    classified_idx = np.nonzero( (y_true & y_pred).sum(axis=1) / y_true.sum(axis=1) >= 1.0 )[0]

elif query == 6:
    classified_idx = np.nonzero(y_true.sum(axis=1) >0 )[0] # fudging the full list
    
elif query == 7:    
    classified_idx = np.nonzero( (y_true.astype('int') - y_pred.astype('int')).min(axis=1) < 0)[0]

else:
    raise ValueError('Invalid query')
   
print(query_options[query])

print(classified_idx.shape[0], 'of', y_pred.shape[0] )
print('proportion: %.2f' % (classified_idx.shape[0] / y_pred.shape[0]))

#filter specific class
if show_class >= 0:
    class_filter = np.nonzero(y_true[:, show_class] == 1)[0] if query != 7 else np.nonzero( (y_pred[:, show_class] - y_true[:, show_class]) == 1)[0] 
    n = classified_idx.shape[0]
    classified_idx = np.intersect1d(classified_idx , class_filter)
    print('%d of %d containing "%s"' % (classified_idx.shape[0], n, test_set.classes[show_class] ))

if is_random:
    pick = np.random.randint(0, len(classified_idx), size=item_count)
else:
    pick = np.arange(item_count)

testset_idx_pick = classified_idx[pick]


#visualizer(test_set, y_pred, y_scores, testset_idx_pick, show_max_pred_labels=10)
HTML(display_HTML(test_set, y_pred, y_scores, testset_idx_pick, show_max_pred_labels=10))<jupyter_output>>=50% INTERSECTION
2305 of 4463
proportion: 0.52
66 of 2305 containing "chien"<jupyter_text>Output to HTML file<jupyter_code>HTML(display_HTML(test_set, y_pred, y_scores, testset_idx_pick, show_max_pred_labels=10, file_name='lions_binary.softmax.FP.html'))<jupyter_output><empty_output><jupyter_text>Output to csv file<jupyter_code>df = pd.DataFrame(columns= ['reference' , 'repr', 'imageFile', 'image', 'label', 'prediction'])

df.reference = [test_set.samples[i][2] for i in testset_idx_pick]
df.repr = [test_set.samples[i][3] for i in testset_idx_pick]
df.imageFile = [test_set.samples[i][0].replace('/', '\\') for i in testset_idx_pick]

df.label = [target_to_text(test_set.classes,test_set.samples[i][1], separator=' | ') for i in testset_idx_pick]    
df.prediction = [prediction_to_text(test_set.classes, scores[i] , separator=' | ') for i in testset_idx_pick]

df.to_csv('lions_multilabel.20.csv', encoding='utf8', sep='\t')
#df<jupyter_output><empty_output><jupyter_text>Scores Distribution<jupyter_code>classified_scores = scores[classified_idx].numpy() * y_pred[classified_idx]
classified_scores = classified_scores.flatten()
classified_scores = classified_scores[classified_scores.nonzero()]
plt.hist(classified_scores)<jupyter_output><empty_output><jupyter_text>Confusion Matrix<jupyter_code>from MonaLIA.util.visualization_helpers import show_confusion_matrix
from MonaLIA.util.visualization_helpers import target_to_text, prediction_to_text

#y_true = np.array(test_set.targets)
#y_pred = decision(scores[:,1:], decision_param).numpy()

tick_labels= test_loader.dataset.classes

if(multi_label):
    y_true_text = [target_to_text(test_loader.dataset.classes, list(x), separator='+') for x in y_true]
    y_pred_text = [target_to_text(test_loader.dataset.classes, list(x), separator='+') for x in y_pred]
    tick_labels = sorted(set(y_true_text).union(y_pred_text))
    
show_confusion_matrix(y_true_text, y_pred_text, test_loader.dataset.classes)<jupyter_output><empty_output><jupyter_text>Precision-Recall curvehttps://machinelearningmastery.com/roc-curves-and-precision-recall-curves-for-classification-in-python/<jupyter_code>import ipywidgets as widgets
dd = widgets.Dropdown(
       options= test_set.classes,
       value=test_set.classes[0],
       description='Category:')
dd
c = test_set.class_to_idx[dd.value]


y_true_c = y_true.take([c], axis=1).ravel()
y_score_c = y_scores.take([c], axis=1).ravel()

average_precision = metrics.average_precision_score(y_true_c, y_score_c)
precision, recall, _ = metrics.precision_recall_curve( y_true_c, y_score_c)

plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, alpha=0.2, color='b', step='post')

plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('{0} Precision-Recall curve: AP={1:0.2f}'.format(test_set.classes[c].upper(),
          average_precision))<jupyter_output><empty_output><jupyter_text>Correlation between number of the training labels and the Average Precision<jupyter_code>from sklearn.linear_model import LinearRegression
import statsmodels.api as sm

X = np.array(list(train_set.labels_count.values())).reshape(-1,1)#[:-1]
y = ap#[:-1]

reg = LinearRegression().fit(X, y)
reg.score(X, y)

X2 = sm.add_constant(X)
est = sm.OLS(y, X2)
est2 = est.fit()
print(est2.summary())
y_pred = reg.predict(X)
plt.scatter(X, y,  color='black')
plt.plot(X, y_pred, color='blue', linewidth=3)
plt.show()
covariance = np.cov(X.squeeze(), y)
covariance
from scipy.stats import pearsonr, spearmanr
corr, _ = pearsonr(X.squeeze(), y)
print('Pearsons correlation: %.3f' % corr)

# calculate spearman's correlation
corr, _ = spearmanr(X.squeeze(), y)
print('Spearmans correlation: %.3f' % corr)<jupyter_output>Pearsons correlation: 0.483
Spearmans correlation: 0.483<jupyter_text>Aggregated Binary Ensemble Model Test<jupyter_code>net_b = model.load_net(model_name = model_name, class_count=2)

model_checkpoint_file = 'output/Inception_v3_Joconde_40_classes.%s.1000.4.checkpoint.pth.tar'

output = pd.DataFrame(columns= ['class', 'test', 'recall', 'precision', 'f1-score', 'AP', 'f1 macroavg','acc', 'mAP', 'sim', 'threshold','time'])

pcut_threshold = True
if pcut_threshold:
    th_dict = dict(zip(checkpoint['classes'], checkpoint['threshold'].round(2)))
else:
    th_dict = dict(zip(checkpoint['classes'], [0.5] * len(checkpoint['classes'])))

ref = [s[2] for s in test_set.samples]
scores_df = pd.DataFrame(index= ref, columns=test_set.classes)

for clss in test_set.classes:
    #cls_subset = BinarySubset(test_set, clss) #for small balance datasets
    cls_subset = transform_dataset(copy.deepcopy(test_set), clss)
    
    subset_loader = torch.utils.data.DataLoader(dataset=cls_subset,
                                        batch_size=batch_size,
                                        shuffle=False,
                                        num_workers=2)
    
    check_point_file = model_checkpoint_file % clss
    checkpoint = torch.load(check_point_file)

    print(clss , len(subset_loader.dataset)//2, check_point_file)

    net_b.load_state_dict(checkpoint['state_dict'])
    
    sc = model.predict(net_b, subset_loader, torch.softmax, model.decision_by_threshold, th_dict[clss])
    
    
    sc = sc.cpu()
    

    y_tr = np.array(cls_subset.targets , dtype = np.dtype('B')) 
    y_pr = model.decision_by_threshold(sc, th_dict[clss]).numpy().astype(np.dtype('B'))
    y_sc = sc.numpy()
    
    prec, rec, fs, sup = metrics.precision_recall_fscore_support(y_true= y_tr, y_pred= y_pr)

    acc = metrics.accuracy_score(y_true= y_tr,  y_pred= y_pr)

    sim = metrics.jaccard_score(y_true= y_tr, y_pred= y_pr, average='samples')

    mAP =  metrics.average_precision_score(y_true= y_tr, y_score=y_sc, average='macro')
    
    ap =  metrics.average_precision_score(y_true= y_tr, y_score=y_sc, average=None)
    
    output = output.append({'class': clss,
                   'test': sup[1],
                   'recall': rec[1],
                   'precision': prec[1],
                   'f1-score': fs[1],
                   'AP': ap[1],
                   'f1 macroavg': np.average(fs), 
                   'acc': acc,
                   'mAP': mAP,
                   'sim': sim,
                   'threshold': th_dict[clss],
                   'time': checkpoint['elapsed_time']}, ignore_index=True)
    
    #scores_df.loc [[s[2] for s in cls_subset.samples if s[1][1] == 1], clss] = sc.chunk(2)[0][:,1]
        
    #sc_full = model.score(net_b, test_loader, torch.sigmoid, print_frequency=2000)
    #scores_df.loc [:, clss] = sc_full.cpu()[:,1]
    
    scores_df.loc[:, clss] = y_sc[:, 1]

scores_df.fillna(0.0, inplace = True)

pd.options.display.float_format = '{:,.2f}'.format
output
y_scores_b = scores_df.to_numpy()
#y_pred_b = model.decision_by_threshold(torch.Tensor(y_scores_b), 0.5).numpy().astype(np.dtype('B'))
y_pred_b = model.decision_by_class_threshold(torch.Tensor(y_scores_b), th).numpy().astype(np.dtype('B'))
#testset_idx_pick = np.random.randint(0, len(test_set.samples), size=4) 
#testset_idx_pic = np.arange(100)
testset_idx_pick = [3834, 2335, 320]

HTML(display_compare_HTML(test_set,
                          y_pred, y_scores,
                          y_pred_b, y_scores_b,
                          testset_idx_pick, show_max_pred_labels=10))<jupyter_output><empty_output><jupyter_text>Save to HTML<jupyter_code>HTML(display_compare_HTML(test_set,
                          y_pred, y_scores,
                          y_pred_b, y_scores_b, 
                          np.arange(4463), show_max_pred_labels=10, file_name='40 class classification comparision.0.5.html'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12408649702057489, 'fraction_numerical': 0.021961548551512198, 'mean_word_length': 3.344513187886682, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 66, 'https://': 6, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '268761', 'license': 'CC0-1.0', 'n_tokens_mistral': 9494, 'n_tokens_neox': 8904, 'n_words': 1651, 'path': 'Notebooks 2.0/Pipeline/MonaLIA.STEP 1.3.Test Classifier.ipynb', 'repo_name': 'Wimmics/MonaLIA', 'size': '26683'}