<jupyter_start><jupyter_text>teex Evaluation of explanation quality: saliency mapsIn this notebook, we are going to explore how we can use **teex** to compare image explanation methods.  1. Evaluating synthetic image data explanations In this section, we are going to 1. Generate image data with available g.t. explanations using the 'seneca' method.2. Create and train a pytorch classifier that will learn to recognize the pattern in the images.3. Generate explanations with some model agnostic methods and evaluate them w.r.t. the ground truth explanations.<jupyter_code>import matplotlib.pyplot as plt

import numpy as np
import pandas as pd

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

import PIL
import pickle

from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, f1_score

from math import floor

from captum.attr import GradientShap, IntegratedGradients, Occlusion, DeepLift, Lime, GuidedBackprop, GuidedGradCam

from teex.saliencyMap.data import SenecaSM
from teex.saliencyMap.eval import saliency_map_scores

from teex._utils._arrays import _minmax_normalize_array<jupyter_output><empty_output><jupyter_text>1.1. Generating the data<jupyter_code>cellH, cellW = 4, 4
imH, imW = 32, 32

generator = SenecaSM(imageH=imH, imageW=imW, 
                     cellH=cellH, cellW=cellW, 
                     nSamples=5000, randomState=7)

X, y, exps = generator[:]
pattern = generator.pattern
i = 2
fig, axs = plt.subplots(1, 3, figsize=(8, 8))
axs[0].imshow(pattern)
axs[0].set_title('Pattern')
axs[1].imshow(X[i])
axs[1].set_title('Generated image')
axs[2].imshow(exps[i])
axs[2].set_title('Explanation')<jupyter_output><empty_output><jupyter_text>1.2. Declaring and training the model Declare a simple LeNet variant and its training routine.<jupyter_code>class FCNN(nn.Module):
    """ Basic NN for image classification. """

    def __init__(self, imH, imW, cellH, randomState=1):
        super(FCNN, self).__init__()
        stride = 1
        kSize = 5
        torch.manual_seed(randomState)
        self.conv1 = nn.Conv2d(3, 6, kernel_size=kSize, stride=stride)
        self.H1, self.W1 = floor(((imH - kSize) / stride) + 1), floor(((imW - kSize) / stride) + 1)
        self.conv2 = nn.Conv2d(6, 3, kernel_size=kSize, stride=stride)
        self.H2, self.W2 = floor(((self.H1 - kSize) / stride) + 1), floor(((self.W1 - kSize) / stride) + 1)
        self.conv3 = nn.Conv2d(3, 1, kernel_size=kSize, stride=stride)
        self.H3, self.W3 = floor(((self.H2 - kSize) / stride) + 1), floor(((self.W2 - kSize) / stride) + 1)
        self.fc1 = nn.Linear(self.H3 * self.W3, 100)
        self.fc2 = nn.Linear(100, 2)

    def forward(self, x):
        if len(x.shape) == 3:
            # single instance, add the batch dimension
            x = x.unsqueeze(0)
        x = nn.ReLU()(self.conv1(x))
        x = nn.ReLU()(self.conv2(x))
        x = nn.ReLU()(self.conv3(x))
        x = nn.ReLU()(self.fc1(x.view(x.shape[0], -1)))
        x = self.fc2(x)
        return x
    
import copy
# sample training function for classification
def train_net(model, data, criterion, optimizer, device, batchSize, nEpochs, randomState=888):
    """ data: dict with 'train' and 'val' entries. Each entry is a list with X: FloatTensor, y: LongTensor """
    torch.manual_seed(randomState)
    bestValAcc = -np.inf
    bestModelWeights = copy.deepcopy(model.state_dict())
    for epoch in range(nEpochs):
        for phase in ['train', 'val']:
            if phase == 'train':
                model.train()
            else:
                model.eval()
                
            lossVal = .0
            corrects = 0
            
            for batch in range(int(len(data[phase][0]) / batchSize)):
                XBatch = data[phase][0][batch:batch + batchSize].to(device)
                yBatch = data[phase][1][batch:batch + batchSize].to(device)
                
                model.zero_grad()
                
                with torch.set_grad_enabled(phase == 'train'):

                    out = model(XBatch)
                    loss = criterion(out, yBatch)

                    if phase == 'train':
                        loss.backward()
                        optimizer.step()
                    
                    _, preds = torch.max(out, 1)
                
                lossVal += loss.item() * XBatch.size(0)
                corrects += torch.sum(preds == yBatch.data)
            
            epochLoss = lossVal / len(data[phase][0])
            epochAcc = corrects.double() / len(data[phase][0])
            print(f'{phase} Loss: {round(epochLoss, 4)} Acc: {round(epochAcc.item(), 4)}')
        
            if phase == 'val' and epochAcc > bestValAcc:
                bestValAcc = epochAcc
                bestModelWeights = copy.deepcopy(model.state_dict())
        
        model.load_state_dict(bestModelWeights)
        
    return model, bestValAcc<jupyter_output><empty_output><jupyter_text>We cast the images to torch.Tensor type and get train, validation and test splits.<jupyter_code>XTrain, XTest, yTrain, yTest, expsTrain, expsTest = train_test_split(X, y, exps, train_size=0.8, random_state=7)
XTrain, XVal, yTrain, yVal, expsTrain, expsVal = train_test_split(XTrain, yTrain, expsTrain, train_size=0.75, random_state=7)

XTrain = torch.FloatTensor(XTrain).permute(0, 3, 1, 2)
yTrain = torch.LongTensor(yTrain)
XVal = torch.FloatTensor(XVal).permute(0, 3, 1, 2)
yVal = torch.LongTensor(yVal)
XTest = torch.FloatTensor(XTest).permute(0, 3, 1, 2)
yTest = torch.LongTensor(yTest)
print(f'Data proportions -> Train: {round(len(XTrain) / len(X), 3)}, Val: {round(len(XVal) / len(X), 3)}, Test: {round(len(XTest) / len(X), 3)}')
print(f'Positive label proportions -> Train: {round((sum(yTrain) / len(yTrain)).item(), 3)}, \
Val: {round((sum(yVal) / len(yVal)).item(), 3)}, Test: {round((sum(yTest) / len(yTest)).item(), 3)}')<jupyter_output>Data proportions -> Train: 0.6, Val: 0.2, Test: 0.2
Positive label proportions -> Train: 0.489, Val: 0.526, Test: 0.507<jupyter_text>and train the network<jupyter_code>nFeatures = len(XTrain[0].flatten())
criterion = nn.CrossEntropyLoss()
model = FCNN(imH=32, imW=32, cellH=4)
optimizer = optim.Adam(model.parameters(), lr=1e-3)
data = {'train': [XTrain, yTrain], 'val': [XVal, yVal]}
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model, valAcc = train_net(model, data, criterion, optimizer, device, batchSize=10, nEpochs=10, randomState=7)
print(f'Validation F1: {round(f1_score(yVal, F.softmax(model(torch.FloatTensor(XVal)), dim=-1).argmax(dim=1).detach().numpy()), 3)}')
print(f'Test F1: {round(f1_score(yTest, F.softmax(model(torch.FloatTensor(XTest)), dim=-1).argmax(dim=1).detach().numpy()), 3)}')<jupyter_output>Validation F1: 0.99
Test F1: 0.984<jupyter_text>1.3. Generating and evaluating explanations With the model trained on the synthetic images, we generate explanations (with Captum, but feel free to use other methods!). First, declare the explainers:<jupyter_code>layer = [layer for _, layer in model.named_modules()][-3]

gradShap = GradientShap(model)
intGrad = IntegratedGradients(model)
occlusion = Occlusion(model)
deepLift = DeepLift(model)
guidedBackProp = GuidedBackprop(model)
guidedGradCAM = GuidedGradCam(model, layer)<jupyter_output><empty_output><jupyter_text>And define a function to obtain the explanations from different methods:<jupyter_code>def get_attributions(data, targets, explainer, params=None):
    """  
    :param data: (Tensor) data to explain
    :param targets: (Tensor) class labels w.r.t which we want to compute the attributions
    :param explainer: (captum.attr method) initialised explainer
    :param params: (dict) parameters for the .attribute method of the explainer 
    :return: ndarray of shape with attributions
    """
    
    if params is None:
        params = {}
    attributions = []
    for image, target in zip(data, targets):
        image = image.unsqueeze(0)
        attr = explainer.attribute(image, target=target, **params).squeeze().cpu().detach().numpy()
        # mean pool channel attributions
        attr = np.mean(attr, axis=0)
        # viz._normalize_image_attr(tmp, 'absolute_value', 10)
        attributions.append(_minmax_normalize_array(attr))

    return np.array(attributions)
predTargets[:5]
# use predicted labels
obsToExplain = XTest[:5]
expsToCompare = expsTest[:5]
predTargets = F.softmax(model(torch.FloatTensor(obsToExplain)), dim=-1).argmax(dim=1)[:5]
z = torch.LongTensor([1 if e == 0 else 0 for e in predTargets])
# z = torch.zeros(len(predTargets), dtype=torch.int)
# takes some minutes to run
gradShapExpsTest = get_attributions(obsToExplain, predTargets, gradShap, {'baselines': torch.zeros((1, 3, imH, imW))})
intGradExpsTest = get_attributions(obsToExplain, predTargets, intGrad)
deepLiftExpsTest = get_attributions(obsToExplain, predTargets, deepLift)
occlusionExpsTest = get_attributions(obsToExplain, predTargets, occlusion, {'baselines': 0, 'sliding_window_shapes': (3, cellH*2, cellW*2)})
gBackPropExpsTest = get_attributions(obsToExplain, predTargets, guidedBackProp)
gGradCAMExpsTest = get_attributions(obsToExplain, predTargets, guidedGradCAM)

with open('docs/demos/temp/expsSynth.pickle', 'wb') as handle:
    pickle.dump([gradShapExpsTest, intGradExpsTest, deepLiftExpsTest, occlusionExpsTest, gBackPropExpsTest, gGradCAMExpsTest], handle)
F.softmax(model(torch.FloatTensor(obsToExplain)), dim=-1)
with open('temp/expsSynth.pickle', 'rb') as handle:
    gradShapExpsTest, intGradExpsTest, deepLiftExpsTest, occlusionExpsTest,\
        gBackPropExpsTest, gGradCAMExpsTest = pickle.load(handle)
i = 3

fig, axs = plt.subplots(1, 8, figsize=(15,15))
axs[0].imshow(obsToExplain[i].permute(1, 2, 0))
axs[0].set_title('Image')
axs[1].imshow(expsToCompare[i])
axs[1].set_title('g.t. explanation')
axs[2].imshow(intGradExpsTest[i])
axs[2].set_title('integratedGradient')
axs[3].imshow(gradShapExpsTest[i])
axs[3].set_title('gradientSHAP')
axs[4].imshow(deepLiftExpsTest[i])
axs[4].set_title('deepLift')
axs[5].imshow(occlusionExpsTest[i])
axs[5].set_title('occlusion')
axs[6].imshow(gBackPropExpsTest[i])
axs[6].set_title('Guided Backprop.')
axs[7].imshow(gGradCAMExpsTest[i])
axs[7].set_title('Guided gradCAM.')
gGradCAMExpsTest[i]<jupyter_output><empty_output><jupyter_text>And we can evaluate the explanations. We set the binarization threshold for the generated explanations to 0.5 because we only want high values to count.<jupyter_code>metrics = ['auc', 'fscore', 'prec', 'rec', 'cs']
gradShapScores = saliency_map_scores(expsTest[yTest == 1], gradShapExpsTest, metrics=metrics, binThreshold=0.5)
intGradScores = saliency_map_scores(expsTest[yTest == 1], intGradExpsTest, metrics=metrics, binThreshold=0.5)
deepLiftScores = saliency_map_scores(expsTest[yTest == 1], deepLiftExpsTest, metrics=metrics, binThreshold=0.5)
occlusionScores = saliency_map_scores(expsTest[yTest == 1], occlusionExpsTest, metrics=metrics, binThreshold=0.5)
gBackPropScores = saliency_map_scores(expsTest[yTest == 1], gBackPropExpsTest, metrics=metrics, binThreshold=0.5)
gGradCAMScores = saliency_map_scores(expsTest[yTest == 1], gGradCAMExpsTest, metrics=metrics, binThreshold=0.5)

scores = pd.DataFrame(data=[gradShapScores, intGradScores, deepLiftScores, 
                            occlusionScores, gBackPropScores, gGradCAMScores], columns=metrics)
scores['technique'] = ['gradSHAP', 'intGrad', 'deepLift', 'occlusion', 'guidedBackProp', 'guidedGradCAM']
scores<jupyter_output>/Users/master/Google Drive/U/4t/TFG/teex/venv/lib/python3.8/site-packages/teex/featureImportance/eval.py:79: UserWarning: A binary prediction contains uniform values, so one entry has been randomly flipped for the metrics to be defined.
  warnings.warn('A binary prediction contains uniform values, so one entry has been randomly flipped '
/Users/master/Google Drive/U/4t/TFG/teex/venv/lib/python3.8/site-packages/teex/featureImportance/eval.py:85: UserWarning: A prediction does not contain values != 0, so 1e-4 has been added to one random entry in both.
  warnings.warn('A prediction does not contain values != 0, so 1e-4 has been added to one random entry '
/Users/master/Google Drive/U/4t/TFG/teex/venv/lib/python3.8/site-packages/teex/featureImportance/eval.py:79: UserWarning: A binary prediction contains uniform values, so one entry has been randomly flipped for the metrics to be defined.
  warnings.warn('A binary prediction contains uniform values, so one entry has been randomly flipped [...]<jupyter_text>Note how the warnings tell us that the predictions from one of the techniques did not contain any relevant values. From here, we can build a more complex explanation evaluation pipeline. Suppose that, given some explainer architecture and model, we want to measure how the influence of some explainer hyperparameters influence the quality of their generated explanations.<jupyter_code>def eval_explainers(
        explainers, 
        explainerConfigs, 
        data, 
        targets, 
        trueExps, 
        metrics, 
        binThreshold: float = .5) -> dict:
    """ 
    :param explainers: (dict of captum.attr explainers) explainers to use
    :param explainerConfigs: (dict of list of dict) hyperparameter values to use for each explainer (see Captum docs)
    :param data: (Tensor) data to explain
    :param targets: (Tensor) labels w.r.t. which we compute the explanations
    :param trueExps: (ndarray) ground truth explanations
    :param metrics: (array-like of str) metrics to compute
    :param float binThreshold: threshold to use when binarizing for the computation of classification metrics.
    """
    allScores = {explainer: {met: [] for met in metrics} for explainer in explainers.keys()}
    for explainerName, explainer in explainers.items():
        for config in explainerConfigs[explainerName]:
            exps = get_attributions(data, targets, explainer, config)
            evals = saliency_map_scores(trueExps, exps, metrics=metrics, binThreshold=binThreshold)
            for i, score in enumerate(evals):
                allScores[explainerName][metrics[i]].append(score)
    return allScores<jupyter_output><empty_output><jupyter_text>for example, given these gradSHAP and guidedGradCAM configurations:<jupyter_code>configs = {
    'gradSHAP': [{'baselines': torch.zeros((1, 3, imH, imW)), 'n_samples': 10, 'stdevs': 0.1},
                 {'baselines': torch.zeros((1, 3, imH, imW)), 'n_samples': 15, 'stdevs': 0.15}],
    'gradCAM': [{'interpolate_mode': 'nearest'},
                {'interpolate_mode': 'area'}]
}

explainers = {'gradSHAP': gradShap, 'gradCAM': guidedGradCAM}

metrics = ['auc', 'fscore', 'prec', 'rec', 'cs']
# takes some minutes to run
scores = eval_explainers(explainers, configs, XTest[yTest==1], yTest[yTest==1], expsTest[yTest==1], metrics)

with open('temp/synthScores.pickle', 'wb') as f:
    pickle.dump(scores, f)
with open('temp/synthScores.pickle', 'rb') as f:
    scores = pickle.load(f)<jupyter_output><empty_output><jupyter_text>For each explainer and configuration, we have a score:<jupyter_code>scores<jupyter_output><empty_output><jupyter_text>With these metrics, then, we can evaluate the performance of explainers.  2. Evaluating kahikatea image explanations teex includes real datasets with available ground truth explanations. For example, the Kahikatea dataset contains 519 images, and the task is to tell whether each observation contains Kahikatea trees or not. There are 232 positive observations and 287 negative ones.In teex, the non-artificial datasets are implemented as classes, similarly to PyTorch. After instancing the class, the data itself will be downloaded if it has not been used before. Once done, one can slice it to obtain observations. Each observation contains the data point, the label and the ground truth explanation.<jupyter_code>from teex.saliencyMap.data import Kahikatea
nClasses = 2
kahikateaData = Kahikatea()
kData, kLabels, kExps = kahikateaData[:]
i = 0
fig, axs = plt.subplots(1, 2, figsize=(15,15))
axs[0].imshow(kData[i])
axs[0].set_title('Image')
axs[1].imshow(kExps[i])
axs[1].set_title('g.t. explanation')
kahikateaData.classMap<jupyter_output><empty_output><jupyter_text>Let's fine-tune a pre-trained squeezenet for our particular task.<jupyter_code>torch.hub._validate_not_a_forked_repo=lambda a,b,c: True  # 
sqznet = torch.hub.load('pytorch/vision:v0.9.0', 'squeezenet1_0', pretrained=True, progress=False)
torch.manual_seed(7)<jupyter_output>Using cache found in /Users/master/.cache/torch/hub/pytorch_vision_v0.9.0
Downloading: "https://download.pytorch.org/models/squeezenet1_0-a815701f.pth" to /Users/master/.cache/torch/hub/checkpoints/squeezenet1_0-a815701f.pth<jupyter_text>And modify its architecture so the shape of the output conforms to our 2-class problem instead of the 1000-class ImageNet.<jupyter_code>sqznet.classifier[1] = nn.Conv2d(512, nClasses, kernel_size=(1,1), stride=(1,1))
sqznet.num_classes = nClasses
inputSize = 224<jupyter_output><empty_output><jupyter_text>Define the required transform for the input images<jupyter_code>from torchvision import transforms

inputTransform = transforms.Compose([transforms.Resize((inputSize, inputSize)),
                                     transforms.ToTensor(),
                                     transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
resizeTransform = transforms.Compose([transforms.Resize((inputSize, inputSize)),
                                     transforms.ToTensor()])<jupyter_output><empty_output><jupyter_text>Transform the data to torch tensors and create the splits:<jupyter_code>kData = torch.stack([inputTransform(img) for img in kData])
kExps = torch.stack([resizeTransform(img) if isinstance(img, PIL.Image.Image) else torch.zeros((3, inputSize, inputSize)) for img in kExps]).permute(0, 2, 3, 1)
kExps = kExps.numpy().astype(np.float32)  # we need g.t. explanations to be numpy arrays for the evaluation
kLabels = torch.LongTensor(kLabels)

kTrain, kTest, kTrainLabels, kTestLabels, kExpsTrain, kExpsTest = train_test_split(kData, kLabels, kExps, train_size=0.8, random_state=7)
kTrain, kVal, kTrainLabels, kValLabels, kExpsTrain, kExpsVal = train_test_split(kTrain, kTrainLabels, kExpsTrain, train_size=0.75, random_state=7)
print(f'Data proportions -> Train: {round(len(kTrain) / len(kData), 3)}, Val: {round(len(kVal) / len(kData), 3)}, Test: {round(len(kTest) / len(kData), 3)}')
print(f'Positive label proportions -> Train: {round((sum(kTrainLabels) / len(kTrainLabels)).item(), 3)}, \
Val: {round((sum(kValLabels) / len(kValLabels)).item(), 3)}, Test: {round((sum(kTestLabels) / len(kTestLabels)).item(), 3)}')
opt = optim.SGD(sqznet.parameters(), lr=1e-3)
crit = nn.CrossEntropyLoss()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
batchSize = 30
nEpochs = 15
data = {'train': [kTrain, kTrainLabels], 'val': [kVal, kValLabels]}

sqznet, valAcc = train_net(sqznet, data, crit, opt, device, batchSize, nEpochs, randomState=7)

with open('temp/sqznetTrained.pickle', 'wb') as f:
    pickle.dump(sqznet.state_dict(), f)
with open('temp/sqznetTrained.pickle', 'rb') as f:
    sqznet.load_state_dict(pickle.load(f))
print(f'Validation F1: {round(f1_score(kValLabels, F.softmax(sqznet(kVal), dim=-1).argmax(dim=1).detach().numpy()), 3)}')
print(f'Test F1: {round(f1_score(kTestLabels, F.softmax(sqznet(kTest), dim=-1).argmax(dim=1).detach().numpy()), 3)}')<jupyter_output>/Users/master/Google Drive/U/4t/TFG/teex/venv/lib/python3.8/site-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  ../c10/core/TensorImpl.h:1156.)
  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)<jupyter_text>Let us get some explanations as samples<jupyter_code>gradShap = GradientShap(sqznet)
intGrad = IntegratedGradients(sqznet)
occlusion = Occlusion(sqznet)
deepLift = DeepLift(sqznet)
guidedBackProp = GuidedBackprop(sqznet)
guidedGradCAM = GuidedGradCam(sqznet, sqznet.features[12])

# use predicted labels
obsToExplain = kTest[kTestLabels == 1][:2]
expsToCompare = kExpsTest[kTestLabels == 1][:2]
predTargets = F.softmax(sqznet(torch.FloatTensor(obsToExplain)), dim=-1).argmax(dim=1)

# takes some minutes to run
gradShapExpsTest = get_attributions(obsToExplain, predTargets, gradShap, {'baselines': torch.zeros((1, 3, inputSize, inputSize))})
intGradExpsTest = get_attributions(obsToExplain, predTargets, intGrad)
deepLiftExpsTest = get_attributions(obsToExplain, predTargets, deepLift)
occlusionExpsTest = get_attributions(obsToExplain, predTargets, occlusion, {'baselines': 0, 'sliding_window_shapes': (3, inputSize, inputSize)})
gBackPropExpsTest = get_attributions(obsToExplain, predTargets, guidedBackProp)
gGradCAMExpsTest = get_attributions(obsToExplain, predTargets, guidedGradCAM)
# we binarize the results for easier visualization (this step is implicitly done by teex)
#   when computing metrics.
from teex._utils._arrays import _binarize_arrays

gsh = _binarize_arrays(gradShapExpsTest, method='abs', threshold=0.45)
ig = _binarize_arrays(intGradExpsTest, method='abs', threshold=0.45)
dl = _binarize_arrays(deepLiftExpsTest, method='abs', threshold=0.45)
oc = _binarize_arrays(occlusionExpsTest, method='abs', threshold=0.45)
gbp = _binarize_arrays(gBackPropExpsTest, method='abs', threshold=0.45)
ggc = _binarize_arrays(gGradCAMExpsTest, method='abs', threshold=0.45)

i = 1

fig, axs = plt.subplots(1, 8, figsize=(15,15))
axs[0].imshow(obsToExplain[i].permute(1, 2, 0))
axs[0].set_title('Image')
axs[1].imshow(expsToCompare[i])
axs[1].set_title('g.t. explanation')
axs[2].imshow(ig[i])
axs[2].set_title('integratedGradient')
axs[3].imshow(gsh[i])
axs[3].set_title('gradientSHAP')
axs[4].imshow(dl[i])
axs[4].set_title('deepLift')
axs[5].imshow(oc[i])
axs[5].set_title('occlusion')
axs[6].imshow(gbp[i])
axs[6].set_title('Guided Backprop.')
axs[7].imshow(ggc[i])
axs[7].set_title('Guided gradCAM.')<jupyter_output>Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).<jupyter_text>From this visualization, it is clear that the threshold level is an important hyperparameter to consider. Now, let's evaluate the quality of the explanations:<jupyter_code>kExplainers = {'gradSHAP': GradientShap(sqznet), 
               'gradCAM': GuidedGradCam(sqznet, sqznet.features[12]),
               'deepLift': DeepLift(sqznet),
               'backProp': GuidedBackprop(sqznet),
               'occlusion': Occlusion(sqznet),
               'intGrad': IntegratedGradients(sqznet)}

kConfigs = {
    'gradSHAP': [{'baselines': torch.zeros((1, 3, inputSize, inputSize)), 'n_samples': 5, 'stdevs': 0.1},
                 {'baselines': torch.zeros((1, 3, inputSize, inputSize)), 'n_samples': 5, 'stdevs': 0.2},
                 {'baselines': torch.zeros((1, 3, inputSize, inputSize)), 'n_samples': 5, 'stdevs': 0.3},
                 {'baselines': torch.zeros((1, 3, inputSize, inputSize)), 'n_samples': 15, 'stdevs': 0.1},
                 {'baselines': torch.zeros((1, 3, inputSize, inputSize)), 'n_samples': 25, 'stdevs': 0.1}],
    'gradCAM': [{'interpolate_mode': 'nearest'},
                {'interpolate_mode': 'area'}],
    'deepLift': [{}],
    'backProp': [{}],
    'occlusion': [{'baselines': 0, 'sliding_window_shapes': (3, round(inputSize/2), round(inputSize/2))},
                  {'baselines': 0, 'sliding_window_shapes': (3, round(inputSize/3), round(inputSize/3))},
                  {'baselines': 0, 'sliding_window_shapes': (3, round(inputSize/4), round(inputSize/4))},
                  {'baselines': 0, 'sliding_window_shapes': (3, round(inputSize/5), round(inputSize/5))}],
    'intGrad': [{'method': 'riemann_right'}, {'method': 'riemann_left'}, {'method': 'riemann_middle'}, 
                {'method': 'riemann_trapezoid'}, {'method': 'gausslegendre'}]
}

metrics = ['auc', 'fscore', 'prec', 'rec', 'cs']

binThresholds = [.1, .2, .3, .4, .5, .6]<jupyter_output><empty_output><jupyter_text>Evaluate positive test explanations. Note that teex implicitly handles the conversion of the RGB masks into 0-1 normalised grayscale masks (the shape of the g.t.s need to be (imH, imW, 3) for it to happen).<jupyter_code># use predicted labels
obsToExplain = kTest[kTestLabels == 1]
expsToCompare = kExpsTest[kTestLabels == 1]
predTargets = F.softmax(sqznet(torch.FloatTensor(obsToExplain)), dim=-1).argmax(dim=1)

for binThres in binThresholds:
    # takes hours to run
    scoresK = eval_explainers(kExplainers, kConfigs, obsToExplain, predTargets, 
                              expsToCompare, metrics, binThreshold=binThres)
    fileName = f'kahikateaScoresThres{binThres}.pickle'
    with open('/temp/'+fileName, 'wb') as f:
        pickle.dump(scoresK, f)
with open('docs/demos/temp/'+fileName, 'wb') as f:
        pickle.dump(scoresK, f)
with open('teex/_demos/notebooks/temp/kahikateaScores.pickle', 'rb') as f:
    scoresK = pickle.load(f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13215928922735207, 'fraction_numerical': 0.021339044899254322, 'mean_word_length': 4.5340210711150135, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 79, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '264782', 'license': 'MIT', 'n_tokens_mistral': 9015, 'n_tokens_neox': 8542, 'n_words': 2041, 'path': 'docs/demos/eval_saliency_map.ipynb', 'repo_name': 'chus-chus/teex', 'size': '25212'}