<jupyter_start><jupyter_text>Exploration of image interpretable representations<jupyter_code>import os

_base_path = ('https://raw.githubusercontent.com/fat-forensics/resources/'
              'master/surrogates_overview/scripts/{}')
_scripts = ['__init__.py', 'image_classifier.py', 'imagenet_label_map.py']
! mkdir -p scripts
for i in _scripts:
    i_ = _base_path.format(i)
    if not os.path.exists(f'scripts/{i}'):
        ! wget $i_ -O scripts/$i

! mkdir -p _figures
# NBVAL_IGNORE_OUTPUT
import fatf

import fatf.utils.data.segmentation as fatf_segmentation
import fatf.utils.data.occlusion as fatf_occlusion

import scripts.image_classifier as imgclf

from helpers import plot_bar_exp
from PIL import Image

import numpy as np

import matplotlib
import matplotlib.pyplot as plt
%matplotlib inline

matplotlib.rc('text', usetex=True)
plt.style.use('seaborn')  # 'classic'
def imshow(img):
    plt.grid(None)
    plt.axis('off')
    plt.imshow(img)
    plt.show()
husky = Image.open('img/husky.jpeg')
husky_array = np.asarray(husky)
husky_mask = Image.open('img/husky_mask.jpeg')
husky_mask_array = np.asarray(husky_mask)
husky
husky_mask
clf = imgclf.ImageClassifier()

clf.proba2tuple(
    clf.predict_proba([husky_array]),
    labels_no=5
)<jupyter_output><empty_output><jupyter_text>Segmentation<jupyter_code>fatf.setup_random_seed(42)

segmenter = fatf_segmentation.QuickShift(
    husky_array,
    segmentation_mask=husky_mask_array,
    ratio=0.3,
    kernel_size=9,
    max_dist=30)
print(f'Number of segments: {segmenter.segments_number}')
husky_segmented = segmenter.mark_boundaries(colour=(255, 255, 0))

imshow(husky_segmented)
plt.imsave('_figures/husky_segments.jpeg', husky_segmented)<jupyter_output><empty_output><jupyter_text>Occlusion<jupyter_code>occluder = fatf_occlusion.Occlusion(husky_array, segmenter.segments)
imshow(
    segmenter.number_segments(image=husky_segmented, colour=(255, 0, 0))
)
occlusion_mask = np.zeros(segmenter.segments_number, dtype=np.int8)
occlusion_mask[3] = 1<jupyter_output><empty_output><jupyter_text>Mean<jupyter_code>husky_occlusion_mean = occluder.occlude_segments_vectorised(
    occlusion_mask, colour='mean')

imshow(husky_occlusion_mean)
plt.imsave('_figures/husky_occlusion_mean.jpeg', husky_occlusion_mean)
clf.proba2tuple(
    clf.predict_proba([husky_occlusion_mean]),
    labels_no=5
)<jupyter_output><empty_output><jupyter_text>Black<jupyter_code>husky_occlusion_black = occluder.occlude_segments_vectorised(
    occlusion_mask, colour='black')

imshow(husky_occlusion_black)
plt.imsave('_figures/husky_occlusion_black.jpeg', husky_occlusion_black)
clf.proba2tuple(
    clf.predict_proba([husky_occlusion_black]),
    labels_no=5
)<jupyter_output><empty_output><jupyter_text>Random<jupyter_code>fatf.setup_random_seed(42)

husky_occlusion_random = occluder.occlude_segments_vectorised(
    occlusion_mask, colour='random-patch')

imshow(husky_occlusion_random)
plt.imsave('_figures/husky_occlusion_random.jpeg', husky_occlusion_random)
clf.proba2tuple(
    clf.predict_proba([husky_occlusion_random]),
    labels_no=5
)<jupyter_output><empty_output><jupyter_text>Mosaic<jupyter_code>segments_no = [30, 75, 100]

segments_step = []
for i in segments_no:
    fatf.setup_random_seed(42)
    s = fatf_segmentation.Slic(husky_array, n_segments=i)
    o = fatf_occlusion.Occlusion(husky_array, s.segments, colour='mean')
    segments_step.append((s, o))
def find_dog_prediction(tuple_list):
    classes = ('Siberian husky', 'Eskimo dog', 'malamute')
    return [i for i in tuple_list if i[0] in classes]
for i, (s, o) in zip(segments_no, segments_step):
    print(f'Segments #: {s.segments_number} (set to {i})')

    all_mask = np.zeros(s.segments_number, dtype=np.int8)
    img_occ = o.occlude_segments_vectorised(all_mask)

    img_pred = clf.proba2tuple(clf.predict_proba([img_occ]), labels_no=1000)[0]
    print('Top 5 predictions: ', img_pred[:5])
    print('Dog predictions: ', find_dog_prediction(img_pred))

    imshow(img_occ)
    plt.imsave(f'_figures/husky_mosaic_{s.segments_number:d}.jpeg', img_occ)<jupyter_output>Segments #: 17 (set to 30)
Top 5 predictions:  [('diaper', 0.11473429, 529), ('bearskin', 0.10694315, 439), ('caldron', 0.0605456, 469), ('coffee mug', 0.055410918, 504), ('cauliflower', 0.04704182, 938)]
Dog predictions:  [('malamute', 0.00010096867, 249), ('Siberian husky', 6.077184e-05, 250), ('Eskimo dog', 4.2534924e-05, 248)]<jupyter_text>Mock Surrogate Explanation -- Image<jupyter_code>mock_explanation = [
    -0.01, -0.04, 0.01, 0.41, 0.01, -0.01, 0.08, 0.33, -0.02, 0.02, 0.04]
top_n = 5

highlight_mask = []
for i, v in enumerate(mock_explanation):
    c = (255, 0, 0) if v < 0 else (0, 255, 0)
    c_ = 'red' if v < 0 else 'green'
    highlight_mask.append((i + 1, v, c, c_))
highlight_mask = sorted(highlight_mask, key=lambda i: abs(i[1]), reverse=True)

seg, col = [], []
for s, _, c, _ in highlight_mask[:top_n]:
    seg.append(s)
    col.append(c)
husky_explanation = segmenter.highlight_segments(seg, colour=col)
husky_explanation_ = segmenter.mark_boundaries(
    image=husky_explanation, colour=(255, 255, 0))

imshow(husky_explanation_)

plt.imsave('_figures/husky_explanation.pdf', husky_explanation_)# jpeg
# NBVAL_IGNORE_OUTPUT
# fontsize=14 # labelsize=16
# xlim = (-0.001, 0.017) # xlim = (-0.01, 0.76)
plot_bar_exp(mock_explanation, savepath='_figures/husky_bar.pdf', onesided=True,
             label_fmt='\\(x_{{{:d}}}^\star\\)')<jupyter_output><empty_output><jupyter_text>Mock Surrogate Explanation -- Text<jupyter_code>mock_explanation_txt = [
    0, 0.1, 0, 0, 0.7, -0.1, 0, -0.3, 0]
#   0.7, 0, -0.1, 0, 0, 0, -0.3, 0.1, 0
# NBVAL_IGNORE_OUTPUT
# fontsize=14 # labelsize=16
# xlim = (-0.001, 0.017) # xlim = (-0.01, 0.76)
plot_bar_exp(
    mock_explanation_txt,
    savepath='_figures/text_bar.pdf',
    onesided=True,
    feature_no=4,
    label_fmt='\\(x_{{{:d}}}^\star\\)')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14056969492668128, 'fraction_numerical': 0.04618236979605596, 'mean_word_length': 5.697516930022573, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 36, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '559394', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2605, 'n_tokens_neox': 2364, 'n_words': 378, 'path': 'DAMI_2021/exploration_image.ipynb', 'repo_name': 'So-Cool/bLIMEy', 'size': '5933'}