<jupyter_start><jupyter_text>Create image given a dlatentWe choose the index of one of the 250 preloaded dlatents from which we generate an image<jupyter_code>import torch
import numpy as np
from matplotlib import pyplot as plt
import os
import IPython.display
import pickle
import requests
import collections

from tfrecord.torch.dataset import TFRecordDataset

img_index = 10

# Choose dlatent and make it into [1,num_layers,514]
imgs, lbl = next(dataloader)
imgs = imgs.to(config.device)

dlatents = Trained_StylEx.get_latent(imgs).to(config.device)
expanded_dlatent_tmp = dlatents[None, :, :].expand(Trained_StylEx.n_gen_blocks, -1, -1)

# Feed dlatent into the generator
gen_output = generator(expanded_dlatent_tmp.to(config.device), Trained_StylEx.get_noise(1)).detach()

img_out = torch.maximum(torch.minimum(gen_output, torch.Tensor([1]).to(config.device)), torch.Tensor([-1]).to(config.device)).detach()
print('Real')
show_images(imgs[img_index].unsqueeze(0).detach().to('cpu'))
print('Reconstruction')
show_images(img_out[img_index].unsqueeze(0).detach().to('cpu'))<jupyter_output>Real<jupyter_text>Generate Style space per index (min max)<jupyter_code>values_per_index = collections.defaultdict(list)
for dlatent in dlatents:
    # Get the style vector:
    expanded_dlatent_tmp = dlatent.unsqueeze(0)[None, :, :].expand(Trained_StylEx.n_gen_blocks, -1, -1)
    
    s_img = torch.cat(generator.style_vector_calculator(
        expanded_dlatent_tmp)[1], dim=1).detach().to('cpu').numpy()[0]
    for i, s_val in enumerate(s_img):
        values_per_index[i].append(s_val)

values_per_index = dict(values_per_index)
s_indices_num = len(values_per_index.keys())
minimums = [min(values_per_index[i]) for i in range(s_indices_num)] 
maximums = [max(values_per_index[i]) for i in range(s_indices_num)] 
print(len(minimums))
%mkdir ../../../../tmp/mnist
%mkdir ../../../../tmp/mnist/data
#@title SSpace calculation (this is also very heavy, skip to next cell to load precomputed ones) {form-width: '20%'}
import tensorflow as tf
import tqdm

def get_classifier_results(model: StylEx, latent_vec):
  noise = model.get_noise(latent_vec.shape[1])
  return model.classifier(model.generator(latent_vec, noise))

s_shift_size = .01 # @param
data_path = '../../../../tmp/mnist/data/examples_1.tfrecord' #@param {type: 'string'}
b_prob = []
c_res = []
lat = []
s_vec = []

with tf.io.TFRecordWriter(data_path) as writer:  
  dl = dlatents[0:5]
  for dlatent_index, dlatent in enumerate(dl): 
    expanded_dlatent = dlatent.unsqueeze(0)[None, :, :].expand(
        Trained_StylEx.n_gen_blocks, -1, -1)
    
    base_prob = get_classifier_results(Trained_StylEx, expanded_dlatent).detach().to('cpu')
    s_tmp = generator.style_vector_calculator(
        expanded_dlatent)
    s_rgbs = s_tmp[0]
    style_vectors = s_tmp[1]
    tot_idx = 0
    n_s = len(minimums)

    # n_styles x n_classes x n_dir 
    classifier_results = torch.zeros((n_s, 2, 2))
    s_v = torch.zeros(n_s)
    for i in range(len(style_vectors)):
      style_vector = style_vectors[i].detach()
      len_s = style_vector.shape[1]
      for sindex in tqdm.tqdm(range(0, len_s)):
        s_shift_down = (minimums[tot_idx] - style_vector[0,sindex]) * s_shift_size
        s_shift_up = (maximums[tot_idx] - style_vector[0,sindex]) * s_shift_size
        noise = Trained_StylEx.get_noise(1)

        s_v[tot_idx] = style_vector[0, sindex].item()

        # Shift up
        s = style_vector[:, sindex] + s_shift_up
        style_vectors[i] = s
        c_up = classifier(generator.synthesis(expanded_dlatent, style_vectors, noise, s_rgb = s_rgbs)).detach().to('cpu')

        # Shift down
        s = style_vector[:, sindex] - s_shift_up - s_shift_down
        style_vectors[i] = s
        c_down = classifier(generator.synthesis(expanded_dlatent, style_vectors, noise, s_rgb = s_rgbs)).detach().to('cpu')
        
        # Return to base
        s = style_vector[:, sindex] + s_shift_down
        style_vectors[i] = s

        classifier_results[tot_idx,:, 0] = c_down
        classifier_results[tot_idx,:, 1] = c_up

        tot_idx +=1

        # Get the style vector.
    s_vec.append(s_v.unsqueeze(0))
    feature = {}
    b_prob.append(base_prob.unsqueeze(0).to('cpu'))
    lat.append(dlatent.unsqueeze(0).to('cpu'))
    c_res.append(classifier_results.detach().unsqueeze(0).to('cpu'))
#@title Load effect data from the tfrecord {form-width: '20%'}

style_change_effect = []
dlatents = []
base_probs = []

base_probs = torch.cat(b_prob)
style_change_effect = torch.cat(c_res)
dlatents = torch.cat(lat)

expanded_dlatent_tmp = torch.tile(dlatents, [1, num_layers, 1])
W_values, style_change_effect, base_probs = dlatents.squeeze(), style_change_effect.squeeze(), base_probs.squeeze()
print(len(s_vec))
all_style_vectors = torch.cat(s_vec, dim = 0)
if len(style_change_effect.shape) == 3:
  style_change_effect.unsqueeze_(0)
  all_style_vectors.unsqueeze_(0)
  base_probs.unsqueeze_(0)
  W_values.unsqueeze_(0)
style_change_effect = style_change_effect.permute((0, 3, 1, 2))
print(torch.min(torch.abs(style_change_effect)))
style_change_effect = filter_unstable_images(style_change_effect, effect_threshold=10)
print(style_change_effect)
print(torch.max(torch.abs(style_change_effect)))

style_min = torch.Tensor(minimums)
style_max = torch.Tensor(maximums)

all_style_vectors_distances = np.zeros((all_style_vectors.shape[0], all_style_vectors.shape[1], 2))
all_style_vectors_distances[:,:, 0] = all_style_vectors - torch.tile(style_min, (all_style_vectors.shape[0], 1))
all_style_vectors_distances[:,:, 1] = torch.tile(style_max, (all_style_vectors.shape[0], 1)) - all_style_vectors
all_style_vectors.shape
stop
W_values.shape<jupyter_output><empty_output><jupyter_text>Run the extraction step of AttFind<jupyter_code>#@title Split by class
all_labels = np.argmax(base_probs, axis=1)
style_effect_classes = {}
W_classes = {}
style_vectors_distances_classes = {}
all_style_vectors_classes = {}
for img_ind in range(label_size):
  img_inx = np.array([i for i in range(all_labels.shape[0]) 
  if all_labels[i] == img_ind])
  curr_style_effect = np.zeros((len(img_inx), style_change_effect.shape[1], 
                                style_change_effect.shape[2], style_change_effect.shape[3]))
  curr_w = np.zeros((len(img_inx), W_values.shape[1]))
  curr_style_vector_distances = np.zeros((len(img_inx), style_change_effect.shape[2], 2))
  for k, i in enumerate(img_inx):
    curr_style_effect[k, :, :] = style_change_effect[i, :, :, :]
    curr_w[k, :] = W_values[i, :].detach()
    curr_style_vector_distances[k, :, :] = all_style_vectors_distances[i, :, :]
  style_effect_classes[img_ind] = curr_style_effect
  W_classes[img_ind] = curr_w
  style_vectors_distances_classes[img_ind] = curr_style_vector_distances
  all_style_vectors_classes[img_ind] = all_style_vectors[img_inx]
  print(f'Class {img_ind}, {len(img_inx)} images.')
#@title Significant S values - combined {form-width: '20%'}
label_size_clasifier = 2 #@param
num_indices =  8 #@param
effect_threshold = 0.2 #@param
use_discriminator = False #@param {type: 'boolean'}
discriminator_model = None
s_indices_and_signs_dict = {}
print(style_effect_classes[1])
for class_index in [0, 1]:
  split_ind = 1 - class_index
  all_s = style_effect_classes[split_ind]
  all_w = W_classes[split_ind]

  # Find s indicies
  if all_s.shape[0]:
    s_indices_and_signs = find_significant_styles(
      style_change_effect=all_s,
      num_indices=num_indices,
      class_index=class_index,
      discriminator=discriminator_model,
      generator=generator,
      classifier=classifier,
      all_dlatents=all_w,
      style_min=style_min,
      style_max=style_max,
      max_image_effect=effect_threshold*5,
      label_size=label_size_clasifier,
      discriminator_threshold=0.2,
      sindex_offset=0)

    s_indices_and_signs_dict[class_index] = s_indices_and_signs
  else:
    s_indices_and_signs_dict[class_index] = []

# Combine the style indicies for the two classes.

sindex_class_0 = [sindex for _, sindex in s_indices_and_signs_dict[0]]

all_sindex_joined_class_0 = [(1 - direction, sindex) for direction, sindex in 
                            s_indices_and_signs_dict[1] if sindex not in sindex_class_0]
all_sindex_joined_class_0 += s_indices_and_signs_dict[0]

scores = []
for direction, sindex in all_sindex_joined_class_0:
  other_direction = 1 if direction == 0 else 0
  curr_score = torch.mean(style_change_effect[:, direction, sindex, 0]) + torch.mean(style_change_effect[:, other_direction, sindex, 1])
  scores.append(curr_score)

s_indices_and_signs = [all_sindex_joined_class_0[i] for i in np.argsort(scores)[::-1]]

print('Directions and style indices for moving from class 1 to class 0 = ', s_indices_and_signs[:num_indices])
print('Use the other direction to move for class 0 to 1.')
max_images = 8 #@param
sindex = 800 #@param
class_index = 0 #@param {type: "integer"} 
shift_sign = "1" #@param [0, 1]
wsign_index = int(shift_sign)

shift_size =  1 #@param
effect_threshold = 0 #@param
split_by_class = False #@param {type:"boolean"}
select_images_by_s_distance = True #@param {type:"boolean"}
draw_probabilities_on_image = True #@param {type:"boolean"}

if split_by_class:
  split_ind = 1 if class_index == 0 else 0
  all_s = style_effect_classes[split_ind]
  all_w = W_classes[split_ind]
  all_s_distances = style_vectors_distances_classes[split_ind]
else:
  all_s = style_change_effect
  all_w = W_values
  all_s_distances = all_style_vectors_distances

font_file = './arialuni.ttf'
if not os.path.exists(font_file):
  r = requests.get('https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/ipwn/arialuni.ttf')
  open(font_file, 'wb').write(r.content)
stop
if not select_images_by_s_distance:
  yy = visualize_style(generator, 
                       classifier,
                       all_w,
                       all_s,
                       style_min,
                       style_max,
                       sindex,
                       wsign_index,
                       max_images=max_images,
                       shift_size=shift_size,
                       font_file=font_file,
                       label_size=label_size,
                       class_index=class_index,
                       effect_threshold=effect_threshold,
                       draw_results_on_image=draw_probabilities_on_image)
    
else:
  yy = visualize_style_by_distance_in_s(
    generator,
    classifier,
    all_w,
    all_s_distances,
    style_min,
    style_max,
    sindex,
    wsign_index,
    max_images=max_images,
    shift_size=shift_size,
    font_file=font_file,
    label_size=label_size,
    class_index=class_index,
    effect_threshold=effect_threshold,
    draw_results_on_image=draw_probabilities_on_image)

if yy.size > 0:
  show_image(yy)
else:
  print('no images found')
#@title Show animation {form-width: '20%'}

import matplotlib.animation 
from IPython.display import HTML

ani = make_animation(yy, resolution)

HTML(ani.to_jshtml())
#Show the 4 top attributes

index_to_naming = {1: "Skin Pigminatation", 2: "Eyebrow Thickness", 3: "Add/Remove Glasses", 4: "Dark/White Hair"}
images_list = [[5, 3], [3, 0], [0, 6], [4, 1]]
shift_sizes = shift_sizes = [(2, 1.5),(1, 1),(1, 1),(1.5, 2)]
effect_threshold = 0

print('Original images are on the first row, the probabilities displayed are for the other class - left column for being old, right column for being young)')

for i, (direction, sindex) in enumerate(s_indices_and_signs[:4]):
  images_s = np.zeros((resolution * 2, resolution * 2, 3)).astype(np.uint8)
  for d in [direction, 1 - direction]:
    class_index = 0 if d == direction else 1
    split_ind = 1 if d == direction else 0
    all_s = style_effect_classes[split_ind]
    all_s_distances = style_vectors_distances_classes[split_ind]
    
    # Generate images
    yy = visualize_style_by_distance_in_s(
      generator,
      classifier,
      all_w,
      all_s_distances,
      style_min,
      style_max,
      sindex,
      d,
      max_images=max_images,
      shift_size=shift_sizes[i][class_index],
      font_file=font_file,
      label_size=label_size,
      class_index=class_index,
      effect_threshold=effect_threshold,
      draw_results_on_image=draw_probabilities_on_image)
    
    for n in range(2):
      images_s[n * resolution: (n + 1) * resolution, class_index * resolution: (class_index + 1) * resolution, :] = yy[(images_list[i][class_index]) * resolution: (images_list[i][class_index] + 1) * resolution, n * resolution: (n + 1) * resolution, :]
  
  print(f'Attribute {i + 1}: {index_to_naming[i+1]}')
  show_image(images_s)<jupyter_output><empty_output><jupyter_text>Image-specific explanationsIn this section, we implement the **Independent** selection strategy, to find the top 5 attributes for explaining a perceived-age classifier for a **specific image**<jupyter_code>#Choose latent index to find image's top 5 attributes
latent_index = 0
print(base_probs.shape)

if len(base_probs.shape) == 1:
  base_probs.unsqueeze_(0)
  style_change_effect.unsqueeze(0)
  W_values[latent_index].unsqueeze(0)
  all_style_vectors.unsqueeze(0)

one_latent = torch.unsqueeze(W_values[latent_index], 0)
one_base_prob = torch.unsqueeze(base_probs[latent_index], 0)
one_style_change_effect = torch.unsqueeze(style_change_effect[latent_index], 0)

expanded_dlatent_tmp = torch.tile(one_latent, [1, num_layers, 1])

style_min,_ = torch.min(all_style_vectors, dim=0)
style_max,_ = torch.max(all_style_vectors, dim=0)
all_style_vectors_distances = np.zeros((all_style_vectors.shape[0], all_style_vectors.shape[1], 2))
all_style_vectors_distances[:,:, 0] = all_style_vectors - torch.tile(style_min, (all_style_vectors.shape[0], 1))
all_style_vectors_distances[:,:, 1] = torch.tile(style_max, (all_style_vectors.shape[0], 1)) - all_style_vectors
# Find s indicies
class_index = 0# @param {type: "integer"} 
label_size_clasifier = 2 #@param
num_indices = 5 #@param
effect_threshold = 0.5 #@param
use_discriminator = False #@param {type: 'boolean'}
discriminator_model = None

s_indices_and_signs = find_significant_styles(
    style_change_effect=one_style_change_effect.numpy(),
    num_indices=num_indices,
    class_index=class_index,
    discriminator=discriminator_model,
    generator=generator,
    classifier=classifier,
    all_dlatents=one_latent,
    style_min=style_min,
    style_max=style_max,
    max_image_effect=effect_threshold*5,
    label_size=label_size_clasifier,
    discriminator_threshold=0.2,
    sindex_offset=0)

print(f'Directions and style indices for moving from class 1 (yound) to class {class_index} (old) = ', s_indices_and_signs[:num_indices])
shift_sign = "0" #@param [0, 1]
wsign_index = int(shift_sign)

shift_size = 2#@param
effect_threshold = 0#@param

shift_sizes = [1, -1.5, -1, 1.5, 2.5]
index_to_naming = {1: "Add Glasses", 2: "White Hair", 3: "Neck Coverage", 4: "Receding Hairline", 5: "Glowy White Hair"}

 
for i, (direction, sindex) in enumerate(s_indices_and_signs[:5]): 
  yy = visualize_style_by_distance_in_s(
      generator,
      classifier,
      one_latent,
      all_style_vectors_distances,
      style_min,
      style_max,
      sindex,
      direction,
      max_images=max_images,
      shift_size=shift_sizes[i],
      font_file=font_file,
      label_size=label_size,
      class_index=class_index,
      effect_threshold=effect_threshold,
      draw_results_on_image=draw_probabilities_on_image,
      images_idx=[0])

  print(f'Attribute {i + 1}: {index_to_naming[i + 1]}')

  if yy.size > 0:
    show_image(yy)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11613316261203585, 'fraction_numerical': 0.014980793854033291, 'mean_word_length': 4.418314255983351, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '518663', 'license': 'MIT', 'n_tokens_mistral': 5847, 'n_tokens_neox': 5618, 'n_words': 1057, 'path': 'train/Trained_StylEx_Test.ipynb', 'repo_name': 'irenepap2/FACT_UvA_2022', 'size': '15620'}