<jupyter_start><jupyter_text><jupyter_code>#!pip install seaborn
#!pip install tensorflow
#!pip install tf_keras_vis
#Load some packages
import numpy as np
import matplotlib.pyplot as plt

import tensorflow as tf
from tensorflow import keras

print("Tensorflow Edition:", tf.__version__)
seed = 2020
np.random.seed(seed)
tf.random.set_seed(seed)
print("Packages Loaded!")
#inspect GPU
physical_devices = tf.config.experimental.list_physical_devices('GPU')
n_GPUs = len(physical_devices)
print("Num_GPUs_Available:", n_GPUs)
if n_GPUs>0:
    tf.config.experimental.set_memory_growth(physical_devices[0], True)
!nvidia-smi
#load data from Google Drive
from google.colab import drive
drive.mount("/content/drive", force_remount=True)
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import os
class encode_booster():
  
  def __init__(self, model_list, img_process_list, data_path, save_path, name_list):
    self.model_list = model_list
    self.img_process_list = img_process_list
    self.save_path = save_path
    self.data_path = data_path
    self.name_list = name_list


  def load_data(self, part, index, batch_size=16):
    single_model = self.model_list[index]
    single_process = self.img_process_list[index]
    input_shape = single_model.layers[0].input_shape[0][1:3]

    image_loader = ImageDataGenerator(preprocessing_function = single_process)
    aug_loader = ImageDataGenerator(
      preprocessing_function = single_process,
      #rescale=1./255,
      rotation_range=40,
      #width_shift_range=0.2,
      #height_shift_range=0.2,
      #shear_range=0.2,
      zoom_range=0.2,
      horizontal_flip=True,
      fill_mode='nearest',
      vertical_flip=True,
      channel_shift_range=10
      )
    
    if part==0:
      print('Trainingset:')
      loader = image_loader.flow_from_directory(directory=self.data_path+'train', shuffle=False,
                            target_size=input_shape, batch_size=batch_size)
    if part==1:
      print('Validation dataset:')
      loader = image_loader.flow_from_directory(directory=self.data_path+'val', shuffle=False,
                            target_size=input_shape, batch_size=batch_size) 
    if part==2:
      print('Testingset:')
      loader = image_loader.flow_from_directory(directory= self.data_path+'test', shuffle=False,
                            target_size=input_shape, batch_size=16)
    
    return loader


  def print_test(self, index):
    single_model = self.model_list[index]
    single_process = self.img_process_list[index]
    input_shape = single_model.layers[0].input_shape[0][1:3]
    image_loader = ImageDataGenerator(preprocessing_function = single_process)

    test_loader = image_loader.flow_from_directory(directory= self.data_path+'test', target_size=input_shape, batch_size=16)
    print('Testing model_{} ...'.format(index))
    results = single_model.evaluate(test_loader)
    print('model_{}: test_loss:{:.3f}, test_auc:{:.3f}\n'.format(index, results[0], results[1]))  


  def plot_decompose(self, encode, index, labels, part):
    model_name = 'Part '+str(part)+': model_'+str(index)+'_'+self.name_list[index]
    fig = plt.figure(index*3+part, figsize=[3*7, 7])
    axes = fig.subplots(1, 3)

    from sklearn.manifold import TSNE
    decomposer_tsne = TSNE(n_components=2)
    de_tsne = decomposer_tsne.fit_transform(encode)
    scatter = axes[0].scatter(x=de_tsne[:,0], y=de_tsne[:,1], c=labels)
    legend1 = axes[0].legend(*scatter.legend_elements(), loc="best", title="Classes")
    axes[0].set_title(model_name+'---Tsne')
    axes[0].add_artist(legend1)

    from sklearn.manifold import Isomap
    decomposer_iso = Isomap(n_components=2, n_neighbors=10)
    de_iso = decomposer_iso.fit_transform(encode)
    scatter = axes[1].scatter(x=de_iso[:,0], y=de_iso[:,1], c=labels)
    legend1 = axes[1].legend(*scatter.legend_elements(), loc="best", title="Classes")
    axes[1].set_title(model_name+'---Isomap')
    axes[1].add_artist(legend1)

    from sklearn.decomposition import PCA
    decomposer_pca = PCA(n_components=2)
    de_pca = decomposer_pca.fit_transform(encode)
    scatter = axes[2].scatter(x=de_pca[:,0], y=de_pca[:,1], c=labels)
    legend1 = axes[2].legend(*scatter.legend_elements(), loc="best", title="Classes")
    axes[2].set_title(model_name+'---PCA')
    axes[2].add_artist(legend1)


  def get_decompose(self, index, data_part):
    single_model = self.model_list[index]
    single_process = self.img_process_list[index]
    input_shape = single_model.layers[0].input_shape[0][1:3]
    loader = self.load_data(data_part, index)

    labels = np.array([])
    for step in range(len(loader)):
      X, y = loader.next()
      labels = np.append(labels, y.argmax(axis=1))
    
    encoder = keras.Model(inputs=[self.model_list[index].input],
              outputs=[self.model_list[index].layers[-2].output])
    single_model = encoder
    single_process = self.img_process_list[index]
    input_shape = single_model.layers[0].input_shape[0][1:3]

    model_encode = single_model.predict(loader)
    self.plot_decompose(model_encode, index, labels, data_part)

    return np.append(model_encode, labels.reshape(-1, 1), axis=1)

SAVE_PATH = '/content/drive/My Drive/PAPER/model_trained/'
DATA_PATH = '/content/drive/My Drive/PAPER/data/'
NAME_LIST = ['VGG', 'Xcept', 'Efcb5', 'Res50']
'''
#downloading model
model_VGG19 = keras.applications.vgg19
model_XCEP = keras.applications.xception
model_EFCb5 = keras.applications.efficientnet
model_RES50 = keras.applications.resnet

model_list = [model_VGG19, model_XCEP, model_EFCb5, model_RES50]
img_process = [model.preprocess_input for model in model_list]
model_list = [model_VGG19.VGG19(), model_XCEP.Xception(), model_EFCb5.EfficientNetB5(), model_RES50.ResNet50()]
'''
'''
#load model
model_loaded = []
for i in range(4):
  model = keras.models.load_model(SAVE_PATH+'model_'+str(i))
  model_loaded.append(model)
'''
'''
#generating encode
test_model = encode_booster(model_loaded, img_process, DATA_PATH, SAVE_PATH, NAME_LIST)
for data_part in range(3):
  for model_no in range(4):
    decompose = test_model.get_decompose(model_no, data_part)
    file_name = SAVE_PATH+'model_'+str(model_no)+'/part'+str(data_part)+'_encode.csv'
    np.savetxt(file_name, decompose, delimiter=",")
'''
encoded_list = []
labels_list = []
for model_no in range(4):
  for data_part in range(3):
    file_name = SAVE_PATH+'model_'+str(model_no)+'/part'+str(data_part)+'_encode.csv'
    single_encode = np.genfromtxt(file_name, delimiter=',')
    encoded_list.append(single_encode[:, :-1])
    labels_list.append(single_encode[:, -1])

def plot_encode(encode, index, labels, part):
  model_name = 'Part '+str(part)+': model_'+str(index)+'_'+NAME_LIST[index]
  fig = plt.figure(index*3+part, figsize=[3*7, 7])
  axes = fig.subplots(1, 3)

  from sklearn.manifold import TSNE
  decomposer_tsne = TSNE(n_components=2)
  de_tsne = decomposer_tsne.fit_transform(encode)
  scatter = axes[0].scatter(x=de_tsne[:,0], y=de_tsne[:,1], c=labels)
  legend1 = axes[0].legend(*scatter.legend_elements(), loc="best", title="Classes")
  axes[0].set_title(model_name+'---Tsne')
  axes[0].add_artist(legend1)

  from sklearn.manifold import Isomap
  decomposer_iso = Isomap(n_components=2, n_neighbors=10)
  de_iso = decomposer_iso.fit_transform(encode)
  scatter = axes[1].scatter(x=de_iso[:,0], y=de_iso[:,1], c=labels)
  legend1 = axes[1].legend(*scatter.legend_elements(), loc="best", title="Classes")
  axes[1].set_title(model_name+'---Isomap')
  axes[1].add_artist(legend1)

  from sklearn.decomposition import PCA
  decomposer_pca = PCA(n_components=2)
  de_pca = decomposer_pca.fit_transform(encode)
  scatter = axes[2].scatter(x=de_pca[:,0], y=de_pca[:,1], c=labels)
  legend1 = axes[2].legend(*scatter.legend_elements(), loc="best", title="Classes")
  axes[2].set_title(model_name+'---PCA')
  axes[2].add_artist(legend1)
  print('Done!')

  file_name = SAVE_PATH+'model_'+str(model_no)+'/part'+str(data_part)
  np.savetxt(file_name+'_tsne.csv', de_tsne, delimiter=",")
  np.savetxt(file_name+'_iso.csv', de_iso, delimiter=",")
  np.savetxt(file_name+'_pca.csv', de_pca, delimiter=",")
for model_no in range(4):
  for data_part in range(3):
    part = data_part
    index = model_no
    file_name = SAVE_PATH+'model_'+str(model_no)+'/part'+str(data_part)+'_'
    model_name = 'Part '+str(part)+': model_'+str(index)+'_'+NAME_LIST[index]
    fig = plt.figure(index*3+part, figsize=[3*7, 7])
    axes = fig.subplots(1, 3)
    labels = labels_list[index*3+part]

    from sklearn.manifold import TSNE
    de_tsne = np.genfromtxt(file_name+'tsne.csv', delimiter=',')
    scatter = axes[0].scatter(x=de_tsne[:,0], y=de_tsne[:,1], c=labels)
    legend1 = axes[0].legend(*scatter.legend_elements(), loc="best", title="Classes")
    axes[0].set_title(model_name+'---Tsne')
    axes[0].add_artist(legend1)

    from sklearn.manifold import Isomap
    de_iso = np.genfromtxt(file_name+'iso.csv', delimiter=',')
    scatter = axes[1].scatter(x=de_iso[:,0], y=de_iso[:,1], c=labels)
    legend1 = axes[1].legend(*scatter.legend_elements(), loc="best", title="Classes")
    axes[1].set_title(model_name+'---Isomap')
    axes[1].add_artist(legend1)

    from sklearn.decomposition import PCA
    de_pca = np.genfromtxt(file_name+'pca.csv', delimiter=',')
    scatter = axes[2].scatter(x=de_pca[:,0], y=de_pca[:,1], c=labels)
    legend1 = axes[2].legend(*scatter.legend_elements(), loc="best", title="Classes")
    axes[2].set_title(model_name+'---PCA')
    axes[2].add_artist(legend1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14852408078715693, 'fraction_numerical': 0.019782496116002073, 'mean_word_length': 4.782035928143713, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '345061', 'license': 'MIT', 'n_tokens_mistral': 3662, 'n_tokens_neox': 3516, 'n_words': 537, 'path': 'ploting_decompose.ipynb', 'repo_name': 'DefaultaideN/EncodeBoosting', 'size': '9655'}