<jupyter_start><jupyter_text>Basic AutoencoderThis notebook is to train the basic autoencoder model and then perform clustering on the encoder part.  Imports<jupyter_code>!pip install "torch==1.4" "torchvision==0.5.0"
from fastai import *
import torch
from torch import nn
import imageio
from fastai.vision import *
import os
import torch.nn.functional as F
import numpy as np
from automatic_asset_classification import baseline_model
from automatic_asset_classification import model_layers
from google.colab import drive
drive.mount('/content/drive', force_remount = False)
%cd "/content/drive/My Drive/automatic-asset-classification"
%ls<jupyter_output>Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly

Enter your authorization code:
··········
Mounted at /content/drive
/content/drive/My Drive/automatic-asset-classification
ae_res_1_new    ae_res_3_torch  ae_res_5_torch       final_model  test.png
ae_res_1_torch  ae_res_4_new    ae_res_6_new         model1.pt
ae_res_2_new    ae_res_4-new    ae_res_6_torch       model2.pt
ae_res_2_torch  ae_res_4_torch  basic_animation.mp4  [0m[01;34mmodels[0m/
ae_res_3_new    ae_res_5_new    [01;34mdata[0m/                [01;34moutputs[0m/<jupyter_text>Data ImportData is imported using a fastai ImageImageList. This allows both an input and output to be an image.<jupyter_code>np.random.seed(3333)
torch.manual_seed(3333)
image_path = "data/final_dataset"
size = 224
batchsize = 32

tfms = get_transforms(do_flip = True)
src = (ImageImageList.from_folder(image_path).split_by_rand_pct().label_from_func(lambda x: x))
data = (src.transform(tfms, size=size, tfm_y=True)
        .databunch(bs=batchsize)
        .normalize(imagenet_stats, do_y = False))
print("imported")
data<jupyter_output><empty_output><jupyter_text>Training  Initiate the model<jupyter_code>autoencoder=baseline_model.Autoencoder()
learn = Learner(data, autoencoder, loss_func = F.mse_loss)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>learn.fit_one_cycle(5)
learn.lr_find()
learn.recorder.plot(suggestion=True)
learn.fit_one_cycle(10, max_lr=1e-02)
learn.show_results(ds_type=DatasetType.Train,rows=1)
learn.show_results(ds_type=DatasetType.Valid,rows=1)<jupyter_output><empty_output><jupyter_text>Save the model<jupyter_code>torch.save(learn.model, "model1.pt")<jupyter_output>/usr/local/lib/python3.6/dist-packages/torch/serialization.py:360: UserWarning: Couldn't retrieve source code for container of type autoencoder. It won't be checked for correctness upon loading.
  "type " + obj.__name__ + ". It won't be checked "
/usr/local/lib/python3.6/dist-packages/torch/serialization.py:360: UserWarning: Couldn't retrieve source code for container of type encoder. It won't be checked for correctness upon loading.
  "type " + obj.__name__ + ". It won't be checked "
/usr/local/lib/python3.6/dist-packages/torch/serialization.py:360: UserWarning: Couldn't retrieve source code for container of type convblock. It won't be checked for correctness upon loading.
  "type " + obj.__name__ + ". It won't be checked "
/usr/local/lib/python3.6/dist-packages/torch/serialization.py:360: UserWarning: Couldn't retrieve source code for container of type downsamp. It won't be checked for correctness upon loading.
  "type " + obj.__name__ + ". It won't be checked "
/usr/local/lib/python[...]<jupyter_text>Clustering  Create an encoder model<jupyter_code>class Encoder(nn.Module):
    def __init__(self):
        super(Encoder, self).__init__()
        #Load the trained model and keep the encoder part
        self.encoder = torch.load("model1.pt").encoder
        
    def encode(self,x): return self.encoder(x)
            
    def forward(self, x):
        encoded = self.encoder(x)
        return encoded
ec = Encoder1()
enc1=Learner(data, ec,loss_func=F.mse_loss)
preds,acts=enc1.get_preds()<jupyter_output><empty_output><jupyter_text>Convert the prediction of the validation set into a numpy array.<jupyter_code>cluster_data = preds.numpy()<jupyter_output><empty_output><jupyter_text>Standardise the data ready for the PCA.<jupyter_code>cluster_data_std = (cluster_data - cluster_data.mean())/cluster_data.std()
from sklearn.decomposition import PCA
pca = PCA(n_components=50)
cluster_data_pca = pca.fit_transform(cluster_data_std)<jupyter_output><empty_output><jupyter_text>Find optimal number of clusters<jupyter_code>linked = linkage(cluster_data_pca, method = 'ward')
fig = plt.figure(figsize = (12,8))
dendrogram(linked,
            orientation='top',
            #labels=labelList,
            distance_sort='descending',
            show_leaf_counts=True)
plt.show()<jupyter_output><empty_output><jupyter_text>Data only splits well into two clusters. Create a TSNE representation of the encoded data.<jupyter_code>from sklearn.manifold import TSNE
X_embedded = TSNE(n_components=2, perplexity = 5).fit_transform(cluster_data_pca)
X_embedded2 = TSNE(n_components=2, perplexity = 30).fit_transform(cluster_data_pca)
X_embedded3 = TSNE(n_components=2, perplexity = 50).fit_transform(cluster_data_pca)
from sklearn.cluster import AgglomerativeClustering
cluster = AgglomerativeClustering(n_clusters=2, affinity='euclidean', linkage='ward')
clusters = cluster.fit_predict(cluster_data_pca)<jupyter_output><empty_output><jupyter_text>Plot the TSNE representation with clusters coloured:<jupyter_code>plt.figure(figsize=(10, 7))
plt.scatter(X_embedded[:,0], X_embedded[:,1], c=cluster.labels_, cmap='rainbow')
plt.figure(figsize=(10, 7))
plt.scatter(X_embedded2[:,0], X_embedded2[:,1], c=cluster.labels_, cmap='rainbow')
plt.figure(figsize=(10, 7))
plt.scatter(X_embedded3[:,0], X_embedded3[:,1], c=cluster.labels_, cmap='rainbow')<jupyter_output><empty_output><jupyter_text>Save the images of each cluster into folders for use:<jupyter_code>one_idx = []
two_idx = []

for i in range(len(clusters)):
    if clusters_2[i] == 0:
        one_idx.append(i)
    if clusters_2[i] == 1:
        two_idx.append(i)
for idx in one_idx:
    data.valid_ds[idx][0].save("/content/drive/My Drive/automatic-asset-classification/outputs/basic_autoencoder"
                               "/class1/"+str(idx)+".png")
for idx in two_idx:
    data.valid_ds[idx][0].save("/content/drive/My Drive/automatic-asset-classification/outputs/basic_autoencoder"
                               "/class2/"+str(idx)+".png")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11550376161675763, 'fraction_numerical': 0.0274376751733294, 'mean_word_length': 5.673228346456693, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 47, 'https://': 1, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '279547', 'license': 'MIT', 'n_tokens_mistral': 2509, 'n_tokens_neox': 2352, 'n_words': 439, 'path': 'notebooks/Basic_Autoencoder.ipynb', 'repo_name': 'henriwoodcock/automatic-asset-classification', 'size': '6779'}