<jupyter_start><jupyter_text>Import libraries<jupyter_code>import sys
import os
sys.path.append("..")  # Adds higher directory to python modules path.
#from img_to_vec import Img2Vec
from PIL import Image
from sklearn.metrics.pairwise import cosine_similarity
import pandas as pd
from skimage import io
import numpy as np
from PyQt5.QtWidgets import QApplication, QWidget, QInputDialog, QLineEdit, QFileDialog
from matplotlib import pyplot as plt
import timeit
#import cv2
#from cv2_rolling_ball import subtract_background_rolling_ball<jupyter_output><empty_output><jupyter_text>User Input<jupyter_code>import sys
from PyQt5 import QtCore, QtWidgets
from PyQt5.QtWidgets import QApplication, QWidget, QFileDialog

app = QtCore.QCoreApplication.instance()
if app is None:
    app = QtWidgets.QApplication(sys.argv)
input_path = QFileDialog.getExistingDirectory(caption = 'Select files')
print("Active directory: ",input_path)
File_filter = ".tif"
Auto_write = True
Run_with_cuda = True<jupyter_output><empty_output><jupyter_text>Guts<jupyter_code>#Img2Vec from GitHub, https://github.com/christiansafka/img2vec

import torch
import torch.nn as nn
import torchvision.models as models
import torchvision.transforms as transforms

class Img2Vec():

    def __init__(self, cuda=False, model='resnet-18', layer='default', layer_output_size=512):
        """ Img2Vec
        :param cuda: If set to True, will run forward pass on GPU
        :param model: String name of requested model
        :param layer: String or Int depending on model.  See more docs: https://github.com/christiansafka/img2vec.git
        :param layer_output_size: Int depicting the output size of the requested layer
        """
        self.device = torch.device("cuda" if cuda else "cpu")
        self.layer_output_size = layer_output_size
        self.model_name = model
        
        self.model, self.extraction_layer = self._get_model_and_layer(model, layer)

        self.model = self.model.to(self.device)

        self.model.eval()

        self.scaler = transforms.Resize((224, 224))
        self.RandomRotation = transforms.RandomRotation(degrees=90) #Added rotational invariance for this application
        self.normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                              std=[0.229, 0.224, 0.225])
        self.to_tensor = transforms.ToTensor()

    def get_vec(self, img, tensor=False):
        """ Get vector embedding from PIL image
        :param img: PIL Image
        :param tensor: If True, get_vec will return a FloatTensor instead of Numpy array
        :returns: Numpy ndarray
        """
        image = self.normalize(self.to_tensor(self.scaler(img))).unsqueeze(0).to(self.device)

        if self.model_name == 'alexnet':
            my_embedding = torch.zeros(1, self.layer_output_size)
        else:
            my_embedding = torch.zeros(1, self.layer_output_size, 1, 1)

        def copy_data(m, i, o):
            my_embedding.copy_(o.data)

        h = self.extraction_layer.register_forward_hook(copy_data)
        h_x = self.model(image)
        h.remove()

        if tensor:
            return my_embedding
        else:
            if self.model_name == 'alexnet':
                return my_embedding.numpy()[0, :]
            else:
                return my_embedding.numpy()[0, :, 0, 0]

    def _get_model_and_layer(self, model_name, layer):
        """ Internal method for getting layer from model
        :param model_name: model name such as 'resnet-18'
        :param layer: layer as a string for resnet-18 or int for alexnet
        :returns: pytorch model, selected layer
        """
        if model_name == 'resnet-18':
            model = models.resnet18(pretrained=True)
            if layer == 'default':
                layer = model._modules.get('avgpool')
                self.layer_output_size = 512
            else:
                layer = model._modules.get(layer)

            return model, layer

        elif model_name == 'alexnet':
            model = models.alexnet(pretrained=True)
            if layer == 'default':
                layer = model.classifier[-2]
                self.layer_output_size = 4096
            else:
                layer = model.classifier[-layer]

            return model, layer

        else:
            raise KeyError('Model %s was not found' % model_name)
            
#Set Self
img2vec = Img2Vec(cuda=Run_with_cuda, model='resnet-18', layer='default', layer_output_size=2048)
start_time_all = timeit.default_timer()

pics = {}
count = 0
for subdir, dirs, files in os.walk(input_path):
    for file in files:
        start_time = timeit.default_timer()
        filepath = os.path.join(subdir, file)
        
        if filepath.endswith(File_filter) and not 'thumb' in filepath.lower():
            count = count + 1
            #Open images ending with filter
            filename = os.fsdecode(file)
            img = io.imread(filepath)
            #img, background = subtract_background_rolling_ball(img, 100, light_background=False, use_paraboloid=True, do_presmooth=True)
            
            #linearly Converts 16-bit images to 3 chan 8-bit, img2vec error w/ 16-bit input
            nimg = np.uint8(np.float32(img)/65535*255)
            arr = np.zeros((img.shape[0], img.shape[1], 3), dtype=np.uint8)
            arr[:,:,0], arr[:,:,1], arr[:,:,2] = nimg, nimg, nimg
            out_img = Image.fromarray(arr, 'RGB')
            
            #Performs neural embedding, Image -> Vector
            vec = img2vec.get_vec(out_img)
            pics[filename] = vec
            elapsed = timeit.default_timer() - start_time
            print("Finished: ",filepath,"     ",elapsed)
        
    if Auto_write:
        os.chdir(subdir)
        df = pd.DataFrame(pics)
        df = df.T
        df.index.name ='File_Name'
        if len(df)>0:
            df.to_csv('DeepHTS.csv')
            pics = {}
            
elapsed_all = timeit.default_timer() - start_time_all
print("Finished analyzing ", count, ' images. The total runtime was ',elapsed_all/60,' minutes')<jupyter_output>Finished:  U:/MX ImageStore/MDA-Nikhil/20201013-growth test_Proj\2020-10-13\ch01_A_1_t1.tif       2.6455145
Finished:  U:/MX ImageStore/MDA-Nikhil/20201013-growth test_Proj\2020-10-13\ch01_A_2_t1.tif       0.6039423999999998
Finished:  U:/MX ImageStore/MDA-Nikhil/20201013-growth test_Proj\2020-10-13\ch01_A_3_t1.tif       0.4515184000000003
Finished:  U:/MX ImageStore/MDA-Nikhil/20201013-growth test_Proj\2020-10-13\ch01_A_4_t1.tif       0.6884783000000003
Finished:  U:/MX ImageStore/MDA-Nikhil/20201013-growth test_Proj\2020-10-13\ch01_A_5_t1.tif       0.6097216999999997
Finished:  U:/MX ImageStore/MDA-Nikhil/20201013-growth test_Proj\2020-10-13\ch01_A_6_t1.tif       0.5652866000000003
Finished:  U:/MX ImageStore/MDA-Nikhil/20201013-growth test_Proj\2020-10-13\ch01_A_7_t1.tif       0.48093310000000056
Finished:  U:/MX ImageStore/MDA-Nikhil/20201013-growth test_Proj\2020-10-13\ch01_A_8_t1.tif       0.8000912000000007
Finished:  U:/MX ImageStore/MDA-Nikhil/20201013-growth test_Proj\2020-10[...]<jupyter_text>Write data<jupyter_code>df = pd.DataFrame(pics)
df = df.T
df.shape
df.head()
os.getcwd()
os.chdir(input_path)
df.to_csv('data.csv')
os.getcwd()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10390678546949966, 'fraction_numerical': 0.05716244002741604, 'mean_word_length': 2.7511568123393317, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 7, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '150452', 'license': 'MIT', 'n_tokens_mistral': 2697, 'n_tokens_neox': 2385, 'n_words': 522, 'path': 'Code/2) RESNET-embedding.ipynb', 'repo_name': 'ReidTPowell/Organoid-analysis', 'size': '7295'}