<jupyter_start><jupyter_text>Time appears to be represented as fractional days since Jan 1, 1900 (43798 days is ~119.9 years)<jupyter_code>cur = con.cursor()
cur.execute("SELECT Val FROM GENERAL WHERE Par = 'Fertilization'")
fert = float(cur.fetchone()[0])
fert
df["datetime"] = pd.to_datetime(df.Time, origin=datetime(1900,1,1), unit="D").dt.floor("s")
df["datetime"].describe()
df["Time since fertilisation (minutes)"] = (df.Time - fert) * 1440
df["Time since fertilisation (minutes)"].describe()
def images_to_sprite(data):
    """
    Creates the sprite image along with any necessary padding
    Source : https://github.com/tensorflow/tensorflow/issues/6322
    Args:
      data: NxHxW[x3] tensor containing the images.
    Returns:
      data: Properly shaped HxWx3 image with any necessary padding.
    """
    if len(data.shape) == 3:
        data = np.tile(data[...,np.newaxis], (1,1,1,3))
    data = data.astype(np.float32)
    min = np.min(data.reshape((data.shape[0], -1)), axis=1)
    data = (data.transpose(1,2,3,0) - min).transpose(3,0,1,2)
    max = np.max(data.reshape((data.shape[0], -1)), axis=1)
    data = (data.transpose(1,2,3,0) / max).transpose(3,0,1,2)
    
    n = int(np.ceil(np.sqrt(data.shape[0])))
    padding = ((0, n ** 2 - data.shape[0]), (0, 0),
            (0, 0)) + ((0, 0),) * (data.ndim - 3)
    data = np.pad(data, padding, mode='constant',
            constant_values=0)
    # Tile the individual thumbnails into an image.
    data = data.reshape((n, n) + data.shape[1:]).transpose((0, 2, 1, 3)
            + tuple(range(4, data.ndim + 1)))
    data = data.reshape((n * data.shape[1], n * data.shape[3]) + data.shape[4:])
    data = (data * 255).astype(np.uint8)
    return data


def populate_img_arr(images, target_size=(100,100),should_preprocess= False):
    """
    Get an array of images for a list of image paths
    Args:
        target_size: the size of image , in pixels 
        should_preprocess: if the images should be processed (according to InceptionV3 requirements)
    Returns:
        arr: An array of the loaded images
    """
    arr = []
    for i,img_bytes in enumerate(images):
        img = Image.open(io.BytesIO(img_bytes))
        img = img.convert('RGB')
        img = img.resize(target_size, Image.NEAREST)
        x = image.img_to_array(img)
        arr.append(x)
    arr = np.array(arr)
    if should_preprocess:
        arr = preprocess_input(arr)
    return arr<jupyter_output><empty_output><jupyter_text>Model Definition If you want to use another model, you can change it here<jupyter_code>model = InceptionV3(include_top=False,pooling='avg')
model.summary()
sample = df.sample(1000)
sample
%%time
img_arr = populate_img_arr(sample.Image,target_size=(100,100),should_preprocess=True)
preds = model.predict(img_arr,batch_size=64)
preds.tofile("./oss_data/tensor.bytes")
del img_arr,preds
%%time
raw_imgs = populate_img_arr(sample.Image, target_size=(100,100),should_preprocess=False)
sprite = Image.fromarray(images_to_sprite(raw_imgs).astype(np.uint8))
sprite.save('./oss_data/sprites.png')
del raw_imgs
sample["label"] = "unknown"
sample[["Well", "Run", "datetime", "Time since fertilisation (minutes)", "label"]].to_csv('./oss_data/metadata.tsv',sep='\t',index_label="index")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15661182205971969, 'fraction_numerical': 0.03443022547227301, 'mean_word_length': 3.974242424242424, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '149823', 'license': 'MIT', 'n_tokens_mistral': 1222, 'n_tokens_neox': 1152, 'n_words': 254, 'path': 'Feature-extractor.ipynb', 'repo_name': 'UoA-eResearch/TSNE-UMAP-Embedding-Visualisation', 'size': '3282'}