<jupyter_start><jupyter_text>Retrieve Data for FishyFish Training  Dependencies<jupyter_code>import numpy as np
import tensorflow as tf
from importlib import reload
import os
import pickle
import fish_data as fd
from datetime import datetime
import pandas as pd
from scipy import misc<jupyter_output><empty_output><jupyter_text>Load label_dictionary<jupyter_code>with open('label_dictionary.pickle', 'rb') as handle :
    label_dictionary = pickle.load(handle)
print(len(label_dictionary))<jupyter_output>3777<jupyter_text>Create DataFrame of annotated fovea embeddings<jupyter_code>labels = pd.DataFrame([], columns = ['ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT', 'NoF'], index = [])
box_preds = pd.DataFrame([], columns = ['scale', 'y_offset', 'x_offset'], index = [])
annotated_boxes = pd.DataFrame([], columns = ['scale', 'y_offset', 'x_offset'], index = [])
FiNoF_prob = pd.Series([], index = [])
for key in list(label_dictionary.keys()) :
    #print(label_dictionary.get(key))
    #print(np.expand_dims(label_dictionary.get(key).get('box_preds'), 0))
    labels = labels.append(pd.DataFrame(label_dictionary.get(key).get('onehot'), index = [key], columns =['ALB', 'BET', 'DOL', 'LAG', 'OTHER', 'SHARK', 'YFT', 'NoF']) )
    box_preds = box_preds.append( pd.DataFrame(np.expand_dims(label_dictionary.get(key).get('box_preds'), 0), index = [key] , columns = ['scale', 'y_offset', 'x_offset'] ))
    
    if label_dictionary.get(key).get('scale') is not None :
        annotated_boxes = annotated_boxes.append(pd.DataFrame(np.expand_dims(np.concatenate([label_dictionary.get(key).get('scale'),label_dictionary.get(key).get('coord') ], 0),0), 
                                                             columns = ['scale', 'y_offset', 'x_offset'], index = [key]))
    FiNoF_prob = FiNoF_prob.append(pd.Series(label_dictionary.get(key).get('FiNoF'), index = [key]))
embedding_df = pd.read_pickle('embedding_dataframe.pickle')
print(embedding_df.shape)
labels.to_pickle('onehot_df.pickle')
box_preds.to_pickle('box_preds.pickle')
annotated_boxes.to_pickle('annotated_boxes.pickle')
FiNoF_prob.to_pickle('FiNoF_prob.pickle')<jupyter_output><empty_output><jupyter_text>Store Annotated Fovea in appropriate directories<jupyter_code>for key in list(label_dictionary.keys()) :
    if label_dictionary.get(key).get('scale') is not None :
        scale = label_dictionary.get(key).get('scale')
        yxcoord = label_dictionary.get(key).get('coord')
        fov = fd.retrieve_fovea(key, yxcoord, scale[0], fov_dim = 72)
        new_path = 'data/annotated_fovea_train/'+key[11:]
        misc.imsave(new_path, fov, format = 'JPEG' )<jupyter_output><empty_output><jupyter_text>Store Predicted Fovea in appropriate directories<jupyter_code>for key in list(label_dictionary.keys()) :
    box = label_dictionary.get(key).get('box_preds')
    scale = box[0]
    yxcoord = box[1:]
    fov = fd.retrieve_fovea(key, yxcoord, scale, fov_dim = 72)
    new_path = 'data/predicted_fovea_train/'+key[11:]
    misc.imsave(new_path, fov, format = 'JPEG' )
os.listdir('data/annotated_fovea_train/')<jupyter_output><empty_output><jupyter_text>Test Stage 2 Images  Get all filenames<jupyter_code>test_fnames = fd.generate_filenames_list('data/test_stg2/', False)
print(len(test_fnames))

test_fnames.remove('data/test_stg2/.DS_Store')
print(len(test_fnames))<jupyter_output>12153<jupyter_text>Run through FishFinder and store FiNoF , box specs in pandas dataframe<jupyter_code>version_ID = 'v1.2'
initiate_FishFinder = False 

wd = os.getcwd()
md = wd+'/FishFinder/'+version_ID
if not os.path.exists(md) :
    os.makedirs(md)
tensorboard_path = md+'/Tensorboard_logs'

%run -i 'FishFinder/FishFinder_PARAMS.py'
%run -i 'FishFinder/FishFinder_GRAPH.py'
with tf.Session(graph = fish_finder) as session :
    if 'meta_dictionary.pickle' in os.listdir(md) and initiate_FishFinder != True:
        print("Loading FishFinder_MT version {}".format(version_ID))
        with open(md+'/meta_dictionary.pickle', 'rb') as  handle :
            meta_dict = pickle.load(handle)
        print("Metadata dictionary loaded!")
        total_fovea = meta_dict.get(np.max([key for key in meta_dict])).get('fovea_trained')
        epochs_completed = meta_dict.get(np.max([key for key in meta_dict])).get('Num_epochs')
        restorer = tf.train.Saver()
        print("Initializing restorer...")
        restorer.restore(session, tf.train.latest_checkpoint(md))
        print("Weights and biases retrieved!  Picking up at {} epochs completed : {} training images observed".format(epochs_completed, total_fovea))
    print("Running Predictor on Test Stage 2 images...")
    
    keys_list = test_fnames.copy()
    t_embedding_arr = np.zeros([len(keys_list), 32])
    t_FiNoF = np.zeros([len(keys_list), 1])
    t_box_preds = np.zeros([len(keys_list),3])
    cursor = 0
    while len(keys_list) > batch_size :
        for i in range(batch_size) :
            coarse = misc.imresize(misc.imread(keys_list.pop(0), mode = 'RGB'), size = [64, 112,3], mode = 'RGB')
            if i == 0 :
                coarse_arr = np.expand_dims(coarse, 0)
            else :
                coarse_arr = np.concatenate([coarse_arr, np.expand_dims(coarse,0)], 0)
        
        feed_dict = {coarse_images_for_prediction : coarse_arr}
        FiNoF_Probability, Box_Predictions, coarse_embedding = session.run([stack_FishNoF_preds, stack_box_preds, stack_dense_output], feed_dict = feed_dict)
        
        for i in range(batch_size) :
            t_FiNoF[cursor, :] = FiNoF_Probability[i]
            t_box_preds[cursor, :] = Box_Predictions[i,:]
            t_embedding_arr[cursor, :] = coarse_embedding[i, :]
            if (cursor % 1024) == 0 :
                print("{} images embedded".format(cursor))
                print("Length of prediction_keys : {}".format(len(keys_list)))
            cursor += 1
                
    for i in range(len(keys_list)) :
        coarse = misc.imresize(misc.imread(keys_list.pop(0), mode = 'RGB'), size = [64, 112,3], mode = 'RGB')
        if i == 0 :
            coarse_arr = np.expand_dims(coarse, 0)
        else :
            coarse_arr = np.concatenate([coarse_arr, np.expand_dims(coarse,0)], 0)
        
    feed_dict = {coarse_images_for_prediction : coarse_arr}
    FiNoF_Probability, Box_Predictions, coarse_embedding = session.run([stack_FishNoF_preds, stack_box_preds, stack_dense_output], feed_dict = feed_dict)

    for i in range(batch_size) :
        t_FiNoF[cursor, :] = FiNoF_Probability[i]
        t_box_preds[cursor, :] = Box_Predictions[i,:]
        t_embedding_arr[cursor, :] = coarse_embedding[i, :]
        cursor += 1<jupyter_output>Loading FishFinder_MT version v1.2
Metadata dictionary loaded!
Initializing restorer...
Weights and biases retrieved!  Picking up at 665 epochs completed : 2230912 training images observed
Running Predictor on Test Stage 2 images...
0 images embedded
Length of prediction_keys : 12025
1024 images embedded
Length of prediction_keys : 11001
2048 images embedded
Length of prediction_keys : 9977
3072 images embedded
Length of prediction_keys : 8953
4096 images embedded
Length of prediction_keys : 7929
5120 images embedded
Length of prediction_keys : 6905
6144 images embedded
Length of prediction_keys : 5881
7168 images embedded
Length of prediction_keys : 4857
8192 images embedded
Length of prediction_keys : 3833
9216 images embedded
Length of prediction_keys : 2809
10240 images embedded
Length of prediction_keys : 1785
11264 images embedded
Length of prediction_keys : 761<jupyter_text>Note that is ok.  Made mistake in number of iterations for the final garbage loop<jupyter_code>t_FiNoF.shape
t_box_preds.shape
t_embedding_arr.shape
t_FiNoF_df = pd.DataFrame(t_FiNoF, columns = ['FiNoF'], index = test_fnames.copy())
print(t_FiNoF_df.head())
t_box_preds_df = pd.DataFrame(t_box_preds, columns = ['scale', 'y_off', 'x_off'], index = test_fnames.copy())
t_embedding_df = pd.DataFrame(t_embedding_arr, columns = list(range(32)), index = test_fnames.copy())
t_embedding_df.head()
t_box_preds_df.head()
t_FiNoF_df.to_pickle('test_FiNoF_dataframe.pickle')
t_embedding_df.to_pickle('test_embeddings_df.pickle')
t_box_preds_df.to_pickle('test_box_preds_df.pickle')<jupyter_output><empty_output><jupyter_text>Make Fovea for Test Stage 2<jupyter_code>for key in test_fnames :
    scale = t_box_preds_df['scale'].loc[key]
    yxcoord = np.array(t_box_preds_df.loc[key, ['y_off', 'x_off']])

    fov = fd.retrieve_fovea(key, yxcoord, scale, fov_dim = 72)
    new_path = 'data/predicted_fovea_test_stg2/'+key[15:]

    misc.imsave(new_path, fov, format = 'JPEG' )
key
'img'+key[20:]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12738191477075875, 'fraction_numerical': 0.021134080147823073, 'mean_word_length': 4.481012658227848, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '626051', 'license': 'MIT', 'n_tokens_mistral': 3236, 'n_tokens_neox': 3038, 'n_words': 597, 'path': 'FishyFish Data Retrieval.ipynb', 'repo_name': 'CCThompson82/Kaggle_FF3', 'size': '8659'}