<jupyter_start><jupyter_text>Note - This model is actually used for this project and uses the Darknet-19 forward prop outlined in the YOLO_v2 paper.  YOLO_v2 inspired model- This model takes in input images of shape (448,448,3) and produces output of shape (14,14,14). In looking at the output of my k-means clustering step in my minibatch data cleaning notebook, I decided to have 2 (1,1,7) encodings such that the model can learn the difference between the dimensions of open and closed palms.- I will be using stochastic gradient descent with adam optimization, but with a manually interatively decreasing learning rate, unlike the set diminishing learning rate promoted by the YOLO_v2 model.- This model is essentially the Darknet-19 model, which I chose over the Darknet-53 resnet model strictly to decrease the time needed for model training.<jupyter_code>import numpy as np
import tensorflow as tf
import pandas as pd
from keras import backend as K
import matplotlib.pyplot as plt
import latex
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split

%matplotlib inline
import warnings
warnings.filterwarnings('ignore')
# Loading data
X = np.load("../../data/dinorunner/images_448.npy")
y = np.load("../../data/dinorunner/encodings_448.npy")
print(X.shape)
print(y.shape)
# Creating testing set
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.05, random_state=1)
print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
print(y_test.shape)<jupyter_output>(376, 448, 448, 3)
(376, 14, 14, 7)
(20, 448, 448, 3)
(20, 14, 14, 7)<jupyter_text>Tensorflow Placeholders<jupyter_code># Placeholder values for input X,y data
def get_placeholders(x_h,x_w,x_c,y_h,y_w,y_c):
    """
    x_h: Height for x input 
    x_w: Width for x input
    x_c: Channels for x input
    y_h: Height for y input
    y_w: Width for y input
    y_c: Channels for y input
    """
    X = tf.placeholder(tf.float32, name="X", shape=(None,x_h,x_w,x_c))
    y = tf.placeholder(tf.float32, name="y", shape=(None,y_h,y_w,y_c))
    return X,y
# Testing placeholders
tf.reset_default_graph()
with tf.Session() as sess:
    X,y = get_placeholders(448,448,3,14,14,7)
    print("X shape:",X.shape)
    print("y shape:",y.shape)<jupyter_output>X shape: (?, 448, 448, 3)
y shape: (?, 14, 14, 7)<jupyter_text>Tensorflow Forward Propagation<jupyter_code># Defining constant layer for 2d convolution, batch norm, and activation
def conv(the_input,layer,f,ks):
    """
    the_input: the layer which will be used as input in conv layer
    layer: specifies the layer number for naming sections of graph
    f (filters): the number of filters to be used for conv layer
    ks (kernel_size): kernel size for conv2d layer
    Note - conv2d layers all use padding
    """
    layer = str(layer)
    Z = tf.layers.conv2d(the_input,filters=f,kernel_size=[ks,ks],strides=(1,1),padding="same",name="Z"+layer,kernel_initializer=tf.contrib.layers.xavier_initializer(seed=0))
    Bn = tf.layers.batch_normalization(Z,name="Bn"+layer)
    A = tf.nn.leaky_relu(Bn,alpha=0.1,name="A"+layer)
    return A
# Building the forward pass based on Darket-19
# Note - forward pass will use leaky_relu
def forward_pass(X):
    input_layer = tf.reshape(X,[-1,448,448,3]) # Input shape of images
    S1 = conv(input_layer,1,32,3)
    P1 = tf.layers.max_pooling2d(S1,pool_size=[2,2],strides=2,padding="valid",name="P1") # 224x224
    S2 = conv(P1,2,64,3)
    P2 = tf.layers.max_pooling2d(S2,pool_size=[2,2],strides=2,padding="valid",name="P2") # 112x112
    S3 = conv(P2,3,128,3)
    S4 = conv(S3,4,64,1)
    S5 = conv(S4,5,128,3)
    P5 = tf.layers.max_pooling2d(S5,pool_size=[2,2],strides=2,padding="valid",name="P5") # 56x56
    S6 = conv(P5,6,256,3)
    S7 = conv(S6,7,128,1)
    S8 = conv(S7,8,256,3)
    P8 = tf.layers.max_pooling2d(S8,pool_size=[2,2],strides=2,padding="valid",name="P8") # 28x28
    S9 = conv(P8,9,512,3)
    S10 = conv(S9,10,256,1)
    S11 = conv(S10,11,512,3)
    S12 = conv(S11,12,256,1)
    S13 = conv(S12,13,512,3)
    P13 = tf.layers.max_pooling2d(S13,pool_size=[2,2],strides=2,padding="valid",name="P13") #14x14
    S14 = conv(P13,14,1024,3)
    S15 = conv(S14,15,512,1)
    S16 = conv(S15,16,1024,3)
    S17 = conv(S16,17,512,1)
    S18 = conv(S17,18,2014,3)
    # Final layer - no batch norm, linear activation
    S19 = tf.layers.conv2d(S18,filters=14,kernel_size=[1,1],strides=(1,1),padding="valid",name="S19",activation=None)
    return S19
# Testing forward prop
tf.reset_default_graph()
with tf.Session() as sess:
    np.random.seed(1)
    X,y = get_placeholders(448,448,3,14,14,7)
    Z19 = forward_pass(X) # Computation graph
    init = tf.global_variables_initializer()
    sess.run(init)
    aZ = sess.run(Z19,feed_dict={X:np.random.randn(1,448,448,3),y:np.random.randn(1,14,14,7)})
    print("Z shape:", str(aZ.shape))<jupyter_output>Z shape: (1, 14, 14, 14)<jupyter_text>Tensorflow Cost FunctionUsing the predicted box [shape(1,1,7)] with the highest IoU score to determine the prediction cost for a given image.Note - I deviate from the YOLO function function in that for all of the cells that don't hold the actual prediction for the bounding box, I penalize ALL of the confidence predictions (2 per cell in this case). In order to compensate for this change and not greatly increase the cost from just wrong confidence scores I decrease the lambdanoobj from 0.5 to 0.25. This is due to the fact that I am factoring in 2 times the previous number of confidence scores.<jupyter_code># Returns the sum of all of the confidence scores corresponding w/ cells that don't have prediction
def get_false_conf(Z,mask):
    conf_1 = tf.square(tf.boolean_mask(Z[:,:,:,0:1],mask))
    conf_2 = tf.square(tf.boolean_mask(Z[:,:,:,7:8],mask))
    conf_sum = tf.add(tf.reduce_sum(conf_1),tf.reduce_sum(conf_2))
    return conf_sum
# Returns the values with a specific mask applied to it
def get_box_values(box,mask):
    """
    Index:
    0: confidence there is an object in cell, 1: mid_x, 2: mid_y, 
    3: width, 4: length, 5: prob_open_palm, 6: prob_close_palm
    """
    confidence = tf.boolean_mask(box[:,:,:,0:1],mask)
    mid_x = tf.boolean_mask(box[:,:,:,1:2],mask)
    mid_y = tf.boolean_mask(box[:,:,:,2:3],mask)
    width = tf.boolean_mask(box[:,:,:,3:4],mask)
    height = tf.boolean_mask(box[:,:,:,4:5],mask)
    prob_open = tf.boolean_mask(box[:,:,:,5:6],mask)
    prob_closed = tf.boolean_mask(box[:,:,:,6:7],mask)
    box = {"co":confidence, "mx":mid_x,"my":mid_y,"w":width,"h":height,"d":prob_open,"c":prob_closed}
    return box
# Returns the IoU
def get_iou(box1, box2):
    """
    box1 - coordinates (x1, y1, x2, y2)
    box2 - coordinates (x1, y1, x2, y2)
    """
    xi1 = tf.maximum(box1["x1"], box2["x1"])
    yi1 = tf.maximum(box1["y1"], box2["y1"])
    xi2 = tf.minimum(box1["x2"], box2["x2"])
    yi2 = tf.minimum(box1["y2"], box2["y2"])
    inter_area = tf.maximum((xi2 - xi1),0) * tf.maximum((yi2 - yi1),0) # no neg bounding box

    box1_area = (box1["x2"]-box1["x1"]) * (box1["y2"] - box1["y1"])
    box2_area = (box2["x2"]-box2["x1"]) * (box2["y2"] - box2["y1"])
    union_area = box1_area + box2_area - inter_area
    iou = inter_area / union_area
    
    return tf.reshape(iou,[])
# Returns the (x1,y1),(x2,y2) coordinates for each bounding box as a dict
def get_xy(box_v):
    mid_x = box_v["mx"]
    mid_y = box_v["my"]
    width = box_v["w"]
    height = box_v["h"]
    width = width * 448
    height = height * 448
    mid_x = mid_x * 32 + 224
    mid_y = mid_y * 32 + 224
    
    x1 = mid_x - (1/2*width)
    x2 = mid_x + (1/2*width)
    y1 = mid_y - (1/2*height)
    y2 = mid_y + (1/2*height)
    
    box_xy = {"x1":x1,"x2":x2,"y1":y1,"y2":y2}
    return box_xy
# This returns the information associated with the bounding box prediction with the max IoU
def get_max_iou(box1, box2, y, mask):
    """
    b1_v: bounding box values associated with box1
    b2_v: bounding box values associated with box2
    y_v: bounding box values associated with y
    """
    b1_v = get_box_values(box1,mask)
    b2_v = get_box_values(box2,mask)
    y_v = get_box_values(y,mask)
    # These new coordinates will be used to get the IoU
    b1_xy = get_xy(b1_v)
    b2_xy = get_xy(b2_v)
    y_xy = get_xy(y_v)
    # Getting the Iou for each bounding box prediction
    b1_iou = get_iou(b1_xy, y_xy)
    b2_iou = get_iou(b2_xy, y_xy)
    # Comparing the ious to determine which guess is the ground truth prediction
    def b1(): return b1_v, b2_v
    def b2(): return b2_v, b1_v
    highest_iou_values, lowest_values = tf.cond(tf.less(b1_iou,b2_iou), b2, b1)
    return highest_iou_values, lowest_values
# Stochastic gradient descent so input of shape (1,14,14,14)
# Heavily penalizes negative predictions for height and weight
# Bounding box prediction based on guess with the highest IoU
def cost_function(Z,y,coord=5,noobj=0.25):
    """
    Z - shape (?,14,14,14)
    y - shape (?,14,14,7)
    """
    c_mask_true = y[:,:,:,0:1] > 0 # which cell has the encoding
    c_mask_false = y[:,:,:,0:1] < 1
    
    box1 = Z[:,:,:,0:7] # First guess
    box2 = Z[:,:,:,7:14] # Second guess
    y_v = get_box_values(y,c_mask_true) # values for y for cell with object
    m_v,lowest_values = get_max_iou(box1,box2,y,c_mask_true) # values for highest IoU guess
    false_conf_score = get_false_conf(Z,c_mask_false) # sum of false confidence predictions
    #false_conf_score = false_conf_score + tf.reshape(tf.square(lowest_values["co"]),[])
    
    # penalizing width,length predictions if negative
    m_v["w"] = tf.sqrt(tf.maximum(m_v["w"],0.0))
    m_v["l"] = tf.sqrt(tf.maximum(m_v["h"],0.0))
    
    y_v["w"] = tf.sqrt(y_v["w"])
    y_v["l"] = tf.sqrt(y_v["h"])
    
    # correspond to individual summations of the cost function:
    part1 = coord * tf.reduce_sum(tf.square(y_v["mx"]-m_v["mx"])+tf.square(y_v["my"]-m_v["my"]))
    part2 = coord * tf.reduce_sum(tf.square(y_v["w"]-m_v["w"])+tf.square(y_v["l"]-m_v["l"]))
    part3 = tf.reduce_sum(tf.square(y_v["co"]-m_v["co"]))
    part4 = noobj * false_conf_score
    part5 = tf.reduce_sum(tf.add(tf.square(y_v["d"]-m_v["d"]),tf.square(y_v["c"]-m_v["c"])))
    total_cost = part1 + part2 + part3 + part4 + part5

    return total_cost
# Testing cost function
# predicted cost w/ rounding error is 1.4396
ay = np.zeros((1,14,14,7))
ay[0,0,0,:] = np.array([1,0.5,0.5,0.25,0.25,1,0]) # top left corner
az = np.zeros((1,14,14,14))
az[0,0,0,0:7] = np.array([0.8,0.35,0.35,0.2,0.2,0.8,0.2]) # PRED 1
az[0,0,0,7:] = np.array([0.5,0.4,0.4,0.22,0.22,0.8,0.2]) # PRED 2
az[0,0,1,0] = 1
az[0,1,0,0] = 1
az[0,1,0,7] = 1
az[0,2,0,7] = 1

with tf.Session() as sess:
    y = tf.placeholder(tf.float32,shape=(None,14,14,7))
    Z = tf.placeholder(tf.float32,shape=(None,14,14,14))
    aCost = cost_function(Z,y)
    init = tf.global_variables_initializer()
    sess.run(init)
    tot = sess.run(aCost,feed_dict={Z:az,y:ay})
    print(tot)<jupyter_output>1.4395843<jupyter_text>Tensorflow Model<jupyter_code># Building and training YOLO model using stochastic gradient descent
def model(X_train,y_train,lr=0.001,num_epochs=50,print_cost=True):
    tf.reset_default_graph() # resetting graph
    tf.set_random_seed(1)
    seed=0
    costs=[]
    x_h = X_train[0].shape[0]
    x_w = X_train[0].shape[1]
    x_c = X_train[0].shape[2]
    y_h = y_train[0].shape[0]
    y_w = y_train[0].shape[1]
    y_c = y_train[0].shape[2]
    m = X_train.shape[0]
    
    X,y = get_placeholders(x_h,x_w,x_c,y_h,y_w,y_c)
    Z = forward_pass(X)
    cost = cost_function(Z,y)
    optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(cost)
    
    init = tf.global_variables_initializer()
    saver = tf.train.Saver() # to save/load model
    with tf.Session() as sess:
        # Loading saved model
        saver = tf.train.import_meta_graph("../../data/dinorunner/models_stochastic/yolo_model.ckpt.meta")
        saver.restore(sess, "../../data/dinorunner/models_stochastic/yolo_model.ckpt")
        #sess.run(init) # DON'T RUN INIT IF LOADING MODEL
        
        for epoch in range(num_epochs):
            running_cost = 0
            # shuffling training set order each iteration
            seed += 1
            X_train = shuffle(X_train, random_state=seed) 
            y_train = shuffle(y_train, random_state=seed)
            
            for i in range(X_train.shape[0]):
                aX = X_train[i]
                aX.shape = (1,448,448,3)
                aY = y_train[i]
                aY.shape = (1,14,14,7)
                _,temp_cost = sess.run([optimizer,cost], feed_dict={X:aX,y:aY})
                running_cost += temp_cost
                #print(running_cost)
                
            costs.append(running_cost)
            if print_cost and epoch % 1 == 0:
                print("Cost at epoch {}: {}".format(epoch+1,running_cost))
                
        loc = saver.save(sess, "../../data/dinorunner/models_stochastic/yolo_model.ckpt")
        return costs
print(X_train.shape)
print(y_train.shape)
acosts = model(X_train,y_train,lr=0.0001,num_epochs=20,print_cost=True)
acosts2 = model(X_train,y_train,lr=0.0001,num_epochs=20,print_cost=True)
acosts3 = model(X_train,y_train,lr=0.00005,num_epochs=20,print_cost=True)
acosts4 = model(X_train,y_train,lr=0.00001,num_epochs=20,print_cost=True)
acosts4 = model(X_train,y_train,lr=0.000001,num_epochs=10,print_cost=True)
acosts5 = model(X_train,y_train,lr=0.0000001,num_epochs=5,print_cost=True)
final_cost = acosts + acosts2 + acosts3 + acosts4 + acosts5
x_iter = [i for i in range(len(final_cost))]
plt.plot(x_iter,final_cost)
plt.title("cost vs epoch")
plt.xlabel("epoch #")
plt.ylabel("cost")
plt.show()<jupyter_output><empty_output><jupyter_text>Here's an example of the model running for an image in the test setNote - it is quite likely that this model has overfitted to the specific types of images I trained it on, i.e. all of the images are from the same building and essentially all from a similar angle and have the exact same background. I wanted to collect all of the data over a short period of time at a hackathon so didn't look to add training examples from different distributions.<jupyter_code># Getting image encoding
tf.reset_default_graph()
X = tf.placeholder(tf.float32,shape=(None,448,448,3))
Z = forward_pass(X)
saver = tf.train.Saver()
with tf.Session() as sess:
    saver = tf.train.import_meta_graph("../../data/dinorunner/models_stochastic/yolo_model.ckpt.meta")
    saver.restore(sess, "../../data/dinorunner/models_stochastic/yolo_model.ckpt")
    aX = X_test[1]
    aX.shape = (1,448,448,3)
    aZ = sess.run(Z,feed_dict={X:aX})
# Returns the string key
def get_key_value(enc):
    if enc[0] > enc[1]:
        return "up"
    else:
        return "down"
# Handling printing the image with a bounding box
import matplotlib.patches as patches

def print_img(values,img,left,top):
    """
    left: offset in number of cells from far left of image
    top: offset in number of cells from top of image
    """
    x_mid = values[0] * 32
    y_mid = values[1] * 32
    width = values[2] * 448
    height = values[3] * 448
    xmin = x_mid + (32 * left)
    ymin = y_mid + (32 * top)
    
    fig,ax = plt.subplots(1)
    ax.imshow(img)
    rect = patches.Rectangle((xmin,ymin),width,height,linewidth=1,edgecolor='r',facecolor='none')
    ax.add_patch(rect)
    plt.imshow(img)
    plt.show()
# Getting the prediction for the image and printing it out
img = X_test[1]
encoding = aZ
encoding.shape = (14,14,14)
sec_1 = encoding[:,:,0:7]
sec_2 = encoding[:,:,7:14]
indic1 = np.argmax(sec_1[:,:,0])
indic2 =  np.argmax(sec_2[:,:,0])
conf_score_1 = sec_1[indic1//14,indic1-(indic1//14*14),0]
conf_score_2 = sec_2[indic2//14,indic2-(indic2//14*14),0]

if conf_score_1 > conf_score_2: # Use sec_1 encoding
    key = get_key_value(sec_1[indic1//14,indic1-(indic1//14*14),5:7])
    print_img(sec_1[indic1//14,indic1-(indic1//14*14),1:5],img,indic1//14,indic1-(indic1//14*14))
    print(sec_1[indic1//14,indic1-(indic1//14*14),5:7])
else: # Use sec_2 encoding
    key = get_key_value(sec_2[indic2//14,indic2-(indic2//14*14),5:7])
    print_img(sec_2[indic2//14,indic2-(indic2//14*14),1:5],img,indic2//14,indic2-(indic2//14*14))
    print(sec_2[indic2//14,indic2-(indic2//14*14),5:7])
print(key)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1471688853444574, 'fraction_numerical': 0.061039200049076744, 'mean_word_length': 3.9325264750378213, 'pattern_counts': {'":': 11, '<': 22, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '619948', 'license': 'MIT', 'n_tokens_mistral': 6828, 'n_tokens_neox': 6395, 'n_words': 1404, 'path': 'dino_runner_yolo/dino_yolo_model_stochastic.ipynb', 'repo_name': 'nateandre/machine_learning', 'size': '16301'}