<commit_before>#!/usr/bin/env python3
import os
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
import numpy as np
np.set_printoptions(threshold=np.nan)
import tensorflow as tf
import time

def convolve_inner_layers(x, W, b):
    y = tf.nn.conv2d(x, W, strides = [1,1,1,1], padding='SAME')
    y = tf.nn.bias_add(y, b)
    return tf.nn.relu(y)

def convolve_ouput_layer(x, W, b):
    y = tf.nn.conv2d(x, W, strides = [1,1,1,1], padding='SAME')
    y = tf.nn.bias_add(y, b)
    return y

def conv_net(x, W, b):
    conv1 = convolve_inner_layers(x, W['weights1'], b['bias1'])
    conv2 = convolve_inner_layers(conv1, W['weights2'], b['bias2'])
    conv3 = convolve_inner_layers(conv2, W['weights3'], b['bias3'])
    output = convolve_ouput_layer(conv3, W['weights_out'], b['bias_out'])
    return output

def main():
    # parameters
    filter_dim = 11
    number_images = 100
    image_dim = 3
    input_layer = 2
    first_layer = 50
    second_layer = 25
    third_layer = 10
    output_layer = 1
    initializer_scale = 1

    # seeding for debug purposes --- dont forget to remove
    SEED = 12345
    np.random.seed(SEED)
    tf.set_random_seed(SEED)

    print('generating random images ... ')
    # train images
    rand_img_train_1 = np.random.random_sample((number_images,image_dim**2))
    rand_img_train_2 = np.random.random_sample((number_images,image_dim**2))
    difference_train = abs(rand_img_train_1 - rand_img_train_2)

    # test image
    rand_img_test_1 = np.random.random_sample((number_images,image_dim**2))
    rand_img_test_2 = np.random.random_sample((number_images,image_dim**2))
    difference_test = abs(rand_img_test_1 - rand_img_test_2)

    # stacking & reshaping images
    train_data = np.reshape(np.dstack((rand_img_train_1, rand_img_train_2)), [number_images,image_dim,image_dim,2])
    test_data = np.reshape(np.dstack((rand_img_test_1, rand_img_test_2)), [number_images,image_dim,image_dim,2])
    target_data_train = np.reshape(difference_train, [number_images,image_dim,image_dim,1])
    target_data_test = np.reshape(difference_test, [number_images,image_dim,image_dim,1])

    # initializing variables --- fan in
    weights = {
        'weights1': tf.Variable(tf.random_normal([filter_dim,filter_dim,input_layer,first_layer],stddev=(1.0/(initializer_scale*filter_dim*filter_dim*input_layer)))),
        'weights2': tf.Variable(tf.random_normal([filter_dim,filter_dim,first_layer,second_layer],stddev=(1.0/(initializer_scale*filter_dim*filter_dim*first_layer)))),
        'weights3': tf.Variable(tf.random_normal([filter_dim,filter_dim,second_layer,third_layer],stddev=(1.0/(initializer_scale*filter_dim*filter_dim*second_layer)))),
        'weights_out': tf.Variable(tf.random_normal([filter_dim,filter_dim,third_layer,output_layer],stddev=(1.0/(initializer_scale*filter_dim*filter_dim*third_layer))))
    }
    biases = {
        'bias1': tf.Variable(tf.random_normal([first_layer],stddev=(1.0/(initializer_scale*filter_dim*filter_dim*input_layer)))),
        'bias2': tf.Variable(tf.random_normal([second_layer],stddev=(1.0/(initializer_scale*filter_dim*filter_dim*first_layer)))),
        'bias3': tf.Variable(tf.random_normal([third_layer],stddev=(1.0/(initializer_scale*filter_dim*filter_dim*second_layer)))),
        'bias_out': tf.Variable(tf.random_normal([output_layer],stddev=(1.0/(initializer_scale*filter_dim*filter_dim*third_layer))))
    }

    # tf Graph input
    x = tf.placeholder(tf.float32, [None, image_dim, image_dim, 2])
    y = tf.placeholder(tf.float32, [None, image_dim, image_dim, 1])

    # paramaters
    learning_rate = .001
    epochs = 1000

    # model
    prediction = conv_net(x, weights, biases)

    # loss and optimization
    cost = tf.reduce_mean(tf.square(tf.subtract(prediction, y)))
    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

    # session
    init = tf.global_variables_initializer()
    with tf.Session() as sess:
        sess.run(init)
        epoch_count = 0
        start_time = time.time()
        print("starting training ... ")
        while epoch_count < epochs:
            x_data_train, y_data_train = train_data, target_data_train
            sess.run(optimizer, feed_dict={x : x_data_train, y : y_data_train})
            loss = sess.run(cost, feed_dict={x : x_data_train, y : y_data_train})
            print("  -  training global_step {}. current error: {}. ".format(epoch_count, loss))
            epoch_count+=1
        print('optimization finished!')
        print('\nstarting testing...')
        score = sess.run(cost, feed_dict={x: test_data, y: target_data_test})
        pred = sess.run(prediction, feed_dict={x: test_data})
        for i in range(image_dim):
            print(rand_img_test_1[0][i],rand_img_test_2[0][i], pred[0][0][i], difference_test[0][i])
        print('---- score : {} ----'.format(score))

if __name__ == '__main__':
    main()<commit_msg>adding variance and percent error metric<commit_after>#!/usr/bin/env python3
import os
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
import numpy as np
np.set_printoptions(threshold=np.nan)
import tensorflow as tf
import time

def convolve_inner_layers(x, W, b):
    y = tf.nn.conv2d(x, W, strides = [1,1,1,1], padding='SAME')
    y = tf.nn.bias_add(y, b)
    return tf.nn.relu(y)

def convolve_ouput_layer(x, W, b):
    y = tf.nn.conv2d(x, W, strides = [1,1,1,1], padding='SAME')
    y = tf.nn.bias_add(y, b)
    return y

def conv_net(x, W, b):
    conv1 = convolve_inner_layers(x, W['weights1'], b['bias1'])
    conv2 = convolve_inner_layers(conv1, W['weights2'], b['bias2'])
    conv3 = convolve_inner_layers(conv2, W['weights3'], b['bias3'])
    output = convolve_ouput_layer(conv3, W['weights_out'], b['bias_out'])
    return output

def get_variance(training_target):
    all_pixels = training_target.flatten()
    return all_pixels.var()

def main():
    # parameters
    filter_dim = 11
    number_images = 100
    image_dim = 96
    input_layer = 2
    first_layer = 50
    second_layer = 25
    third_layer = 10
    output_layer = 1
    initializer_scale = float(10.0)

    # seeding for debug purposes --- dont forget to remove
    SEED = 12345
    np.random.seed(SEED)
    tf.set_random_seed(SEED)

    print('generating random images ... ')
    # train images
    rand_img_train_1 = np.random.random_sample((number_images,image_dim**2))
    rand_img_train_2 = np.random.random_sample((number_images,image_dim**2))
    difference_train = abs(rand_img_train_1 - rand_img_train_2)

    # test image
    rand_img_test_1 = np.random.random_sample((number_images,image_dim**2))
    rand_img_test_2 = np.random.random_sample((number_images,image_dim**2))
    difference_test = abs(rand_img_test_1 - rand_img_test_2)

    # stacking & reshaping images
    train_data = np.reshape(np.dstack((rand_img_train_1, rand_img_train_2)), [number_images,image_dim,image_dim,2])
    test_data = np.reshape(np.dstack((rand_img_test_1, rand_img_test_2)), [number_images,image_dim,image_dim,2])
    target_data_train = np.reshape(difference_train, [number_images,image_dim,image_dim,1])
    target_data_test = np.reshape(difference_test, [number_images,image_dim,image_dim,1])

    # initializing variables --- fan in
    weights = {
        'weights1': tf.Variable(tf.random_normal([filter_dim,filter_dim,input_layer,first_layer],stddev=(1.0/(initializer_scale*filter_dim*filter_dim*input_layer)))),
        'weights2': tf.Variable(tf.random_normal([filter_dim,filter_dim,first_layer,second_layer],stddev=(1.0/(initializer_scale*filter_dim*filter_dim*first_layer)))),
        'weights3': tf.Variable(tf.random_normal([filter_dim,filter_dim,second_layer,third_layer],stddev=(1.0/(initializer_scale*filter_dim*filter_dim*second_layer)))),
        'weights_out': tf.Variable(tf.random_normal([filter_dim,filter_dim,third_layer,output_layer],stddev=(1.0/(initializer_scale*filter_dim*filter_dim*third_layer))))
    }
    biases = {
        'bias1': tf.Variable(tf.random_normal([first_layer],stddev=(1.0/(initializer_scale*filter_dim*filter_dim*input_layer)))),
        'bias2': tf.Variable(tf.random_normal([second_layer],stddev=(1.0/(initializer_scale*filter_dim*filter_dim*first_layer)))),
        'bias3': tf.Variable(tf.random_normal([third_layer],stddev=(1.0/(initializer_scale*filter_dim*filter_dim*second_layer)))),
        'bias_out': tf.Variable(tf.random_normal([output_layer],stddev=(1.0/(initializer_scale*filter_dim*filter_dim*third_layer))))
    }

    # tf Graph input
    x = tf.placeholder(tf.float32, [None, image_dim, image_dim, 2])
    y = tf.placeholder(tf.float32, [None, image_dim, image_dim, 1])

    # paramaters
    learning_rate = .001
    epochs = 1000

    # model
    prediction = conv_net(x, weights, biases)

    # get variance to normalize error terms during training
    variance = get_variance(difference_train)

    # loss and optimization
    cost = tf.reduce_mean(tf.square(tf.subtract(prediction, y)))
    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

    # session
    init = tf.global_variables_initializer()
    with tf.Session() as sess:
        sess.run(init)
        epoch_count = 0
        start_time = time.time()
        print("starting training ... ")
        while epoch_count < epochs:
            x_data_train, y_data_train = train_data, target_data_train
            sess.run(optimizer, feed_dict={x : x_data_train, y : y_data_train})
            loss = sess.run(cost, feed_dict={x : x_data_train, y : y_data_train})
            percent_error = 100*loss/variance
            print("  -  training global_step {}. current error: {}. percent error: {}%.".format(epoch_count, loss, percent_error))
            epoch_count+=1
        print('optimization finished!')
        print('\nstarting testing...')
        score = sess.run(cost, feed_dict={x: test_data, y: target_data_test})
        pred = sess.run(prediction, feed_dict={x: test_data})
        for i in range(image_dim):
            print(rand_img_test_1[0][i],rand_img_test_2[0][i], pred[0][0][i], difference_test[0][i])
        print('---- score : {} ----'.format(score))

if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.13747534516765286, 'fraction_numerical': 0.021893491124260357, 'mean_word_length': 4.2435367114788, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3298675', 'n_tokens_mistral': 3662, 'n_tokens_neox': 3541, 'n_words': 549}