<commit_before>import tensorflow.python.platform

import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt

import random

from humanIntuitionUtils import graphHelpers
from humanIntuitionUtils import extract_data
from humanIntuitionUtils import variable_summaries
from humanIntuitionUtils import init_weights
from humanIntuitionUtils import multilayer_perceptron

# Original from https://github.com/jasonbaldridge/try-tf/

# set seed if required
random.seed(15)
tf.set_random_seed(15)

# Global variables.
BATCH_SIZE = 1  # The number of training examples to use per training step. We use 1 to simulate an individual updating their personal neural networks one example at a time
PERCENT_TESTING = 0.5;

# Define the flags useable from the command line.
tf.app.flags.DEFINE_string('data','./server/exports/mlData.json', 'File containing the data, labels, features.')
tf.app.flags.DEFINE_integer('num_epochs', 1, 'Number of examples to separate from the training data for the validation set.')
tf.app.flags.DEFINE_boolean('verbose', False, 'Produce verbose output.')

FLAGS = tf.app.flags.FLAGS


def main(argv=None):
    # Be verbose?
    verbose = FLAGS.verbose

    # Get the data.
    data_filename = FLAGS.data

    # Extract it into numpy arrays.
    data, labels, testData, testLabels, labels_map = extract_data(data_filename, PERCENT_TESTING)

    data_size, num_features = data.shape
    testData_size, num_testData_features = testData.shape

    num_labels = len(labels_map)

    print "data shape: " + str(num_features)
    print "train data rows: " + str(data_size)
    print "test data rows: " + str(testData_size)
    print "total labels: " + str(num_labels)

    # Get the number of epochs for training.
    num_epochs = FLAGS.num_epochs

    # Get the size of layer one.
    hidden_layer_size_1 = num_features
    hidden_layer_size_2 = num_features

    # This is where training samples and labels are fed to the graph.
    # These placeholder nodes will be fed a batch of training data at each
    # training step using the {feed_dict} argument to the Run() call below.
    x = tf.placeholder("float", shape=[None, num_features], name="x")
    y_ = tf.placeholder("float", shape=[None, num_labels], name="y")

    # For the test data, hold the entire dataset in one constant node.
    data_node = tf.constant(data)

    # Define and initialize the network.


    # =========== SINGLE HIDDEN LAYER TRACK ============

    w_hidden = init_weights('w_hidden', [num_features, num_labels], 'uniform')
    b_hidden = init_weights('b_hidden', [1, num_labels], 'zeros')

    # The hidden layer.
    hidden_single = tf.matmul(x,w_hidden) + b_hidden;


    # =========== MULTI LAYER TRACK ============

    weights = {
        'h1': init_weights('w1', [num_features, hidden_layer_size_1], 'uniform'),
        'h2': init_weights('w2', [hidden_layer_size_1, hidden_layer_size_2], 'uniform'),
        'out': init_weights('wOut', [hidden_layer_size_2, num_labels], 'uniform')
    }
    biases = {
        'b1': init_weights('b1', [1, hidden_layer_size_1], 'zeros'),
        'b2': init_weights('b2', [1, hidden_layer_size_2], 'zeros'),
        'out': init_weights('bOut', [1, num_labels], 'zeros')
    }

    hidden_multi = multilayer_perceptron(x, weights, biases)


    hidden_combined = tf.concat(0, [hidden_single, hidden_multi])


    # The output layer.
    y = tf.nn.softmax(hidden_combined);

    # Optimization.
    cross_entropy = -tf.reduce_sum(y_*tf.log(y))
    train_step = tf.train.GradientDescentOptimizer(0.03).minimize(cross_entropy)

    # Evaluation.
    correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))

    tf.summary.scalar('accurarcy', accuracy)

    #summary
    summary_op = tf.summary.merge_all()

    # Create a local session to run this computation.
    with tf.Session() as sess:

        # Run all the initializers to prepare the trainable parameters.
        tf.global_variables_initializer().run()

        writer = tf.train.SummaryWriter('./logs', sess.graph)

        # Iterate and train.
        for step in xrange(num_epochs * data_size // BATCH_SIZE):
            if verbose:
                print step,

            offset = (step * BATCH_SIZE) % data_size<commit_msg>cleaning up notation for multi track<commit_after>import tensorflow.python.platform

import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt

import random

from humanIntuitionUtils import graphHelpers
from humanIntuitionUtils import extract_data
from humanIntuitionUtils import variable_summaries
from humanIntuitionUtils import init_weights
from humanIntuitionUtils import multilayer_perceptron

# Original from https://github.com/jasonbaldridge/try-tf/

# set seed if required
random.seed(15)
tf.set_random_seed(15)

# Global variables.
BATCH_SIZE = 1  # The number of training examples to use per training step. We use 1 to simulate an individual updating their personal neural networks one example at a time
PERCENT_TESTING = 0.5;

# Define the flags useable from the command line.
tf.app.flags.DEFINE_string('data','./server/exports/mlData.json', 'File containing the data, labels, features.')
tf.app.flags.DEFINE_integer('num_epochs', 1, 'Number of examples to separate from the training data for the validation set.')
tf.app.flags.DEFINE_boolean('verbose', False, 'Produce verbose output.')

FLAGS = tf.app.flags.FLAGS


def main(argv=None):
    # Be verbose?
    verbose = FLAGS.verbose

    # Get the data.
    data_filename = FLAGS.data

    # Extract it into numpy arrays.
    data, labels, testData, testLabels, labels_map = extract_data(data_filename, PERCENT_TESTING)

    data_size, num_features = data.shape
    testData_size, num_testData_features = testData.shape

    # Matrix dimensions
    num_labels = len(labels_map)
    hidden_layer_size_1 = num_features
    hidden_layer_size_2 = num_features

    print "data shape: " + str(num_features)
    print "train data rows: " + str(data_size)
    print "test data rows: " + str(testData_size)
    print "total labels: " + str(num_labels)

    # Get the number of epochs for training.
    num_epochs = FLAGS.num_epochs

    # This is where training samples and labels are fed to the graph.
    # These placeholder nodes will be fed a batch of training data at each
    # training step using the {feed_dict} argument to the Run() call below.
    x = tf.placeholder("float", shape=[None, num_features], name="x")
    y_ = tf.placeholder("float", shape=[None, num_labels], name="y")

    # For the test data, hold the entire dataset in one constant node.
    data_node = tf.constant(data)


    # =========== SINGLE HIDDEN LAYER TRACK ============

    single_weights = init_weights('single_weights', [num_features, num_labels], 'uniform')
    single_biases = init_weights('single_biases', [1, num_labels], 'zeros')

    hidden_single = tf.matmul(x, single_weights) + single_biases;


    # =========== MULTI LAYER TRACK ============

    multi_weights = {
        'h1': init_weights('w1', [num_features, hidden_layer_size_1], 'uniform'),
        'h2': init_weights('w2', [hidden_layer_size_1, hidden_layer_size_2], 'uniform'),
        'out': init_weights('wOut', [hidden_layer_size_2, num_labels], 'uniform')
    }
    multi_biases = {
        'b1': init_weights('b1', [1, hidden_layer_size_1], 'zeros'),
        'b2': init_weights('b2', [1, hidden_layer_size_2], 'zeros'),
        'out': init_weights('bOut', [1, num_labels], 'zeros')
    }

    hidden_multi = multilayer_perceptron(x, multi_weights, multi_biases)


    # =========== MERGE MULTIPLE TRACKS ============

    hidden_combined = tf.concat(0, [hidden_single, hidden_multi])

    # The output layer.
    y = tf.nn.softmax(hidden_combined);

    # Optimization.
    cross_entropy = -tf.reduce_sum(y_*tf.log(y))
    train_step = tf.train.GradientDescentOptimizer(0.03).minimize(cross_entropy)

    # Evaluation.
    correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))

    tf.summary.scalar('accurarcy', accuracy)

    #summary
    summary_op = tf.summary.merge_all()

    # Create a local session to run this computation.
    with tf.Session() as sess:

        # Run all the initializers to prepare the trainable parameters.
        tf.global_variables_initializer().run()

        writer = tf.train.SummaryWriter('./logs', sess.graph)

        # Iterate and train.
        for step in xrange(num_epochs * data_size // BATCH_SIZE):
            if verbose:
                print step,

            offset = (step * BATCH_SIZE) % data_size
{'fraction_non_alphanumeric': 0.11430219146482122, 'fraction_numerical': 0.008073817762399077, 'mean_word_length': 3.7800441014332966, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3296440', 'n_tokens_mistral': 2808, 'n_tokens_neox': 2689, 'n_words': 746}