<commit_before>import datasets
import ets
import models
import tensorflow as tf


# Meta-meta-parameters and debugging knobs.
training_epochs = 10000
half_life = 1500
display_step = 100
gradient_clip = 2.0
hot_ticket = 1
cost_growth = 1.5
base_learn = 0.01
peak_param_cost = 10.0


# Meta-parameters for the optimization, themselves governed by the meta-meta-parameters above.
global_step = tf.Variable(0)
learning_rate = tf.minimum(
    tf.train.exponential_decay(base_learn * 2, global_step, half_life * hot_ticket, 0.5),
    tf.train.exponential_decay(base_learn * (2**hot_ticket), global_step, half_life, 0.5))
cost_weight = tf.train.exponential_decay(peak_param_cost / (cost_growth**hot_ticket), global_step, half_life, cost_growth)


# Set up the input data.
ys = datasets.ukcars
data = tf.placeholder('float', ys.shape)


# And then build the model.
state0, varz, cost = models.QuarterlySeasonal(cost_weight)(data)


# Gradient descent, with decaying learning rate, and with gradient clipping.
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost, global_step=global_step)
optimize_init0 = tf.train.AdamOptimizer(1.0).minimize(cost, global_step=global_step, var_list=[state0])


# Dump the state to the screen.
# TODO: does repeatedly calling sess.run(...) perturb the state?
def PrintDiagnostics(sess, ys, title):
    c = sess.run(cost, feed_dict={data: ys})
    print title, \
        "cost=", "{:.9f}".format(c), \
        "state0=", sess.run(tf.reshape(state0, [5])), \
        "varz=", dict((k, sess.run(v)) for k, v in varz.iteritems())


# Initializing the variables
init = tf.initialize_all_variables()


# Launch the graph
with tf.Session() as sess:
    sess.run(init)
    PrintDiagnostics(sess, ys, 'Init')

    for epoch in range(1000):
        sess.run(optimize_init0, feed_dict={data: ys})
        if epoch % display_step == 0:
            PrintDiagnostics(sess, ys, 'Init0 training: %04d' % epoch)

    PrintDiagnostics(sess, ys, 'Initial state')

    # Fit the data.    
    for epoch in range(training_epochs):
        sess.run(optimizer, feed_dict={data: ys})

        # Display logs per epoch step
        if epoch % display_step == 0:
            PrintDiagnostics(sess, ys, 'Epoch: %04d' % epoch)

    print "Optimization Finished!"
    sess.run(optimizer, feed_dict={data: ys})
    PrintDiagnostics(sess, ys, 'Training')
<commit_msg>small meta-parameter tweaks for `ukcars` fit<commit_after>import datasets
import ets
import models
import tensorflow as tf


# Meta-meta-parameters and debugging knobs.
training_epochs = 10000
half_life = 1000
display_step = 100
gradient_clip = 2.0
hot_ticket = 1
cost_growth = 1.5
base_learn = 0.01
peak_param_cost = 70000.0


# Meta-parameters for the optimization, themselves governed by the meta-meta-parameters above.
global_step = tf.Variable(0)
learning_rate = tf.minimum(
    tf.train.exponential_decay(base_learn * 2, global_step, half_life * hot_ticket, 0.5),
    tf.train.exponential_decay(base_learn * (2**hot_ticket), global_step, half_life, 0.5))
cost_weight = tf.train.exponential_decay(peak_param_cost / (cost_growth**hot_ticket), global_step, half_life, cost_growth)


# Set up the input data.
ys = datasets.ukcars
data = tf.placeholder('float', ys.shape)


# And then build the model.
state0, varz, cost = models.QuarterlySeasonal(cost_weight)(data)


# Gradient descent, with decaying learning rate, and with gradient clipping.
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost, global_step=global_step)
optimize_init0 = tf.train.AdamOptimizer(1.0).minimize(cost, global_step=global_step, var_list=[state0])


# Dump the state to the screen.
# TODO: does repeatedly calling sess.run(...) perturb the state?
def PrintDiagnostics(sess, ys, title):
    c = sess.run(cost, feed_dict={data: ys})
    print title, \
        "cost=", "{:.9f}".format(c), \
        "state0=", sess.run(tf.reshape(state0, [5])), \
        "varz=", dict((k, sess.run(v)) for k, v in varz.iteritems())


# Initializing the variables
init = tf.initialize_all_variables()


# Launch the graph
with tf.Session() as sess:
    sess.run(init)
    PrintDiagnostics(sess, ys, 'Init')

    for epoch in range(1000):
        sess.run(optimize_init0, feed_dict={data: ys})
        if epoch % display_step == 0:
            PrintDiagnostics(sess, ys, 'Init0 training: %04d' % epoch)

    PrintDiagnostics(sess, ys, 'Initial state')

    # Fit the data.    
    for epoch in range(training_epochs):
        sess.run(optimizer, feed_dict={data: ys})

        # Display logs per epoch step
        if epoch % display_step == 0:
            PrintDiagnostics(sess, ys, 'Epoch: %04d' % epoch)

    print "Optimization Finished!"
    sess.run(optimizer, feed_dict={data: ys})
    PrintDiagnostics(sess, ys, 'Training')

{'fraction_non_alphanumeric': 0.12081239530988275, 'fraction_numerical': 0.02198492462311558, 'mean_word_length': 4.055026455026455, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '457165', 'n_tokens_mistral': 1738, 'n_tokens_neox': 1608, 'n_words': 413}