<commit_before>""" Train
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tqdm

import tensorflow as tf

from utils import start_threads
from data_loader import DATA_PATH, queue_context
from model import Model

flags = tf.flags
logging = tf.logging

logging.set_verbosity(logging.INFO)

flags.DEFINE_string("corpus_name", "ptb", (
    "Corpus name."))
flags.DEFINE_integer("batch_size", 32, (
    "Batch size for dequeue."))
flags.DEFINE_integer("embedding_dim", 128, (
    "Hidden dimensions for embedding."))
flags.DEFINE_integer("rnn_hidden_dim", 128, (
    "Hidden dimensions for RNN hidden vectors."))

FLAGS = flags.FLAGS


if __name__ == "__main__":
    path = DATA_PATH[FLAGS.corpus_name]["train"]
    model = Model(path)

    with tf.Session() as sess:
        sess.run(tf.local_variables_initializer())
        sess.run(tf.global_variables_initializer())

        start_threads(model.enqueue_data, (sess, ))

        with queue_context(sess):
            epoch_size = 10000
            for _ in tqdm.tqdm(range(epoch_size)):
                sess.run(model.train_op)<commit_msg>train works for now for generator<commit_after>""" Train
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tqdm

import tensorflow as tf

from utils import start_threads
from data_loader import DATA_PATH, queue_context
from model import Model

flags = tf.flags
logging = tf.logging

logging.set_verbosity(logging.INFO)

flags.DEFINE_string("corpus_name", "ptb", (
    "Corpus name."))
flags.DEFINE_integer("batch_size", 32, (
    "Batch size for dequeue."))
flags.DEFINE_integer("embedding_dim", 128, (
    "Hidden dimensions for embedding."))
flags.DEFINE_integer("rnn_hidden_dim", 128, (
    "Hidden dimensions for RNN hidden vectors."))

FLAGS = flags.FLAGS


def set_train_op(loss, **opts):
    cost = tf.reduce_mean(loss)

    optim = tf.train.AdamOptimizer(learning_rate=0.005)
    train_op = optim.minimize(cost)
    return train_op


if __name__ == "__main__":
    path = DATA_PATH[FLAGS.corpus_name]["train"]
    model = Model(path)

    loss = model.g_tensors.loss
    train_op = set_train_op(loss)

    with tf.Session() as sess:
        sess.run(tf.local_variables_initializer())
        sess.run(tf.global_variables_initializer())

        start_threads(model.enqueue_data, (sess, ))

        with queue_context(sess):
            epoch_size = 10000
            for _ in tqdm.tqdm(range(epoch_size)):
                sess.run(train_op)
{'fraction_non_alphanumeric': 0.10550814584949574, 'fraction_numerical': 0.011636927851047323, 'mean_word_length': 3.8660377358490567, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5571283', 'n_tokens_mistral': 865, 'n_tokens_neox': 818, 'n_words': 193}