<commit_before>'''
To view tensorboard train and validation loss
In the 1st terminal, run the command line:
  --> tensorboard --logdir=./train_logs
  Then open http://0.0.0.0:6006/ into your web browser
In the 2nd terminal, run the command line:
  --> tensorboard --logdir=./val_logs --port=8008"
  Then open http://0.0.0.0:8008/ into your web browser
'''
import os
import os.path
import tensorflow as tf
import datagen
import model
import numpy as np

LOGDIR = './save'
CKPT_FILE = './save/model.ckpt'
TRAIN_TENSORBOARD_LOG = './train_logs'
VAL_TENSORBOARD_LOG = './val_logs'

sess = tf.InteractiveSession()

loss = tf.reduce_mean(tf.square(tf.sub(model.y_, model.y)))
train_step = tf.train.AdamOptimizer(1e-4).minimize(loss)

saver = tf.train.Saver()

train_summary = tf.scalar_summary("train_loss", loss)
val_summary = tf.scalar_summary("val_loss", loss)

if not os.path.exists(LOGDIR):
  os.makedirs(LOGDIR)

if os.path.isfile(CKPT_FILE):
  saver.restore(sess, CKPT_FILE)
else:
  sess.run(tf.initialize_all_variables())

if not os.path.exists(TRAIN_TENSORBOARD_LOG):
  os.makedirs(TRAIN_TENSORBOARD_LOG)
if not os.path.exists(VAL_TENSORBOARD_LOG):
  os.makedirs(VAL_TENSORBOARD_LOG)

train_summary_writer = tf.train.SummaryWriter(TRAIN_TENSORBOARD_LOG, graph=tf.get_default_graph())
val_summary_writer = tf.train.SummaryWriter(VAL_TENSORBOARD_LOG, graph=tf.get_default_graph())

batch_size = 100

for i in range(int(datagen.num_images * 3)):
  xs_train, ys_train = datagen.LoadTrainBatch(batch_size)
  train_step.run(feed_dict={model.x: xs_train, model.y_: ys_train, model.keep_prob: 0.8})

  if i % 10 == 0:
    xs_val, ys_val = datagen.LoadValBatch(batch_size)
    # Write logs at every iteration
    train_loss = train_summary.eval(feed_dict={model.x:xs_train, model.y_: ys_train, model.keep_prob: 1.0})
    val_loss = val_summary.eval(feed_dict={model.x:xs_val, model.y_: ys_val, model.keep_prob: 1.0})
    train_summary_writer.add_summary(train_loss, i)
    val_summary_writer.add_summary(val_loss, i)
    train_loss = loss.eval(feed_dict={model.x:xs_train, model.y_: ys_train, model.keep_prob: 1.0})
    val_loss = loss.eval(feed_dict={model.x:xs_val, model.y_: ys_val, model.keep_prob: 1.0})
    print("step: %d, loss: %g, val loss: %g" % (i, train_loss, val_loss))

  if i % 100 == 0:
    checkpoint_path = os.path.join(LOGDIR, "model.ckpt")
    filename = saver.save(sess, checkpoint_path)
print("Model saved in file: %s" % filename)<commit_msg>increased dropout rate<commit_after>'''
To view tensorboard train and validation loss
In the 1st terminal, run the command line:
  --> tensorboard --logdir=./train_logs
  Then open http://0.0.0.0:6006/ into your web browser
In the 2nd terminal, run the command line:
  --> tensorboard --logdir=./val_logs --port=8008"
  Then open http://0.0.0.0:8008/ into your web browser
'''
import os
import os.path
import tensorflow as tf
import datagen
import model
import numpy as np

LOGDIR = './save'
CKPT_FILE = './save/model.ckpt'
TRAIN_TENSORBOARD_LOG = './train_logs'
VAL_TENSORBOARD_LOG = './val_logs'

sess = tf.InteractiveSession()

loss = tf.reduce_mean(tf.square(tf.sub(model.y_, model.y)))
train_step = tf.train.AdamOptimizer(1e-4).minimize(loss)

saver = tf.train.Saver()

train_summary = tf.scalar_summary("train_loss", loss)
val_summary = tf.scalar_summary("val_loss", loss)

if not os.path.exists(LOGDIR):
  os.makedirs(LOGDIR)

if os.path.isfile(CKPT_FILE):
  saver.restore(sess, CKPT_FILE)
else:
  sess.run(tf.initialize_all_variables())

if not os.path.exists(TRAIN_TENSORBOARD_LOG):
  os.makedirs(TRAIN_TENSORBOARD_LOG)
if not os.path.exists(VAL_TENSORBOARD_LOG):
  os.makedirs(VAL_TENSORBOARD_LOG)

train_summary_writer = tf.train.SummaryWriter(TRAIN_TENSORBOARD_LOG, graph=tf.get_default_graph())
val_summary_writer = tf.train.SummaryWriter(VAL_TENSORBOARD_LOG, graph=tf.get_default_graph())

batch_size = 100

for i in range(int(datagen.num_images * 3)):
  xs_train, ys_train = datagen.LoadTrainBatch(batch_size)
  train_step.run(feed_dict={model.x:xs_train, model.y_:ys_train, model.keep_prob:0.5})

  if i % 10 == 0:
    xs_val, ys_val = datagen.LoadValBatch(batch_size)
    # Write logs at every iteration
    train_loss = train_summary.eval(feed_dict={model.x:xs_train, model.y_:ys_train, model.keep_prob:1.0})
    val_loss = val_summary.eval(feed_dict={model.x:xs_val, model.y_:ys_val, model.keep_prob:1.0})
    train_summary_writer.add_summary(train_loss, i)
    val_summary_writer.add_summary(val_loss, i)
    train_loss = loss.eval(feed_dict={model.x:xs_train, model.y_:ys_train, model.keep_prob:1.0})
    val_loss = loss.eval(feed_dict={model.x:xs_val, model.y_:ys_val, model.keep_prob:1.0})
    print("step: %d, loss: %g, val loss: %g" % (i, train_loss, val_loss))

  if i % 100 == 0:
    checkpoint_path = os.path.join(LOGDIR, "model.ckpt")
    filename = saver.save(sess, checkpoint_path)
print("Model saved in file: %s" % filename)
{'fraction_non_alphanumeric': 0.1339031339031339, 'fraction_numerical': 0.018315018315018316, 'mean_word_length': 5.098014888337469, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6995912', 'n_tokens_mistral': 1969, 'n_tokens_neox': 1894, 'n_words': 366}