<jupyter_start><jupyter_text>Imports<jupyter_code>import tensorflow as tf
import tensorflow_hub as hub
import numpy as np
import re
import random<jupyter_output><empty_output><jupyter_text>Model  v1 - Current* One layer of LSTM* Non-batch train  Dataset Initialization<jupyter_code># Dataset
embed = hub.Module("https://tfhub.dev/google/nnlm-en-dim50/1")
string2idtable = tf.contrib.lookup.index_table_from_file(vocabulary_file="vocabulary-shakespeare.txt", num_oov_buckets=0)
id2stringtabel = tf.contrib.lookup.index_to_string_table_from_file(vocabulary_file="vocabulary-shakespeare.txt")
def _insertSpace(sentence):
    sentence = sentence.decode()
    sentence = sentence.lower()
    sentence = re.sub(r'([\W\d])', r' \1 ', sentence)
    return sentence
def _getLabel(sentence):
    splited = tf.string_split([sentence]).values
    sentence = splited
    sentence = sentence[0:-1]
    ids = string2idtable.lookup(splited)
    ids = tf.one_hot(ids, 11405)
    return sentence, ids
filenames = ["poems/shakespeare/sonnets.txt"]
dataset = tf.data.TextLineDataset(filenames)
dataset = dataset.map(lambda sentence: tf.py_func(_insertSpace, [sentence], tf.string))
dataset = dataset.map(_getLabel)
dataset = dataset.shuffle(buffer_size=10000)
# dataset = dataset.batch(4)
dataset = dataset.repeat()
iterator = dataset.make_initializable_iterator()
next_poem = iterator.get_next()<jupyter_output><empty_output><jupyter_text>Model initialization and Graph building<jupyter_code># with tf.Graph().as_default():
# Graph
# Variables
softmax_w = tf.get_variable("softmax_w", shape=[400, 11405], initializer=tf.random_normal_initializer)
softmax_b = tf.get_variable("softmax_b", shape=[1, 11405], initializer=tf.random_normal_initializer)

# TODO Use tuple for state
lstm = tf.contrib.rnn.LSTMCell(400, state_is_tuple=False, initializer=tf.random_normal_initializer, reuse=tf.AUTO_REUSE, name="LSTM1")

# Model
sentence, label = next_poem
sentence = embed(sentence)
sentence = tf.concat([tf.zeros([1, 50]), sentence], 0, name="Insert_X_0")

state = tf.placeholder(shape=[1, lstm.state_size], dtype=tf.float32, name="Previous_State")

# Input and Label
x = tf.placeholder(shape=[1, 50], dtype=tf.float32, name="Input_Word")
y = tf.placeholder(shape=[1, 11405], dtype=tf.int32, name="Target_Word")

input_word = tf.reshape(x, [1, 50])
output, out_state = lstm(input_word, state)
logits = tf.add(tf.matmul(output, softmax_w), softmax_b)
possibility = tf.nn.softmax(logits=logits, name="Possibilities")
word_index = tf.argmax(possibility, axis=-1, name="Predict_Word_index")
word = id2stringtabel.lookup(word_index)
loss_op = tf.losses.softmax_cross_entropy(onehot_labels=y, logits=logits)
tf.summary.scalar(name="loss", tensor=loss_op)
merged = tf.summary.merge_all()
# loss_op = tf.nn.softmax_cross_entropy_with_logits_v2(labels=y, logits=logits, name="Loss")

#total_loss = tf.get_variable(shape=[], dtype=tf.float32, name="Total_Cost", initializer=tf.zeros_initializer)

# Train Ops
optimizer = tf.train.AdamOptimizer()
train_op = optimizer.minimize(loss_op)

# Global step as variable for persistent tensorboard output
global_step = tf.get_variable("Global_Steps", [], initializer=tf.ones_initializer, dtype=tf.int64)

# Variables saver
saver = tf.train.Saver()

sess = tf.Session()
# with tf.Session() as sess:
try:
    saver.restore(sess, "tmp/model/model.ckpt")
except tf.errors.NotFoundError:
    pass
sess.run(tf.global_variables_initializer())
tf.tables_initializer().run(session=sess)
sess.run(iterator.initializer)

# Trainning log
# TODO writer = tf.summary.FileWriter("tmp/log/"+Modelv1+embedding_version, sess.graph)
writer = tf.summary.FileWriter("tmp/log/", sess.graph)<jupyter_output><empty_output><jupyter_text>Train Module<jupyter_code>total_poems = 10000
for _ in range(total_poems):
    wordsVec, targets = sess.run([sentence, label])
    # TODO How to Access timestep?
    # Initial
    _, pre_state = sess.run([output, out_state], feed_dict={x: [wordsVec[0]], y: [targets[0]], state: np.zeros([1, lstm.state_size], dtype=float)})
    for i in range(1, wordsVec.shape[0]):
        # total_loss += loss_op
        # print(wordsVec[i].shape)
        g_steps = global_step.eval(session=sess)
        _, pre_state, loss, summary, out = sess.run([train_op, out_state, loss_op, merged, word], feed_dict={x: [wordsVec[i]], y: [targets[i]], state: pre_state})
        writer.add_summary(summary, global_step=g_steps)
        # TODO Should Average loss
        tf.logging.log_every_n(tf.logging.INFO, "Loss: %s | local step: %s | global step: %s | Output: %s", 100, loss, i, g_steps, out)
        if g_steps%500 == 0:
            # Store Variables Every 10000 times
            saver.save(sess, "tmp/model/model.ckpt")
        global_step += 1                             
print("Trainning poems: ", total_poems)
saver.save(sess, "tmp/model/model.ckpt")<jupyter_output><empty_output><jupyter_text>Predict Module<jupyter_code>def pretty(input_str):
    return re.sub("xxxnewlinexxx", "\n", input_str)
random_word = id2stringtabel.lookup(tf.constant([random.randint(0, 11404)], dtype=tf.int64))
word_predict, prev_state = sess.run([random_word, out_state], feed_dict={x: np.zeros([1, 50]), state: np.zeros([1, lstm.state_size], dtype=float)})
print(word_predict)
poem = [word_predict[0].decode()]
while word_predict[0].decode() != "xxxendxxx":
    word_predict = sess.run(embed(word_predict))
    word_predict, prev_state = sess.run([word, out_state], feed_dict={x:word_predict, state: prev_state})
    poem.append(word_predict[0].decode())
    print(pretty(" ".join(poem)))
# Dump to generated dir
generated_poem = pretty(" ".join(poem))
filename = "-".join(poem[:5])
with open("generated/"+filename+".txt", "w") as f:
    f.write(generated_poem)
# Write graph to file
writer.close()<jupyter_output><empty_output><jupyter_text>Save Model<jupyter_code># TODO<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14370647188720911, 'fraction_numerical': 0.019024970273483946, 'mean_word_length': 5.822711471610661, 'pattern_counts': {'":': 1, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '178607', 'license': 'Apache-2.0', 'n_tokens_mistral': 2171, 'n_tokens_neox': 2073, 'n_words': 378, 'path': 'main.ipynb', 'repo_name': 'William-An/Machine-Poet', 'size': '5887'}