<jupyter_start><jupyter_text>SIMPLE CHAR-RNN<jupyter_code>from __future__ import print_function
import tensorflow as tf
import numpy as np
from tensorflow.contrib import rnn
tf.set_random_seed(0)  
print ("TENSORFLOW VERSION IS %s" % (tf.__version__))<jupyter_output>TENSORFLOW VERSION IS 1.0.1<jupyter_text>DEFINE TRAINING SEQUENCE<jupyter_code>sentence = ("if you want to build a ship, don't drum up people together to "
            "collect wood and don't assign them tasks and work, but rather "
            "teach them to long for the endless immensity of the sea.")
print ("FOLLOWING IS OUR TRAINING SEQUENCE:")
print (sentence)<jupyter_output>FOLLOWING IS OUR TRAINING SEQUENCE:
if you want to build a ship, don't drum up people together to collect wood and don't assign them tasks and work, but rather teach them to long for the endless immensity of the sea.<jupyter_text>DEFINE VOCABULARY AND DICTIONARY<jupyter_code>char_set = list(set(sentence))
char_dic = {w: i for i, w in enumerate(char_set)}
print ("VOCABULARY: ")
print (char_set)
print ("DICTIONARY: ")
print (char_dic)<jupyter_output>VOCABULARY: 
[' ', "'", ',', '.', 'a', 'c', 'b', 'e', 'd', 'g', 'f', 'i', 'h', 'k', 'm', 'l', 'o', 'n', 'p', 's', 'r', 'u', 't', 'w', 'y']
DICTIONARY: 
{' ': 0, "'": 1, ',': 2, '.': 3, 'a': 4, 'c': 5, 'b': 6, 'e': 7, 'd': 8, 'g': 9, 'f': 10, 'i': 11, 'h': 12, 'k': 13, 'm': 14, 'l': 15, 'o': 16, 'n': 17, 'p': 18, 's': 19, 'r': 20, 'u': 21, 't': 22, 'w': 23, 'y': 24}<jupyter_text>CONFIGURE NETWORK<jupyter_code>data_dim    = len(char_set)
num_classes = len(char_set)
hidden_size     = 64
sequence_length = 10  # Any arbitrary number<jupyter_output><empty_output><jupyter_text>SET TRAINING BATCHES<jupyter_code>dataX = []
dataY = []
for i in range(0, len(sentence) - sequence_length):
    x_str = sentence[i:i + sequence_length]
    y_str = sentence[i + 1: i + sequence_length + 1]
    x = [char_dic[c] for c in x_str]  # x str to index
    y = [char_dic[c] for c in y_str]  # y str to index
    dataX.append(x)
    dataY.append(y)
    print ("[%3d/%3d] [%s]=>[%s]" % (i, len(sentence), x_str, y_str))
    print ("%s%s=>%s" % (' '*10, x, y))
ndata      = len(dataX)
batch_size = 20
print ("     'NDATA' IS %d" % (ndata))
print ("'BATCH_SIZE' IS %d" % (batch_size))<jupyter_output>'NDATA' IS 170
'BATCH_SIZE' IS 20<jupyter_text>DEFINE PLACEHOLDERS<jupyter_code>X = tf.placeholder(tf.int32, [None, sequence_length])
Y = tf.placeholder(tf.int32, [None, sequence_length])
X_OH = tf.one_hot(X, num_classes)
print ("'sequence_length' IS [%d]" % (sequence_length))
print ("    'num_classes' IS [%d]" % (num_classes))
print("'X' LOOKS LIKE \n   [%s]" % (X))  
print("'X_OH' LOOKS LIKE \n   [%s]" % (X_OH))<jupyter_output>'sequence_length' IS [10]
    'num_classes' IS [25]
'X' LOOKS LIKE 
   [Tensor("Placeholder:0", shape=(?, 10), dtype=int32)]
'X_OH' LOOKS LIKE 
   [Tensor("one_hot:0", shape=(?, 10, 25), dtype=float32)]<jupyter_text>DEFINE MODEL<jupyter_code>with tf.variable_scope('CHAR-RNN', reuse=False):
    cell = rnn.BasicLSTMCell(hidden_size, state_is_tuple=True)
    cell = rnn.MultiRNNCell([cell] * 2, state_is_tuple=True)
    # DYNAMIC RNN WITH FULLY CONNECTED LAYER
    _outputs, _states = tf.nn.dynamic_rnn(cell, X_OH, dtype=tf.float32)
    _outputs  = tf.contrib.layers.fully_connected(_outputs, num_classes, activation_fn=None)
    # RESHAPE FOR SEQUNCE LOSS
    outputs = tf.reshape(_outputs, [batch_size, sequence_length, num_classes])
print ("OUTPUTS LOOKS LIKE [%s]" % (outputs))
print ("MODEL DEFINED.")<jupyter_output>OUTPUTS LOOKS LIKE [Tensor("CHAR-RNN/Reshape:0", shape=(20, 10, 25), dtype=float32)]
MODEL DEFINED.<jupyter_text>DEFINE TF FUNCTIONS<jupyter_code># EQUAL WEIGHTS
weights = tf.ones([batch_size, sequence_length])
seq_loss = tf.contrib.seq2seq.sequence_loss(
    logits=outputs, targets=Y, weights=weights)
loss = tf.reduce_mean(seq_loss)
optm  = tf.train.AdamOptimizer(learning_rate=0.1).minimize(loss)
print ("FUNCTIONS DEFINED.")<jupyter_output>FUNCTIONS DEFINED.<jupyter_text>OPTIMIZE<jupyter_code>sess = tf.Session()
sess.run(tf.global_variables_initializer())
for i in range(500):
    randidx = np.random.randint(low=0, high=ndata, size=batch_size)
    batchX = [dataX[iii] for iii in randidx]
    batchY = [dataY[iii] for iii in randidx]
    feeds = {X: batchX, Y: batchY}
    _, loss_val, results = sess.run(
        [optm, loss, outputs], feed_dict=feeds)
    if (i%100) == 0:
        for j, result in enumerate(results):
            index = np.argmax(result, axis=1)
            print(i, j, ''.join([char_set[t] for t in index]), loss_val)<jupyter_output>0 0 yyyyyyluuu 3.21928
0 1 uuu''ylluu 3.21928
0 2 ukukkkklkk 3.21928
0 3 uuuuuuuuuu 3.21928
0 4 ktuttlllll 3.21928
0 5 ukukkkklkk 3.21928
0 6 yygguggggu 3.21928
0 7 nuggggsuuu 3.21928
0 8 uuuuu'uuu' 3.21928
0 9 dddd'''''' 3.21928
0 10 uluuuuh''' 3.21928
0 11 ukkkllllll 3.21928
0 12 sshuuuuuuu 3.21928
0 13 uuulukkssu 3.21928
0 14 duuuu'uuuu 3.21928
0 15 ywwsllkell 3.21928
0 16 d..huuuuuu 3.21928
0 17 uuuuuuhuuu 3.21928
0 18  ll''''''' 3.21928
0 19 bblllllluu 3.21928
100 0  ether te  0.368646
100 1 tather tea 0.368646
100 2 gity of th 0.368646
100 3 tnd won't  0.368646
100 4 thip, don' 0.368646
100 5 hem teach  0.368646
100 6 im to coll 0.368646
100 7 hem te col 0.368646
100 8 hach them  0.368646
100 9 g to build 0.368646
100 10 t wood and 0.368646
100 11 or themend 0.368646
100 12  ether te  0.368646
100 13 hto cuild  0.368646
100 14  uosks and 0.368646
100 15 maogether  0.368646
100 16 t them to  0.368646
100 17  ople toge 0.368646
100 18 cd won't a 0.368646
100 19 hem teach  0.368646
[...]<jupyter_text>SAMPLING FUNCTION<jupyter_code>LEN = 1;
# XL = tf.placeholder(tf.int32, [None, LEN])
XL     = tf.placeholder(tf.int32, [None, 1])
XL_OH  = tf.one_hot(XL, num_classes)
with tf.variable_scope('CHAR-RNN', reuse=True):
    cell_L = rnn.BasicLSTMCell(hidden_size, state_is_tuple=True)
    cell_L = rnn.MultiRNNCell([cell_L] * 2, state_is_tuple=True)
    istate = cell_L.zero_state(batch_size=1, dtype=tf.float32)
    # DYNAMIC RNN WITH FULLY CONNECTED LAYER
    _outputs_L, states_L = tf.nn.dynamic_rnn(cell_L, XL_OH
                                , initial_state=istate, dtype=tf.float32)
    _outputs_L  = tf.contrib.layers.fully_connected(
        _outputs_L, num_classes, activation_fn=None)
    # RESHAPE FOR SEQUNCE LOSS
    outputs_L = tf.reshape(_outputs_L, [LEN, 1, num_classes])
print (XL)<jupyter_output>Tensor("Placeholder_6:0", shape=(?, 1), dtype=int32)<jupyter_text>SAMPLE<jupyter_code># BURNIN
prime = "if you "
istateval = sess.run(cell_L.zero_state(1, tf.float32))
for c in prime[:-1]:
    index = char_dic[c]
    inval = [[index]]
    outval, stateval = sess.run([outputs_L, states_L]
                        , feed_dict={XL:inval, istate:istateval})
    istateval = stateval
# SAMPLE
inval  = [[char_dic[prime[-1]]]]
outval, stateval = sess.run([outputs_L, states_L]
                    , feed_dict={XL:inval, istate:istateval})
index = np.argmax(outval)
char  = char_set[index]
chars = ''
for i in range(500):
    inval = [[index]]
    outval, stateval = sess.run([outputs_L, states_L]
                        , feed_dict={XL:inval, istate:istateval})
    istateval = stateval
    index = np.argmax(outval)
    char  = char_set[index]
    chars += char

print ("SAMPLED SETENCE: \n %s" % (prime+chars))
print ("\nORIGINAL SENTENCE: \n %s" % (sentence))<jupyter_output>SAMPLED SETENCE: 
 if you nd work, but rather teather teather teach them to long for the endless immensity of the secple together tollect wood and don't assign them to long for the endless immensity of the secple together tollect wood and don't assign them to long for the endless immensity of the secple together tollect wood and don't assign them to long for the endless immensity of the secple together tollect wood and don't assign them to long for the endless immensity of the secple together tollect wood and don't assig

ORIGINAL SENTENCE: 
 if you want to build a ship, don't drum up people together to collect wood and don't assign them tasks and work, but rather teach them to long for the endless immensity of the sea.
{'fraction_non_alphanumeric': 0.13430082400688723, 'fraction_numerical': 0.06542860656745787, 'mean_word_length': 4.236316806181584, 'pattern_counts': {'":': 1, '<': 35, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '355917', 'license': 'MIT', 'n_tokens_mistral': 3630, 'n_tokens_neox': 3152, 'n_words': 898, 'path': 'tensorflow/advanced-tensorflow/char-rnn/char-rnn-simple.ipynb', 'repo_name': 'gopala-kr/ds-notebooks', 'size': '8131'}