<jupyter_start><jupyter_text>TRAIN HANGUL-RNN<jupyter_code># -*- coding: utf-8 -*-
# Import Packages
import numpy as np
import tensorflow as tf
import collections
import argparse
import time
import os
from six.moves import cPickle
from TextLoader import *
from Hangulpy import *
print ("Packages Imported")<jupyter_output>Packages Imported<jupyter_text>LOAD DATASET WITH TEXTLOADER<jupyter_code>corpus_name = "invisible_dragon" # "nine_dreams"

data_dir    = "data/" + corpus_name
batch_size  = 10
seq_length  = 100
data_loader = TextLoader(data_dir, batch_size, seq_length)<jupyter_output>reading text file<jupyter_text>VOCAB AND CHARS<jupyter_code>vocab_size = data_loader.vocab_size
vocab = data_loader.vocab
chars = data_loader.chars
print ( "type of 'data_loader.vocab' is %s, length is %d" 
       % (type(data_loader.vocab), len(data_loader.vocab)) )
print ( "type of 'data_loader.chars' is %s, length is %d" 
       % (type(data_loader.chars), len(data_loader.chars)) )<jupyter_output>type of 'data_loader.vocab' is <type 'dict'>, length is 84
type of 'data_loader.chars' is <type 'tuple'>, length is 84<jupyter_text>VOCAB: DICTIONARY (CHAR->INDEX)<jupyter_code>print (data_loader.vocab)<jupyter_output>{u'_': 81, u'6': 63, u'|': 83, u'\n': 6, u'\r': 7, u',': 74, u'x': 75, u';': 73, u'[': 71, u'\u3144': 60, u'!': 28, u' ': 2, u'#': 68, u'"': 34, u'\u1d25': 0, u"'": 65, u')': 50, u'(': 51, u'+': 76, u'*': 82, u']': 72, u'\u3133': 58, u'/': 45, u'.': 24, u'\u3131': 5, u'0': 27, u'3': 54, u'2': 36, u'5': 61, u'\u3134': 4, u'\u3137': 11, u'\u3136': 49, u'\u3139': 8, u'\u3138': 31, u'\u3156': 52, u':': 40, u'\u313c': 67, u'?': 44, u'4': 59, u'\u3141': 14, u'\u3140': 77, u'\u3143': 57, u'\u3142': 21, u'\u3145': 15, u'7': 47, u'\u3147': 1, u'\u3146': 22, u'\u3149': 38, u'\u3148': 16, u'\u314b': 26, u'\u314a': 30, u'\u314d': 33, u'\u314c': 25, u'\u314f': 3, u'\u314e': 19, u'\u3151': 32, u'\u3150': 18, u'\u3153': 13, u'\u3152': 69, u'\u3155': 20, u'\u3154': 23, u'\u3157': 12, u'8': 46, u'\u3159': 62, u'\u3158': 41, u'\u315b': 29, u'\u315a': 53, u'\u315d': 48, u'\u315c': 17, u'\u315f': 39, u'^': 64, u'\u3161': 10, u'\u3160': 55, u'\u3163': 9, u'\u3162': 43, u'k': 78, u'9': 35, u'\u313a': 70, u'[...]<jupyter_text>CHARS: LIST (INDEX->CHAR)<jupyter_code>print (data_loader.chars)
# USAGE
print (data_loader.chars[0])<jupyter_output>(u'\u1d25', u'\u3147', u' ', u'\u314f', u'\u3134', u'\u3131', u'\n', u'\r', u'\u3139', u'\u3163', u'\u3161', u'\u3137', u'\u3157', u'\u3153', u'\u3141', u'\u3145', u'\u3148', u'\u315c', u'\u3150', u'\u314e', u'\u3155', u'\u3142', u'\u3146', u'\u3154', u'.', u'\u314c', u'\u314b', u'0', u'!', u'\u315b', u'\u314a', u'\u3138', u'\u3151', u'\u314d', u'"', u'9', u'2', u'~', u'\u3149', u'\u315f', u':', u'\u3158', u'\u3132', u'\u3162', u'?', u'/', u'8', u'7', u'\u315d', u'\u3136', u')', u'(', u'\u3156', u'\u315a', u'3', u'\u3160', u'1', u'\u3143', u'\u3133', u'4', u'\u3144', u'5', u'\u3159', u'6', u'^', u"'", u'%', u'\u313c', u'#', u'\u3152', u'\u313a', u'[', u']', u';', u',', u'x', u'+', u'\u3140', u'k', u'<', u'}', u'_', u'*', u'|')
ᴥ<jupyter_text>TRAINING BATCH (IMPORTANT!!)<jupyter_code>x, y = data_loader.next_batch()
print ("Type of 'x' is %s. Shape is %s" % (type(x), x.shape,))
print ("x looks like \n%s" % (x))
print
print ("Type of 'y' is %s. Shape is %s" % (type(y), y.shape,))
print ("y looks like \n%s" % (y))<jupyter_output>Type of 'x' is <type 'numpy.ndarray'>. Shape is (10, 100)
x looks like 
[[ 7  6  1 20  0  8 13  0 21 10  4  0  2 16 18  0  5  3  0  2 11 10  0 11
   9  0  1 13  0  2  5 10  8  0  1 10  8  0  2 22 10 14  0  4  9  0 11  3
   0 37  7  6 14  3  4  0  1  9  0  2 21 41  0 16 17  0 15 18  0  1 29  0
  37  7  6  4 18  0  1 29  1  0  1 10  4  0  2 30 53  0  5  3  1  0  1 43
   0  2 11 10]
 [23  0 34  7  6  7  6  2  7  6  7  6  5 10  0  8 13  0 16  3  0  2 16 13
   0  5 10  8  0  8  9  1  0  5 41  0 19  9  0 11 10  0  8  3  0 11 10  8
   0  1 10  4  0  2 11 17  0  8 20  0  1 17 14  0  1 23  0 14 12 14  0  1
  10  8  0  2 31 13  8  0  1 13 22  0 11  3  0  7  6  5 10  4  0 11 23  0
   1  3  4  0]
 [ 1 13 21  0 15 13 22  0 11  3  0  7  6 25 17  0 14 20  1  0 19 18  0 15
  13  0  1  3  4  0 21 12  0  1 20  0 15 13  0  2  5 10  0  8 18  0 15 13
   0  2 11 13  0 14 17  0 15 13  0  1 48  0 15 13  0 16 12  8  0  8  3  0
   2 14 12 14  0  1 10  8  0  2 14  3  5  0  2 14 17  0 15 13  0  1 48  0
  15 13  0[...]<jupyter_text>DEFINE A MULTILAYER LSTM NETWORK<jupyter_code>rnn_size   = 128
num_layers = 2
grad_clip  = 5. # <= GRADIENT CLIPPING (PRACTICALLY IMPORTANT)
vocab_size = data_loader.vocab_size

# SELECT RNN CELL (MULTI LAYER LSTM)
def unit_cell():
    return tf.contrib.rnn.BasicLSTMCell(rnn_size,state_is_tuple=True,reuse=tf.get_variable_scope().reuse)
cell = tf.contrib.rnn.MultiRNNCell([unit_cell() for _ in range(num_layers)])

# Set paths to the graph
input_data = tf.placeholder(tf.int32, [batch_size, seq_length])
targets    = tf.placeholder(tf.int32, [batch_size, seq_length])
initial_state = cell.zero_state(batch_size, tf.float32)

# Set Network
with tf.variable_scope('rnnlm'):
    softmax_w = tf.get_variable("softmax_w", [rnn_size, vocab_size])
    softmax_b = tf.get_variable("softmax_b", [vocab_size])
    with tf.device("/cpu:0"):
        embedding = tf.get_variable("embedding", [vocab_size, rnn_size])
        inputs = tf.split(tf.nn.embedding_lookup(embedding, input_data), seq_length, 1)
        inputs = [tf.squeeze(input_, [1]) for input_ in inputs]
print ("Network ready")<jupyter_output>Network ready<jupyter_text>Define functions<jupyter_code># Output of RNN
outputs, last_state = tf.contrib.rnn.static_rnn(cell,inputs, initial_state, 
                                                scope='rnnlm')

output = tf.reshape(tf.concat(outputs,1), [-1, rnn_size])
logits = tf.nn.xw_plus_b(output, softmax_w, softmax_b)

# Next word probability
probs = tf.nn.softmax(logits)
print ("FUNCTIONS READY")<jupyter_output>FUNCTIONS READY<jupyter_text>DEFINE LOSS FUNCTION<jupyter_code>loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example([logits], # Input
    [tf.reshape(targets, [-1])], # Target
    [tf.ones([batch_size * seq_length])], # Weight
    vocab_size)
print ("LOSS FUNCTION")<jupyter_output>LOSS FUNCTION<jupyter_text>DEFINE COST FUNCTION<jupyter_code>cost = tf.reduce_sum(loss) / batch_size / seq_length

# GRADIENT CLIPPING ! 
lr = tf.Variable(0.0, trainable=False) # <= LEARNING RATE 
tvars = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars), grad_clip)
_optm = tf.train.AdamOptimizer(lr)
optm = _optm.apply_gradients(zip(grads, tvars))

final_state = last_state
print ("NETWORK READY")<jupyter_output>NETWORK READY<jupyter_text>OPTIMIZE NETWORK WITH LR SCHEDULING<jupyter_code>num_epochs    = 5000
save_every    = 2000
learning_rate = 0.001
decay_rate    = 0.999

save_dir = 'data/' + corpus_name
sess = tf.InteractiveSession()

sess.run(tf.initialize_all_variables())

summary_writer = tf.summary.FileWriter(save_dir
                    , graph=sess.graph)
saver = tf.train.Saver(tf.all_variables())
for e in range(num_epochs): # for all epochs
    # LEARNING RATE SCHEDULING 
    sess.run(tf.assign(lr, learning_rate * (decay_rate ** e)))

    data_loader.reset_batch_pointer()
    state = sess.run(initial_state)
    for b in range(data_loader.num_batches):
        start = time.time()
        x, y = data_loader.next_batch()
        feed = {input_data: x, targets: y, initial_state: state}
        # Train!
        train_loss, state, _ = sess.run([cost, final_state, optm], feed)
        end = time.time()
        # PRINT 
        if b % 100 == 0:
            print ("%d/%d (epoch: %d), loss: %.3f, time/batch: %.3f"  
                   % (e * data_loader.num_batches + b
                    , num_epochs * data_loader.num_batches
                    , e, train_loss, end - start))
        # SAVE MODEL
        if (e * data_loader.num_batches + b) % save_every == 0:
            checkpoint_path = os.path.join(save_dir, 'model.ckpt')
            saver.save(sess, checkpoint_path
                       , global_step = e * data_loader.num_batches + b)
            print("model saved to {}".format(checkpoint_path))<jupyter_output>WARNING:tensorflow:From <ipython-input-12-36cc09519909>:4: all_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.
Instructions for updating:
Please use tf.global_variables instead.
0/325000 (epoch: 0), loss: 4.437, time/batch: 0.968
model saved to data/invisible_dragon/model.ckpt
65/325000 (epoch: 1), loss: 3.019, time/batch: 0.074
130/325000 (epoch: 2), loss: 2.194, time/batch: 0.073
195/325000 (epoch: 3), loss: 1.977, time/batch: 0.072
260/325000 (epoch: 4), loss: 1.880, time/batch: 0.087
325/325000 (epoch: 5), loss: 1.791, time/batch: 0.073
390/325000 (epoch: 6), loss: 1.725, time/batch: 0.071
455/325000 (epoch: 7), loss: 1.669, time/batch: 0.071
520/325000 (epoch: 8), loss: 1.621, time/batch: 0.072
585/325000 (epoch: 9), loss: 1.576, time/batch: 0.093
650/325000 (epoch: 10), loss: 1.538, time/batch: 0.085
715/325000 (epoch: 11), loss: 1.502, time/batch: 0.081
780/325000 (epoch: 12), loss: 1.469, time/batch: 0.071
845/325000 (epoch: [...]
{'fraction_non_alphanumeric': 0.17271739130434782, 'fraction_numerical': 0.13434782608695653, 'mean_word_length': 3.3814285714285712, 'pattern_counts': {'":': 1, '<': 41, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '88390', 'license': 'MIT', 'n_tokens_mistral': 4740, 'n_tokens_neox': 3937, 'n_words': 1072, 'path': 'Practice6/4_kor_char_rnn_train.ipynb', 'repo_name': 'kiseyno92/SNU_ML', 'size': '9200'}