<jupyter_start><jupyter_text>From the above, we see the first sentense isx = [0, 6, 3513, 7, 155, 794, 25, 223, 8, 32, 20, 202, 5025, 350, 91, 6, 66, 207, 5, 2]where each number represent one word, and we will vectorize them later (i.e. $x_t$).And y = [6, 3513, 7, 155, 794, 25, 223, 8, 32, 20, 202, 5025, 350, 91, 6, 66, 207, 5, 2, 1]that is to say, our model will generate the next word conditioned on the previous words.The formulars are:$\begin{aligned}s_t &= \tanh(Ux_t + Ws_{t-1}) \\o_t &= \mathrm{softmax}(Vs_t)\end{aligned}$<jupyter_code>def softmax2D(x):
    assert len(x.shape) == 2, 'x.shape: {0} do not match.'.format(x.shape)
    x = np.asarray(x, dtype='float32')
    dim = x.shape[1]
    x_tile = x[:, :, np.newaxis] # expand the last dim
    x_tile = np.tile(x_tile, (1, 1, dim)) # repeat the last dim
    
    x_sub = x[:, np.newaxis, :] # expand the second dim
    x_out = x_tile - x_sub
    
    g = 1. / np.sum(np.exp(-x_out), axis=1)
    return g

def softmax(x):
    assert isinstance(x, np.ndarray), 'type: {0} do not match.'.format(type(x))
    
    if len(x.shape) == 1:
        x_new = np.reshape(x, (1, -1))
        g = softmax2D(x_new)
        g = g.ravel()
    elif len(x.shape) == 2:
        g = softmax2D(x)
    else:
        raise ValueError('Input array have x.shape == {0}, which is not allowed.'.format(x.shape))
    return g

class RNNNumpy(object):
    def __init__(self, word_dim, hidden_dim, bptt_truncate=4):
        t1 = time.time()
        self.word_dim = word_dim
        self.hidden_dim = hidden_dim
        self.bptt_truncate = bptt_truncate
        
        s_1 = np.sqrt(1./word_dim)
        s_2 = np.sqrt(1./hidden_dim)
        self.U = np.random.uniform(-s_1, s_1, size=(hidden_dim, word_dim))
        self.W = np.random.uniform(-s_2, s_2, size=(hidden_dim, hidden_dim))
        self.V = np.random.uniform(-s_2, s_2, size=(word_dim, hidden_dim))
        t2 = time.time()
        print('Initial time: {0}'.format(t2 - t1))
    
    def forward_prop(self, x):
        t1 = time.time()
        T = len(x)
        s = np.zeros((T + 1, self.hidden_dim), dtype='float32')
        o = np.zeros((T, self.word_dim), dtype='float32')
        for t in range(T):
            s[t] = np.tanh(self.U[:, x[t]] + np.dot(self.W, s[t-1]))
            o[t] = softmax(np.dot(self.V, s[t]))
        t2 = time.time()
        print('forward time: {0}'.format(t2 - t1))
        return o, s
    
    def predict(self, x):
        o, s = self.forward_prop(x)
        y_pred = np.argmax(o, axis=1)
        return y_pred
rnn = RNNNumpy(vocabulary_size, 100)
print(X_train[0])
o_sent, s_sent = rnn.forward_prop(X_train[0])
#print(o_sent.shape, s_sent.shape)
print(o_sent.shape, s_sent.shape)
print([len(x) for x in X_train[:10]])
print([len(y) for y in y_train[:10]])
max_sent_len = 30
feat_dim = 100
dtype_INT = tf.int32
dtype_FLOAT = tf.float32
input_tensor = tf.placeholder(shape=(None, max_sent_len), dtype=dtype_INT)
class RNNTF(object):
    def __init__(self, embedded_words, ):
        pass

    
def embedding(input_tensor, vocab_size, word_dim):
    """
    inputs
    -------
    input_tensor: 2D tensor with shape=(batch_size, max_sentence_len), indices of words
    vocab_size: integer, total num of unqiue words, max(input_tensor) == vocab_size - 1
    word_dim: integer, length of vectorized word
    
    return
    -------
    embedded_words: 3D tensor with shape=(batch_size, max_sentence_len, word_dim)
    """
    shape = (vocab_size, word_dim)
    scale = np.sqrt(1./word_dim)
    initializer = tf.random_uniform_initializer(-scale, scale, dtype=dtype_FLOAT)
    
    with tf.variable_scope('embedding'):
        params = tf.get_variable('params', shape, initializer=initializer)
    embedded_words = tf.gather(params, input_tensor)
    
    return embedded_words
embedded_words = embedding(input_tensor, max_sent_len, feat_dim)
embedded_words
unstack_sent = tf.unstack(embedded_words)
unstack_sent<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1472145197882531, 'fraction_numerical': 0.039072346861608265, 'mean_word_length': 2.9879396984924624, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '338368', 'license': 'MIT', 'n_tokens_mistral': 1612, 'n_tokens_neox': 1507, 'n_words': 362, 'path': 'jupyter_tf/RNN.ipynb', 'repo_name': 'huizhuzhao/jupyter_notebook', 'size': '3967'}