<jupyter_start><jupyter_text>Introduction  Imports<jupyter_code>import os
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
gpu_options = tf.GPUOptions(allow_growth=True)  # init TF ...
config=tf.ConfigProto(gpu_options=gpu_options)  # w/o taking ...
with tf.Session(config=config): pass            # all GPU memory
import pdb<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>human_dates_raw = []
human_vocab = set()
machine_dates_raw = []
machine_vocab = set()
with open(os.path.expanduser('~/dates_train.csv'), 'r') as f:
    for line in f:
        h, m = line.strip().split(',')
        human_dates_raw.append(h)
        human_vocab.update(h)
        machine_dates_raw.append(m)
        machine_vocab.update(m)
        
#human_vocab = ['<pad>', *sorted(human_vocab)]
#machine_vocab = sorted(machine_vocab)
print(human_dates_raw[:4])
print(machine_dates_raw[:4])
print('human_vocab:  ', sorted(human_vocab))
print('machine_vocab:', sorted(machine_vocab))
human_sym2idx = {c : i for i, c in enumerate(sorted(human_vocab), 1)}  # 0 is for padding
print(human_sym2idx)
machine_sym2idx = {c : i for i, c in enumerate(sorted(machine_vocab))}
machine_idx2sym = {i : c for c, i in machine_sym2idx.items()}
print('machine_sym2idx:', machine_sym2idx)
print('machine_idx2sym:', machine_idx2sym)
human_min_len = len(min(human_dates_raw, key=len))
human_max_len = len(max(human_dates_raw, key=len))
machine_min_len = len(min(machine_dates_raw, key=len))
machine_max_len = len(max(machine_dates_raw, key=len))
assert machine_min_len == machine_max_len == 10  # sanity check
print('Human dates min-max len:', human_min_len, '-', human_max_len)
def tokenize(list_of_examples, sym2idx):
    result_tokenized = []
    for list_of_symbols in list_of_examples:
        result_tokenized.append( [sym2idx[s] for s in list_of_symbols] )
    return result_tokenized
human_dates_tok = tokenize(human_dates_raw, human_sym2idx)
display(human_dates_tok[:3])
machine_dates_tok = tokenize(machine_dates_raw, machine_sym2idx)
display(machine_dates_tok[:3])
def pad(list_of_examples, target_len, dtype):
    result_np = np.zeros(shape=(len(list_of_examples), target_len), dtype=dtype)
    
    for i in range(len(list_of_examples)):
        example = list_of_examples[i]
        result_np[i, 0:len(example)] = example
        
    return result_np
human_dates_pad = pad(human_dates_tok, human_max_len, dtype=int)
print('shape:', human_dates_pad.shape)
for i in range(3):
    print(human_dates_pad[i][:20])
machine_dates_pad = np.array(machine_dates_tok)  # nothing to pad
print('shape:', machine_dates_pad.shape)
for i in range(3):
    print(machine_dates_pad[i])
def onehot_2d(array, nb_class):
    assert len(array.shape) == 2
    
    result = np.zeros((*array.shape, nb_class))
    for i in range(len(array)):
        for j in range(array.shape[1]):
            k = array[i, j]
            result[i, j, k] = 1
            
    return result
human_max_idx = max(human_sym2idx.values())
human_dates_oh = onehot_2d(human_dates_pad, human_max_idx+1)
machine_max_idx = max(machine_sym2idx.values())
machine_dates_oh = onehot_2d(machine_dates_pad, machine_max_idx+1)
print('date:', machine_dates_raw[0])
print('padded:', machine_dates_pad[0])
print('onehot:\n', machine_dates_oh[0])
n_hseq = human_max_len
n_hout = human_max_idx + 1
n_mseq = machine_max_len
n_mout = machine_max_idx + 1
print('n_hseq', n_hseq)
print('n_hout', n_hout)
print('n_mseq', n_mseq)
print('n_mout', n_mout)
import tensorflow.keras.backend as K
def softmax(x, axis=1):
    """Softmax activation function.
    # Arguments
        x : Tensor.
        axis: Integer, axis along which the softmax normalization is applied.
    # Returns
        Tensor, output of softmax transformation.
    # Raises
        ValueError: In case `dim(x) == 1`.
    """
    ndim = K.ndim(x)
    if ndim == 2:
        return K.softmax(x)
    elif ndim > 2:
        e = K.exp(x - K.max(x, axis=axis, keepdims=True))
        s = K.sum(e, axis=axis, keepdims=True)
        return e / s
    else:
        raise ValueError('Cannot apply softmax to a tensor that is 1D')
# Defined shared layers as global variables
repeator = tf.keras.layers.RepeatVector(n_hseq)
concatenator = tf.keras.layers.Concatenate(axis=-1)
densor1 = tf.keras.layers.Dense(10, activation = "tanh")
densor2 = tf.keras.layers.Dense(1, activation = None) # "relu")
activator = tf.keras.layers.Activation(softmax, name='attention_weights') # custom softmax
dotor = tf.keras.layers.Dot(axes = 1)
# GRADED FUNCTION: one_step_attention

def one_step_attention(a, s_prev):
    """
    Performs one step of attention: Outputs a context vector computed as a dot product of the attention weights
    "alphas" and the hidden states "a" of the Bi-LSTM.
    
    Arguments:
    a -- hidden state output of the Bi-LSTM, numpy-array of shape (m, Tx, 2*n_a)
    s_prev -- previous hidden state of the (post-attention) LSTM, numpy-array of shape (m, n_s)
    
    Returns:
    context -- context vector, input of the next (post-attetion) LSTM cell
    """
    
    # a (?, 27, 2*32)
    # s_prev (?, 64)
    
    ### START CODE HERE ###
    # Use repeator to repeat s_prev to be of shape (m, Tx, n_s) so that you can concatenate it with all hidden states "a" (≈ 1 line)
    s_prev = repeator(s_prev)  # (?, 27, 64)
    # Use concatenator to concatenate a and s_prev on the last axis (≈ 1 line)
    concat = concatenator([a, s_prev])  # (?, 27, 128)
    # Use densor1 to propagate concat through a small fully-connected neural network to compute the "intermediate energies" variable e. (≈1 lines)
    e = densor1(concat)  # (?, 27, 10)
    # Use densor2 to propagate e through a small fully-connected neural network to compute the "energies" variable energies. (≈1 lines)
    energies = densor2(e)  # (?, 27, 1)
    # Use "activator" on "energies" to compute the attention weights "alphas" (≈ 1 line)
    alphas = activator(energies)  # (?, 27, 1)
    # Use dotor together with "alphas" and "a" to compute the context vector to be given to the next (post-attention) LSTM-cell (≈ 1 line)
    context = dotor([alphas, a])  # (?, 1, 64)
    ### END CODE HERE ###
    
    return context, alphas
n_a = 32
n_s = 64
post_activation_LSTM_cell = tf.keras.layers.LSTM(n_s, return_state = True)
output_layer = tf.keras.layers.Dense(n_mout, activation=softmax)
# GRADED FUNCTION: model

def create_model(Tx, Ty, n_a, n_s, human_vocab_size, machine_vocab_size):
    """
    Arguments:
    Tx -- length of the input sequence
    Ty -- length of the output sequence
    n_a -- hidden state size of the Bi-LSTM
    n_s -- hidden state size of the post-attention LSTM
    human_vocab_size -- size of the python dictionary "human_vocab"
    machine_vocab_size -- size of the python dictionary "machine_vocab"

    Returns:
    model -- Keras model instance
    """
    
    # pdb.set_trace()
    
    # Define the inputs of your model with a shape (Tx,)
    # Define s0 and c0, initial hidden state for the decoder LSTM of shape (n_s,)
    X = tf.keras.Input(shape=(Tx, human_vocab_size))  # (?, 27, 36)
    s0 = tf.keras.Input(shape=(n_s,), name='s0')      # (?, 64)
    c0 = tf.keras.Input(shape=(n_s,), name='c0')      # (?, 64)
    s = s0
    c = c0
    
    # Initialize empty list of outputs
    outputs = []
    attentions = []
    
    ### START CODE HERE ###
    
    # Step 1: Define your pre-attention Bi-LSTM. Remember to use return_sequences=True. (≈ 1 line)
    a = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(n_a, return_sequences=True))(X)  # (?, 27, 2*32)
    
    # Step 2: Iterate for Ty steps
    for t in range(Ty):
    
        # Step 2.A: Perform one step of the attention mechanism to get back the context vector at step t (≈ 1 line)
        context, alphas = one_step_attention(a, s)  # (?, 1, 64)
        
        # Step 2.B: Apply the post-attention LSTM cell to the "context" vector.
        # Don't forget to pass: initial_state = [hidden state, cell state] (≈ 1 line)
        s, _, c = post_activation_LSTM_cell(context, initial_state=[s, c]) # (?, 1, 64), (?, 1, 64), (?, 1, 64)
        
        # Step 2.C: Apply Dense layer to the hidden state output of the post-attention LSTM (≈ 1 line)
        out = output_layer(s)  # (?, 11)
        
        # Step 2.D: Append "out" to the "outputs" list (≈ 1 line)
        outputs.append(out)
        attentions.append(alphas)
        
    outputs2 = tf.keras.layers.Concatenate()(outputs)
    outputs3 = tf.keras.layers.Reshape((Ty, machine_vocab_size))(outputs2)
    
    attentions2 = tf.keras.layers.Concatenate()(attentions)
    attentions3 = tf.keras.layers.Permute((2, 1))(attentions2)
    
    # pdb.set_trace()
    
    # Step 3: Create model instance taking three inputs and returning the list of outputs. (≈ 1 line)
    model_train = tf.keras.Model([X, s0, c0], outputs3)
    model_attend = tf.keras.Model([X, s0, c0], [outputs3, attentions3])
    
    ### END CODE HERE ###
    
    return model_train, model_attend
model, model_attend = create_model(n_hseq, n_mseq, n_a, n_s, n_hout, n_mout)
model.summary()
### START CODE HERE ### (≈2 lines)
opt = tf.keras.optimizers.Adam(lr = 0.005, beta_1=0.9, beta_2=0.999, decay = 0.01)
model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])
### END CODE HERE ###
m = len(human_dates_oh)
s0 = np.zeros((m, n_s))
c0 = np.zeros((m, n_s))
#outputs = list(machine_dates_oh.swapaxes(0,1))
outputs = machine_dates_oh<jupyter_output><empty_output><jupyter_text>**Target Output**Epoch xx/xx10000/10000 [==============================] - 15s 2ms/step - loss: 0.0543 - dense_3_loss: 0.0021 - dense_3_acc: 0.9998 - dense_3_acc_1: 1.0000 - dense_3_acc_2: 1.0000 - dense_3_acc_3: 1.0000 - dense_3_acc_4: 1.0000 - dense_3_acc_5: 0.9982 - dense_3_acc_6: 0.9997 - dense_3_acc_7: 1.0000 - dense_3_acc_8: 0.9958 - dense_3_acc_9: 0.9997<jupyter_code>hist = model.fit([human_dates_oh, s0, c0], outputs, epochs=50, batch_size=100)
#model.save_weights('2410_attention_new.h5')
#model.load_weights('2410_attention_new.h5')
EXAMPLES = ['3 May 1979', '5 April 09', '21th of August 2016', 'Tue 10 Jul 2007', 'Saturday May 9 2018', 'March 3 2001', 'March 3rd 2001', '1 March 2001']
CORRECT =  ['1979-05-03', '2009-04-05', '2016-08-21',          '2007-07-10',      '2018-05-09', '2001-03-03', '2001-03-03', '2001-03-01']
for example, correct in zip(EXAMPLES, CORRECT):
    
    source = onehot_2d(pad(tokenize([example.lower()], human_sym2idx), n_hseq, int), n_hout)
    prediction = model.predict([source, s0, c0])
    prediction = np.argmax(prediction, axis=-1)
    output = [machine_idx2sym[i] for i in prediction.ravel()]
    output = ''.join(output)
    
    print("source:", example)
    print("output:", output)
    print('correc:', correct)
    print('is ok: ', 'YES' if output==correct else 'no')
    print()
raise
plt.plot(hist.history['loss'])
plt.grid()
plt.plot(hist.history['acc'])
plt.grid()
attention_map = plot_attention_map(model_attend, human_sym2idx, machine_idx2sym, "Tuesday 09 Oct 1993", num=7, n_s = 64)
human_dates_valid_raw = []
machine_dates_valid_raw = []
with open(os.path.expanduser('~/dates_valid.csv'), 'r') as f:
    for line in f:
        h, m = line.strip().split(',')
        human_dates_valid_raw.append(h)
        machine_dates_valid_raw.append(m)
        
#human_vocab = ['<pad>', *sorted(human_vocab)]
#machine_vocab = sorted(machine_vocab)
count_corret = 0
for i in range(1000):
#for example, correct in zip(human_dates_valid_raw, machine_dates_valid_raw):
    
    example = human_dates_valid_raw[i]
    correct = machine_dates_valid_raw[i]
    
    source = onehot_2d(pad(tokenize([example.lower()], human_sym2idx), n_hseq, int), n_hout)
    prediction = model.predict([source, s0, c0])
    prediction = np.argmax(prediction, axis=-1)
    output = [machine_idx2sym[i] for i in prediction.ravel()]
    output = ''.join(output)
    
#     print("source:", example)
#     print("output:", output)
#     print('correc:', correct)
#     print('is ok: ', 'YES' if output==correct else 'no')
#     print()
    
    if output==correct:
        count_corret += 1
        
print('count_correct', count_corret)
print('n_hseq', n_hseq)
print('n_hout', n_hout)
print('n_mseq', n_mseq)
print('n_mout', n_mout)
def plot_attention_map(model, input_vocabulary, inv_output_vocabulary, text, n_s = 128, num = 6, Tx = 30, Ty = 10):
    """
    Plot the attention map.
  
    """
    attention_mapA = np.zeros((n_mseq, n_hseq))
    Ty, Tx = attention_mapA.shape
    print('Ty, Tx', attention_mapA.shape)
    
    s0 = np.zeros((1, n_s))
    c0 = np.zeros((1, n_s))
#     layer = model.layers[num]

    encoded = onehot_2d(pad(tokenize([text.lower()], human_sym2idx), n_hseq, int), n_hout)
    encoded = encoded.astype(np.float32)
    
    print(Ty)
    print(encoded.shape, encoded.dtype)
    print(s0.shape, s0.dtype)
    print(c0.shape, c0.dtype)

    
#     pdb.set_trace()

#     f = K.function(model.inputs, [layer.get_output_at(t) for t in range(Ty)])
#     r = f([encoded, s0, c0])
    
#     for t in range(Ty):
#         for t_prime in range(Tx):
#             attention_map[t][t_prime] = r[t][0,t_prime,0]



    prediction, attention_map = model.predict([encoded, s0, c0])
    attention_map = attention_map.squeeze()
    #attention_map = attention_map.T
    #print(prediction)
    #pdb.set_trace()
    
    prediction = np.argmax(prediction, axis=-1)
    predicted_text = [machine_idx2sym[i] for i in prediction.ravel()]
    #print(predicted_text)
    
#     predicted_text = []
#     for i in range(len(prediction)):
#         predicted_text.append(int(np.argmax(prediction[i], axis=1)))
        
#     predicted_text = list(predicted_text)
#     print(predicted_text)
#     predicted_text = [machine_idx2sym[i] for i in predicted_text.ravel()]
    #predicted_text = int_to_string(predicted_text, inv_output_vocabulary)
    text_ = list(text)
    
    # get the lengths of the string
    input_length = len(text)
    output_length = Ty
    
    # Plot the attention_map
    plt.clf()
    f = plt.figure(figsize=(8, 8.5))
    ax = f.add_subplot(1, 1, 1)

    # add image
    i = ax.imshow(attention_map, interpolation='nearest', cmap='Blues')

    # add colorbar
    cbaxes = f.add_axes([0.2, 0, 0.6, 0.03])
    cbar = f.colorbar(i, cax=cbaxes, orientation='horizontal')
    cbar.ax.set_xlabel('Alpha value (Probability output of the "softmax")', labelpad=2)

    # add labels
    ax.set_yticks(range(output_length))
    ax.set_yticklabels(predicted_text[:output_length])

    ax.set_xticks(range(input_length))
    ax.set_xticklabels(text_[:input_length], rotation=45)

    ax.set_xlabel('Input Sequence')
    ax.set_ylabel('Output Sequence')

    # add grid and legend
    ax.grid()

    #f.show()
    
    return attention_map
attentions2, attention_map = plot_attention_map(model_attend, human_sym2idx, machine_idx2sym, "Tuesday 09 Oct 1993", num=7, n_s = 64)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13462181502039725, 'fraction_numerical': 0.03223433424730823, 'mean_word_length': 3.6643792888334374, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '214630', 'license': 'MIT', 'n_tokens_mistral': 5816, 'n_tokens_neox': 5480, 'n_words': 1300, 'path': 'KerasNN/2410_Attention_Dates_new.ipynb', 'repo_name': 'marcinbogdanski/ai-sketchpad', 'size': '14953'}