<jupyter_start><jupyter_text>Input attentionAs in [DARNN](https://arxiv.org/pdf/1704.02971.pdf) the input attention is designed to focus on raw features of the k dimensional input of the time series. Given a cell hidden state $h_{t-1}$ and the series features $x_{t-1}$, attention is computed as a deterministic filter, for example a dense layer with output shape of 1, applied to the all batch and the last hidden state of the cell.$$e_t^k = W(h_{t-1} + x^k) + b$$Then we can normalize the weights using a softmax function:$$\alpha_t^k = softmax(e_t^k)$$And compute the candidate output, $\widetilde{x}$ as:$$\widetilde{x_t} = \alpha_t \odot x_t $$<jupyter_code>path= "../tf/rnn/woc-2018-09-21-02-06-21"
config = Logger.get_config(path)
df = load_data(config.dataset)
features = df.columns.tolist()
features
x, index, stats = make_features(df, 
                                seq_len=config.seq_len, 
                                preprocess=config.preprocess,
                                lags=config.lags, 
                                use_x=config.use_x,
                               )
dataset = Dataset(x, index,stats, seq_len=config.seq_len, pred_len=config.pred_len,random_start=True)
it = dataset.__iter__()
feeds, dates = next(it)
tf.reset_default_graph()
model = RNN(dataset.shape, config=config)
trainer = Trainer(model, config=config, path=path)
trainer.init_sess()
trainer.restore(path=path)
sess =trainer.sess
sess.run(tf.global_variables_initializer())
tf.trainable_variables()
h = model.x
attn = InputAttention(output_shape=1, memory=h, name='rnn/encoder/input_attention', use_bias=True)
state = tf.zeros(shape=(8, config.encoder), dtype=tf.float32)
t = 0 
x_tilde = attn.apply(h[:, t,:], state)
feed_dict = trainer._get_dict(feeds)<jupyter_output><empty_output><jupyter_text>First timestep, first batch of random day<jupyter_code>features.pop(0)
alpha = trainer.sess.run(x_tilde, feed_dict)
alpha = alpha.mean(axis=0).reshape((1,-1))
# features importance for each a single sequence at time 0
df = pd.DataFrame(alpha, columns=features)
mif = df[df>0].fillna(0)
mif.mean(axis=0).sort_values(ascending=False)
lif = df[df<0.0].fillna(0)
lif.mean(axis=0).sort_values(ascending=True)<jupyter_output><empty_output><jupyter_text>Single batch over time steps<jupyter_code>cell = tf.contrib.rnn.GRUBlockCellV2(num_units=config.encoder, reuse=tf.AUTO_REUSE, name='rnn/encoder/gru_cell')
states = []
c_xs = []
ws = []
state = tf.zeros(shape=(8, config.encoder), dtype=tf.float32)
for t in range(config.seq_len):
    c_x = attn.apply(h[:, t, :], state)
    ws.append(tf.div(c_x, h[:, t,:]))
    _, state = cell(c_x, state)
    states.append(state)
    c_xs.append(c_x)
ws = trainer.sess.run(ws, feed_dict)
c_xs = trainer.sess.run(c_xs, feed_dict)
c_xs = np.array(c_xs)
avg_c = c_xs.mean(axis=1)
avg_c = (avg_c - avg_c.mean(axis=0))/avg_c.std(axis=0)
x_true = feed_dict[model.x]
avg_c_true = x_true.mean(axis=0)
fig, ax = plt.subplots(6,3, figsize=(20,20), sharex=True, sharey=True)

for idx,a in zip(range(avg_c.shape[-1]), ax.flatten()):
    a.plot(avg_c[:, idx])
    a.plot(avg_c_true[:, idx])
    a.set_title(features[idx])

fig.savefig('input_attn_avg_batch.png')
avg_b = c_xs.mean(axis=0)
fig, ax = plt.subplots(6,3, figsize=(20,20), sharex=True, sharey=True)

for idx,a in zip(range(avg_b.shape[-1]), ax.flatten()):
    a.plot(avg_b[:, idx])
    a.set_title(features[idx])

fig.savefig('input_attn_avg_time')
full_series = c_xs.reshape((-1,len(features)))
t = (full_series - full_series.mean(axis=0)) / full_series.std(axis=0)
full_series_true = x_true.reshape((-1,len(features)))
fig, ax = plt.subplots(6,3, figsize=(20,20), sharex=True, sharey=True)

for idx,a in zip(range(full_series.shape[-1]), ax.flatten()):
    a.plot(t[:, idx])
    a.plot(full_series_true[:, idx])
    a.set_title(features[idx])
fig.savefig('input_attn_full.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1570290413775379, 'fraction_numerical': 0.020046260601387818, 'mean_word_length': 4.648766328011611, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '319750', 'license': 'MIT', 'n_tokens_mistral': 1528, 'n_tokens_neox': 1489, 'n_words': 282, 'path': 'nb/attention.ipynb', 'repo_name': 'd3sm0/ntsa', 'size': '3891'}