<jupyter_start><jupyter_text>A qualitative example of Recurrence for Time Series Classification<jupyter_code>import sys
import os

# get the files if executed in colab
if not os.path.exists("requirements.txt"):
  !git clone https://github.com/MarcCoru/phiweek19 /tmp/repo
  !mv /tmp/repo/* .

!pip install -r requirements.txt

import matplotlib.pyplot as plt
from matplotlib.lines import Line2D
import seaborn as sns
import numpy as np
from rnn import RNN
import torch
import torch.nn.functional as F
import tqdm
from torch import matmul, sigmoid, tanh

sns.set_style("whitegrid")

BANDS = ['B1', 'B10', 'B11', 'B12', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'B8',
       'B8A', 'B9']

# simulate no GPU (for checks like torch.cuda.is_available())
os.environ["CUDA_VISIBLE_DEVICES"] = ""<jupyter_output>Requirement already satisfied: jupyter in /home/marc/miniconda3/envs/pytorch/lib/python3.7/site-packages (from -r requirements.txt (line 1)) (1.0.0)
Requirement already satisfied: matplotlib in /home/marc/miniconda3/envs/pytorch/lib/python3.7/site-packages (from -r requirements.txt (line 2)) (3.1.0)
Requirement already satisfied: seaborn in /home/marc/miniconda3/envs/pytorch/lib/python3.7/site-packages (from -r requirements.txt (line 3)) (0.9.0)
Requirement already satisfied: torch in /home/marc/miniconda3/envs/pytorch/lib/python3.7/site-packages (from -r requirements.txt (line 4)) (1.1.0)
Requirement already satisfied: numpy in /home/marc/miniconda3/envs/pytorch/lib/python3.7/site-packages (from -r requirements.txt (line 5)) (1.14.6)
Requirement already satisfied: tqdm in /home/marc/miniconda3/envs/pytorch/lib/python3.7/site-packages (from -r requirements.txt (line 7)) (4.32.2)
Requirement already satisfied: pandas in /home/marc/miniconda3/envs/pytorch/lib/python3.7/site-packages (fro[...]<jupyter_text>Data Data PartitioningThe models were trained on the trainset of the HOLL region.This notebook loads data exclusively from test partitions of the HOLL region and a spatially different KRUM region. Change the parameter `region=holl` to `region=krum` to change the data region<jupyter_code># Test Partition of the HOLL region (model was trained on the Train Partition on the HOLL region)
region = "holl" # <- change to 'krum' for test data from a different region

if region == "holl":
    X = np.load("data/x.npy")
    Y = np.load("data/y.npy")
    meta = np.load("data/meta.npy")
elif region == "krum":
    # Test Partition of the KRUM region
    X = np.load("data/x_krum.npy")
    Y = np.load("data/y_krum.npy")
    meta = np.load("data/meta_krum.npy")

print(f"X shape: (N, T, D):{X.shape} with N examples, sequencelength T, and D number of features per t")
print(f"y shape: (N, C):{Y.shape} with N examples, sequencelength T (class repeatet T times)")

klassennamen = np.load("data/classnames.npy", allow_pickle=True)
classnames = klassennamen<jupyter_output>X shape: (N, T, D):(100, 50, 13) with N examples, sequencelength T, and D number of features per t
y shape: (N, C):(100, 50) with N examples, sequencelength T (class repeatet T times)<jupyter_text>Initialize the model and load pre-trained weights<jupyter_code>model = RNN(input_dim=13, nclasses=33, hidden_dims=32,
                      num_rnn_layers=4, dropout=0.710883, bidirectional=True)
model.load("rnn.pth")
model.eval()<jupyter_output>loading model from rnn.pth<jupyter_text>Chose one Example for the Analysis<jupyter_code>idx = 5 # <- change this to an index between 0 and N=99

x = X[idx]
y = Y[idx,0]
x = torch.from_numpy(x)

logprobabilities = model.forward(x[None,:,:])<jupyter_output><empty_output><jupyter_text>Get the weights from the model's first layer and split up for respective gates<jupyter_code># get weight tensors of the first layer forward pass (l0)
weight_hh = model.lstm.weight_hh_l0
weight_ih = model.lstm.weight_ih_l0

w_ii, w_if, w_ig, w_io = weight_ih.chunk(4, 0)
w_hi, w_hf, w_hg, w_ho = weight_hh.chunk(4, 0)

w_i = torch.cat([w_ii,w_hi],dim=1)
w_f = torch.cat([w_if,w_hf],dim=1)
w_g = torch.cat([w_ig,w_hg],dim=1)
w_o = torch.cat([w_io,w_ho],dim=1)<jupyter_output><empty_output><jupyter_text>Visualize the weights of the first recurrent layer<jupyter_code>fix, axs = plt.subplots(2,4, figsize=(16,6))
axs[0,0].imshow(w_ii.detach().numpy())
axs[0,0].set_title("input gate weights x")
axs[0,1].imshow(w_if.detach().numpy())
axs[0,1].set_title("forget gate weights x")
axs[0,2].imshow(w_ig.detach().numpy())
axs[0,2].set_title("modulation gate weights x")
axs[0,3].imshow(w_io.detach().numpy())
axs[0,3].set_title("output gate weights x")
axs[1,0].imshow(w_hi.detach().numpy())
axs[1,0].set_title("input gate weights h")
axs[1,1].imshow(w_hf.detach().numpy())
axs[1,1].set_title("forget gate weights h")
axs[1,2].imshow(w_hg.detach().numpy())
axs[1,2].set_title("modulation gate weights h")
axs[1,3].imshow(w_ho.detach().numpy())
axs[1,3].set_title("output gate weights h")

[ax.grid(False) for ax in axs.reshape(-1)]<jupyter_output><empty_output><jupyter_text>Re-implement the LSTM updateIn pytorch, the LSTM loop is optimized away. We have access to the weights and input output features.so, we can re-implement the LSTM cell state update<jupyter_code># for later plotting
i_all = list()
f_all = list()
g_all = list()
o_all = list()
h_all = list()
c_all = list()

h_prev = torch.zeros(32)
c_prev = torch.zeros(32)

for t in range(x.shape[0]):
    
    # append previous information t-1 with current information
    xh = torch.cat([x[t,:],h_prev])

    i = sigmoid(matmul(w_i, xh))
    f = sigmoid(matmul(w_f, xh))
    g = tanh(matmul(w_g, xh))
    o = sigmoid(matmul(w_o, xh))
    
    c = f*c_prev + i*g
    h = o*tanh(c)
    
    # update c, h for next iteration
    h_prev = h
    c_prev = c

    # store gates for later plotting
    i_all.append(i)
    f_all.append(f)
    g_all.append(g)
    o_all.append(o)
    h_all.append(h)
    c_all.append(c)
    
i_all = torch.stack(i_all)
f_all = torch.stack(f_all)
g_all = torch.stack(g_all)
o_all = torch.stack(o_all)
h_all = torch.stack(h_all)
c_all = torch.stack(c_all)<jupyter_output><empty_output><jupyter_text>Visualize the Gate Activations when doing inference on a time series<jupyter_code>import matplotlib.pylab as pl

hidden_dims = 32

colors = pl.cm.tab20(np.linspace(0,1,hidden_dims))

# positive spike for clouds
colors[30] = np.array([0,0,0,0])

fig, axs = plt.subplots(7,1, figsize=(12,13))

# randomize the sequence in which the lines are plotted (on top of each other...)
plotting_idxs = np.arange(hidden_dims)
np.random.shuffle(plotting_idxs)

axs[0].plot(x.detach().numpy())
axs[0].set_ylabel("Input Series")
axs[0].set_title("Class {}".format(classnames[y]))
[axs[1].plot(i_all[:,i].detach().numpy(), color=colors[i], alpha=0.5) for i in plotting_idxs]
axs[1].set_ylabel("Input Gate")
[axs[2].plot(f_all[:,i].detach().numpy(), color=colors[i], alpha=0.5) for i in plotting_idxs]
axs[2].set_ylabel("Forget Gate")
[axs[3].plot(g_all[:,i].detach().numpy(), color=colors[i], alpha=0.5) for i in plotting_idxs]
axs[3].set_ylabel("Modulation Gate")
[axs[4].plot(o_all[:,i].detach().numpy(), color=colors[i], alpha=0.5) for i in plotting_idxs]
axs[4].set_ylabel("Output Gate")
[axs[5].plot(h_all[:,i].detach().numpy(), color=colors[i], alpha=0.5) for i in plotting_idxs]
axs[5].set_ylabel("Output")
[axs[6].plot(c_all[:,i].detach().numpy(), color=colors[i], alpha=0.5, label="hidden dim {}".format(i)) for i in range(hidden_dims)]
axs[6].set_ylabel("Cell State")
#axs[7].legend()

path="/home/marc/projects/Phiweek19_Presentation/images/rnn_examples"

import pandas as pd
os.makedirs(os.path.join(path,str(idx)), exist_ok=True)

df = pd.DataFrame(x.detach().numpy(), columns=BANDS)
df.index.name="t"
df.to_csv(os.path.join(path,str(idx),"x.csv"))
print("writing: "+os.path.join(path,str(idx),"x.csv"))

for name, tensor in zip(["i","f","o","g","h","c"],[i_all,f_all,o_all,g_all,h_all,c_all]):
    df = pd.DataFrame(tensor.detach().numpy())
    df.index.name="t"
    df.to_csv(os.path.join(path,str(idx),name+".csv"))
    print("writing: "+os.path.join(path,str(idx),name+".csv"))<jupyter_output>writing: /home/marc/projects/Phiweek19_Presentation/images/rnn_examples/5/x.csv
writing: /home/marc/projects/Phiweek19_Presentation/images/rnn_examples/5/i.csv
writing: /home/marc/projects/Phiweek19_Presentation/images/rnn_examples/5/f.csv
writing: /home/marc/projects/Phiweek19_Presentation/images/rnn_examples/5/o.csv
writing: /home/marc/projects/Phiweek19_Presentation/images/rnn_examples/5/g.csv
writing: /home/marc/projects/Phiweek19_Presentation/images/rnn_examples/5/h.csv
writing: /home/marc/projects/Phiweek19_Presentation/images/rnn_examples/5/c.csv<jupyter_text>Gradients can be used to analyze the input feature importance![](doc/grad_explained.png)  Do Forward Inference and Gradient Backpropagation1. Add gradients to the input `x_` (new variable with gradients)2. do forward inference3. get the scalar with the highest prediction score: `logprobabilities.exp().max()`4. do gradient backpropagation `.backward()` (to all variables with gradients)5. retrieve the gradients with `.grad`<jupyter_code>x_ = torch.autograd.Variable(x[None,:,:], requires_grad=True)
logprobabilities = model.forward(x_)
maxypred = logprobabilities.exp().max()
maxypred.backward()
dydx = x_.grad<jupyter_output><empty_output><jupyter_text>Visualize Gradients along with input time series<jupyter_code>fig, axs = plt.subplots(2, figsize=(12,10))
axs[0].plot(x_[0].detach().numpy())
axs[0].set_ylabel("x")
axs[0].legend(BANDS)
axs[1].plot(x_.grad[0].numpy(),linewidth=4)
axs[1].set_ylabel("dy/dx")
axs[1].legend(BANDS)

df = pd.DataFrame(x_.grad[0].numpy(), columns=BANDS)
df.index.name="t"
df.to_csv(os.path.join(path,str(idx),"dydx.csv"))
print("writing: "+os.path.join(path,str(idx),"dydx.csv"))
fig, axs = plt.subplots(4, figsize=(12,10))
axs[0].plot(x_[0].detach().numpy())
axs[0].set_ylabel("x")
axs[1].plot(x_.grad[0,:,[0,1,12]].numpy(),linewidth=4)
axs[1].set_ylabel("atmosphere bands")
axs[1].legend(np.array(BANDS)[[0,1,12]])
axs[2].plot(x_.grad[0,:,[2,3]].numpy(),linewidth=4)
axs[2].legend(np.array(BANDS)[[2,3]])
axs[2].set_ylabel("swir bands")
axs[3].plot(x_.grad[0,:,4:12].numpy(),linewidth=4)
axs[3].set_ylabel("dy/dx")
axs[3].legend(np.array(BANDS)[4:12])<jupyter_output><empty_output><jupyter_text>A PGF/Tikzed example of idx=5 (Holl region)![](doc/grad_result.png)  Get the Gradients on the weights<jupyter_code>dweight_hh = model.lstm.weight_hh_l0.grad
dweight_ih = model.lstm.weight_ih_l0.grad

dw_ii, dw_if, dw_ig, dw_io = weight_ih.chunk(4, 0)
dw_hi, dw_hf, dw_hg, dw_ho = weight_hh.chunk(4, 0)

dw_i = torch.cat([w_ii,w_hi],dim=1)
dw_f = torch.cat([w_if,w_hf],dim=1)
dw_g = torch.cat([w_ig,w_hg],dim=1)
dw_o = torch.cat([w_io,w_ho],dim=1)<jupyter_output><empty_output><jupyter_text>Visualize<jupyter_code>fix, axs = plt.subplots(4, figsize=(16,16))
axs[0].imshow(dw_ii.detach().numpy().T)
axs[1].imshow(dw_if.detach().numpy().T)
axs[2].imshow(dw_ig.detach().numpy().T)
axs[3].imshow(dw_io.detach().numpy().T)

for ax, name in zip(axs.reshape(-1),["input","forget","modulation", "output"]):
    ax.set_yticks(np.arange(len(BANDS)))
    ax.set_yticklabels(BANDS)
    ax.set_title("gradients " + name + " gate")
    ax.grid(False)<jupyter_output><empty_output><jupyter_text>Values Gates<jupyter_code>fix, axs = plt.subplots(4, figsize=(16,16))
axs[0].imshow(w_ii.detach().numpy().T)
axs[1].imshow(w_if.detach().numpy().T)
axs[2].imshow(w_ig.detach().numpy().T)
axs[3].imshow(w_io.detach().numpy().T)

for ax, name in zip(axs.reshape(-1),["input","forget","modulation", "output"]):
    ax.set_yticks(np.arange(len(BANDS)))
    ax.set_yticklabels(BANDS)
    ax.set_title(name + " gate")
    ax.grid(False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17052343950132354, 'fraction_numerical': 0.02672700879514986, 'mean_word_length': 5.789565217391305, 'pattern_counts': {'":': 2, '<': 51, '<?xml version=': 0, '>': 49, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '141065', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 4898, 'n_tokens_neox': 4785, 'n_words': 896, 'path': 'Recurrence.ipynb', 'repo_name': 'MarcCoru/phiweek19', 'size': '11711'}