<jupyter_start><jupyter_text>RNN for time-series<jupyter_code># Put these at the top of every notebook, to get automatic reloading and inline plotting
%reload_ext autoreload
%autoreload 2
%matplotlib inline
from fastai.column_data import *
from fastai.text import *<jupyter_output><empty_output><jupyter_text>Generate some data<jupyter_code>N  = 30000 # Number of samples
b, c, tau = 0.1, 0.2, 17
y = [0.9697, 0.9699, 0.9794, 1.0003, 1.0319, 1.0703, 1.1076, 1.1352, 1.1485,
     1.1482, 1.1383, 1.1234, 1.1072, 1.0928, 1.0820, 1.0756, 1.0739, 1.0759]
for n in range(17,N+99):
    y.append(y[n] - b*y[n] + c*y[n-tau]/(1+y[n-tau]**10))
x = np.array(y[100:])
n = np.array(list(range(len(x)))) # n is just for testing purposes
path = 'data/'
plt.plot(x[:200]);<jupyter_output><empty_output><jupyter_text>Create sequences<jupyter_code>seq_len = 3
xs = x.reshape(-1,seq_len)
ns = n.reshape(-1,seq_len)
xs.shape, ns.shape
ys, yn = xs[1:,0].reshape(-1,1), ns[1:,0].reshape(-1,1)
xs, ns = xs[:-1,:], ns[:-1,:]
ns[:10, :], yn[:10]<jupyter_output><empty_output><jupyter_text>For each sequence we want to predict the next value, for example, the first line is $[0, 1, 2, 3]$ and we want to predict $[4]$. We will use $xs$ and $ys$, $ns$ and $yn$ are just to check if the sequences are created as we expect.<jupyter_code># Last 30% of data for validation
val_idx = np.arange(len(xs)//(1/0.7), len(xs)).astype(int);
len(val_idx)/len(xs)<jupyter_output><empty_output><jupyter_text>Define and train the model<jupyter_code># fastai.text
class RNN_Learner(Learner):
    def __init__(self, data, models, **kwargs):
        super().__init__(data, models, **kwargs)

    def _get_crit(self, data): return F.cross_entropy
    def fit(self, *args, **kwargs): return super().fit(*args, **kwargs, seq_first=False)

    def save_encoder(self, name): save_model(self.model[0], self.get_model_path(name))
    def load_encoder(self, name): load_model(self.model[0], self.get_model_path(name))
# Based on Lesson 6 Char3Model (http://course.fast.ai/lessons/lesson6.html @1:33:18)
class TSRnn3steps(nn.Module):
    def __init__(self, in_sz, h_sz, out_sz):
        super().__init__()
        self.lin_in = nn.Linear(in_sz, h_sz)
        self.lin_h = nn.Linear(h_sz, h_sz)
        self.lin_out = nn.Linear(h_sz, out_sz)
    
    def forward(self, x1, x2, x3):
        x1 = F.relu(self.lin_in(x1.view(-1,1)))
        x2 = F.relu(self.lin_in(x2.view(-1,1)))
        x3 = F.relu(self.lin_in(x3.view(-1,1)))
        
        h = torch.zeros(x1.size(), requires_grad=True).cuda()
        h = F.tanh(self.lin_h(x1))
        h = F.tanh(self.lin_h(h+x2))
        h = F.tanh(self.lin_h(h+x3))
        
        return self.lin_out(h)
md = ColumnarModelData.from_arrays(path, val_idx, xs.astype(float), ys.astype(float), bs=100)
# https://github.com/fastai/fastai/blob/master/courses/dl2/translate.ipynb
m = RNN_Learner(md, SingleModel(to_gpu(TSRnn3steps(1,10,1))))
m.crit = nn.L1Loss()
m.lr_find()
m.sched.plot()
lr = 1e-1
m.fit(lr, 1, cycle_len=12, use_clr=(20,10))<jupyter_output><empty_output><jupyter_text>Multi-step predictions<jupyter_code># Generate test data
N  = 103 # Number of samples
b, c, tau = 0.1, 0.2, 17
y = [0.9697, 0.9699, 0.9794, 1.0003, 1.0319, 1.0703, 1.1076, 1.1352, 1.1485,
     1.1482, 1.1383, 1.1234, 1.1072, 1.0928, 1.0820, 1.0756, 1.0739, 1.0759]
y = list(np.random.permutation(y))
for n in range(17,N+99):
    y.append(y[n] - b*y[n] + c*y[n-tau]/(1+y[n-tau]**10))
x = np.array(y[100:])
in_test = [T(o) for o in x[:3]]
out_save = []
while len(out_save) < N:
    out = m.model(*in_test)
    in_test = [*in_test, out[0]][1:]
    out_save.append(out.item())
plt.plot(x)
plt.plot([*x[:3], *out_save])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18328840970350405, 'fraction_numerical': 0.08194070080862534, 'mean_word_length': 4.197478991596639, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '443539', 'license': 'MIT', 'n_tokens_mistral': 1788, 'n_tokens_neox': 1608, 'n_words': 324, 'path': 'timeseries_rnn_01.ipynb', 'repo_name': 'mnpinto/fastai_pytorch_notebooks', 'size': '3710'}