<jupyter_start><jupyter_text>Chapter 6b: Predicting English word version of numbers using RNN<jupyter_code>from fastai.text.all import *
bs = 64

path = untar_data(URLs.HUMAN_NUMBERS)
path.ls()
sl = 40  # better than bptt = sl = 20
lines = L()
with open(path/"train.txt") as f: lines += L(*f.readlines())
with open(path/"valid.txt") as f: lines += L(*f.readlines())

text = " . ".join([l.strip() for l in lines]) # separator
tokens = text.split(" ")
vocab = L(*tokens).unique()
word2idx = {w:i for i, w in enumerate(vocab)}
nums = L(word2idx[i] for i in tokens)


def group_chunks(ds, bs): 
    m = len(ds) // bs
    new_ds = L()
    for i in range(m): new_ds += L(ds[i + m * j] for j in range(bs))
    return new_ds


seqs = L((tensor(nums[i : i + sl]), tensor(nums[i + 1: i + sl + 1]))
        for i in range(0, len(nums) - sl - 1, sl))
cut = int(len(seqs) * 0.8)
dls = DataLoaders.from_dsets(group_chunks(seqs[:cut], bs),
                             group_chunks(seqs[cut:], bs),
                             bs=bs, drop_last=True, shuffle=False)
nv = len(vocab)
nh = 56
nv, nh
def loss_func(inp, targ): 
    return F.cross_entropy(inp.view(-1, len(vocab)), targ.view(-1))<jupyter_output><empty_output><jupyter_text>Adding a GRU.<jupyter_code>class Model5(Module): 
    """One layer GRU"""
    def __init__(self, nv, nh):
        self.i_h = nn.Embedding(nv, nh)
        self.rnn = nn.GRU(nh, nh, 1, batch_first=True)
        self.h_o = nn.Linear(nh, nv)
        self.bn = BatchNorm1dFlat(nh)
        self.h = torch.zeros(1, bs, nh).cuda()

    def forward(self, x, targ):
        res, h = self.rnn(self.i_h(x), self.h)
        self.h = h.detach()
        return self.h_o(self.bn(res))
class TeacherForcing(Callback):
    def __init__(self, end_epoch): self.end_epoch = end_epoch
    
    # def before_batch(self): 
    #     self.learn.xb = (self.x, self.y)
    def before_batch(self): 
        # print(self.x.shape)
        print(self.x + self.y)
        # self.learn.xb = (self.x, self.y)
        self.learn.xb = self.x

    def before_epoch(self):
        self.learn.model.pr_force = 1 - self.learn.epoch / self.end_epoch
learn = Learner(dls, Model5(nv, nh).cuda(), metrics=accuracy, loss_func=loss_func,
cbs=TeacherForcing(5))
learn.fit_one_cycle(10, 1e-2)
learn = Learner(dls, Model5(nv, nh).cuda(), metrics=accuracy, loss_func=loss_func)
learn.fit_one_cycle(10, 1e-2)<jupyter_output><empty_output><jupyter_text>Let's make our own GRUUsing PyTorch's GRUCell. Axis 1 is the time dimension. We want to loop through axis 1.<jupyter_code>def rnn_loop(cell, h, x): 
    res = []
    for x_ in x.transpose(0, 1):
        h = cell(x_, h)
        res.append(h)
    return torch.stack(res, dim=1)
class Model6(Model5):
    def __init__(self, nv, nh):
        super().__init__(nv, nh)
        self.rnnc = nn.GRUCell(nh, nh)
        self.h = torch.zeros(bs, nh).cuda()

    def forward(self, x):
        res = rnn_loop(self.rnnc, self.h, self.i_h(x))
        self.h = res[:, -1].detach()
        return self.h_o(self.bn(res))
learn = Learner(dls, Model6(nv, nh).cuda(), metrics=accuracy, loss_func=loss_func)
learn.fit_one_cycle(10, 1e-2)<jupyter_output><empty_output><jupyter_text>Writing Custom GRUCell<jupyter_code>class custom_GRUCell(Module):
    def __init__(self, ni, nh): 
        super(custom_GRUCell, self).__init__()
        self.ni, self.nh = ni, nh
        self.i2h = nn.Linear(ni, 3 * nh)
        self.h2h = nn.Linear(nh, 3 * nh)

    def forward(self, x, h):
        gate_x = self.i2h(x).squeeze()
        gate_h = self.h2h(x).squeeze()
        i_r, i_u, i_n = gate_x.chunk(3, 1)
        h_r, h_u, h_n = gate_h.chunk(3, 1)

        reset_gate = torch.sigmoid(i_r + h_r)
        update_gate = torch.sigmoid(i_u + h_u)
        new_gate = torch.tanh(i_n + (reset_gate * h_n))
        return update_gate * h + (1 - update_gate) * new_gate
class Model7(Model6):
    def __init__(self, nv, nh):
        super(Model7, self).__init__(nv, nh)
        self.rnnc = custom_GRUCell(nh, nh)
learn = Learner(dls, Model7(nv, nh).cuda(), metrics=accuracy, loss_func=loss_func)
learn.fit_one_cycle(10, 1e-2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15561471442400773, 'fraction_numerical': 0.016214908034849953, 'mean_word_length': 3.055937193326791, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '190753', 'license': 'Apache-2.0', 'n_tokens_mistral': 1735, 'n_tokens_neox': 1653, 'n_words': 353, 'path': 'Chapter6b.ipynb', 'repo_name': 'Wabinab/fastai-coursenlp-v2_rewrite', 'size': '4132'}