<jupyter_start><jupyter_text>Human numbers<jupyter_code>%reload_ext autoreload
%autoreload 2
%matplotlib inline
from fastai.text import *

from fastai import *
bs=64<jupyter_output><empty_output><jupyter_text>Data<jupyter_code>untar_data??
path = untar_data(URLs.HUMAN_NUMBERS)
path.ls()
valid = 'valid.txt'
! head {path}/{valid}
! head {path}/'train.txt'
'  eight'.strip()
def readnums(d): return [', '.join(o.strip() for o in open(path/d).readlines())]
train_txt = readnums('train.txt')
len(train_txt)
train_txt[0][:80]
train_txt[0][:5]
valid_txt = readnums('valid.txt'); valid_txt[0][-80:]
len(valid_txt[0].split(","))
doc(TextList)
path.parts
type(train_txt)
array(train_txt, dtype=object)[0][:5]
train = TextList(train_txt, path=path)
valid = TextList(valid_txt, path=path)

src = ItemLists(path=path, train=train, valid=valid).label_for_lm()
data = src.databunch(bs=bs)
train[0].text[:80]
data.batch_size
data.train_ds.y
data.train_ds[0][0].data
len(data.train_ds[0][0].data)
data.valid_ds[0][0].data<jupyter_output><empty_output><jupyter_text>There are 13,017 tokens in the validation set (including the commas). Tokens are each item in the doc separated by a space<jupyter_code>len(data.valid_ds[0][0].data)
data.bptt, len(data.valid_dl)
data.valid_dl
len(data.train_dl)<jupyter_output><empty_output><jupyter_text>Our dataset of 13k tokens is broken up into 64 segments (since batch size of 64) and then we divide into bptt groups of about 70 (randomized a bit on the size) to give us 3 actual batches from the validation set<jupyter_code>13017/70/bs<jupyter_output><empty_output><jupyter_text>13017 tokens in validation set divided by bptt divided by bs = 3 and we have 3 iters of the validation loader<jupyter_code>it = iter(data.valid_dl)
x1,y1 = next(it)
x2,y2 = next(it)
x3,y3 = next(it)
it.close()
it2 = iter(data.valid_dl)
f1,g1 = next(it2)<jupyter_output><empty_output><jupyter_text>f1 is our independent variables and g1 is our dependent variables. so the g1's are just shifted one over from the f1's since predicting the next word<jupyter_code>f1, g1<jupyter_output><empty_output><jupyter_text>get an error if iterate 4 times since we only have 3 iterations<jupyter_code>f2,g2 = next(it2)
f3,g3 = next(it2)
f4,g4 = next(it2)
torch.numel??
# shows us number of elements in tensor. so for 4x4 matrix, 16 numel<jupyter_output><empty_output><jupyter_text>We have 12,928 tokens in the 3 batches rather than the original 13,017 tokens since the excess didn't fit into the last batch<jupyter_code>x1.numel()+x2.numel()+x3.numel()
x1.numel()
95 * 64
y1.numel()<jupyter_output><empty_output><jupyter_text>64 is our batch size and 95 represents our bptt here. The data loader randomizes it so for each batch it will be around 70 rather than exactly 70<jupyter_code>x1.shape,y1.shape
x2.shape,y2.shape
x1.shape
x1
# gives all rows of column 1

x1[:, 1]
# gives all columns of row 0

x1[0,:]
# this is what we want to pull our tokens in order. we see that y1[0] is offset by 1

x1[0]
y1[0]
# gives all rows of column 0. not what we want

x1[:,0]
y1[:, 0]
v = data.valid_ds.vocab<jupyter_output><empty_output><jupyter_text>The below shows that our minibatches match up. We end x1[0] and then x2[0] picks up from there and then x3[0]. Then x1[1] picks up around where x3[0] ended.<jupyter_code>v.textify(x1[0])
v.textify(y1[0])
v.textify(x2[0])
v.textify(x3[0])
v.textify(x1[1])
v.textify(x2[1])
v.textify(x3[1])
v.textify(x3[-1])
data.show_batch(ds_type=DatasetType.Valid)<jupyter_output><empty_output><jupyter_text>Single fully connected model<jupyter_code>data = src.databunch(bs=bs, bptt=3, max_len=0, p_bptt=1.)
x,y = data.one_batch()
x.shape,y.shape
x
y
nv = len(v.itos); nv
v.itos
nh=64
def loss4(input,target): return F.cross_entropy(input, target[:,-1])
def acc4 (input,target): return accuracy(input, target[:,-1])
accuracy??
F.cross_entropy??<jupyter_output><empty_output><jupyter_text>We're now doing a batch size of 64 with bptt of 3<jupyter_code>x.shape
class Model0(nn.Module):
    def __init__(self):
        super().__init__()
#         input to hidden
        self.i_h = nn.Embedding(nv,nh)  # green arrow
#         hidden to hidden
        self.h_h = nn.Linear(nh,nh)     # brown arrow
#     hidden to output
        self.h_o = nn.Linear(nh,nv)     # blue arrow
        self.bn = nn.BatchNorm1d(nh)
        
    def forward(self, x):
#         print("x.shape!!", x.shape)
        h = self.bn(F.relu(self.i_h(x[:,0])))
#         print("h!!", h)
#         at the end of the batch, could have fewer than 3 words left so need this condition
        if x.shape[1]>1:
#         for all of the first words for the 64 batch, add an input to the hidden layer using that first word 
            h = h + self.i_h(x[:,1])
            h = self.bn(F.relu(self.h_h(h)))
        if x.shape[1]>2:
#         for all of the second words for the 64 batch, add an input to the hidden layer using that second word 
            h = h + self.i_h(x[:,2])
            h = self.bn(F.relu(self.h_h(h)))
        return self.h_o(h)
learn = Learner(data, Model0(), loss_func=loss4, metrics=acc4)
learn.fit_one_cycle(2, 1e-4)<jupyter_output><empty_output><jupyter_text>Same thing with a loop<jupyter_code>class Model1(nn.Module):
    def __init__(self):
        super().__init__()
        self.i_h = nn.Embedding(nv,nh)  # green arrow
        self.h_h = nn.Linear(nh,nh)     # brown arrow
        self.h_o = nn.Linear(nh,nv)     # blue arrow
        self.bn = nn.BatchNorm1d(nh)
        
    def forward(self, x):
#         we change the initialization of h here to all 0s which is improving performance somehow?
        h = torch.zeros(x.shape[0], nh).to(device=x.device)
#     prior h initialization:
#         h = self.bn(F.relu(self.i_h(x[:,0])))
#         print("h!!", h)

# same thing as in prior model but have for loop
        for i in range(x.shape[1]):
            h = h + self.i_h(x[:,i])
            h = self.bn(F.relu(self.h_h(h)))
        return self.h_o(h)
learn = Learner(data, Model1(), loss_func=loss4, metrics=acc4)
learn.fit_one_cycle(2, 1e-4)<jupyter_output><empty_output><jupyter_text>Multi fully connected model<jupyter_code>data = src.databunch(bs=bs, bptt=20)
x,y = data.one_batch()
x.shape,y.shape
x,y = data.one_batch()
x.shape,y.shape
accuracy
data.one_batch??<jupyter_output><empty_output><jupyter_text>Why 45?? If have 20 bptt??  It is never exactly 20, randomizes a bit and looks like first batch is large.<jupyter_code>class Model2(nn.Module):
    def __init__(self):
        super().__init__()
        self.i_h = nn.Embedding(nv,nh)
        self.h_h = nn.Linear(nh,nh)
        self.h_o = nn.Linear(nh,nv)
        self.bn = nn.BatchNorm1d(nh)
        
    def forward(self, x):
        h = torch.zeros(x.shape[0], nh).to(device=x.device)
        res = []
        print("x.shape!!", x.shape)
        for i in range(x.shape[1]):
            h = h + self.i_h(x[:,i])
            h = F.relu(self.h_h(h))
            res.append(self.h_o(self.bn(h)))
        return torch.stack(res, dim=1)<jupyter_output><empty_output><jupyter_text>Now using default accuracy and loss function instead of just checking against one output at the end. Check for every word<jupyter_code>learn = Learner(data, Model2(), metrics=accuracy)
learn.fit_one_cycle(2, 1e-4, pct_start=0.1)<jupyter_output><empty_output><jupyter_text>Maintain state  We move the initialization of the hidden layer to the constructor so we don't have to start from scratch on each batch. Since the batches are in order this works<jupyter_code>class Model3(nn.Module):
    def __init__(self):
        super().__init__()
        self.i_h = nn.Embedding(nv,nh)
        self.h_h = nn.Linear(nh,nh)
        self.h_o = nn.Linear(nh,nv)
        self.bn = nn.BatchNorm1d(nh)
        self.h = torch.zeros(x.shape[0], nh).cuda()
        
    def forward(self, x):
        res = []
        h = self.h
        for i in range(x.shape[1]):
            h = h + self.i_h(x[:,i])
            h = F.relu(self.h_h(h))
            res.append(self.bn(h))
        self.h = h.detach()
        res = torch.stack(res, dim=1)
        res = self.h_o(res)
        return res
learn = Learner(data, Model3(), metrics=accuracy)
learn.fit_one_cycle(1, 3e-3)
sys.path
BatchNorm1dFlat
nn.RNN??<jupyter_output><empty_output><jupyter_text>nn.RNN<jupyter_code>class Model4(nn.Module):
    def __init__(self):
        super().__init__()
        self.i_h = nn.Embedding(nv,nh)
#         nh for input_size and nh for hidden_size
        self.rnn = nn.RNN(nh,nh, batch_first=True)
        self.h_o = nn.Linear(nh,nv)
        self.bn = BatchNorm1dFlat(nh)
        self.h = torch.zeros(1, bs, nh).cuda()
        
    def forward(self, x):
        res,h = self.rnn(self.i_h(x), self.h)
        self.h = h.detach()
        return self.h_o(self.bn(res))
learn = Learner(data, Model4(), metrics=accuracy)
learn.fit_one_cycle(2, 3e-3)
doc(nn.GRU)<jupyter_output><empty_output><jupyter_text>2-layer GRU<jupyter_code>class Model5(nn.Module):
    def __init__(self):
        super().__init__()
        self.i_h = nn.Embedding(nv,nh)
        self.rnn = nn.GRU(nh,nh,2, batch_first=True)
        self.h_o = nn.Linear(nh,nv)
        self.bn = BatchNorm1dFlat(nh)
        self.h = torch.zeros(2, bs, nh).cuda()
        
    def forward(self, x):
        res,h = self.rnn(self.i_h(x), self.h)
        self.h = h.detach()
        return self.h_o(self.bn(res))
learn = Learner(data, Model5(), metrics=accuracy)
learn.fit_one_cycle(10, 1e-2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1364690993461295, 'fraction_numerical': 0.026576671588272517, 'mean_word_length': 3.6101118133203696, 'pattern_counts': {'":': 0, '<': 77, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '533709', 'license': 'Apache-2.0', 'n_tokens_mistral': 3745, 'n_tokens_neox': 3626, 'n_words': 850, 'path': 'nbs/dl1/lesson7-human-numbers-Copy1.ipynb', 'repo_name': 'LaurenSpiegel/course-v3', 'size': '9482'}