<jupyter_start><jupyter_text>Funnynet-Three character modelSpecial thanks to taivop for providing the [dataset](https://github.com/taivop/joke-dataset).This notebook is heavily inspired by [fastai NLP work](https://github.com/fastai/fastai/blob/master/courses/dl2/imdb.ipynb).This is the initial experiment in which we build a model which considers the previous three characters at a time to predict the next.<jupyter_code>import pdb
import json
from pathlib import Path
import numpy as np
import matplotlib.pyplot as plt
import math, random
import preprocessing<jupyter_output><empty_output><jupyter_text>Run the first line in the cell below to create the data set for this notebook the first time.After, run the second line in the cell below to reload the data for this notebook.<jupyter_code>#idx, char_indices, indices_char, chars, vocab_size = preprocessing.save_data_to_pickle("data/800000char.pickle", 800000)
idx, char_indices, indices_char, chars, vocab_size = preprocessing.load_data("data/800000char.pickle")
cs = 3
c1_data = c2_data = c3_data = c4_data = []
for i in range(0, len(idx)-cs, cs):
    c1_data.append(idx[i])
    c2_data.append(idx[i+1])
    c3_data.append(idx[i+2])
    c4_data.append(idx[i+3])<jupyter_output><empty_output><jupyter_text>Create the input and outputs of our RNN<jupyter_code>x1 = np.stack(c1_data)
x2 = np.stack(c2_data)
x3 = np.stack(c3_data)
y  = np.stack(c4_data)<jupyter_output><empty_output><jupyter_text>Create and train model<jupyter_code>n_hidden = 256
embeddings_sz = 42 # size of embeddings matrix
# These libraries require some setup, try the pip install git+https.github.com/... trick
from fastai.io import *
from fastai.conv_learner import *

from fastai.column_data import *
class ThreeCharRNN(nn.Module):
    def __init__(self, vocab_size, embeddings_sz):
        super().__init__()
        self.e = nn.Embedding(vocab_size, embeddings_sz)
        self.l_in = nn.Linear(embeddings_sz, n_hidden)
        self.l_hidden = nn.Linear(n_hidden, n_hidden)
        self.l_out = nn.Linear(n_hidden, vocab_size)
        
    def forward(self, c1, c2, c3):
        in1 = F.relu(self.l_in(self.e(c1))) # Why relu?
        in2 = F.relu(self.l_in(self.e(c2)))
        in3 = F.relu(self.l_in(self.e(c3)))
        
        h = V(torch.zeros(in1.size()).cuda())
        h = F.tanh(self.l_hidden(h+in1))
        h = F.tanh(self.l_hidden(h+in2))
        h = F.tanh(self.l_hidden(h+in3))
        
        return F.log_softmax(self.l_out(h))
md = ColumnarModelData.from_arrays('.', [-1], np.stack([x1,x2,x3], axis=1), y, bs=512)
model = ThreeCharRNN(vocab_size, embeddings_sz).cuda()
train_iterator = iter(md.trn_dl)
*xs,yt = next(train_iterator)
t = model(*V(xs))
optimizer = optim.Adam(model.parameters(), lr=1e-2)
fit(model, md, n_epochs=1, opt=optimizer, crit=F.nll_loss) # The negative log likelihood loss
def get_next(input):
   running_indicies = []
   indicies = []
   for char in input:
       running_indicies.append(char_indices[char])
   for i in range(10):
       indicies = np.array(running_indicies[-3:])
       indicies = T(indicies)
       prediction = model(*VV(indicies))
       pred_idx = np.argmax(to_np(prediction))
       running_indicies.append(pred_idx)
   result_chars = []
   for index in running_indicies:
       result_chars.append(chars[index])
   return result_chars
get_next('y. ')
get_next('the')
get_next('blo')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12287390029325514, 'fraction_numerical': 0.020821114369501466, 'mean_word_length': 4.3213728549141965, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '203530', 'license': 'MIT', 'n_tokens_mistral': 1274, 'n_tokens_neox': 1252, 'n_words': 270, 'path': 'funnynet-3char.ipynb', 'repo_name': 'thedch/funnynet', 'size': '3410'}