<jupyter_start><jupyter_text>Thinking in tensors in PyTorchHands-on training  by [<NAME>](https://p.migdal.pl) (2019).  Text generation* [The Unreasonable Effectiveness of Recurrent Neural Networks](http://karpathy.github.io/2015/05/21/rnn-effectiveness/) by <NAME>* [RecurrentJS](http://cs.stanford.edu/people/karpathy/recurrentjs) - an in-browser demo by <NAME>* [Unsupervised sentiment neuron by OpenAI](https://openai.com/blog/unsupervised-sentiment-neuron/)* [Generating Magic cards using deep, recurrent neural networks](https://www.mtgsalvation.com/forums/magic-fundamentals/custom-card-creation/612057-generating-magic-cards-using-deep-recurrent-neural)Other* [Training a Keras model to generate colors](https://heartbeat.fritz.ai/how-to-train-a-keras-model-to-generate-colors-3bc79e54971b) Various practical links* [What is the best way to remove accents in a Python unicode string?](https://stackoverflow.com/questions/517923/what-is-the-best-way-to-remove-accents-in-a-python-unicode-string)<jupyter_code>!pip install unidecode
import numpy as np
import pandas as pd
from collections import Counter
from unidecode import unidecode
from sklearn.model_selection import train_test_split

import torch
from torch import nn
from torch import optim
from torch.utils.data import TensorDataset, DataLoader

from livelossplot import PlotLosses
names = pd.read_csv("https://www.dropbox.com/s/nu2y0p3i2jvwfki/surnames.csv?dl=1")
names.info()
names.sample(5)
names['language'].value_counts()
100 * names['language'].value_counts(normalize=True)
names['name'].apply(len).value_counts().sort_index()
letters_all = Counter()
for name in names['name']:
    letters_all.update(name)
letters_all.most_common()
letters = Counter()
for name in names['name']:
    letters.update(unidecode(name))
letters.most_common()
char2id = {c: i for i, (c, v) in enumerate(letters.items())}
id2char = {i: c for i, (c, v) in enumerate(letters.items())}
char2id
lang2id = {lang: i for i, lang in enumerate(names['language'].value_counts().index)}
lang2id
max_len = 10
BEGIN_ID = len(char2id)
AFTER_ID = len(char2id) + 1

X = np.zeros((len(names), max_len), dtype=np.int64)
X[:,:] = AFTER_ID
X[:,0] = BEGIN_ID

Y = np.zeros(len(names), dtype=np.int64)

for i, (name, lang) in names.iterrows():
    Y[i] = lang2id[lang]
    for j, c in enumerate(unidecode(name)):
        if j + 1 >= max_len:
            break
        X[i, j + 1] = char2id[c]
chars = [c for i, c in id2char.items()] + [">", "<"]
def encode(name, end=True):
    code = [char2id[c] for c in name.lower()]
    if end:
        return torch.tensor([BEGIN_ID] + code + [AFTER_ID]).unsqueeze(0)
    else:
        return torch.tensor([BEGIN_ID] + code).unsqueeze(0)
encode("Kow")
X.shape
X[:5]
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.25, random_state=42)
trainloader = DataLoader(TensorDataset(torch.from_numpy(X_train).long(),
                                       torch.from_numpy(Y_train).long()),
                         batch_size=128, shuffle=True)
testloader = DataLoader(TensorDataset(torch.from_numpy(X_test).long(),
                                      torch.from_numpy(Y_test).long()),
                         batch_size=128, shuffle=False)

dataloaders = {
    "train": trainloader,
    "validation": testloader
}
# train on cuda if available
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

def train_model(model, criterion, optimizer, num_epochs=10):
    liveloss = PlotLosses()
    model = model.to(device)
    
    for epoch in range(num_epochs):
        logs = {}
        for phase in ['train', 'validation']:
            if phase == 'train':
                model.train()
            else:
                model.eval()

            running_loss = 0.0
            running_corrects = 0

            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device)
                labels = labels.to(device)

                outputs = model(inputs)
                loss = criterion(outputs, labels)

                if phase == 'train':
                    optimizer.zero_grad()
                    loss.backward()
                    optimizer.step()

                _, preds = torch.max(outputs, 1)
                running_loss += loss.detach() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)

            epoch_loss = running_loss / len(dataloaders[phase].dataset)
            epoch_acc = running_corrects.float() / len(dataloaders[phase].dataset)
            
            prefix = ''
            if phase == 'validation':
                prefix = 'val_'

            logs[prefix + 'log loss'] = epoch_loss.item()
            logs[prefix + 'accuracy'] = epoch_acc.item()
        
        liveloss.update(logs)
        liveloss.draw()
# train on cuda if available
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

def train_model_gener(model, criterion, optimizer, num_epochs=10):
    liveloss = PlotLosses()
    model = model.to(device)
    
    for epoch in range(num_epochs):
        logs = {}
        for phase in ['train', 'validation']:
            if phase == 'train':
                model.train()
            else:
                model.eval()

            running_loss = 0.0
            running_corrects = 0

            for inputs_full, labels_class in dataloaders[phase]:
                
                # here are changes!
                inputs = inputs_full[:, :-1].to(device)
                labels = inputs_full[:, 1:].to(device)

                outputs = model(inputs)
                
                loss = criterion(outputs, labels)

                if phase == 'train':
                    optimizer.zero_grad()
                    loss.backward()
                    optimizer.step()

                _, preds = torch.max(outputs, 1)
                running_loss += loss.detach() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)

            epoch_loss = running_loss / len(dataloaders[phase].dataset)
            epoch_acc = running_corrects.float() / len(dataloaders[phase].dataset)
            
            prefix = ''
            if phase == 'validation':
                prefix = 'val_'

            logs[prefix + 'log loss'] = epoch_loss.item()
            logs[prefix + 'accuracy'] = epoch_acc.item()
        
        liveloss.update(logs)
        liveloss.draw()
class RecurrentLSTM(nn.Module):
    def __init__(self, embedding_size, hidden_size):
        super().__init__()
        self.emb = nn.Embedding(len(char2id) + 2, embedding_size)
        self.lstm = nn.LSTM(input_size=embedding_size, hidden_size=hidden_size)
        # note: input size is the numer of channels/embedding dim, NOT length
        self.fc = nn.Linear(hidden_size, len(lang2id))

    def forward(self, x):
        x = self.emb(x)
        x = x.permute(1, 0, 2)  # BLC -> LBC
        output, (hidden, cell) = self.lstm(x)
        res = self.fc(cell).squeeze(0)
        return res
class GenerativeLSTM(nn.Module):
    def __init__(self, embedding_size, hidden_size):
        super().__init__()
        dictionary_len = len(char2id) + 2
        self.emb = nn.Embedding(dictionary_len, embedding_size)
        self.lstm = nn.LSTM(input_size=embedding_size, hidden_size=hidden_size)
        # note: input size is the numer of channels/embedding dim, NOT length
        self.fc = nn.Linear(hidden_size, dictionary_len)

    def forward(self, x):
        x = self.emb(x)
        x = x.permute(1, 0, 2)  # BLC -> LBC
        output, (hidden, cell) = self.lstm(x)
        res = self.fc(output)
        return res.permute(1, 2, 0) #  LBC -> BCL
X_example = torch.from_numpy(X_train[:3]).long().to(device)
model = RecurrentLSTM(embedding_size=5, hidden_size=16).to(device)
model
model(X_example).size()
model = RecurrentLSTM(embedding_size=5, hidden_size=16)

optimizer = optim.Adam(model.parameters(), lr=0.01)
criterion = nn.CrossEntropyLoss()

train_model(model, criterion, optimizer, num_epochs=10)
model = GenerativeLSTM(embedding_size=5, hidden_size=16)
model(X_example).size()
model(encode("Kowalski"))
model = GenerativeLSTM(embedding_size=5, hidden_size=16)

optimizer = optim.Adam(model.parameters(), lr=0.01)
criterion = nn.CrossEntropyLoss()

train_model_gener(model, criterion, optimizer, num_epochs=51)
pred = model(encode("Kowalsk", end=False))
pred.size()
next_letter_prob = pred[0,:,-1].softmax(dim=0).cpu().detach().numpy()
next_letter_prob
100 * pd.Series(next_letter_prob, index=chars).sort_values(ascending=False)
next_char_id = np.random.choice(len(next_letter_prob), 1, p=next_letter_prob)[0]
next_char_id
chars[next_char_id]
def generate(start="", next_chars=10, model=model, temperature=1.):
    word = start

    for i in range(next_chars):
        pred = model(encode(word, end=False))
        next_letter_prob = pred[0,:,-1].softmax(dim=0).cpu().detach().numpy()
  
        next_letter_prob = next_letter_prob**(1/temperature)
        next_letter_prob = next_letter_prob / next_letter_prob.sum()

        next_char_id = np.random.choice(len(next_letter_prob), 1, p=next_letter_prob)[0]

        word += chars[next_char_id]

        if chars[next_char_id] == '<':
            break

    return word<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12901487389523605, 'fraction_numerical': 0.016059495580944168, 'mean_word_length': 2.7811735941320292, 'pattern_counts': {'":': 2, '<': 10, '<?xml version=': 0, '>': 13, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '377766', 'license': 'MIT', 'n_tokens_mistral': 3152, 'n_tokens_neox': 2982, 'n_words': 616, 'path': 'rnns/Text generation.ipynb', 'repo_name': 'soumendukrg/thinking-in-tensors-writing-in-pytorch', 'size': '9302'}