<jupyter_start><jupyter_text>Fasttext RNN<jupyter_code>import warnings
warnings.filterwarnings('ignore')
from time import time
from random import random, choice
from collections import Counter
from os import system

import numpy as np
import pandas as pd

import gensim
import numpy as np
from sklearn.metrics import accuracy_score, f1_score

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import sru

from torch.autograd import Variable
from torch.utils.data.sampler import SubsetRandomSampler

import torchtext

from tensorboardX import SummaryWriter
from tqdm import tqdm as tqdm

from torchqrnn import QRNN

np.random.seed(42)
CUDA = torch.cuda.is_available()

CUDA
BATCH_SIZE = 32
VALID_SIZE = 0.1

NOISE_LEVEL = 0.1

NOISE_LEVELS = [0, 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.125, 0.15, 0.175, 0.2]<jupyter_output><empty_output><jupyter_text>Data preparation<jupyter_code>%%time
# 5 min
fasttext = gensim.models.FastText.load_fasttext_format('/media/data/nlp/data/wiki.en.bin')
# fasttext = gensim.models.FastText.load_fasttext_format('/media/data/nlp/data/wiki.ru.bin')
# yes, spacy really helps
fasttext.similarity('mixed', 'mixed,')<jupyter_output>/home/phobos_aijun/.virtualenvs/pytorch-env/lib/python3.5/site-packages/ipykernel_launcher.py:2: DeprecationWarning: Call to deprecated `similarity` (Method will be removed in 4.0.0, use self.wv.similarity() instead).<jupyter_text>IMDB<jupyter_code>MAXLEN = 256  # now it's words, not symbols

# without spacy tokenizer it's commas all after the words =(

text_field = torchtext.data.Field(
    lower=True, include_lengths=False, fix_length=MAXLEN, tensor_type=torch.FloatTensor, batch_first=True,
    use_vocab=False, tokenize='spacy'
)
label_field = torchtext.data.Field(sequential=False, use_vocab=False)
%%time
# Do not need this!
train, test = torchtext.datasets.imdb.IMDB.splits(text_field, label_field)
def preprocess_text_without_noise(text):
    vectors = []
    
    _text_tensor = torch.zeros([MAXLEN, fasttext.vector_size])

    for i, token in enumerate(text):
        if i >= MAXLEN:
            break

        if token in fasttext:
            token_vec = fasttext[token]
        else:
            token_vec = fasttext['unk']  # is this real <UNK> token?
        token_tensor = torch.FloatTensor(token_vec)
        _text_tensor[i, :] = token_tensor

    return _text_tensor

class FastTextNoisedIMDB(torchtext.datasets.imdb.IMDB):
    """
    Zero vector used for padding
    """
    noise_level = 0
    # alphabet from the paper
    # https://papers.nips.cc/paper/5782-character-level-convolutional-networks-for-text-classification.pdf
    alphabet = ['\n'] + [s for s in """ abcdefghijklmnopqrstuvwxyz0123456789-,;.!?:’’’/\|_@#$%ˆ&* ̃‘+-=<>()[]{}"""]

    def __getitem__(self, idx):
        item = super(FastTextNoisedIMDB, self).__getitem__(idx)
        vectors = []
        # indicies orded different from previous models — (word_vec, word_num) instead of (word_num, word_vec)
        _text_tensor = torch.zeros([MAXLEN, fasttext.vector_size])

        for i, token in enumerate(item.text):
            if i >= MAXLEN:
                break

            token = self.noise_generator(token)  # это плохо?
            if token in fasttext:
                token_vec = fasttext[token]
            else:
                token_vec = fasttext['unk']  # is this real <UNK> token?
            token_tensor = torch.FloatTensor(token_vec)
            _text_tensor[i, :] = token_tensor

        label = int(item.label == 'pos')
        return _text_tensor, label

    def noise_generator(self, string):
        # removed '' symbol from alphabet for safety on word vectors
        noised = ""
        for c in string:
            if random() > self.noise_level:
                noised += c
            if random() < self.noise_level:
                noised += choice(self.alphabet)
        return noised

def get_train_valid_loader(dataset, valid_size, batch_size, random_seed=42, shuffle=True, num_workers=4):

    len_dataset = len(dataset)
    indices = list(range(len_dataset))

    if shuffle:
        np.random.seed(random_seed)
        np.random.shuffle(indices)

    val_actual_size = int(len_dataset * valid_size)

    train_idx, valid_idx = indices[:-val_actual_size], indices[-val_actual_size:]

    train_sampler = SubsetRandomSampler(train_idx)
    valid_sampler = SubsetRandomSampler(valid_idx)

    train_loader = torch.utils.data.DataLoader(
        dataset, batch_size=batch_size, sampler=train_sampler, num_workers=4
    )
    valid_loader = torch.utils.data.DataLoader(
        dataset, batch_size=batch_size, sampler=valid_sampler, num_workers=4
    )

    return train_loader, valid_loader

%%time
train, test = FastTextNoisedIMDB.splits(text_field, label_field)<jupyter_output>CPU times: user 1min 8s, sys: 976 ms, total: 1min 9s
Wall time: 1min 14s<jupyter_text>Mokoron<jupyter_code>from random import random
from pymystem3 import Mystem
from nltk.tokenize import word_tokenize

# MAX_TEXT_LEN = MAXLEN = 32
MAX_TEXT_LEN = MAXLEN = 128

ALPHABET = ['<UNK>'] + ['\n'] + [s for s in """ 0123456789-,;.!?:'’’/\|_@#$%ˆ&* ̃‘+-=<>()[]{}"""]
# ALPHABET += [s for s in 'абвгдеёжзийклмнопрстуфхцчщъыьэюя']
ALPHABET += [s for s in 'abcdefghijklmnopqrstuvwxyz']

# ALPHABET = [s for s in ALPHABET if s not in ('(', ')')]

ALPHABET_LEN = len(ALPHABET)
char2int = {s: i for s, i in zip(ALPHABET, range(ALPHABET_LEN))}


class FastTextMokoron(torch.utils.data.Dataset):
    """
    Zero vector used for padding
    """
    noise_level = 0
    alphabet = ALPHABET

    def __init__(self, filepath, text_field, maxlen=MAX_TEXT_LEN):
        self.alphabet = ALPHABET

        self.mystem = Mystem()
        self.text_field = text_field
        self.data = pd.read_csv(filepath)
        self.maxlen = maxlen
        self.char2int = {s: i for s, i in zip(self.alphabet, range(len(self.alphabet)))}

    def __len__(self):
        return len(self.data)

    def _tokenize(self, text):
        return word_tokenize(text)
#         return [res['text'] for res in self.mystem.analyze(text) if res['text'] != ' ']

    def __getitem__(self, idx):
        line = self.data.iloc[idx]
        text = line[self.text_field].lower()
        label = int(line.sentiment == 1.)

        if self.noise_level > 0:
            text = self._noise_generator(text)

        text = self._tokenize(text)
        text = self._preprocess(text)
        return text, label

    def _preprocess(self, text):
        vectors = []
        # indicies orded different from previous models — (word_vec, word_num) instead of (word_num, word_vec)
        _text_tensor = torch.zeros([MAXLEN, fasttext.vector_size])

        for i, token in enumerate(text):
            if i >= MAXLEN:
                break

            if token in fasttext:
                token_vec = fasttext[token]
            else:
                token_vec = fasttext['unk']  # is this real <UNK> token?
            token_tensor = torch.FloatTensor(token_vec)
            _text_tensor[i, :] = token_tensor

        return _text_tensor

    def _noise_generator(self, string):
        noised = ""
        for c in string:
            if random() > self.noise_level:
                noised += c
            if random() < self.noise_level:
                noised += choice(self.alphabet)
        return noised

def get_metrics(model, test_data, noise_level=None):
    """
    :param test_data: dataset or dataloader

    Moder will be in TRAIN mode after that
    """
    model.eval()

    predictions = []
    lables = []
    
    if isinstance(test_data, torch.utils.data.Dataset):
        if noise_level is not None:
            test_data.noise_level = noise_level

        test_dataloader = torch.utils.data.DataLoader(
            test_data, batch_size=BATCH_SIZE
        )
    else:
        assert isinstance(test_data, torch.utils.data.DataLoader)
        test_dataloader = test_data

    for text, label in test_dataloader:
        if CUDA:
            text = Variable(text.cuda())
        else:
            text = Variable(text)

        text = text.permute(1, 0, 2)  # (1, 0, 2) for RNN
        prediction = model(text)

        _, idx = torch.max(prediction, 1)
        predictions += idx.data.tolist()
        lables += label.tolist()

    acc = accuracy_score(lables, predictions)
    f1 = f1_score(lables, predictions)
    model.train()
    return {'accuracy': acc, 'f1': f1}<jupyter_output><empty_output><jupyter_text>IMDB<jupyter_code>dataloader, val_dataloader = get_train_valid_loader(train, VALID_SIZE, BATCH_SIZE)
test_dataloader = torch.utils.data.DataLoader(
    test, batch_size=BATCH_SIZE
)<jupyter_output><empty_output><jupyter_text>Mokoron<jupyter_code>basepath = '/media/data/nlp/sentiment/ru-mokoron/splits/'
basepath = '/media/data/nlp/sentiment/IMDB/splits/'
train = FastTextMokoron(basepath + 'train.csv', 'text_original')
valid = FastTextMokoron(basepath + 'validation.csv', 'text_original')
test = FastTextMokoron(basepath + 'test.csv', 'text_original')

test_original = FastTextMokoron(basepath + 'test.csv', 'text_original')

dataloader = torch.utils.data.DataLoader(train, BATCH_SIZE, shuffle=True, num_workers=4)
val_dataloader = torch.utils.data.DataLoader(valid, BATCH_SIZE, shuffle=True, num_workers=4)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>class SimpleRNN(nn.Module):
    def __init__(self, hidden_dim, num_layers=1, dropout=0.5, type_='GRU'):
        super(SimpleRNN, self).__init__()
        self.hidden_dim = hidden_dim
        self.num_layers = num_layers
        self.dropout_prob = dropout

        if type_ == 'SRU':
            self.rnn = sru.SRU(fasttext.vector_size, hidden_dim, num_layers=num_layers)
        elif type_ == 'GRU':
            self.rnn = nn.GRU(fasttext.vector_size, hidden_dim, num_layers=num_layers)
        elif type_ == 'QRNN':
            self.rnn = QRNN(fasttext.vector_size, hidden_dim, num_layers=num_layers)
        elif type_ == 'LSTM': 
            self.rnn = nn.LSTM(fasttext.vector_size, hidden_dim, num_layers=num_layers)
        else:
            raise ValueError('Wrong type_', type_)
        self.dropout = nn.Dropout(self.dropout_prob)
        self.projector = nn.Linear(hidden_dim, 2)

    def forward(self, x):
        x, c_states = self.rnn(x)
        x = self.dropout(x[-1])
        x = self.projector(x)
        return x

def model_params_num(model):
    return sum(np.prod(list(p.size())) for p in model.parameters())

def mk_dataline(model_type, epochs, lr, noise_level_train, noise_level_test, acc_train, acc_test,
                f1_train, f1_test, dropout, model, run_name, task, init_function=None):
    return {
        'task': task,
        'model_type': model_type,
        'trainable_params': model_params_num(model), 'dropout': dropout, 'init_function': init_function,
        'epochs': epochs, 'lr': lr,
        'noise_level_train': noise_level_train, 'noise_level_test': noise_level_test,
        'acc_train': acc_train, 'acc_test': acc_test,
        'f1_train': f1_train, 'f1_test': f1_test,
        'model_desc': str(model),
        'run_name': run_name,
        'data_desc': 'Maxlen %s' % MAXLEN
    }
results = []
def run_model_with(noise_level, hidden, lr=1e-4, dropout=0.5, layers=1,
                   epochs=30, type_='GRU', _model=None, comment='', log_every=1):
    start_time = time()
#     CharIMDB.noise_level = noise_level
    task='IMDB binary classification'
    FastTextMokoron.noise_level = noise_level
#     task='Mokoron binary classification'

    if _model is None:
        model = SimpleRNN(hidden, num_layers=layers, dropout=dropout, type_=type_)
        if CUDA:
            model.cuda()
        model.train()
    
    else:
        model = _model
        
    model_name = '_FastText%s_lr%s_dropout%s_embed_layers%s_hidden%s_noise_level%s' %(
        type_, int(-np.log10(lr)), dropout, layers, hidden, noise_level
    ) + comment

    if '(' not in ALPHABET:
        model_name += '_no_emoji'

    writer = SummaryWriter(comment=model_name)
    if len(list(writer.all_writers.keys())) > 1:
        print('More than one writer! 0_o')
        print(list(writer.all_writers.keys()))

    run_name = list(writer.all_writers.keys())[0]
    print('Writer: %s' % run_name)

    optimizer = optim.Adam(params=model.parameters(), lr=lr)
    optimizer.zero_grad()
    
    global_step = 0

    loss_f = F.cross_entropy

    for epoch in range(epochs):

        for batch_idx, (text, label) in enumerate(dataloader):
            optimizer.zero_grad()

            if CUDA:
                text = Variable(text.cuda())
                label = Variable(torch.LongTensor(label).cuda())
            else:
                text = Variable(text)
                label = Variable(torch.LongTensor(label))

            text = text.permute(1, 0, 2)
            prediction = model(text)
            loss = loss_f(prediction, label)

            writer.add_scalar('loss', loss.data[0], global_step=global_step)

            loss.backward()        
            torch.nn.utils.clip_grad_norm(model.parameters(), 1e-1)
            optimizer.step()

            if CUDA:
                torch.cuda.synchronize()
            global_step += 1

        # evaluation
        if epoch % log_every == 0:
            print('Epoch %s. Global step %s. T=%s min' % (epoch, global_step, (time() - start_time) / 60.))
            print('Loss               : %s' % loss.data[0])

        # in-batch
        _, idx = torch.max(prediction, 1)
        _labels = label.data.tolist()
        _predictions = idx.data.tolist()
        acc = accuracy_score(_labels, _predictions)
        f1 = f1_score(_labels, _predictions)
        writer.add_scalar('accuracy_train', acc, global_step=global_step)
        writer.add_scalar('f1_train', f1, global_step=global_step)
        if epoch % log_every == 0:
            print('In-batch accuracy  :', acc)

        # validation
        metrics = get_metrics(model, val_dataloader)
        if epoch % log_every == 0:
            print('Validation accuracy: %s, f1: %s' % (metrics['accuracy'], metrics['f1']))
            print()

        writer.add_scalar('accuracy_val', metrics['accuracy'], global_step=global_step)
        writer.add_scalar('f1_val', metrics['f1'], global_step=global_step)

    with open('models/%s.torch' % run_name.split('/')[-1], 'wb') as f:
        try:
            torch.save(model, f)
        except Exception as e:
            print(e)
            print('Continuing (probably) without saving')
        
    # Test
    model.eval()

    metrics_test = None

    print('Calculating validation metrics... Time %s min' % ((time() - start_time) / 60.))
    metrics_train = get_metrics(model, dataloader)
    acc_train = metrics_train['accuracy']
    f1_train = metrics_train['f1']

    for test_noise in tqdm(NOISE_LEVELS):
        metrics = get_metrics(model, test, test_noise)
        if test_noise == noise_level:
            metrics_test = metrics

        acc_test = metrics['accuracy']
        f1_test = metrics['f1']
        results.append(mk_dataline(
            model_type='FasttextGRU', epochs=epochs, lr=lr,
            noise_level_train=noise_level, acc_train=acc_train, f1_train=f1_train,
            noise_level_test=test_noise, acc_test=acc_test, f1_test=f1_test,
            dropout=dropout, model=model,
            init_function=None,
            run_name=run_name,
            task=task
        ))
    
    # test original
    metrics = get_metrics(model, test_original)
    results.append(mk_dataline(
        model_type='FasttextGRU', epochs=epochs, lr=lr,
        noise_level_train=noise_level, acc_train=acc_train, f1_train=f1_train,
        noise_level_test=-1, acc_test=metrics['accuracy'], f1_test=metrics['f1'],
        dropout=dropout, model=model,
        init_function=None,
        run_name=run_name,
        task=task
    ))
    
    print('Original dataset: acc %s, f1 %s' % (metrics['accuracy'], metrics['f1']))
    writer.add_scalar('accuracy_test_original', metrics['accuracy'], global_step=global_step)
    writer.add_scalar('f1_test_original', metrics['f1'], global_step=global_step)

    print('Final test metrics: %s, Time %s min' % (metrics_test, ((time() - start_time) / 60.)))
    if metrics_test is not None:
        writer.add_scalar('accuracy_test_final', metrics_test['accuracy'], global_step=global_step)
        writer.add_scalar('f1_test_final', metrics_test['f1'], global_step=global_step)
    print()
    # model is in EVAL mode!
    return model<jupyter_output><empty_output><jupyter_text>Main exp<jupyter_code>test.text_field, MAXLEN
test.data.sample()
test._tokenize('Hello, apple.inc!')
for noise_level in tqdm(NOISE_LEVELS):
    model = run_model_with(
        noise_level=noise_level, hidden=256, lr=1e-4, dropout=0.5, layers=1,
        log_every=10, comment='_IMDB'
    )
    pd.DataFrame(results).to_csv('results/FastTextGRU_IMDB.csv')
model = run_model_with(
    noise_level=0, hidden=256, lr=1e-4, dropout=0.5, layers=1,
    log_every=10, comment='_IMDB_unchanged'
)
pd.DataFrame(results[-12:])
model = run_model_with(
    noise_level=0, hidden=256, lr=1e-4, dropout=0.5, layers=1,
    log_every=10, comment='_mokoron'
)
pd.DataFrame(results[13:]).to_csv('results/FastTextGRU_mokoron_dropout_0.csv')
pd.DataFrame(results).to_csv('results/FastTextGRU_mokoron.csv')
%%time
# GRU
model = run_model_with(noise_level=0, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=1, comment='_test')
%%time
# GRU
model = run_model_with(noise_level=0, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=10)
%%time
# SRU
model = run_model_with(noise_level=0, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=10)
%%time
# SRU
model = run_model_with(noise_level=0, hidden=256, lr=1e-4, dropout=0, layers=2, epochs=10)
%%time
# QRNN
model = run_model_with(noise_level=0, hidden=256, lr=1e-4, dropout=0, layers=2, epochs=10)
model = run_model_with(noise_level=0, hidden=256, lr=1e-4, dropout=0, layers=2, epochs=10, _model=model)
model = run_model_with(noise_level=0.05, hidden=256, lr=1e-4, dropout=0, layers=2, epochs=20)
model = run_model_with(noise_level=0.1, hidden=256, lr=1e-4, dropout=0, layers=2, epochs=20)
%%time
# GRU
model = run_model_with(noise_level=0, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=15)
%%time
# GRU
model = run_model_with(noise_level=0.1, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=15)
%%time
# GRU
model = run_model_with(noise_level=0.05, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=15)
%%time
# GRU
model = run_model_with(noise_level=0.01, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=15)
%%time
# GRU
model = run_model_with(noise_level=0.025, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=15)
%%time
# GRU
model = run_model_with(noise_level=0.075, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=15)
%%time
# GRU
model = run_model_with(noise_level=0.125, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=15)
%%time
# GRU
model = run_model_with(noise_level=0.15, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=15)
%%time
# GRU
model = run_model_with(noise_level=0.175, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=15)
%%time
# GRU
model = run_model_with(noise_level=0.2, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=15)
%%time
# LSTM
model = run_model_with(noise_level=0, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=15, type_='LSTM')
%%time
# LSTM
model = run_model_with(noise_level=0.05, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=15, type_='LSTM')
%%time
# LSTM
model = run_model_with(noise_level=0.1, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=15, type_='LSTM')
%%time
# SRU
model = run_model_with(noise_level=0, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=15, type_='SRU')
%%time
# QRNN
model = run_model_with(noise_level=0, hidden=256, lr=1e-4, dropout=0, layers=1, epochs=15, type_='QRNN')<jupyter_output>Global step: 704
Loss after epoch 0: 0.7177828550338745
In-batch accuracy: 0.25
Validation accuracy: 0.518<jupyter_text>Results  GRU and LSTM данные захадкожены, потому что доставать их из tensorboard не то, чтобы удобно<jupyter_code>x_noise = [0, 0.1, 0.05, 0.01, 0.025, 0.075, 0.125, 0.15, 0.175, 0.2]
y_acc = [0.8658, 0.7634, 0.8094, 0.8572, 0.8122, 0.7668, 0.7303, 0.7190, 0.6927, 0.6565]
gru_vals = sorted(zip(x_noise, y_acc), key=lambda x: x[1])
gru_noise, gru_acc = zip(*gru_vals)
lstm_noise = [0, 0.05, 0.1]
lstm_acc = [0.8259, 0.7590, 0.6667]
import matplotlib.pyplot as plt
import seaborn
%pylab inline
seaborn.set()
plt.plot(gru_noise, gru_acc)
plt.plot(lstm_noise, lstm_acc, 'r')
model<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1267790353135474, 'fraction_numerical': 0.0296303492495264, 'mean_word_length': 3.349883794633425, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 47, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '295572', 'license': 'MIT', 'n_tokens_mistral': 7787, 'n_tokens_neox': 7338, 'n_words': 1470, 'path': 'notebooks/10_fasttext-rnn.ipynb', 'repo_name': 'Guitaricet/text_classification', 'size': '20587'}