<jupyter_start><jupyter_text>SentiPy Tutorial *An Application for Twitter Sentiment Analytics***SentiPy** provides models to analyze user's sentiments from tweets. The models are based on **Word Embeddings** and **Convolutional Neural Network** (CNN).<jupyter_code>import matplotlib.pyplot as plt

# This is a bit of magic to make matplotlib figures appear inline in the notebook
# rather than in a new window.
%matplotlib inline
plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.style.use('seaborn-darkgrid')

# Some more magic so that the notebook will reload external python modules;
# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython
%load_ext autoreload
%autoreload 2<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload<jupyter_text>1. Model Implementation 1.1. Preprocessing the dataThe dataset used for this models is taken from [Sentiment140](http://www.sentiment140.com/) dataset which is composed of 1.6b tweets. The dataset is availble at [Stanford University](http://cs.stanford.edu/people/alecmgo/trainingandtestdata.zip). More help can be found at [this link](http://help.sentiment140.com/for-students/).The custom torch dataset *Sentiment140* extract tweets and labels from the csv dataset, and download it in case you don't have it. Because *Sentiment140* class inherits from Dataset class, you have access to splits, iters and others default methods.<jupyter_code>import torch
from torchtext import data

from sentipy.datasets import Sentiment140
from sentipy.tokenizer import tokenizer_tweets
import numpy as np
SEED = 2020
np.random.seed(SEED)
torch.manual_seed(SEED)
TEXT = data.Field(tokenize = tokenizer_tweets, batch_first = True)
LABEL = data.LabelField(preprocessing = lambda y: y//2, dtype = torch.float)

# Get the training data / validation data / test data
train_data, test_data = Sentiment140.splits(TEXT, LABEL, keepneutral=False, size=50000, shuffle=True)
train_data, valid_data = train_data.split(0.8)
print("Labels")
print([vars(train_data[i])["label"] for i in range(10)])

print(vars(train_data[0]))<jupyter_output><empty_output><jupyter_text>Now, let's take a look at the loaded dataset.<jupyter_code>print(f'Number of training examples: {len(train_data)}')
print(f'Number of testing examples: {len(test_data)}')
print("Sanity Check\nTraining dataset length: {}".format(len(train_data) + len(valid_data)))
print("\nExample n°0: {}".format(vars(train_data[0])))<jupyter_output>Sanity Check
Training dataset length: 50000

Example n°0: {'label': 2, 'id': 1759774113, 'date': 'Sun May 10 19:56:48 PDT 2009', 'query': 'NO_QUERY', 'user': 'melux', 'text': ['playing', 'medal', 'of', 'honor', 'with', 'my', 'brother']}<jupyter_text>1.2. Processing the data<jupyter_code>VOCAB_SIZE = 20000

# Create the vocabulary for words embeddings
TEXT.build_vocab(train_data, 
                 max_size = VOCAB_SIZE, 
                 vectors = "glove.6B.100d", 
                 # vectors = "glove.twitter.27B.100d", 
                 unk_init = torch.Tensor.normal_)
LABEL.build_vocab(train_data)
print(LABEL.vocab.stoi)
print(TEXT.vocab.freqs.most_common(20))
print(TEXT.vocab.itos[:10])
glove_vectors = TEXT.vocab.vectors
print(glove_vectors)
print(f'There are {len(glove_vectors)} words in the vocabulary\n')

idx = TEXT.vocab.itos.index("<user>")
print(f"vector: {glove_vectors[idx][:10]}, size: {len(glove_vectors[idx])}")
BATCH_SIZE = 64
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

train_iterator = data.BucketIterator(train_data, 
                             shuffle=True,
                             batch_size = BATCH_SIZE, 
                             device = device)
valid_iterator = data.BucketIterator(valid_data,
                             shuffle=True,
                             batch_size = BATCH_SIZE, 
                             device = device)
test_iterator = data.BucketIterator(test_data,
                             shuffle=True,
                             batch_size = BATCH_SIZE, 
                             device = device)
print([vars(valid_data[i])["label"] for i in range(10)])
for (idx, batch) in enumerate(train_iterator):
    print(batch.label)
    break
for (idx, batch) in enumerate(test_iterator):
    print(batch.label)
    break<jupyter_output>[0, 2, 2, 0, 0, 0, 2, 2, 0, 2]
tensor([1., 1., 1., 1., 1., 0., 1., 1., 0., 0., 1., 1., 0., 1., 1., 1., 0., 0.,
        0., 1., 0., 0., 1., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 0.,
        1., 1., 0., 0., 1., 1., 1., 1., 0., 1., 0., 0., 0., 0., 1., 0., 1., 1.,
        1., 0., 1., 1., 1., 0., 0., 1., 1., 0.])
tensor([0., 1., 1., 1., 1., 1., 1., 0., 1., 0., 1., 0., 1., 1., 1., 1., 0., 0.,
        0., 0., 0., 0., 1., 1., 1., 1., 0., 1., 1., 0., 0., 1., 0., 0., 0., 0.,
        1., 1., 0., 0., 1., 1., 1., 0., 0., 0., 0., 1., 0., 0., 1., 0., 1., 0.,
        0., 0., 1., 1., 1., 0., 1., 1., 1., 1.])<jupyter_text>1.3. Creating the model<jupyter_code>import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

from sentipy.model import CNN

# Hyperparameters
EPOCHS = 50
EMBEDDING_DIM = 100
N_FITLERS = 100
FILTER_SIZES = [2, 3, 4, 5, 8]
OUTPUT_DIM = 2
DROPOUT = 0.5
ACTIVATION_LAYER = F.relu
ACTIVATION_OUTPUT = F.sigmoid
LR = 0.1
WEIGHT_DECAY = 0.001
# Model
INPUT_DIM = len(TEXT.vocab)
PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]   
model = CNN(INPUT_DIM, EMBEDDING_DIM, N_FITLERS, FILTER_SIZES, 
            OUTPUT_DIM, DROPOUT, pad_idx = PAD_IDX, 
            activation_layer = ACTIVATION_LAYER,
            activation_output = ACTIVATION_OUTPUT)

# Optimization
optimizer = optim.Adadelta(model.parameters(), lr = LR, weight_decay = WEIGHT_DECAY)
criterion = nn.CrossEntropyLoss()
model = model.to(device)
criterion = criterion.to(device)<jupyter_output><empty_output><jupyter_text>1.4. Training the model<jupyter_code>from sentipy.optim.performer import Performer

performer = Performer(model, criterion, optimizer)
performer.run(train_iterator, valid_iterator, epochs = 10)<jupyter_output>Epoch     :   1/10<jupyter_text>1.5. Testing the model<jupyter_code>performer.test(test_iterator)
performer.results_test
performer.test(test_iterator, thresholds=(.8, .8), addneutral=True)
performer.results_test
performer.test(train_iterator, thresholds=(.8, .8), addneutral=True)
performer.results_test
TEXT = data.Field(tokenize = tokenizer_tweets, batch_first = True)
LABEL = data.LabelField(preprocessing = lambda y: y//2, dtype = torch.float)

# Get the training data / validation data / test data
train_data, test_data = Sentiment140.splits(TEXT, LABEL, keepneutral=False, size=10000, shuffle=True)
train_data, valid_data = train_data.split(0.8)
VOCAB_SIZE = 20000

# Create the vocabulary for words embeddings
TEXT.build_vocab(train_data, 
                 max_size = VOCAB_SIZE, 
                 vectors = "glove.6B.100d", 
                 # vectors = "glove.twitter.27B.100d", 
                 unk_init = torch.Tensor.normal_)
LABEL.build_vocab(train_data)
BATCH_SIZE = 64
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

train_iterator = data.BucketIterator(train_data, 
                             shuffle=True,
                             batch_size = BATCH_SIZE, 
                             device = device)
valid_iterator = data.BucketIterator(valid_data,
                             shuffle=True,
                             batch_size = BATCH_SIZE, 
                             device = device)
test_iterator = data.BucketIterator(test_data,
                             shuffle=True,
                             batch_size = BATCH_SIZE, 
                             device = device)
performer.test(test_iterator, thresholds=(.8, .8), addneutral=True)<jupyter_output>Test     :   100% | [==================================================] | Time : 0m 1s
{'fraction_non_alphanumeric': 0.13902193093017393, 'fraction_numerical': 0.0403327451474666, 'mean_word_length': 3.163168940188877, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 33, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '355126', 'license': 'MIT', 'n_tokens_mistral': 2962, 'n_tokens_neox': 2626, 'n_words': 695, 'path': 'sentipy_tutorial.ipynb', 'repo_name': 'arthurdjn/sentipy', 'size': '7934'}