<jupyter_start><jupyter_text>Same thing as `set_format`, but not in-place.<jupyter_code>train_data[0]<jupyter_output><empty_output><jupyter_text>Use `output_all_columns=True` to keep non-converted columns.<jupyter_code>class NBoW(nn.Module):
    def __init__(self, vocab_size, embedding_dim, output_dim, pad_index):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx=pad_index)
        self.fc = nn.Linear(embedding_dim, output_dim)

    def forward(self, text):
        # text = [batch size, seq len]
        embedded = self.embedding(text)
        # embedded = [batch size, seq len, embedding dim]
        pooled = embedded.mean(dim=1)
        # pooled = [batch size, embedding dim]
        prediction = self.fc(pooled)
        # prediction = [batch size, output dim]
        return prediction
vocab_size = len(vocab)
embedding_dim = 300
output_dim = len(train_data.unique('label'))

model = NBoW(vocab_size, embedding_dim, output_dim, pad_index)
vectors = torchtext.vocab.FastText()
hello_vector = vectors.get_vecs_by_tokens('hello')
hello_vector.shape
hello_vector
pretrained_embedding = vectors.get_vecs_by_tokens(vocab.get_itos())
pretrained_embedding.shape
model.embedding.weight
pretrained_embedding
model.embedding.weight.data = pretrained_embedding
model.embedding.weight
optimizer = optim.Adam(model.parameters())
criterion = nn.CrossEntropyLoss()
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

device
model = model.to(device)
criterion = criterion.to(device)
def collate(batch, pad_index):
    batch_ids = [i['ids'] for i in batch]
    batch_ids = nn.utils.rnn.pad_sequence(batch_ids, padding_value=pad_index, batch_first=True)
    batch_labels = [i['label'] for i in batch]
    batch_labels = torch.stack(batch_labels)
    batch = {'ids': batch_ids,
             'label': batch_labels}
    return batch
batch_size = 512

collate = functools.partial(collate, pad_index=pad_index)

train_dataloader = torch.utils.data.DataLoader(train_data, 
                                               batch_size=batch_size, 
                                               collate_fn=collate, 
                                               shuffle=True)

valid_dataloader = torch.utils.data.DataLoader(valid_data, batch_size=batch_size, collate_fn=collate)
test_dataloader = torch.utils.data.DataLoader(test_data, batch_size=batch_size, collate_fn=collate)
def train(dataloader, model, criterion, optimizer, device):

    model.train()
    epoch_loss = 0
    epoch_accuracy = 0

    for batch in dataloader:
        tokens = batch['ids'].to(device)
        label = batch['label'].to(device)
        prediction = model(tokens)
        loss = criterion(prediction, label)
        accuracy = get_accuracy(prediction, label)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        epoch_loss += loss.item()
        epoch_accuracy += accuracy.item()

    return epoch_loss / len(dataloader), epoch_accuracy / len(dataloader)
def evaluate(dataloader, model, criterion, device):
    
    model.eval()
    epoch_loss = 0
    epoch_accuracy = 0

    with torch.no_grad():
        for batch in dataloader:
            tokens = batch['ids'].to(device)
            label = batch['label'].to(device)
            prediction = model(tokens)
            loss = criterion(prediction, label)
            accuracy = get_accuracy(prediction, label)
            epoch_loss += loss.item()
            epoch_accuracy += accuracy.item()

    return epoch_loss / len(dataloader), epoch_accuracy / len(dataloader)
def get_accuracy(prediction, label):
    batch_size = prediction.shape[0]
    predicted_classes = prediction.argmax(dim=-1)
    correct_predictions = predicted_classes.eq(label).sum()
    accuracy = correct_predictions / batch_size
    return accuracy
n_epochs = 10
best_valid_loss = float('inf')

for epoch in range(n_epochs):

    train_loss, train_acc = train(train_dataloader, model, criterion, optimizer, device)
    valid_loss, valid_acc = evaluate(valid_dataloader, model, criterion, device)

    if valid_loss < best_valid_loss:
        best_valid_loss = valid_loss
        torch.save(model.state_dict(), 'nbow.pt')
    
    print(f'epoch: {epoch+1}')
    print(f'train_loss: {train_loss:.3f}, train_acc: {train_acc:.3f}')
    print(f'valid_loss: {valid_loss:.3f}, valid_acc: {valid_acc:.3f}')
model.load_state_dict(torch.load('nbow.pt'))

test_loss, test_acc = evaluate(test_dataloader, model, criterion, device)

print(f'test_loss: {test_loss:.3f}, test_acc: {test_acc:.3f}')
def predict_sentiment(text, model, tokenizer, vocab, device):
    tokens = tokenizer(text)
    ids = [vocab[t] for t in tokens]
    tensor = torch.LongTensor(ids).unsqueeze(dim=0).to(device)
    prediction = model(tensor).squeeze(dim=0)
    probability = torch.softmax(prediction, dim=-1)
    predicted_class = prediction.argmax(dim=-1).item()
    predicted_probability = probability[predicted_class].item()
    return predicted_class, predicted_probability
text = "This film is terrible!"

predict_sentiment(text, model, tokenizer, vocab, device)
text = "This film is great!"

predict_sentiment(text, model, tokenizer, vocab, device)
text = "This film is not terrible, it's great!"

predict_sentiment(text, model, tokenizer, vocab, device)
text = "This film is not great, it's terrible!"

predict_sentiment(text, model, tokenizer, vocab, device)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11196628183983874, 'fraction_numerical': 0.004947773501924134, 'mean_word_length': 3.7585004359197907, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '641229', 'license': 'MIT', 'n_tokens_mistral': 1796, 'n_tokens_neox': 1766, 'n_words': 359, 'path': '1_nbow.ipynb', 'repo_name': 'Abednego97/pytorch-sentiment-analysis', 'size': '5457'}