<jupyter_start><jupyter_text>Data Preparation<jupyter_code>text = Field(
    lower=True,
    tokenize="spacy",
    batch_first=True,
)
label = Field(
    is_target=True,
    batch_first=True
)
# download the dataset
train, val = imdb.IMDB.splits(text_field=text, label_field=label)
# build vocabulary
text.build_vocab(train, min_freq=2)
label.build_vocab(train)
# create data loaders
BATCH_SIZE = 64

train_loader, val_loader = BucketIterator.splits(
    datasets=(train, val),
    batch_sizes=(BATCH_SIZE, BATCH_SIZE),
    device=device
)
x, y = next(iter(train_loader))
print(x.shape, y.shape)
for batch in train_loader:
    print(batch.text.shape)
    print(batch.label.shape)
    break
len(train_loader)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>class GRU(nn.Module):
    
    def __init__(self, vocab_size, embedding_dim, hidden_size, n_classes = 4, dropout = 0.15, num_layers = 4):
        
        super(GRU, self).__init__()
        
        self.embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=embedding_dim)
        self.gru = nn.GRU(
            input_size = embedding_dim, 
            hidden_size = hidden_size, 
            num_layers = num_layers,
            dropout = dropout,
            bidirectional = True,
            batch_first =  True
        )
        
        self.fc = nn.Linear(in_features=hidden_size, out_features=n_classes)
        
    
    
    def forward(self, x, hidden=None):
        # a better initialization of initial hidden state could be very useful
        embedded = self.embedding(x)
        
        _, hidden = self.gru(embedded)
        # since gru is multilayer bidirectional , hidden shape will [2*num_layers, batch, hidden_size]
        
        hidden_mean = hidden.mean(dim=0)
        
        out = F.softmax(self.fc(hidden_mean), dim=1)
        
        return out<jupyter_output><empty_output><jupyter_text>Training<jupyter_code># !pip install pytorch-lightning
import pytorch_lightning as pl
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
def accuracy(y, y_):
    
    correct = 0
    for i in range(y.shape[0]):
        if y[i]==y_[i]:
            correct += 1
    return torch.tensor([correct/y.shape[0]])
class Model(pl.LightningModule):
    
    
    def __init__(self, vocab_size, embedding_dim, hidden_size, n_classes = 4, dropout = 0.15, num_layers = 4):

        super(Model, self).__init__()
        # initialize model
        self.model = GRU(
            vocab_size=vocab_size, 
            embedding_dim=embedding_dim, 
            hidden_size=hidden_size,
            n_classes=n_classes,
            dropout=dropout,
            num_layers=num_layers
        )
    
    
    def forward(self, x):
        outputs = self.model(x)
        return outputs
        
    
    def configure_optimizers(self):
        return optim.Adam(params=self.parameters(), lr=1e-3)
    
    def train_dataloader(self):
        return train_loader
    
    def training_step(self, batch, batch_idx):
        x, y = batch.text, batch.label
        outputs = self(x)
        loss =  F.cross_entropy(outputs, y.view(-1))
        acc = accuracy(y, torch.argmax(outputs, dim=1))
        tensorboard_logs = {'train_loss': loss, 'train_acc': acc}
        return {'loss': loss, 'acc':acc, 'log': tensorboard_logs}
    
    def val_dataloader(self):
        return val_loader
    
    def validation_step(self, batch, batch_idx):
        x =  batch.text
        y = batch.label
        outputs = self(x)    
        loss =  F.cross_entropy(outputs, y.view(-1))
        acc = accuracy(y, torch.argmax(outputs, dim=1))
        return {"val_loss":loss, "val_acc":acc}
    
    def validation_epoch_end(self, outputs):
        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
        avg_acc = [torch.stack([x['val_acc'] for x in outputs]).mean()]
        tensorboard_logs = {'val_loss': avg_loss}
        return {'val_loss': avg_loss, 'log': tensorboard_logs}  
model = Model(vocab_size=len(text.vocab),
    embedding_dim=100,
    hidden_size=64)
early_stopping = EarlyStopping(monitor='val_loss', min_delta=0.05)
checkpoint = ModelCheckpoint(filepath="./model.pth")

trainer = pl.Trainer(
    max_epochs=5, 
    gpus=[0], 
    show_progress_bar=True, 
    checkpoint_callback=checkpoint, 
    early_stop_callback=early_stopping
  )
trainer.fit(model)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10936789449749887, 'fraction_numerical': 0.008185538881309686, 'mean_word_length': 2.795513373597929, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '540019', 'license': 'MIT', 'n_tokens_mistral': 1506, 'n_tokens_neox': 1471, 'n_words': 279, 'path': '15. Simple-Sentiment-Analysis.ipynb', 'repo_name': 'macabdul9/deep-learning-with-pytorch', 'size': '4398'}