<jupyter_start><jupyter_text>Character-based language model with AllenNLP<jupyter_code>import allennlp
print(allennlp.__version__)
# standard libraries
import numpy as np
import pandas as pd

# torch libraries
import torch
import torch.optim as optim

# standard libraries
from matplotlib import pyplot as plt
from collections import Counter
from typing import Dict, List, Tuple, Set, Iterable

# AllenNLP libraries
from allennlp.common.file_utils import cached_path
from allennlp.common.util import START_SYMBOL, END_SYMBOL
from allennlp.data.fields import TextField
from allennlp.data.instance import Instance
from allennlp.data.dataloader import DataLoader
from allennlp.data.dataset_readers.dataset_reader import AllennlpDataset
from allennlp.data.token_indexers import TokenIndexer, SingleIdTokenIndexer
from allennlp.data.tokenizers import Token, CharacterTokenizer
from allennlp.data.vocabulary import Vocabulary, DEFAULT_PADDING_TOKEN
from allennlp.models import Model
from allennlp.modules.seq2seq_encoders import PytorchSeq2SeqWrapper
from allennlp.modules.text_field_embedders import TextFieldEmbedder, BasicTextFieldEmbedder
from allennlp.modules.token_embedders import Embedding
from allennlp.nn.util import get_text_field_mask, sequence_cross_entropy_with_logits
from allennlp.training.trainer import Trainer
full_data = pd.read_csv('/home/adelard/ml/manning/dataset/stackexchange_full_data_tokenized.csv.gz',
                  compression='gzip').sample(frac = 1, random_state = 42).reset_index(drop = True)
# split string into a list of characters
text = ''.join(full_data.text.values).lower()
characters = [char for char in text]
unique_characters = np.unique(characters)
print(len(unique_characters))
print()
print(unique_characters[:100])
# define the vocabulary by keeping the most common characters in the dataset
char_count = Counter(characters)
print(len(char_count.most_common()))
print(char_count.most_common())
# MAX_VOCAB_SIZE = 50
MAX_VOCAB_SIZE = 50
chars_most_common = [char[0] for char in char_count.most_common(MAX_VOCAB_SIZE)]
chars_most_common.sort()<jupyter_output><empty_output><jupyter_text>Training data<jupyter_code># reducing the dataset 
POSTS_TYPE = 'title'
SAMPLE_COUNT = 10000
# subsample the full dataset
small_data = full_data[(full_data.category == POSTS_TYPE)].sample(SAMPLE_COUNT).reset_index(drop=True)
print("data shape: ", small_data.shape)
print(small_data.text.sample(5).values)<jupyter_output><empty_output><jupyter_text>AllenNLP structure InstancesAn instance composed of the input and output objects. ModelModel uses a forward() method that takes tensor inputs and produces a dict of tensor outputs that includes the loss used to train the model. Tokenizer and instancesThe AllenNLP CharacterTokenizer is used to splitting the sentences into list of characters.<jupyter_code>tokenizer = CharacterTokenizer()
# train set is composed of tokenized sequences of the original text
train_data = small_data.text.apply(lambda txt: tokenizer.tokenize(txt.lower())).values
# function to take a list of tokens and an indexer as input and returns 
# an instance composed of the input and output tokens.

def tokens_to_instance(toks: List[Token], token_indexers: Dict[str, TokenIndexer]):
    tokens = list(toks)
    tokens.insert(0, Token(START_SYMBOL))
    tokens.append(Token(END_SYMBOL))
    
    input_field = TextField(tokens[:-1], token_indexers)
    output_field = TextField(tokens[1:], token_indexers)
    return Instance({'input_tokens': input_field, 'output_tokens': output_field})
# The SingleIdTokenIndexer takes care of the mapping between the
# tokens and their unique index in the vocabulary. 
token_indexers = {'tokens': SingleIdTokenIndexer()}
instances = [tokens_to_instance(tokens, token_indexers) for tokens in train_data]<jupyter_output><empty_output><jupyter_text>Vocabulary<jupyter_code>token_counts = {char: 1 for char in chars_most_common}
vocab = Vocabulary({'tokens': token_counts})<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>EMBEDDING_SIZE = 32
HIDDEN_SIZE = 256
BATCH_SIZE = 128
class RNNModel(Model):
    
    def __init__(self, embedder: TextFieldEmbedder,
                hidden_size: int,
                max_len: int,
                vocab: Vocabulary) -> None:
        super().__init__(vocab)
        
        self.embedder = embedder
        
        # initialize a seq2seq encodeer, LSTM
        self.rnn = PytorchSeq2SeqWrapper(
        torch.nn.LSTM(EMBEDDING_SIZE, HIDDEN_SIZE, batch_first=True))
        
        self.hidden2out = torch.nn.Linear(in_features=self.rnn.get_output_dim(),
                                         out_features=vocab.get_vocab_size('tokens'))
        self.hidden_size = hidden_size
        self.max_len = max_len
        
    def forward(self, input_tokens, output_tokens):
        mask = get_text_field_mask(input_tokens)
        embeddings = self.embedder(input_tokens)
        rnn_hidden = self.rnn(embeddings, mask)
        out_logits = self.hidden2out(rnn_hidden)
        loss = sequence_cross_entropy_with_logits(out_logits, output_tokens['tokens'], mask)
        
        return {'loss': loss}
    
    def generate_text(self)-> Tuple[List[Token], torch.tensor]:
        
        start_symbol_index = self.vocab.get_token_index(START_SYMBOL, 'tokens')
        end_symbol_index = self.vocab.get_token_index(END_SYMBOL, 'tokens')
        padding_symbol_index = self.vocab.get_token_indexn(DEFAULT_PADDING_TOKEN, 'tokens')
        
        log_likelihood = 0.
        result_words = []
        state = (torch.zeros(1, 1, self.hidden_size), torch.zeros(1, 1, self.hidden_size))
        
        word_index = start_symbol_index
        
        for i in range(self.max_len):
            tokens = torch.tensor([[word_index]])
            
            embeddings = self.embedder({'tokens': tokens})
            output, state = self.rnn._module(embeddings, state)
            output = self.hidden2out(output)
            
            log_prob = torch.log_softmax(output[0, 0], dim=0)
            
            dist = torch.exp(log_prob)
            word_index = start_symbol_index
            
            while word_index in {start_symbol_index, padding_symbol_index}:
                word_index = torch.multinomial(dist, num_samples=1, replacement=False).item()
                
            log_likelihood += log_prob[word_index]
            
            if word_index == end_symbol_index:
                break
                
            token = Token(text-self.vocab.get_token_from_index(word_index, 'tokens'))
            words.append(token)
            
        return result_words, log_likelihood
# instantiate the embeddings, the rnn model, the optimizer
# the trainer
token_embedding = Embedding(num_embeddings=vocab.get_vocab_size('tokens'), 
                            embedding_dim=EMBEDDING_SIZE)
embedder = BasicTextFieldEmbedder({"tokens": token_embedding})

model = RNNModel(embedder=embedder, hidden_size=HIDDEN_SIZE, max_len=80, vocab=vocab)
dataset = AllennlpDataset(instances, vocab)
loader = DataLoader(dataset, batch_size=BATCH_SIZE)
loader = next(iter(loader))
optimizer = optim.Adam(rnn_model.parameters(), lr=5.e-3)
trainer = Trainer(model=model,
                 optimizer=optimizer,
                 iterator=loader,
                 train_dataset=instances_tokens,
                 num_epochs=20)
def predict(txt: str, model: Model) -> float:
    tokenizer = CharacterTokenizer()
    tokens = tokenizer.tokenizer(txt)
    
    token_indexers = {'tokens': SingleIdTokenIndexer()}
    instance = tokens_to_instance(tokens, token_indexers)
    output = model.forward_on_instance(instance)
    print(output)<jupyter_output><empty_output><jupyter_text>Text generation<jupyter_code>for _ in range(25):
    tokens, _ = model.generate()
    print(''.join(token.text for token in tokens))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08663617886178862, 'fraction_numerical': 0.0067327235772357726, 'mean_word_length': 3.920625, 'pattern_counts': {'":': 1, '<': 25, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '117949', 'license': 'MIT', 'n_tokens_mistral': 2451, 'n_tokens_neox': 2404, 'n_words': 528, 'path': '04_character_based_language_model_with_allenNLP.ipynb', 'repo_name': 'abanza/building_domain_specific_nlp_models', 'size': '7872'}