<jupyter_start><jupyter_text>Keras ELMo TutorialThis IPython Notebook contains an example how the ELMo embeddings from the paper *[Deep contextualized word representations (Peters et al., 2018)](http://arxiv.org/abs/1802.05365)* can be used for document classification. As the computation of the embeddings is computationally expensive, we will include it into a preprocessing step:* We read in the dataset (here the IMDB dataset)* Text is tokenized and truncated to a fix length* Each text is fed as a sentence to the AllenNLP ElmoEmbedder to get a 1024 dimensional embedding for each word in the document* These embeddings are then fed to our neural network that we trainComputing the embeddings once in the pre-processing significantly reduces the overall computational time. Otherwise, they would be computed for each epoch. However, this requires that enough memory is available, as our transformed dataset will constist of $\text{number_of_tokens} \cdot 1024$ float32 numbers.**Note:** Our simple tokenization process ignores sentence boundaries and the complete document is fed as one single sentence to the ElmoEmbedder. As the ELMo embeddings are defined sentence wise, it would be better to first identify the sentences in a document and to process the document sentence by sentence to get the correct embeddings.**Requirements**:* Python 3.6 - lower versions of Python do not work* AllenNLP 0.5.1 - to compute the ELMo representations* Keras 2.2.0 - For the creation of BiLSTM-CNN-CRF architectureSee [https://github.com/UKPLab/elmo-bilstm-cnn-crf](https://github.com/UKPLab/elmo-bilstm-cnn-crf) how to install these requirements. Code<jupyter_code>import keras
import os
import sys
from allennlp.commands.elmo import ElmoEmbedder
import numpy as np
import random
from keras.models import Sequential
from keras.layers import Dense, Conv1D, GlobalMaxPooling1D, GlobalAveragePooling1D, Activation, Dropout
# Load all files from a directory into dictionaries
def load_directory_data(directory, label):
    data = []
    for file_path in os.listdir(directory):
        with open(os.path.join(directory, file_path), "r") as f:
            data.append({"text": f.read().replace("<br />", " "), "label": label})
    return data

# Load the positive and negative examples from the dataset
def load_dataset(directory):
    pos_data = load_directory_data(os.path.join(directory, "pos"), 1)
    neg_data = load_directory_data(os.path.join(directory, "neg"), 0)
    return pos_data+neg_data

# Download and process the IMDB dataset
def download_and_load_datasets(force_download=False):
    dataset = keras.utils.get_file(
      fname="aclImdb.tar.gz", 
      origin="http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz", 
      extract=True)

    train_data = load_dataset(os.path.join(os.path.dirname(dataset), "aclImdb", "train"))
    test_data = load_dataset(os.path.join(os.path.dirname(dataset), "aclImdb", "test"))

    return train_data, test_data

train_data, test_data = download_and_load_datasets()

random.shuffle(train_data)
random.shuffle(test_data)
#Tokenize text. Note, it would be better to first split it into sentences.
def tokenize_text(documents, max_tokens):
    for document in documents:
        document['tokens'] = keras.preprocessing.text.text_to_word_sequence(document['text'], lower=False)
        document['tokens'] = document['tokens'][0:max_tokens]
 
max_tokens = 100
tokenize_text(train_data, max_tokens)
tokenize_text(test_data, max_tokens)
# Lookup the ELMo embeddings for all documents (all sentences) in our dataset. Store those
# in a numpy matrix so that we must compute the ELMo embeddings only once.
def create_elmo_embeddings(elmo, documents, max_sentences = 1000):
    num_sentences = min(max_sentences, len(documents)) if max_sentences > 0 else len(documents)
    print("\n\n:: Lookup of "+str(num_sentences)+" ELMo representations. This takes a while ::")
    embeddings = []
    labels = []
    tokens = [document['tokens'] for document in documents]
    
    documentIdx = 0
    for elmo_embedding in elmo.embed_sentences(tokens):  
        document = documents[documentIdx]
        # Average the 3 layers returned from ELMo
        avg_elmo_embedding = np.average(elmo_embedding, axis=0)
             
        embeddings.append(avg_elmo_embedding)        
        labels.append(document['label'])
            
        # Some progress info
        documentIdx += 1
        percent = 100.0 * documentIdx / num_sentences
        line = '[{0}{1}]'.format('=' * int(percent / 2), ' ' * (50 - int(percent / 2)))
        status = '\r{0:3.0f}%{1} {2:3d}/{3:3d} sentences'
        sys.stdout.write(status.format(percent, line, documentIdx, num_sentences))
        
        if max_sentences > 0 and documentIdx >= max_sentences:
            break
            
    return embeddings, labels


elmo = ElmoEmbedder(cuda_device=1) #Set cuda_device to the ID of your GPU if you have one
train_x, train_y = create_elmo_embeddings(elmo, train_data, 1000)
test_x, test_y  = create_elmo_embeddings(elmo, test_data, 1000)
# :: Pad the x matrix to uniform length ::
def pad_x_matrix(x_matrix):
    for sentenceIdx in range(len(x_matrix)):
        sent = x_matrix[sentenceIdx]
        sentence_vec = np.array(sent, dtype=np.float32)
        padding_length = max_tokens - sentence_vec.shape[0]
        if padding_length > 0:
            x_matrix[sentenceIdx] = np.append(sent, np.zeros((padding_length, sentence_vec.shape[1])), axis=0)

    matrix = np.array(x_matrix, dtype=np.float32)
    return matrix

train_x = pad_x_matrix(train_x)
train_y = np.array(train_y)

test_x = pad_x_matrix(test_x)
test_y = np.array(test_y)

print("Shape Train X:", train_x.shape)
print("Shape Test Y:", test_x.shape)
# Simple model for sentence / document classification using CNN + global max pooling
model = Sequential()
model.add(Conv1D(filters=250, kernel_size=3, padding='same'))
model.add(GlobalMaxPooling1D())
model.add(Dense(250, activation='relu'))
model.add(Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.fit(train_x, train_y, validation_data=(test_x, test_y), epochs=10, batch_size=32)


model.summary()<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv1d_1 (Conv1D)            (None, 100, 250)          768250    
_________________________________________________________________
global_max_pooling1d_1 (Glob (None, 250)               0         
_________________________________________________________________
dense_1 (Dense)              (None, 250)               62750     
_________________________________________________________________
dense_2 (Dense)              (None, 1)                 251       
=================================================================
Total params: 831,251
Trainable params: 831,251
Non-trainable params: 0
_________________________________________________________________
{'fraction_non_alphanumeric': 0.1101072840203275, 'fraction_numerical': 0.020892151326933936, 'mean_word_length': 4.259836674090572, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '390723', 'license': 'Apache-2.0', 'n_tokens_mistral': 2137, 'n_tokens_neox': 1990, 'n_words': 613, 'path': 'Keras_ELMo_Tutorial.ipynb', 'repo_name': 'psorianom/elmo-bilstm-cnn-crf', 'size': '7084'}