<jupyter_start><jupyter_text>Pad the dataEach sentece has different lenghts. But our CNN models needs a fixed-size input. Hence, we include zeros at the ende so each sentence will have the same number of words.<jupyter_code>def max_words(sentences):
    """ Return the maximum number of words in the dataset """
    max_num_words = -1
    for sentence in sentences:
        len_sentence = len(sentence)
        if len_sentence > max_num_words:
            max_num_words =len_sentence
    return max_num_words

def pad(sentences, max_sequence=None):
    """ Pad all the sententences in order to have sequences of same length """
    if not max_sequence:
        max_sequence = max_words(sentences)
        
    padded_dataset = []
    for sentence in sentences:
        sentence = sentence.split()
        len_padding = max_sequence - len(sentence)
        padded_sentence = sentence + ['</pad>'] * len_padding
        assert len(padded_sentence) == max_sequence
        padded_dataset.append(padded_sentence)
    
    assert len(padded_dataset) == len(sentences)
    return padded_dataset, max_sequence

train_pad, max_sequence = pad(X_train)
test_pad, _ = pad(X_test, max_sequence)
X_train[0]
' '.join(train_pad[0])<jupyter_output><empty_output><jupyter_text>Create the lookup matrix and the vocabulary<jupyter_code>from collections import Counter
from itertools import chain

def build_vocab(dataset, to_lower=False):
    """ Create a lookup table and a list with the vocabulary.

    Args:
        dataset (list): a matrix with the words from the dataset

    Returns:
        vocab_sorted (list): list of the words sorted by its frequency
        lookup (dict): a dictionary with the lookup table. The keys are
            the words and the values are the indexes.
    """
    # Count how many times a word appear in the dataset
    word_counts = Counter(chain(*dataset))
    # Create a list with the most common words sorted.
    # The position will be the index of the lookup table.
    vocab_sorted = []
    for word, _ in word_counts.most_common():
        if to_lower:
            word = word.lower()
        if word not in vocab_sorted:
            vocab_sorted.append(word)
    vocab_sorted.append('<oov>')
    # Create a lookup table using a dictionary. Map each index with a word
    lookup = {word: index for index, word in enumerate(vocab_sorted)}

    # TODO: Move this to the unittest
    assert len(list(lookup.keys())) == len(vocab_sorted)

    return vocab_sorted, lookup
vocab_sorted, lookup = build_vocab(train_pad)
print("Vocabulary Size: {:d}".format(len(vocab_sorted)))
def to_indexes(sentences, vocabulary):
    """ Convert a list of sentences to its corresponding indices.

    Args:
        sentences: a list where each element is a lists of words (sentences)
        vocabulary: a dictionary of words and its corresponding index

    Returns:
        a list where each element is a list of indexes (sentences)

    """
    senteces_idx = []
    for sentence in sentences:
        aux_idx = []
        for word in sentence:
            # If the word is in the vocabulary get its index otherwise use the <oov> index
            if word in vocabulary:
                word_idx = vocabulary[word]
            else:
                word_idx = vocabulary['<oov>']
            aux_idx.append(word_idx)
        senteces_idx.append(aux_idx)
    return np.array(senteces_idx)
train_idx = to_indexes(train_pad, lookup)
test_idx = to_indexes(test_pad, lookup)
train_idx[0]
# Model Hyperparameters
sequence_length = max_sequence
embedding_dim = 50
num_filters = 100
dropout_prob = (0.5,)
hidden_dims = (1024, 128)
filters_h = (3, 4, 5)

batch_size = 100
num_epochs = 10

output_classes =  1

def cnn_model_rand(sequence_length, embedding_dim, num_filters, filtes_h, 
              dropout_prob, hidden_dims, output_neurons, verbose=False):
    # Input
    embedding_weights = None
    input_shape = (sequence_length,)
    model_input = Input(shape=input_shape)
    embedding_input = Embedding(len(vocab_sorted), 
                                embedding_dim, 
                                input_length=sequence_length, 
                                weights=embedding_weights, name="embedding")(model_input)

    print("Embedding dim:", len(vocab_sorted), embedding_dim)
        
    # CNNs
    convs = []
    for fh in filtes_h:
        conv = Convolution1D(filters=num_filters,
                             kernel_size=fh,
                             padding='valid',
                             activation='relu',
                             strides=1)(embedding_input)
        pool = MaxPooling1D(pool_size=2)(conv)
        flatten = Flatten()(pool)
        convs.append(flatten)

    concat_layer = Concatenate(axis=-1)(convs)
    dropout_cnn = Dropout(dropout_prob[0])(concat_layer)

    h1 = Dense(hidden_dims[0], activation="relu")(dropout_cnn)
    h2 = Dense(hidden_dims[1], activation="relu")(h1)
    model_output = Dense(output_neurons, activation="sigmoid")(h2)

    model = Model(inputs=model_input, outputs=model_output)
    adam = Adam(lr=1e-4, beta_1=0.9, beta_2=0.999, epsilon=1e-08)

    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])
    if verbose:
        print(model.summary())
    return model
verbose = True
model = cnn_model_rand(sequence_length, embedding_dim, num_filters, filters_h, 
                         dropout_prob, hidden_dims, output_classes, verbose)
model.fit(train_idx, np.array(y_train), batch_size=batch_size, epochs=num_epochs)
loss, acc = model.evaluate(test_idx, np.array(y_test), verbose=0)
print(acc)<jupyter_output>0.626760560862
{'fraction_non_alphanumeric': 0.08766637089618456, 'fraction_numerical': 0.011712511091393079, 'mean_word_length': 2.985855728429986, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '376874', 'license': 'MIT', 'n_tokens_mistral': 1790, 'n_tokens_neox': 1716, 'n_words': 469, 'path': 'notebooks/The Bechdel Test Deep Learning Tutorial.ipynb', 'repo_name': 'maigimenez/DLwoGibberish', 'size': '5635'}