<jupyter_start><jupyter_text>Model trainingThis notebook trains a meme generation model based on: https://github.com/dylanwenzlau/ml-scripts/blob/master/meme_text_gen_convnet/train.py<jupyter_code>import json
import random
import argparse
import numpy as np

from keras import layers
from keras import Sequential
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.callbacks import ModelCheckpoint
from keras.models import load_model
BASE_PATH = "."
TRAINING_DATA_PATH = f"{BASE_PATH}/drake_hotline_bling.json"
SEQUENCE_LENGTH = 128
EMBEDDING_DIM = 16
NUM_EPOCHS = 60
BATCH_SIZE = 256
VAL_RATIO = 0.25
memes = json.loads(open(TRAINING_DATA_PATH).read())['memes']
texts = []
labels_vocabulary = {}
current_labels_vocabulary_key = 0
labels = []

def is_ascii(s):
    return all(ord(c) < 128 for c in s)

for i, meme in enumerate(memes):
    box_index = 0
    meme_id = "1".zfill(8)
    meme_text = f"{meme['text']};"

    if not is_ascii(meme_text):
        continue

    for j in range(1, len(meme_text)):
        character = meme_text[j]
        texts.append(f"{meme_id} {box_index} {meme_text[0:j]}")

        if character not in labels_vocabulary:
            labels_vocabulary[character] = current_labels_vocabulary_key
            current_labels_vocabulary_key += 1

        character_label = labels_vocabulary[character]
        labels.append(character_label)

        if character == ";":
          box_index += 1
"""
    Tokenization
"""
tk = Tokenizer(num_words=0, char_level=True)
tk.fit_on_texts(texts)

vocabulary = tk.word_index
sequences = tk.texts_to_sequences(texts)
"""
    Prepare the sequences and split the dataset in training and validation
"""
data = pad_sequences(sequences, maxlen=SEQUENCE_LENGTH)
labels = np.asarray(labels)

indices = np.arange(data.shape[0])
np.random.shuffle(indices)
data = data[indices]
labels = labels[indices]

val_samples_count = int(VAL_RATIO * data.shape[0])
training_samples_count = data.shape[0] - val_samples_count

X_train = data[:training_samples_count]
y_train = labels[:training_samples_count]

X_val = data[-val_samples_count:]
y_val = labels[-val_samples_count:]
with open(f"{BASE_PATH}/params.json", "w") as params_file:
  params_file.write(json.dumps({
      'sequence_length': SEQUENCE_LENGTH,
      'embedding_dim': EMBEDDING_DIM,
      'samples_count': len(memes),
      'num_epochs': NUM_EPOCHS,
      'batch_size': BATCH_SIZE,
      'vocabulary': vocabulary,
      'labels_vocabulary': labels_vocabulary
  }))
"""
    Setup the model
"""
model = Sequential()
model.add(layers.Embedding(len(vocabulary) + 1, EMBEDDING_DIM, input_length=SEQUENCE_LENGTH))
model.add(layers.Conv1D(1024, 5, activation='relu', padding='same'))
model.add(layers.BatchNormalization())
model.add(layers.MaxPooling1D(2))
model.add(layers.Dropout(0.25))
model.add(layers.Conv1D(1024, 5, activation='relu', padding='same'))
model.add(layers.BatchNormalization())
model.add(layers.MaxPooling1D(2))
model.add(layers.Dropout(0.25))
model.add(layers.Conv1D(1024, 5, activation='relu', padding='same'))
model.add(layers.BatchNormalization())
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dropout(0.25))
model.add(layers.Dense(1024, activation='relu'))
model.add(layers.BatchNormalization())
model.add(layers.Dropout(0.25))
model.add(layers.Dense(len(labels_vocabulary), activation='softmax'))

model.compile(loss='sparse_categorical_crossentropy', optimizer='rmsprop', metrics=['acc'])
model.summary()
# Create a directory for saved models
!mkdir out_models
"""
    Setup checkpoints
"""
checkpointer = ModelCheckpoint(filepath='out_models/model.h5', verbose=1, save_best_only=True)
"""
    Train
"""
history = model.fit(
    X_train,
    y_train,
    validation_data=(X_val, y_val),
    epochs=NUM_EPOCHS,
    batch_size=BATCH_SIZE,
    callbacks=[checkpointer])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1275012827090816, 'fraction_numerical': 0.017957927142124165, 'mean_word_length': 5.423393739703459, 'pattern_counts': {'":': 1, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '656873', 'license': 'MIT', 'n_tokens_mistral': 1428, 'n_tokens_neox': 1370, 'n_words': 244, 'path': 'train_convnet.ipynb', 'repo_name': 'WalterSimoncini/memegen', 'size': '3898'}