<jupyter_start><jupyter_text><jupyter_code>!git clone https://github.com/Aaronsom/poem-generation
%cd poem-generation
%mkdir models
from tensorflow.keras.callbacks import ModelCheckpoint, CSVLogger
import tensorflow.train as optimizer
from poem_generator.dataGenerator import TupleDataGenerator
import poem_generator.data_prepocessing as dp
import poem_generator.embedding as embedding_loader
from poem_generator.global_constants import TRAINING_DATA, EMBEDDING_DIMENSION, EMBEDDING_BINARY, MODELS_DICT
from poem_generator.transformer import transformer
from tensorflow.contrib.tpu import keras_to_tpu_model, TPUDistributionStrategy
from tensorflow.contrib.cluster_resolver import TPUClusterResolver
from tensorflow.keras import Model
import os


ns = [25]
epochs = 20
batch_size = 512
validation_split = 0.9

poems = dp.characterize_poems(TRAINING_DATA)
words = sorted(list(set([token for poem in poems for token in poem])))

#Save embedding for generator
embedding, dictionary = embedding_loader.get_char_embedding(words, save=True)

#model = load_model(MODELS_DICT+"/5model.hdf5", custom_objects={"PositionalEncoding": PositionalEncoding, "Attention": Attention})
model = transformer(100, embedding, len(dictionary), True, train_embedding=True, input_sequence_length=25)
tpu_model = keras_to_tpu_model(
    model,
    strategy=TPUDistributionStrategy(
        TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])
    )
)
tpu_model.compile(optimizer=optimizer.AdamOptimizer(),
            loss="categorical_crossentropy", metrics=["accuracy"])

generator = TupleDataGenerator(poems[:int(validation_split*len(poems))], ns, dictionary, 0.1, batch_size, single=True)
validation_generator = TupleDataGenerator(poems[int(validation_split*len(poems)):], ns, dictionary, 0, batch_size, single=True)
callbacks = [ModelCheckpoint(MODELS_DICT+"/char-model.hdf5", save_best_only=True),
           CSVLogger(MODELS_DICT+"/char-log.csv", append=True, separator=';')]
tpu_model.fit_generator(
  generator, epochs=epochs, callbacks=callbacks, validation_data=validation_generator, workers=4)
#donwload model
%mkdir generated
from poem_generator.character_generator import generate_poems
n = 25
generate_poems(1000, n, "/generated/"+str(n)+"-char-poems.zip", MODELS_DICT+"/char-model.hdf5")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10353753235547886, 'fraction_numerical': 0.012079378774805867, 'mean_word_length': 7.953667953667954, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '212872', 'license': 'MIT', 'n_tokens_mistral': 776, 'n_tokens_neox': 749, 'n_words': 144, 'path': 'colab_poem_generator_character.ipynb', 'repo_name': 'Aaronsom/poem-generation', 'size': '2318'}