<jupyter_start><jupyter_text>Check Points<jupyter_code>checkpoint_dir = './training_checkpoints'
checkpoint_prefix = os.path.join(checkpoint_dir,"ckpt_{epoch}")

checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
    filepath=checkpoint_prefix,
    save_weights_only=True
)
EPOCHS = 20
history = model.fit(dataset, epochs=EPOCHS, callbacks=[checkpoint_callback])
class OneStep(tf.keras.Model):
    def __init__(self, model, chars_from_ids, ids_from_chars, temperature=1.0):
        super().__init__()
        self.temperature = temperature
        self.model = model
        self.chars_from_ids = chars_from_ids
        self.ids_from_chars = ids_from_chars

        skip_ids = self.ids_from_chars(['[UNK]'])[:, None]
        sparse_mask = tf.SparseTensor(
            values=[-float('inf')]*len(skip_ids),
            indices=skip_ids,
            dense_shape=[len(ids_from_chars.get_vocabulary())])
        self.prediction_mask = tf.sparse.to_dense(sparse_mask)

    @tf.function
    def generate_one_step(self, inputs, states=None):
        
        input_chars = tf.strings.unicode_split(inputs, 'UTF-8')
        input_ids = self.ids_from_chars(input_chars).to_tensor()

        predicted_logits, states = self.model(inputs=input_ids, states=states,
                                          return_state=True)
        
        predicted_logits = predicted_logits[:, -1, :]
        predicted_logits = predicted_logits/self.temperature
        
        predicted_logits = predicted_logits + self.prediction_mask
        
        predicted_ids = tf.random.categorical(predicted_logits, num_samples=1)
        predicted_ids = tf.squeeze(predicted_ids, axis=-1)
        
        predicted_chars = self.chars_from_ids(predicted_ids)

        return predicted_chars, states
one_step_model = OneStep(model, chars_from_ids, ids_from_chars)
import time
start = time.time()
states = None
next_char = tf.constant(['Jon'])
results = [next_char]

for n in range(1000):
    next_char, states = one_step_model.generate_one_step(next_char, states=states)
    results.append(next_char)

results = tf.strings.join(results)
end = time.time()
print(results[0].numpy().decode("utf-8"), "\n\n"+ "_"*80)
print("\nRun time: ", end - start)<jupyter_output>Jon agreed. Those are horses and I would not  eat the loose of the other Landing? Yet they were owh if I drew some line. He saw a bull in hobbing, "I'm not condemned, and that will have  a farpetifice alone. Certain of your head," said Late. The Light-gave again, even Halfmaer’s son. The Old Tongue did have  the clothing as it think. Their out the dirt and the cages. How long home, nor to stop his.  The second friend, he thinks  of our trees say, and Bronn brings the greatsword feral want, a goodfrey in the ships too. What afternoon was  she'd best be boiling into fiery liege  lot at each pity. All turn to the clanks, Dany thought.  And it is still apart. The  knight belonged a hundred Prince Column and Dulk, the man no more to be able. The crown had  had the fellow. This settling throble greater had supposed to be a baly from hip.  When he was made a dozen Dany thought, squeal, <NAME>.  “Trusted that too my widow, and some rankled sigil of your father, <NAME>?” avoid  on a few[...]
{'fraction_non_alphanumeric': 0.09419394688079061, 'fraction_numerical': 0.004941321803582459, 'mean_word_length': 3.517433751743375, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '354345', 'license': 'MIT', 'n_tokens_mistral': 1027, 'n_tokens_neox': 1003, 'n_words': 295, 'path': 'Game of Thrones.ipynb', 'repo_name': 'Chirag-Juneja/Game-of-Thrones', 'size': '3246'}