<jupyter_start><jupyter_text>Visualizations of the training process<jupyter_code>import matplotlib.pyplot as plt
import numpy as np<jupyter_output><empty_output><jupyter_text>Model 3: With pre-training<jupyter_code>train_loss = [
    4.618,  # 1
    3.840,
    3.583,  # 3
    3.872,
    3.617,  # 5
    3.487,
    3.393,
    3.320,
    3.260,  # 9
]
val_loss = [
    3.739,  # 1
    3.350,
    3.241,  # 3
    3.562,
    3.494,  # 5
    3.475,
    3.462,
    3.452,
    3.466,  # 9
]
plt.figure(figsize=(10,3))

# Declare the figure's contents
yticks = range(1,len(train_loss)+1)
plt.plot(yticks, train_loss, label='Training Loss', color='blue')
plt.axhline(train_loss[-1], color='blue', linestyle='dotted')
plt.plot(yticks, val_loss, label='Validation Loss', color='green')
plt.axhline(val_loss[-2], color='green', linestyle='dotted')
plt.axvline(len(val_loss)-1, color='red')

# Adjust how the figure is displayed
plt.legend()
plt.title("Loss (with fine-tuning) - lower is better")
plt.xlabel("epoch")
plt.ylabel("loss")
plt.gca().xaxis.grid(True)
plt.xlim(left=1, right=len(train_loss))
plt.tight_layout()
plt.savefig('notebook_images/train_loss_with')
plt.show()
gleu = [
    9.34,  # 1
    12.06,
    13.38,  # 3
    11.93,
    12.82,  # 5
    11.95,
    13.28,
    13.27,
    13.95,  # 9
]
plt.figure(figsize=(10,3))

# Declare the figure's contents
yticks = range(1,len(gleu)+1)
plt.plot(yticks, gleu, label='Validation GLEU', color='green')
plt.axhline(gleu[-2], color='green', linestyle='dotted')
plt.axvline(len(gleu)-1, color='red')

# Adjust how the figure is displayed
plt.legend()
plt.title("GLEU (with fine-tuning) - higher is better")
plt.xlabel("epoch")
plt.ylabel("GLEU")
plt.gca().xaxis.grid(True)
plt.xlim(left=1, right=len(gleu))
plt.tight_layout()
plt.savefig('notebook_images/gleu_with')
plt.show()<jupyter_output><empty_output><jupyter_text>Model 1: Without pre-training<jupyter_code>train_loss = [
    5.665,  # 1
    4.987,
    4.636,
    4.272,
    3.999,  # 5
    3.799,
    3.643,
    3.519,
    3.420,
    3.339,  # 10
    3.273,
    3.216,
    3.166,  # 13
]
val_loss = [
    5.127,  # 1
    4.838,
    4.485,
    4.223,
    4.075,  # 5
    3.970,
    3.880,
    3.811,
    3.771,
    3.734,  # 10
    3.700,
    3.683,
    3.686,  # 13
]
plt.figure(figsize=(10,3))

# Declare the figure's contents
yticks = range(1,len(train_loss)+1)
plt.plot(yticks, train_loss, label='Training Loss', color='blue')
plt.axhline(train_loss[-1], color='blue', linestyle='dotted')
plt.plot(yticks, val_loss, label='Validation Loss', color='green')
plt.axhline(val_loss[-2], color='green', linestyle='dotted')
plt.axvline(len(val_loss)-1, color='red')

# Adjust how the figure is displayed
plt.legend()
plt.title("Loss (without fine-tuning) - lower is better")
plt.xlabel("epoch")
plt.ylabel("loss")
plt.gca().xaxis.grid(True)
plt.xlim(left=1, right=len(train_loss))
plt.tight_layout()
plt.savefig('notebook_images/train_loss_without')
plt.show()
gleu = [
    3.41,  # 1
    3.92,
    7.27,  # 3
    10.38,
    11.25,  # 5
    11.97,
    13.05,
    13.46,
    13.09,
    13.39,  # 10
    13.71,
    14.15,
    12.98,  # 13
]
plt.figure(figsize=(10,3))

# Declare the figure's contents
yticks = range(1,len(train_loss)+1)
plt.plot(yticks, gleu, label='Validation GLEU', color='green')
plt.axhline(gleu[-2], color='green', linestyle='dotted')
plt.axvline(len(gleu)-1, color='red')

# Adjust how the figure is displayed
plt.legend()
plt.title("GLEU (without fine-tuning) - higher is better")
plt.xlabel("epoch")
plt.ylabel("GLEU")
plt.gca().xaxis.grid(True)
plt.xlim(left=1, right=len(gleu))
plt.tight_layout()
plt.savefig('notebook_images/gleu_without')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1797570850202429, 'fraction_numerical': 0.08825910931174089, 'mean_word_length': 3.6792929292929295, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '318683', 'license': 'MIT', 'n_tokens_mistral': 1795, 'n_tokens_neox': 1553, 'n_words': 281, 'path': 'training_images.ipynb', 'repo_name': 'RubenPants/TransPoemer', 'size': '3705'}