<jupyter_start><jupyter_text>Custom Analysis of Training ResultsNotebook demonstrates two methods for plotting training results.  First method uses Ludwig's visualization api.  Second method illustrates converting Ludwig training statistics into a pandas dataframe and plotting data with seaborn package.This notebook is dependent on running the multiple model training example beforehand.  To run the mulitple model training example, enter this command:``` python multiple_model_training.py```  Import required libraries<jupyter_code>from ludwig.utils.data_utils import load_json
from ludwig.visualize import learning_curves
import pandas as pd
import numpy as np
import os.path
import matplotlib.pyplot as plt
import seaborn as sns<jupyter_output><empty_output><jupyter_text>Generate Annotated Learning Curves Using Ludwig Visualization API<jupyter_code># retrieve training statistics
list_of_stats = []
list_of_models = []

for model in ['model1', 'model2']:
    experiment_model_dir = './results/multiple_experiment_' + model 
    train_stats = load_json(os.path.join(experiment_model_dir,'training_statistics.json'))
    list_of_stats.append(train_stats)
    list_of_models.append(model)
    

# generating learning curves from training
learning_curves(list_of_stats, 'Survived',
                model_names=list_of_models,
                output_directory='./visualizations',
                file_format='png')<jupyter_output><empty_output><jupyter_text>Generate Annotated Learning Curves Using seaborn package Helper function to collect training statistics<jupyter_code># function to generate pandas data frame from training statistcs
# Parameter:
#   experiment_model_dir: directory containing the training statistics for a specific model training experiment
#
# Returns: pandas dataframe containing the performance metric and loss
#

def extract_training_stats(experiment_model_dir):
    list_of_splits = ['training', 'validation', 'test']
    list_of_df = []
    for split in list_of_splits:
        train_stats = load_json(os.path.join(experiment_model_dir,'training_statistics.json'))
        df = pd.DataFrame(train_stats[split]['combined'])
        df.columns = [split + '_' + c for c in df.columns]
        list_of_df.append(df)
        
    df = pd.concat(list_of_df, axis=1)
    df['epoch'] = df.index + 1
        
    return df<jupyter_output><empty_output><jupyter_text>Retrieve training results<jupyter_code>model1 = extract_training_stats('./results/multiple_experiment_model1')
model1.name = 'model1'
model2 = extract_training_stats('./results/multiple_experiment_model2')
model2.name = 'model2'
model1.head()<jupyter_output><empty_output><jupyter_text>Helper function to generate plot ready data<jupyter_code># create pandas dataframe suitable for plotting learning curves
# Parameters
#   train_df_list: list of pandas datatframe containing training statistics
#
# Returns: plot ready pandas dataframe

def create_plot_ready_data(list_of_train_stats_df):
    # holding ready for plot ready data
    plot_ready_list = []
    
    # consolidate the multiple training statistics dataframes
    for df in list_of_train_stats_df:
        for col in ['training', 'validation']:
            df2 = df[['epoch', col + '_loss']].copy()
            df2.columns = ['epoch', 'loss']
            df2['type'] = col
            df2['model'] = df.name
            plot_ready_list.append(df2)

    return pd.concat(plot_ready_list, axis=0, ignore_index=True)<jupyter_output><empty_output><jupyter_text>Plot learning curves<jupyter_code># create plot ready data
learning_curves = create_plot_ready_data([model1, model2])
# Plot learning curves for the different models
fig = plt.figure(figsize=(10,6))
sns.set_style(style='dark')
ax = sns.lineplot(x='epoch', y='loss',
             style='type',
             hue='model',
             data=learning_curves)
ax.set_title('Learning Curves', fontdict={'fontsize': 16})
fig.savefig('./visualizations/custom_learning_curve.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08932135728542914, 'fraction_numerical': 0.006487025948103792, 'mean_word_length': 4.818577648766328, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '277559', 'license': 'Apache-2.0', 'n_tokens_mistral': 1197, 'n_tokens_neox': 1168, 'n_words': 302, 'path': 'examples/titanic/model_training_results.ipynb', 'repo_name': 'dantreiman/ludwig', 'size': '4008'}