<jupyter_start><jupyter_text>Benchmark result Analysis---This notebook holds the benchmark results analysis for the following results.- baseline    - baseline_run0.pkl    - baseline_run1.pkl    - baseline_run2.pkl- naive (greedy)    - naive_run0.pkl    - naive_run1.pkl    - naive_run2.pkl- hybrid    - hybrid_jump_half_naive_run0.pkl    - hybrid_jump_half_naive_run1.pkl    - hybrid_jump_half_naive_run2.pkl    - hybrid_jump_half_exhaustive_run0.pkl    - hybrid_jump_half_exhaustive_run1.pkl    - hybrid_jump_half_exhaustive_run2.pkl    - hybrid_jump_std_exhaustive_run0.pkl    - hybrid_jump_std_exhaustive_run1.pkl    - hybrid_jump_std_exhaustive_run2.pkl    - hybrid_jump_3std_exhaustive_run0.pkl    - hybrid_jump_3std_exhaustive_run1.pkl    - hybrid_jump_3std_exhaustive_run2.pkl    - hybrid_jump_5std_exhaustive_run0.pkl    - hybrid_jump_5std_exhaustive_run1.pkl    - hybrid_jump_5std_exhaustive_run2.pkl  Read everthing in as pd.DataFrame<jupyter_code>import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
FOLDER_PATH = './results/'

# --------- baseline method --------
baseline_df_1 = pd.read_pickle(FOLDER_PATH + 'baseline_run0.pkl').sort_values('Instance')
baseline_df_2 = pd.read_pickle(FOLDER_PATH + 'baseline_run1.pkl').sort_values('Instance')
baseline_df_3 = pd.read_pickle(FOLDER_PATH + 'baseline_run2.pkl').sort_values('Instance')

# -------- greedy method ---------
greedy_df_1 = pd.read_pickle(FOLDER_PATH + 'naive_run0.pkl').sort_values('Instance')
greedy_df_2 = pd.read_pickle(FOLDER_PATH + 'naive_run1.pkl').sort_values('Instance')
greedy_df_3 = pd.read_pickle(FOLDER_PATH + 'naive_run2.pkl').sort_values('Instance')

# -------- jump-half-greedy --------
hybrid_jump_half_greedy_df_1 = pd.read_pickle(FOLDER_PATH + 'hybrid_jump_half_naive_run0.pkl').sort_values('Instance')
hybrid_jump_half_greedy_df_2 = pd.read_pickle(FOLDER_PATH + 'hybrid_jump_half_naive_run1.pkl').sort_values('Instance')
hybrid_jump_half_greedy_df_3 = pd.read_pickle(FOLDER_PATH + 'hybrid_jump_half_naive_run2.pkl').sort_values('Instance')

# -------- jump-half-exhaustive --------
hybrid_jump_half_exhaustive_df_1 = pd.read_pickle(FOLDER_PATH + 'hybrid_jump_half_exhaustive_run0.pkl').sort_values('Instance')
hybrid_jump_half_exhaustive_df_2 = pd.read_pickle(FOLDER_PATH + 'hybrid_jump_half_exhaustive_run1.pkl').sort_values('Instance')
hybrid_jump_half_exhaustive_df_3 = pd.read_pickle(FOLDER_PATH + 'hybrid_jump_half_exhaustive_run2.pkl').sort_values('Instance')

# -------- jump-std-exhaustive --------
hybrid_jump_std_exhaustive_df_1 = pd.read_pickle(FOLDER_PATH + 'hybrid_jump_std_exhaustive_run0.pkl').sort_values('Instance')
hybrid_jump_std_exhaustive_df_2 = pd.read_pickle(FOLDER_PATH + 'hybrid_jump_std_exhaustive_run1.pkl').sort_values('Instance')
hybrid_jump_std_exhaustive_df_3 = pd.read_pickle(FOLDER_PATH + 'hybrid_jump_std_exhaustive_run2.pkl').sort_values('Instance')

# -------- jump-3std-exhaustive --------
hybrid_jump_3std_exhaustive_df_1 = pd.read_pickle(FOLDER_PATH + 'hybrid_jump_3std_exhaustive_run0.pkl').sort_values('Instance')
hybrid_jump_3std_exhaustive_df_2 = pd.read_pickle(FOLDER_PATH + 'hybrid_jump_3std_exhaustive_run1.pkl').sort_values('Instance')
hybrid_jump_3std_exhaustive_df_3 = pd.read_pickle(FOLDER_PATH + 'hybrid_jump_3std_exhaustive_run2.pkl').sort_values('Instance')

# -------- jump-5std-exhaustive --------
hybrid_jump_5std_exhaustive_df_1 = pd.read_pickle(FOLDER_PATH + 'hybrid_jump_5std_exhaustive_run0.pkl').sort_values('Instance')
hybrid_jump_5std_exhaustive_df_2 = pd.read_pickle(FOLDER_PATH + 'hybrid_jump_5std_exhaustive_run1.pkl').sort_values('Instance')
hybrid_jump_5std_exhaustive_df_3 = pd.read_pickle(FOLDER_PATH + 'hybrid_jump_5std_exhaustive_run2.pkl').sort_values('Instance')

# the dictionary can then be used to iterate for comparison.
df_dict = {}

# -------- baseline method --------
df_dict['baseline_df_1'] = baseline_df_1
df_dict['baseline_df_2'] = baseline_df_2
df_dict['baseline_df_3'] = baseline_df_3

# -------- greedy method ---------
df_dict['greedy_df_1'] = greedy_df_1
df_dict['greedy_df_2'] = greedy_df_2
df_dict['greedy_df_3'] = greedy_df_3

# -------- jump-half-greedy --------
df_dict['hybrid_jump_half_greedy_df_1'] = hybrid_jump_half_greedy_df_1
df_dict['hybrid_jump_half_greedy_df_2'] = hybrid_jump_half_greedy_df_2
df_dict['hybrid_jump_half_greedy_df_3'] = hybrid_jump_half_greedy_df_3

# -------- jump-half-exhaustive --------
df_dict['hybrid_jump_half_exhaustive_df_1'] = hybrid_jump_half_exhaustive_df_1
df_dict['hybrid_jump_half_exhaustive_df_2'] = hybrid_jump_half_exhaustive_df_2
df_dict['hybrid_jump_half_exhaustive_df_3'] = hybrid_jump_half_exhaustive_df_3

# -------- jump-std-exhaustive --------
df_dict['hybrid_jump_std_exhaustive_df_1'] = hybrid_jump_std_exhaustive_df_1
df_dict['hybrid_jump_std_exhaustive_df_2'] = hybrid_jump_std_exhaustive_df_2
df_dict['hybrid_jump_std_exhaustive_df_3'] = hybrid_jump_std_exhaustive_df_3

# -------- jump-3std-exhaustive --------
df_dict['hybrid_jump_3std_exhaustive_df_1'] = hybrid_jump_3std_exhaustive_df_1
df_dict['hybrid_jump_3std_exhaustive_df_2'] = hybrid_jump_3std_exhaustive_df_2
df_dict['hybrid_jump_3std_exhaustive_df_3'] = hybrid_jump_3std_exhaustive_df_3

# -------- jump-5std-exhaustive --------
df_dict['hybrid_jump_5std_exhaustive_df_1'] = hybrid_jump_5std_exhaustive_df_1
df_dict['hybrid_jump_5std_exhaustive_df_2'] = hybrid_jump_5std_exhaustive_df_2
df_dict['hybrid_jump_5std_exhaustive_df_3'] = hybrid_jump_5std_exhaustive_df_3
print('The number of dataframes to compare with is {}.'.format(len(df_dict)))
for key, val in df_dict.items():
    print("  --> There are {:>6} runs in {:<36}".format(len(val), key))<jupyter_output>The number of dataframes to compare with is 21.
  --> There are  12000 runs in baseline_df_1                       
  --> There are  12000 runs in baseline_df_2                       
  --> There are  12000 runs in baseline_df_3                       
  --> There are  12000 runs in greedy_df_1                         
  --> There are  12000 runs in greedy_df_2                         
  --> There are  12000 runs in greedy_df_3                         
  --> There are  12000 runs in hybrid_jump_half_greedy_df_1        
  --> There are  12000 runs in hybrid_jump_half_greedy_df_2        
  --> There are  12000 runs in hybrid_jump_half_greedy_df_3        
  --> There are  12000 runs in hybrid_jump_half_exhaustive_df_1    
  --> There are  12000 runs in hybrid_jump_half_exhaustive_df_2    
  --> There are  12000 runs in hybrid_jump_half_exhaustive_df_3    
  --> There are  12000 runs in hybrid_jump_std_exhaustive_df_1     
  --> There are  12000 runs in hybrid_jump_std_exhaustive_df_2     
[...]<jupyter_text>Sanity CheckCheck whether each JSSP version find the same result as the baseline model.<jupyter_code># Are each instances faster to solve with the given solver when compared with the baseline.
temp_dict = dict((ind, val) for ind, val in df_dict.items() if ind!='baseline_df_1')

for key, val in temp_dict.items():
    print('{:<50} finds the same solutions as baseline: {}'.format(key, np.array_equal(val['makespan'].astype('int').values, baseline_df_1['makespan'].astype('int').values)))<jupyter_output>baseline_df_2                                      finds the same solutions as baseline: True
baseline_df_3                                      finds the same solutions as baseline: True
greedy_df_1                                        finds the same solutions as baseline: True
greedy_df_2                                        finds the same solutions as baseline: True
greedy_df_3                                        finds the same solutions as baseline: True
hybrid_jump_half_greedy_df_1                       finds the same solutions as baseline: True
hybrid_jump_half_greedy_df_2                       finds the same solutions as baseline: True
hybrid_jump_half_greedy_df_3                       finds the same solutions as baseline: True
hybrid_jump_half_exhaustive_df_1                   finds the same solutions as baseline: True
hybrid_jump_half_exhaustive_df_2                   finds the same solutions as baseline: True
hybrid_jump_half_exhaustive_df_3                   finds the[...]<jupyter_text>The total running timeSome observation from the running time are:- The hybrid model is still slower than the baseline model;- hybrid model is slower than naive model for ~2000 seconds which is probably because the naive model does not have probing phase;- with the optimal prediction hybrid model is still slower than the naive model for ~2786 seconds, which once again might be because of the lack of probing phase.<jupyter_code># Total running time of different benchmark
for key, val in df_dict.items():
    print("Total running time of the {:<50}  model is  {:>10} seconds".format(key, int(val['runtime_ms'].sum() / 1000)))
# Total running time of different benchmark
for key, val in df_dict.items():
    print("Total running time of the {:<50}  model is  {:>10} seconds".format(key, int(val[val.Heuristic == 'action']['runtime_ms'].sum() / 1000)))
# Total running time of different benchmark
for key, val in df_dict.items():
    print("Total running time of the {:<50}  model is  {:>10} seconds".format(key, int(val[val.Heuristic == 'chb']['runtime_ms'].sum() / 1000)))
# Total running time of different benchmark
for key, val in df_dict.items():
    print("Total running time of the {:<50}  model is  {:>10} seconds".format(key, int(val[val.Heuristic == 'afc']['runtime_ms'].sum() / 1000)))
sns.set(style="whitegrid")
y = [key for key, val in df_dict.items()]
x = [int(val['runtime_ms'].sum() / 1000) for key, val in df_dict.items()]
plt.xlabel('Total running time for all three heuristics (seconds)')
ax = sns.barplot(x=x, y=y)
HEURISTIC = 'afc'
sns.set(style="whitegrid")
y = [key for key, val in df_dict.items()]
x = [int(val[val.Heuristic == HEURISTIC]['runtime_ms'].sum() / 1000) for key, val in df_dict.items()]
plt.xlabel('Total running time for \''+ HEURISTIC + '\' (seconds)')
ax = sns.barplot(x=x, y=y)
HEURISTIC = 'chb'
sns.set(style="whitegrid")
y = [key for key, val in df_dict.items()]
x = [int(val[val.Heuristic == HEURISTIC]['runtime_ms'].sum() / 1000) for key, val in df_dict.items()]
plt.xlabel('Total running time for \''+ HEURISTIC + '\' (seconds)')
ax = sns.barplot(x=x, y=y)
HEURISTIC = 'action'
sns.set(style="whitegrid")
y = [key for key, val in df_dict.items()]
x = [int(val[val.Heuristic == HEURISTIC]['runtime_ms'].sum() / 1000) for key, val in df_dict.items()]
plt.xlabel('Total running time for \''+ HEURISTIC + '\' (seconds)')
ax = sns.barplot(x=x, y=y)<jupyter_output><empty_output><jupyter_text>The following code block shows out of all the runs in the benchmark, what is the percentage of certain benchmark is faster than the baseline.-<jupyter_code># Are each instances faster to solve with the given solver when compared with the baseline.
temp_dict = dict((ind, val) for ind, val in df_dict.items() if ind!='baseline_df_1')

for key, val in temp_dict.items():
    comparison_arr = val['runtime_ms'].values < baseline_df_1['runtime_ms'].values
    faster_count = comparison_arr.sum()
    print('In benchmark result of {:>50}, {:6} ({:<.3}%) is faster the baseline benchmark.'.format(key, faster_count, 100*faster_count/len(comparison_arr)))
# Are each instances faster to solve with the given solver when compared with the baseline.
temp_dict = dict((ind, val) for ind, val in df_dict.items() if ind!='baseline_df_1')

HEURISTIC = 'afc'

for key, val in temp_dict.items():
    comparison_arr = val[val.Heuristic == HEURISTIC]['runtime_ms'].values < baseline_df_1[baseline_df_1.Heuristic == HEURISTIC]['runtime_ms'].values
    faster_count = comparison_arr.sum()
    print('In benchmark result of {:>50}, {:6} ({:<.3}%) is faster the baseline benchmark.'.format(key, faster_count, 100*faster_count/len(comparison_arr)))
# Are each instances faster to solve with the given solver when compared with the baseline.
temp_dict = dict((ind, val) for ind, val in df_dict.items() if ind!='baseline_df_1')

HEURISTIC = 'action'

for key, val in temp_dict.items():
    comparison_arr = val[val.Heuristic == HEURISTIC]['runtime_ms'].values < baseline_df_1[baseline_df_1.Heuristic == HEURISTIC]['runtime_ms'].values
    faster_count = comparison_arr.sum()
    print('In benchmark result of {:>50}, {:6} ({:<.3}%) is faster the baseline benchmark.'.format(key, faster_count, 100*faster_count/len(comparison_arr)))
# Are each instances faster to solve with the given solver when compared with the baseline.
temp_dict = dict((ind, val) for ind, val in df_dict.items() if ind!='baseline_df_1')

HEURISTIC = 'chb'

for key, val in temp_dict.items():
    comparison_arr = val[val.Heuristic == HEURISTIC]['runtime_ms'].values < baseline_df_1[baseline_df_1.Heuristic == HEURISTIC]['runtime_ms'].values
    faster_count = comparison_arr.sum()
    print('In benchmark result of {:>50}, {:6} ({:<.3}%) is faster the baseline benchmark.'.format(key, faster_count, 100*faster_count/len(comparison_arr)))
# Are each instances faster to solve with the given solver when compared with the baseline.
temp_dict = dict((ind, val) for ind, val in df_dict.items() if ind!='baseline_df_1')

x = []
y = []

for key, val in temp_dict.items():
    comparison_arr = val['runtime_ms'].values < baseline_df_1['runtime_ms'].values
    faster_count = comparison_arr.sum()
#     print('In benchmark result of {:>40}, {:6} ({:<.3}%) is faster the baseline benchmark.'.format(key, faster_count, 100*faster_count/len(comparison_arr)))
    y.append(key)
    x.append(100*faster_count/len(comparison_arr))

sns.set(style="whitegrid")
plt.xlabel('Percentage of faster cases than baseline model for all three heuristics')
ax = sns.barplot(x=x, y=y)
# Are each instances faster to solve with the given solver when compared with the baseline.
temp_dict = dict((ind, val) for ind, val in df_dict.items() if ind!='baseline_df_1')

HEURISTIC = 'action'

x = []
y = []

for key, val in temp_dict.items():
    comparison_arr = val[val.Heuristic == HEURISTIC]['runtime_ms'].values < baseline_df_1[baseline_df_1.Heuristic == HEURISTIC]['runtime_ms'].values
    faster_count = comparison_arr.sum()
#     print('In benchmark result of {:>40}, {:6} ({:<.3}%) is faster the baseline benchmark.'.format(key, faster_count, 100*faster_count/len(comparison_arr)))
    y.append(key)
    x.append(100*faster_count/len(comparison_arr))

sns.set(style="whitegrid")
plt.xlabel('Percentage of faster cases than baseline model for \''+HEURISTIC+'\'')
ax = sns.barplot(x=x, y=y)
# Are each instances faster to solve with the given solver when compared with the baseline.
temp_dict = dict((ind, val) for ind, val in df_dict.items() if ind!='baseline_df_1')

HEURISTIC = 'afc'

x = []
y = []

for key, val in temp_dict.items():
    comparison_arr = val[val.Heuristic == HEURISTIC]['runtime_ms'].values < baseline_df_1[baseline_df_1.Heuristic == HEURISTIC]['runtime_ms'].values
    faster_count = comparison_arr.sum()
#     print('In benchmark result of {:>40}, {:6} ({:<.3}%) is faster the baseline benchmark.'.format(key, faster_count, 100*faster_count/len(comparison_arr)))
    y.append(key)
    x.append(100*faster_count/len(comparison_arr))

sns.set(style="whitegrid")
plt.xlabel('Percentage of faster cases than baseline model for \''+HEURISTIC+'\'')
ax = sns.barplot(x=x, y=y)
# Are each instances faster to solve with the given solver when compared with the baseline.
temp_dict = dict((ind, val) for ind, val in df_dict.items() if ind!='baseline_df_1')

HEURISTIC = 'chb'

x = []
y = []

for key, val in temp_dict.items():
    comparison_arr = val[val.Heuristic == HEURISTIC]['runtime_ms'].values < baseline_df_1[baseline_df_1.Heuristic == HEURISTIC]['runtime_ms'].values
    faster_count = comparison_arr.sum()
#     print('In benchmark result of {:>40}, {:6} ({:<.3}%) is faster the baseline benchmark.'.format(key, faster_count, 100*faster_count/len(comparison_arr)))
    y.append(key)
    x.append(100*faster_count/len(comparison_arr))

sns.set(style="whitegrid")
plt.xlabel('Percentage of faster cases than baseline model for \''+HEURISTIC+'\'')
ax = sns.barplot(x=x, y=y)<jupyter_output><empty_output><jupyter_text>Runtime distribution<jupyter_code>plt.figure(figsize=(12,8))
# sns.distplot(baseline_df['runtime_ms'], hist = False, kde = True, kde_kws = {'linewidth': 2})
for key, val in df_dict.items():
#     print("Total running time of the {:<30}  model is  {:>10} seconds".format(key, int(val['runtime_ms'].sum() / 1000)))
    sns.kdeplot(val['runtime_ms'], shade=True, label=key)

# Plot formatting
plt.legend(prop={'size': 16}, title = 'runtime_ms')
plt.title('Density Plot of running time in ms')
plt.xlabel('rumtime_ms')
plt.ylabel('Density')
pd.options.mode.chained_assignment = None  # default='warn'

to_visualize_df = pd.DataFrame(columns=['Instance', 'runtime_ms', 'type'])

for key, val in df_dict.items():
    temp_df = val[['Instance', 'runtime_ms']]
    temp_df['type'] = str(key[:-5])

    to_visualize_df = pd.concat([to_visualize_df, temp_df])
    
to_visualize_df['runtime_ms_log'] = np.log(to_visualize_df['runtime_ms'])
to_visualize_df
plt.figure(figsize=(26,8))

sns.boxplot(x='type', y='runtime_ms_log', data=to_visualize_df)
sns.set(style="white", rc={"axes.facecolor": (0, 0, 0, 0)})

# Initialize the FacetGrid object
pal = sns.cubehelix_palette(10, rot=-.25, light=.7)
g = sns.FacetGrid(to_visualize_df, row="type", hue="type", aspect=15, height=2, palette=pal)

# Draw the densities in a few steps
g.map(sns.kdeplot, "runtime_ms", clip_on=False, cumulative=True, shade=True, alpha=1, lw=1.5, bw_method=.2)
# g.map(sns.kdeplot, "runtime_ms", clip_on=False, color="w", lw=2, bw=.2)
g.map(plt.axhline, y=0, lw=2, clip_on=False)


# Define and use a simple function to label the plot in axes coordinates
def label(x, color, label):
    ax = plt.gca()
    ax.text(0, .2, label, fontweight="bold", color=color,
            ha="right", va="center", transform=ax.transAxes)


g.map(label, "runtime_ms")

# Set the subplots to overlap
g.fig.subplots_adjust(hspace=-.25)

# Remove axes details that don't play well with overlap
g.set_titles("")
g.set(yticks=[])
g.despine(bottom=True, left=True)
ordered_types = list(to_visualize_df['type'].value_counts().index).sort()
g = sns.FacetGrid(to_visualize_df, row="type", row_order=ordered_types,
                  height=1.7, aspect=4,)
# g.map(sns.distplot, "runtime_ms", hist=False, rug=False);
# g.map(sns.kdeplot, "runtime_ms_log", cumulative=True);
g.map(sns.kdeplot, "runtime_ms", cumulative=True);
plt.figure(figsize=(12,8))
# sns.distplot(baseline_df['runtime_ms'], hist = False, kde = True, kde_kws = {'linewidth': 2})
for key, val in df_dict.items():
    if (key.startswith('baseline')):
        sns.kdeplot(val['runtime_ms'], shade=True, label=key);

# Plot formatting
plt.legend(prop={'size': 16}, title = 'runtime_ms')
plt.title('Density Plot of running time in ms')
plt.xlabel('rumtime_ms')
plt.ylabel('Density')
plt.figure(figsize=(12,8))
# sns.distplot(baseline_df['runtime_ms'], hist = False, kde = True, kde_kws = {'linewidth': 2})
for key, val in df_dict.items():
    if (key.startswith('naive') or key.startswith('baseline_df')):
        sns.kdeplot(val['runtime_ms'], shade=True, label=key);

# Plot formatting
plt.legend(prop={'size': 16}, title = 'runtime_ms')
plt.title('Density Plot of running time in ms')
plt.xlabel('rumtime_ms')
plt.ylabel('Density')
plt.figure(figsize=(12,8))
# sns.distplot(baseline_df['runtime_ms'], hist = False, kde = True, kde_kws = {'linewidth': 2})
for key, val in df_dict.items():
    if (key.startswith('jump') or key.startswith('baseline_df')):
        sns.kdeplot(val['runtime_ms'], shade=True, label=key);

# Plot formatting
plt.legend(prop={'size': 16}, title = 'runtime_ms')
plt.title('Density Plot of running time in ms')
plt.xlabel('rumtime_ms')
plt.ylabel('Density')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13692176615045357, 'fraction_numerical': 0.022653235102490853, 'mean_word_length': 4.792162554426706, 'pattern_counts': {'":': 1, '<': 42, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '554376', 'license': 'MIT', 'n_tokens_mistral': 7531, 'n_tokens_neox': 6949, 'n_words': 1518, 'path': 'DL_model/3_Result_Analysis.ipynb', 'repo_name': 'bwhub/CONVJSSP', 'size': '19953'}