<jupyter_start><jupyter_text>Plots for evaluation  Global imports and variables<jupyter_code># Import for interactive notebook (see:
# https://ipywidgets.readthedocs.io/en/stable/examples/Using%20Interact.html)
from __future__ import division, print_function
from matplotlib import pyplot as plt
import pandas as pd
import numpy as np
# In a notebook environment, display the plots inline
%matplotlib inline

# Set some parameters to apply to all plots. These can be overridden
# in each plot if desired
import matplotlib
# Plot size to 14" x 7"

matplotlib.rcParams['font.serif'] = ['CMU Serif Roman']

matplotlib.rc('figure', figsize = (20, 10))
# Font size to 14
matplotlib.rc('font', size = 40)
# Do not display top and right frame lines
matplotlib.rc('axes.spines', top = False, right = False)
# Remove grid lines
matplotlib.rc('axes', grid = False)
# Set backgound color to white
matplotlib.rc('axes', facecolor = 'white')


from ipywidgets import interact, interactive, fixed, interact_manual
import ipywidgets as widgets
from IPython.display import display
from ipywidgets import Layout

# Import to list files in directories
import glob

# Import for regular expressions
import re

# Imports for path operations
import os
import os.path

# For date operations
from datetime import datetime

import pandas as pd
pd.set_option('max_colwidth', -1)

import numpy as np
import configparser

RESULTS_DIR = "/Users/gomerudo/workspace/thesis_results"

def rettext(text):
    return text

form_item_layout = Layout(
    width="50%"
)<jupyter_output><empty_output><jupyter_text>Selecting the desired results<jupyter_code>import os.path
results_sorted = sorted(glob.glob("{dir}/*".format(dir=RESULTS_DIR)))
dict_widgets = {}
list_widgets = []
for result in results_sorted:
    hbox = widgets.Box()
    dict_widgets[result] = widgets.Checkbox(
        value=False,
        description=os.path.basename(result),
        disabled=False
    )
    list_widgets.append(dict_widgets[result])

half = len(results_sorted)//2

left_box = widgets.VBox(list_widgets[:half])
right_box = widgets.VBox(list_widgets[half:])
display(widgets.HBox([left_box, right_box]))<jupyter_output><empty_output><jupyter_text>Visualize the average through episodes<jupyter_code>matplotlib.rcParams['font.serif'] = ['CMU Serif Roman']

matplotlib.rc('figure', figsize = (20, 10))
# Font size to 14
matplotlib.rc('font', size = 40)
# Do not display top and right frame lines
matplotlib.rc('axes.spines', top = False, right = False)
# Remove grid lines
matplotlib.rc('axes', grid = False)
# Set backgound color to white
matplotlib.rc('axes', facecolor = 'white')

# Get the selected cells
selected_dirs = []

for key, widget in dict_widgets.items():
    if widget.value:
        selected_dirs.append(key)

cdir = selected_dirs[0]
src_dir = "{dir}".format(dir=cdir)

episodes_stats_csv = "{src}/episodes_stats.csv".format(src=src_dir)
epstats_df = pd.read_csv(episodes_stats_csv)
epstats_df.head()

max_eplength = epstats_df['ep_length'].max() + 1
max_accreward = epstats_df['acc_reward'].max() + 1

limits = [ None, 10, 'max']
# print(limits)
mean_n = 100
for dataset in ['aircraft', 'cu_birds']:
    meta_df = epstats_df[epstats_df['dataset'] == dataset]
    random_df = epstats_df[epstats_df['dataset'] == 'random-{d}'.format(d=dataset)]
    dqn_df = epstats_df[epstats_df['dataset'] == 'dqn-{d}'.format(d=dataset)]

    for metric, name_metric, limit in zip(['best_reward', 'ep_length', 'acc_reward'], ['Best reward', 'Episode length', 'Accumulated reward'], limits):
        # Compute the means and std of meta results
        means_meta = meta_df[metric].groupby(meta_df.index // mean_n).mean()
        std_meta = meta_df[metric].groupby(meta_df.index // mean_n).std()
        x_axis_meta = np.array(range(len(means_meta)))*mean_n
        
        # Compute the means and std of the dqn results 
        means_dqn = dqn_df[metric].groupby(dqn_df.index // mean_n).mean()
        std_dqn = dqn_df[metric].groupby(dqn_df.index // mean_n).std()
        x_axis_dqn = np.array(range(len(means_dqn))) *mean_n
        
        # Compute the means and std of the random results
        means_random = random_df[metric].groupby(random_df.index // mean_n).mean()
        std_random = random_df[metric].groupby(random_df.index // mean_n).std()
        x_axis_random = np.array(range(len(means_random))) *mean_n
    
        # Create the plot object
        fig, ax = plt.subplots()
#         if limit is not None:
#             ax.set_ylim(limit)
        if limit == 'max':
            ymax = max(means_meta.max(), means_dqn.max()) + max(std_meta.max(), std_dqn.max())
            ax.set_ylim([0, ymax])
        if isinstance(limit, int):
            ax.set_ylim([0, limit])
        
        # 245932 eval
        # 539caf train
        ax.plot(x_axis_meta, means_meta, lw = 2, color = '#245932', alpha = 1, label="meta-RL".format(dataset=dataset))
        ax.fill_between(x_axis_meta, means_meta - std_meta, means_meta + std_meta, color = '#245932', alpha = 0.4)

        # 878787
        ax.plot(x_axis_random, means_random, lw = 2, color = '#878787', alpha = 1, label="Random Search".format(dataset=dataset))
        ax.fill_between(x_axis_random, means_random - std_random, means_random + std_random, color = '#878787', alpha = 0.4)

        plt.rcParams['font.family'] = 'serif'
        plt.rcParams['font.sans-serif'] = 'CMU Serif Roman'
        #         hfont = {'fontname':'Helvetica'}

        # Label the axes and provide a title
        ax.set_title(
            "Avg. '{metric}' every {n} episodes for '{dataset}'".format(
                metric=name_metric, n=mean_n, dataset=dataset
            ),
            y=1.08
        )
        ax.set_xlabel("Episode")
        ax.set_ylabel(name_metric)

        ax.legend(loc = 'best')
        
        import os
        save_dir = '{root}/figures'.format(root=src_dir)
        if not os.path.exists(save_dir):
            os.mkdir(save_dir);
        fig.savefig(
            '{sdir}/average-{m}-{d}.png'.format(sdir=save_dir, m=metric, d=dataset)
        )<jupyter_output>/Users/gomerudo/workspace/pyvenv/nas-rl2/lib/python3.7/site-packages/matplotlib/font_manager.py:1241: UserWarning: findfont: Font family ['sans-serif'] not found. Falling back to DejaVu Sans.
  (prop.get_family(), self.defaultFamily[fontext]))
/Users/gomerudo/workspace/pyvenv/nas-rl2/lib/python3.7/site-packages/matplotlib/font_manager.py:1241: UserWarning: findfont: Font family ['serif'] not found. Falling back to DejaVu Sans.
  (prop.get_family(), self.defaultFamily[fontext]))<jupyter_text>Visualize the distribution of actions<jupyter_code>matplotlib.rcParams['font.serif'] = ['CMU Serif Roman']

matplotlib.rc('figure', figsize = (20, 10))
# Font size to 14
matplotlib.rc('font', size = 20)
# Do not display top and right frame lines
matplotlib.rc('axes.spines', top = False, right = False)
# Remove grid lines
matplotlib.rc('axes', grid = False)
# Set backgound color to white
matplotlib.rc('axes', facecolor = 'white')
matplotlib.rc('figure', autolayout=True)




# Get the selected cells
selected_dirs = []

for key, widget in dict_widgets.items():
    if widget.value:
        selected_dirs.append(key)

cdir = selected_dirs[0]
src_dir = "{dir}".format(dir=cdir)

actions_dist_csv = "{src}/actions_dist.csv".format(src=src_dir)
actions_dist_df = pd.read_csv(actions_dist_csv)

max_acc = 0
res = {}
fig, ax = plt.subplots()
enable = False
for dataset in ['aircraft', 'cu_birds']:
    dataset_actions_dist_df = actions_dist_df[actions_dist_df['dataset'] == dataset]        
    # Create the plot object
    indices = range(len(dataset_actions_dist_df['action_id']))
#     if enable:
    width = np.min(np.diff(indices))/3
    if not enable:
        ax.bar(indices - width/2., dataset_actions_dist_df['prop'], width, color = '#245932', alpha = 1, label=dataset)
    else:
        ax.bar(indices + width/2., dataset_actions_dist_df['prop'], width, color = '#245932', alpha = 0.4, label=dataset)
    
    ax.set_title("Proportion of actions executed on the evaluation datasets".format(dataset=dataset))
    ax.set_xlabel("Actions")
    ax.set_ylabel("Proportion")
    if not enable:
        enable = True

# ticks = ax.get_xticks()*120 + 5
# ticks[0] = 0
# ticks = np.array(ticks, dtype=np.int)
# print(ticks)
list_names = ["Conv k=1", "Conv k=3", "Conv k=5", "MaxPool p=2", "MaxPool p=3", "AvgPool p=2", "AvgPool p=3", "Terminal"]
ax.set_xticklabels(["0"] + list_names)
# ax.set_xticklabels(["0"] + list(dataset_actions_dist_df['action_id']))
ax.legend(loc = 'best')
ax.tick_params(axis ='x', rotation = 45) 
import os
save_dir = '{root}/figures'.format(root=src_dir)
if not os.path.exists(save_dir):
    os.mkdir(save_dir);
fig.savefig(
    '{sdir}/actions-dist-evaluation.png'.format(sdir=save_dir, dataset=dataset)
)   # save the figure to file<jupyter_output>/Users/gomerudo/workspace/pyvenv/nas-rl2/lib/python3.7/site-packages/matplotlib/figure.py:2369: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.
  warnings.warn("This figure includes Axes that are not compatible "<jupyter_text>The statistical test<jupyter_code># Using wilcoxon (paired data)
from scipy.stats import wilcoxon

aircraft_actions_dist_df = actions_dist_df[actions_dist_df['dataset'] == 'aircraft']
cubirds_actions_dist_df = actions_dist_df[actions_dist_df['dataset'] == 'cu_birds']
w = wilcoxon(cubirds_actions_dist_df['prop'].values - aircraft_actions_dist_df['prop'].values)
# print(round(w.pvalue, 2))
w
# Using 
from scipy.stats import kruskal
aircraft_actions_dist_df = actions_dist_df[actions_dist_df['dataset'] == 'aircraft']
cubirds_actions_dist_df = actions_dist_df[actions_dist_df['dataset'] == 'cu_birds']
k = kruskal(aircraft_actions_dist_df['prop'], cubirds_actions_dist_df['prop'])
print(round(k.pvalue, 2))
k
cubirds_actions_dist_df['prop'].values*100 - aircraft_actions_dist_df['prop'].values*100
cubirds_actions_dist_df['prop'].values
aircraft_actions_dist_df['prop'].values<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13284279606244406, 'fraction_numerical': 0.015809883663120215, 'mean_word_length': 4.285338938518129, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '397498', 'license': 'MIT', 'n_tokens_mistral': 3632, 'n_tokens_neox': 3432, 'n_words': 761, 'path': 'notebooks/VisualizeFinalPlots-Evaluation.ipynb', 'repo_name': 'gomerudo/nas-rl2', 'size': '10057'}