<jupyter_start><jupyter_text>Training<jupyter_code>S3_MODEL_SAVE_PREFIX = 'lidar-advanced-3CNN-1Dense-Wide_5Cars'
stream_name_list =['sim-zxy0zcf42r8b', 'sim-mtzl872zbd3n', 'sim-j5tgkw5gpkpn', 'sim-py7n1zft2r1z', 'sim-d62fl1h900f3', 'sim-hqj15g2ggp3s', 'sim-nf9mtsvl2r8v', 'sim-rh110cdrsym7']
sagemaker_simapp_name_list = ['sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191130-004028']
s3_bucket = 'sahika-lidar-oldcar-tf12-wide-ic'
s3_prefix = 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191130-004028'
simulation_app_arn = "arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191130-004040/1575074441051"

eval_stream_name_list =['sim-mqgrsljdf80r']


# S3_MODEL_SAVE_PREFIX = 'lidar-advanced-3CNN-2Dense-Narrow_5Cars'
# stream_name_list =['sim-tgw6ff3bp8cl', 'sim-g29ghvz69kbr', 'sim-g7zpvcs3mm6f', 'sim-s3chd3dbh3sq', 'sim-ktmvj1rt6m40', 'sim-l9jm6mwjdzk6', 'sim-kg2fyn59lvns', 'sim-z6llrbm21f5c']
# sagemaker_simapp_name_list = ['sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191130-004228']
# s3_bucket = 'sahika-lidar-oldcar-tf12-wide-ic'
# s3_prefix = 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191130-004228'
# simulation_app_arn = "arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191130-004241/1575074561367"

# eval_stream_name_list =['sim-6txfysc730x7']

S3_MODEL_SAVE_PREFIX = 'lidar-advanced-5CNN-1Dense-Wide_5Cars'
stream_name_list =['sim-2dnz2vmmf76t', 'sim-2338dltcmr2f', 'sim-lfrjc9tvhczy', 'sim-731551c348z7', 'sim-1s2cskzffr2f', 'sim-1mx5bbv750hd', 'sim-tf43p72mk5xq', 'sim-8lh609kq1yz9']
sagemaker_simapp_name_list = ['sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191130-004405']
s3_bucket = 'sahika-lidar-oldcar-tf12-wide-ic'
s3_prefix = 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191130-004405'
simulation_app_arn = "arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191130-004420/1575074660783"

eval_stream_name_list =['sim-zz27sdg8gp0c']


S3_MODEL_SAVE_PREFIX = 'lidar-advanced-5CNN-2Dense-Narrow_5Cars'
stream_name_list =['sim-llxqszv47ww6', 'sim-5mzb6dyc3d31', 'sim-5mzb6dyc3d31', 'sim-gxyb4qx6s17k', 'sim-llx77bv822yr', 'sim-3m09gdg6td25', 'sim-7pwv3zz5kv73', 'sim-4drkx49nmkj3']
sagemaker_simapp_name_list = ['sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191130-164820']
s3_bucket = 'sahika-lidar-oldcar-tf12-wide-ic'
s3_prefix = 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191130-164820'
simulation_app_arn = "arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191130-164836/1575132516838"

eval_stream_name_list =['sim-6pc9l6qpx0h9']


S3_MODEL_SAVE_PREFIX = 'lidar-advanced-3CNN-1Dense-Wide_3Cars'
stream_name_list =['sim-qljyb9qt4gq7', 'sim-4gjnhzb9c50q', 'sim-yrv5s3z8c0pm', 'sim-8nwv53dvv661', 'sim-smr0r6fwwh26', 'sim-pnnj5tcvyhnb', 'sim-29qt3dmg6l51', 'sim-8p0wq4xvmqvz']
sagemaker_simapp_name_list = ['sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191130-165345']
s3_bucket = 'sahika-lidar-oldcar-tf12-wide-ic'
s3_prefix = 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191130-165345'
simulation_app_arn = "arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191130-165405/1575132845598"

eval_stream_name_list =['sim-y1184k74bnjt']


S3_MODEL_SAVE_PREFIX = 'lidar-advanced-3CNN-2Dense-Narrow_3Cars'
stream_name_list =['sim-tskdx2zb51cd', 'sim-j9d9zpd64lgh', 'sim-2lb83v4c9jvm', 'sim-rpbt01nl70yl', 'sim-k7pc33fsph65', 'sim-tw2m49v6n27t', 'sim-qxnwppqf5s0h', 'sim-2d170nx43hrb']
sagemaker_simapp_name_list = ['sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191130-220245']
s3_bucket = 'sahika-lidar-oldcar-tf12-wide-ic'
s3_prefix = 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191130-220245'
simulation_app_arn = "arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191130-220300/1575151380403"

eval_stream_name_list =['sim-wbdp9y9wrd7y']


# S3_MODEL_SAVE_PREFIX = 'lidar-advanced-5CNN-1Dense-Wide_3Cars'
# stream_name_list =['sim-blm0sfv0sn3f', 'sim-7qh5rh768nw6', 'sim-t9hf8y2f7mzf', 'sim-xrbzz7lnbs0k', 'sim-0z8w2zsxk3m4', 'sim-0z8w2zsxk3m4', 'sim-jrnrh92g6rrx', 'sim-y33g8nw5fdb0']
# sagemaker_simapp_name_list = ['sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191130-220512']
# s3_bucket = 'sahika-lidar-oldcar-tf12-wide-ic'
# s3_prefix = 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191130-220512'
# simulation_app_arn = "arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191130-220524/1575151524710"

# eval_stream_name_list =['sim-5b80qws7h0bx']


S3_MODEL_SAVE_PREFIX = 'lidar-advanced-5CNN-2Dense-Narrow_3Cars'
stream_name_list =['sim-wfjkx6x1blwz', 'sim-fnfml277nwkp', 'sim-cgt7n8w9k8w1', 'sim-lyrxy099225p', 'sim-fvc107p5qd3t', 'sim-4spmw4d39t01', 'sim-w471qsv01jk7', 'sim-82xgc5418c8z']
sagemaker_simapp_name_list = ['sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191201-043708']
s3_bucket = 'sahika-lidar-oldcar-tf12-wide-ic'
s3_prefix = 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191201-043708'
simulation_app_arn = "arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191201-043729/1575175049924"

eval_stream_name_list =['sim-mrtgr9j538kt']



S3_MODEL_SAVE_PREFIX = 'lidar-advanced-5CNN-2Dense-Narrow_5Cars'
stream_name_list =['sim-s8d6y7gc1tv0', 'sim-6mgryy46nc90', 'sim-94r10v4gv5lr', 'sim-3d5pqwsgk8p1', 'sim-97czsqm7vzdt', 'sim-x1hryct08zpd', 'sim-7xppg69df2sj', 'sim-ktfvbz3z09p7']
sagemaker_simapp_name_list = ['sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191201-161054']
s3_bucket = 'sahika-lidar-oldcar-tf12-wide-ic'
s3_prefix = 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191201-161054'
simulation_app_arn = "arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191201-161106/1575216666131"

eval_stream_name_list =['sim-zm0ywkvyql74']


S3_MODEL_SAVE_PREFIX = 'lidar-advanced-5CNN-2Dense-Narrow_5Cars'
stream_name_list =['sim-y2t8shkc002j', 'sim-zq6y3ktk0d57', 'sim-qwc14bl6frj3', 'sim-cbmbc7bk7gfs', 'sim-wscp32n8c29k', 'sim-19t0l7pmy5d9', 'sim-72k12zbzx7sh', 'sim-0d4xk6n7r8xc']
sagemaker_simapp_name_list = ['sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191201-161426']
s3_bucket = 'sahika-lidar-oldcar-tf12-wide-ic'
s3_prefix = 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191201-161426'
simulation_app_arn = "arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191201-161440/1575216880607"

eval_stream_name_list =['sim-psw77g9p0qyf']


S3_MODEL_SAVE_PREFIX = 'lidar-advanced-5CNN-2Dense-Narrow_5Cars'
stream_name_list =['sim-kjnn0n15w011', 'sim-3yv8cnrsmz1k', 'sim-2hvf943c5zjc', 'sim-fpl9pb3k3t00', 'sim-lmfn1xwrdlkh', 'sim-b3d47bmz2c7y', 'sim-c33x4pzmsv8f', 'sim-9g2smq1lbyyb', 'sim-x080vm3741d1', 'sim-0t20ywm5qxjz', 'sim-f24np903h3s2', 'sim-n9b2szzhlvq5', 'sim-7tmytcxm4dlv', 'sim-8yp7l0zqcyk8', 'sim-lsx58wgc7dyy', 'sim-np5nyxmbckq5']
sagemaker_simapp_name_list = ['sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191128-180322']
s3_bucket = 'sahika-lidar-oldcar-tf12-wide-ic'
s3_prefix = 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191128-180322'
simulation_app_arn = "arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191128-180336/1574964216309"

eval_stream_name_list =['sim-f1ncm4mdqn6v']




S3_MODEL_SAVE_PREFIX = 'DEC3_lidar-advanced-5CNN-2Dense-Narrow_5Cars'
stream_name_list =['sim-k92pmjn4q238', 'sim-0jmrj74mhf7v', 'sim-2p76nl9shj8p', 'sim-8w93w2bx8dg5', 'sim-ps4r7x1fn51p', 'sim-0j42m79cx6sw', 'sim-wg8cbf7y7lr8', 'sim-vvh8gdz9bvsk']
sagemaker_simapp_name_list = ['sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191204-062337']
s3_bucket = 'sahika-lidar-oldcar-tf12-wide-ic'
s3_prefix = 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191204-062337'
simulation_app_arn = "arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191204-062353/1575440633883"

eval_stream_name_list =['sim-vwl590jh2s12']


S3_MODEL_SAVE_PREFIX = 'DEC3_lidar-advanced-5CNN-2Dense-Narrow_5Cars_run2'
stream_name_list =['sim-04f4jcvf0w71', 'sim-cyb2wmk77nd1', 'sim-8pfqsxtxk3bj', 'sim-bxy77j3ypnkg', 'sim-s521yd47swfm', 'sim-nymjf26dgs01', 'sim-vngcvy8wzpzs', 'sim-6tyv1gz6nmj0']
sagemaker_simapp_name_list = ['sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191204-071349']
s3_bucket = 'sahika-lidar-oldcar-tf12-wide-ic'
s3_prefix = 'sahika-lidar-oldcar-tf12-wide-ic-sagemaker-191204-071349'
simulation_app_arn = "arn:aws:robomaker:us-west-2:828753602420:simulation-application/sahika-multicar-oldcar-191204-071404/1575443644209"

eval_stream_name_list =['sim-w3vgqrrwrklf']


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from datetime import datetime
%matplotlib inline

import os
os.getcwd()

def smooth(y, radius, mode='two_sided', valid_only=False):
    '''
    Smooth signal y, where radius is determines the size of the window
    mode='twosided':
        average over the window [max(index - radius, 0), min(index + radius, len(y)-1)]
    mode='causal':
        average over the window [max(index - radius, 0), index]
    valid_only: put nan in entries where the full-sized window is not available
    '''
    assert mode in ('two_sided', 'causal')
    if len(y) < 2*radius+1:
        return np.ones_like(y) * y.mean()
    elif mode == 'two_sided':
        convkernel = np.ones(2 * radius+1)
        out = np.convolve(y, convkernel,mode='same') / np.convolve(np.ones_like(y), convkernel, mode='same')
        if valid_only:
            out[:radius] = out[-radius:] = np.nan
    elif mode == 'causal':
        convkernel = np.ones(radius)
        out = np.convolve(y, convkernel,mode='full') / np.convolve(np.ones_like(y), convkernel, mode='full')
        out = out[:-radius+1]
        if valid_only:
            out[:radius] = np.nan
    return out
from log_analysis import *
import cw_utils
import os
os.getcwd()
if os.path.exists('logs/'):
    !rm -r tmp
!mkdir tmp

for stream_name in stream_name_list:
    fname = 'logs/deepracer-%s.log' %stream_name
    cw_utils.download_log(fname, stream_prefix=stream_name)
    !tail -n 100 $fname
fig = plt.figure(figsize=(12, 6))
ax1 = fig.add_subplot(111)
color = ['r.', 'b.', 'g.', 'c.', 'm.', 'y.', 'k.', 'r.', 'r.', 'b.', 'g.', 'c.', 'm.', 'y.', 'k.', 'r.']

training_time = []

for idx, stream_name in enumerate(stream_name_list):
    fname = 'logs/deepracer-%s.log' %stream_name
    data = load_data(fname)
    df = convert_to_pandas(data, None)
    
    #print(len(df))
    if len(df) > 10:

        t0 = float(df['timestamp'][0])
        t1 = float(df['timestamp'][len(df)-1])
        tdelta = (t1-t0) / 3600.00
        training_time.append('Training time: %.2f hours' % tdelta)
    #     import time
    #     print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(t0)))
    #     print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(t1)))
    #     print('Training time: %.2f hours' % tdelta)


        REWARD_THRESHOLD = 1000

        # reward graph per episode
        min_episodes = np.min(df['episode'])
        max_episodes = np.max(df['episode'])
    #     print('Number of episodes = ', max_episodes)

        total_reward_per_episode = list()
        max_progress_per_episode = list()
        steps_progress_per_episode = list()
        is_crashed_per_episode = list()
        vicinity_ratio_per_episode = list()
        #print(df_slice['mindist2bot'] )

        for epi in range(min_episodes, max_episodes):
            df_slice = df[df['episode'] == epi]
            total_reward_per_episode.append(np.sum(df_slice['reward']))
            max_progress_per_episode.append(np.max(df_slice['progress']))
            is_crashed = len(df_slice[ df_slice['iscrashed'] == True ])
            
            if len(df_slice) > 0:
                steps_progress_per_episode.append(len(df_slice))
                vicinity_ratio_per_episode.append(np.min(df_slice['mindist2bot']))
            else:
                steps_progress_per_episode.append(float('nan'))
                vicinity_ratio_per_episode.append(float('nan'))

        #ax = fig.add_subplot(313)
        ax1.plot(np.arange(len(max_progress_per_episode)), max_progress_per_episode, color[idx])        
        if len(max_progress_per_episode) > 50:
            ax1.plot(np.arange(len(max_progress_per_episode)), smooth(max_progress_per_episode, radius=50), color[idx])
        ax1.set_ylabel('Max progress')
        ax1.set_xlabel('Episode')

        reward_per_step = list()
        for rew, stp, prog in zip(total_reward_per_episode, \
                                  steps_progress_per_episode, max_progress_per_episode):
            if prog > 20:
                reward_per_step.append(float(rew)/float(stp))
            else:
                reward_per_step.append(float('nan'))
        
plt.savefig('figs/%s_%s_0.png' %(eval_stream_name_list[0], S3_MODEL_SAVE_PREFIX))

print(training_time)
for sim_index in range(1): #len(robomaker_simapp_name_list)):

    # sagemaker
    #fname = 'logs/sagemaker-%s.log' %stream_name
    fname = 'logs/sagemaker-%s.log' %sagemaker_simapp_name_list[sim_index]
    cw_utils.download_log(fname, stream_prefix=sagemaker_simapp_name_list[sim_index], 
                          log_group='/aws/sagemaker/TrainingJobs')
    !tail -n 10 $fname
    
    
#    
for sim_index in range(len(sagemaker_simapp_name_list)):#[0, 21]:
    # round robin
#     stream_name = robomaker_simapp_name_list[sim_index] # VGG
#     experiment_name = experiment_name_list[sim_index]

#     print('Experiment: ', experiment_name)
#     print('Robomaker Simapp: ', stream_name)

    # sagemaker
    fname = 'logs/sagemaker-%s.log' %sagemaker_simapp_name_list[sim_index]
  
    episode_training_results = list()
    policy_training_results = list()
    episode = 0
    with open(fname, 'r') as f:
        for line in f:
            if 'Training> Name=main_level/agent' in line:
                for col in line.split(','):
                    if 'Episode' in col:
                        episode = int(col.split('=')[1])
                    if 'Total reward' in col:
                        total_reward = float(col.split('=')[1])
                    if 'Steps' in col:
                        steps = int(col.split('=')[1])
                episode_training_results.append((episode, total_reward, steps))

            if 'Policy training>' in line:
                for col in line.split(','):
                    if 'Surrogate loss' in col:
                        surrogate_loss = float(col.split('=')[1])
                    if 'Entropy' in col:
                        entropy = float(col.split('=')[1])
                # will use the last episode to sync
                policy_training_results.append((surrogate_loss, entropy, episode))
                
    #print(policy_training_results)
                
    smoothing_radius_0 = 2
    smoothing_radius_1 = 50

    fig = plt.figure(figsize=(10, 10))
    ax = fig.add_subplot(211)
    ax.plot([val[2] for val in policy_training_results], 
            smooth([val[1] for val in policy_training_results], radius=smoothing_radius_0))
    ax.plot([val[2] for val in policy_training_results], 
            smooth([val[1] for val in policy_training_results], radius=smoothing_radius_1))
    ax.set_xlabel('Last episode')
    ax.set_ylabel('Raw Policy Entropy')
    #ax.set_title(stream_name + ': ' + experiment_name)

    ax = fig.add_subplot(212)
    ax.plot([val[0] for val in episode_training_results], 
            smooth([val[1] for val in episode_training_results], radius=smoothing_radius_0))
    ax.plot([val[0] for val in episode_training_results], 
            smooth([val[1] for val in episode_training_results], radius=smoothing_radius_1))
    ax.set_xlabel('Episode')
    ax.set_ylabel('Raw Total Reward')
    

plt.savefig('figs/%s_%s_1.png' %(eval_stream_name_list[0], S3_MODEL_SAVE_PREFIX))<jupyter_output><empty_output><jupyter_text>Eval<jupyter_code>#stream_name_list =['sim-gl8k0xhjbnk4']
#eval_stream_name_list = ['sim-wgtvvydg1cpb']
print(eval_stream_name_list)<jupyter_output><empty_output><jupyter_text>Download the desired log file given the simulation ID If you wish to bulk export the logs from Amazon Cloudwatch to Amazon S3 :: https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/S3ExportTasks.html<jupyter_code>if os.path.exists('logs/'):
    !rm -r tmp
!mkdir tmp

for stream_name in eval_stream_name_list:
    fname = 'logs/deepracer-%s.log' %stream_name
    cw_utils.download_log(fname, stream_prefix=stream_name)
    !tail -n 10 $fname
for idx, stream_name in enumerate(eval_stream_name_list):
    fname = 'logs/deepracer-%s.log' %stream_name
    checkpoints, data = load_checkpoints(fname)
    
print(len(checkpoints))
print(len(data))
print(checkpoints[-1])
color = ['r.', 'b.', 'g.', 'c.']
mean_max_progress = list()
std_max_progress = list()
checkpoint_max_progress = list()
path_list = list()
vicinity_ratio = list()
num_pass_success = list()

fname = 'logs/deepracer-%s.log' %stream_name
checkpoints, data_checkpoints = load_checkpoints(fname)


full_lap_idx_per_checkpoint = []

for idxchk in range(20, len(checkpoints) - 1):
    #print(idxchk, checkpoints[idxchk])
    
    checkpoints_id = int(checkpoints[idxchk].split('/')[-1].split('_')[0])
    #print(checkpoints_id)
    
    
    data = data_checkpoints[idxchk]
    df = convert_to_pandas(data, None)

    REWARD_THRESHOLD = 1000

    # reward graph per episode
    min_episodes = np.min(df['episode'])
    max_episodes = np.max(df['episode'])
#     print('Number of episodes = ', max_episodes)

    total_reward_per_episode = list()
    max_progress_per_episode = list()
    steps_progress_per_episode = list()
    path = list()

    is_crashed_per_episode = list()
    vicinity_ratio_per_episode = list()
    num_pass_success_per_episode = list()

    for epi in range(min_episodes, max_episodes):
        df_slice = df[df['episode'] == epi]
        total_reward_per_episode.append(np.sum(df_slice['reward']))
        max_progress_per_episode.append(np.max(df_slice['progress']))
        is_crashed = len(df_slice[ df_slice['iscrashed'] == True ])

        num_passes = 0
        has_pass_started = False
        mindist2bot = smooth(df_slice['mindist2bot'], radius=5)

        for step in range(1, len(mindist2bot)):
            if mindist2bot[step-1] > 0.5 and mindist2bot[step] < 0.5:
                has_pass_stared = True

            if mindist2bot[step-1] < 0.5 and mindist2bot[step] > 0.5 and has_pass_stared:
                num_passes += 1
                has_pass_stared = False


        if len(df_slice) > 0:
            steps_progress_per_episode.append(len(df_slice))
            vicinity_ratio_per_episode.append(100.0*len([val for val in df_slice['mindist2bot'] if val <=0.5]) / float(len(df_slice)))
            num_pass_success_per_episode.append(num_passes)
        else:
            steps_progress_per_episode.append(float('nan'))
            vicinity_ratio_per_episode.append(float('nan'))
            num_pass_success_per_episode.append(float('nan'))

        #if np.max(df_slice['progress']) >= 99:    
        path.append((df_slice['x'], df_slice['y']))    

    checkpoint_max_progress.append(checkpoints_id)
    mean_max_progress.append(np.mean(max_progress_per_episode))    
    std_max_progress.append(np.std(max_progress_per_episode))
    vicinity_ratio.append(np.mean(vicinity_ratio_per_episode))
    num_pass_success.append(np.sum(num_pass_success_per_episode))

    full_lap_idx = np.where(np.array(max_progress_per_episode) >= 100)[0]
    full_lap_idx_per_checkpoint.append(full_lap_idx.shape[0])
    path_list.append({'model':checkpoints_id, 'path': path, 'full_laps': full_lap_idx.shape[0]})


fig = plt.figure(figsize=(20, 2))
ax1 = fig.add_subplot(141)
ax2 = fig.add_subplot(142)
ax3 = fig.add_subplot(143)
ax4 = fig.add_subplot(144)

mean_smooth = smooth(mean_max_progress, radius=5)
ax1.plot(checkpoint_max_progress, mean_max_progress, '.')
ax1.plot(checkpoint_max_progress, mean_smooth, '-', linewidth=3)
#ax1.fill_between(checkpoint_max_progress, mean_smooth - 0.20*smooth(std_max_progress, radius=5), 
#                 mean_smooth + 0.20*smooth(std_max_progress, radius=5), color='grey')

ax1.set_xlabel('Checkpoint')
ax1.set_ylabel('Percent')
ax1.set_title('Mean progress')
ax1.set_ylim((50, 100))
ax1.grid()

ax2.plot(checkpoint_max_progress, std_max_progress, '.')
ax2.plot(checkpoint_max_progress, smooth(std_max_progress, radius=5), '-', linewidth=3)
#ax1.fill_between(checkpoint_max_progress, mean_smooth - 0.20*smooth(std_max_progress, radius=5), 
#                 mean_smooth + 0.20*smooth(std_max_progress, radius=5), color='grey')

ax2.set_xlabel('Checkpoint')
ax2.set_ylabel('Percent')
ax2.set_title('Std progress')
ax2.set_ylim((0, 50))
ax2.grid()

ax3.plot(checkpoint_max_progress, full_lap_idx_per_checkpoint, '.')
ax3.plot(checkpoint_max_progress, smooth(full_lap_idx_per_checkpoint, radius=5), '-', linewidth=3)

ax3.set_xlabel('Checkpoint')
ax3.set_ylabel('Count')
ax3.set_title('Number of Full laps')
ax3.set_ylim((0, 15))
ax3.grid()


ax4.plot(checkpoint_max_progress, num_pass_success, '.')
ax4.plot(checkpoint_max_progress, smooth(num_pass_success, radius=5), '-', linewidth=3)
ax4.set_xlabel('Checkpoint')
ax4.set_ylabel('Percent')
ax4.set_title('Num Passes')
#ax3.set_ylim((0, 3.10))
ax4.grid()

plt.savefig('figs/%s_%s_2.png' %(eval_stream_name_list[0], S3_MODEL_SAVE_PREFIX))
#print(s3://sahika-models/{S3_MODEL_SAVE_PREFIX})
model_list_candidates = list()
for idx, val_max, val_laps in zip(checkpoint_max_progress, mean_max_progress, full_lap_idx_per_checkpoint):
    if val_max > 95 and val_laps > 10:
        model_list_candidates.append(idx)
        print('chkpnt=%d, mean_max_progress=%.2f, laps_100_percent==%.2f' %(idx, val_max, val_laps))
#print(model_list_candidates)
def get_track_waypoints(track_name):
    return np.load("%s.npy" % track_name)

track_name = 'reInvent2018_36inch'
track_name = 'reInvent2019_track'

waypoints = get_track_waypoints(track_name) ### re:invent track
waypoints.shape

# convert to cm
l_inner_border = 100*waypoints[:,2:4]
l_outer_border = 100*waypoints[:,4:6]

#model_list_candidates= [34]
for dval in path_list:
    if dval['model'] in model_list_candidates:
        #print(dval['model'])
        fig = plt.figure(figsize=(7, 5))
        ax1 = fig.add_subplot(111)
        ax1.plot(l_inner_border[:,0], l_inner_border[:,1], 'k-')
        ax1.plot(l_outer_border[:,0], l_outer_border[:,1], 'k-')
        for val in dval['path']:
            ax1.plot(val[0], val[1], '.')
            ax1.set_title('Model id:%d, Full laps=%d' % (dval['model'], dval['full_laps']))
            
        plt.savefig('figs/%s_%s_paths_%d.png' %(eval_stream_name_list[0], S3_MODEL_SAVE_PREFIX, dval['model']))<jupyter_output><empty_output><jupyter_text>Pick the iteration number of the model<jupyter_code>model_list = model_list_candidates
#model_list = [100, 160, 188]

print(model_list)
#from convert_pb import *

if os.path.exists('tmp/'):
    !rm -r tmp
!mkdir tmp

for model_id in model_list:
    model_folder_name = './tmp/model_' + str(model_id)
    os.makedirs(model_folder_name)
    model_pb_name = "model_" + str(model_id) + ".pb"    
    #!aws s3 sync s3://{s3_bucket}/{s3_prefix}/model $model_folder_name --exclude "*" --include $model_pb_name --include "*.json"
    !aws s3 sync s3://{s3_bucket}/{s3_prefix}/model $model_folder_name --exclude "*" --include $model_pb_name 
        
    model_targz_name = model_folder_name + '.tar.gz'
    !tar -czvf $model_targz_name $model_folder_name
    
    #!python convert_pb_rename.py $model_folder_name//$model_pb_name 
    #!rm -r $model_folder_name//$model_pb_name
    
    # extra files for device
    #model_pb_name_0 = model_folder_name + '/model.pb'
    #!cp $model_folder_name//$model_pb_name $model_folder_name//model.pb
    #!cp convert_pb.py $model_folder_name

# upload
!aws s3 rm --recursive s3://sahika-models/{S3_MODEL_SAVE_PREFIX}
!aws s3 cp --recursive tmp s3://sahika-models/{S3_MODEL_SAVE_PREFIX}
# refactor model if not stereo_cameras
#!mkdir model_refactor
#!aws s3 sync s3://sahika-models/lidar-advanced-mediumStereo-shallowLidar-60-FAST-2019-NARROW/model_128 model_refactor
#!python convert_pb_layer_refactor.py 'model_refactor/model_128.pb'
#!tar -czvf 'model_refactor/lidar_128.tar.gz' 'model_refactor/model_refactor.pb' 'model_refactor/model_metadata.json'
#!aws s3 cp 'model_refactor/lidar_128.tar.gz' s3://sahika-tar-files-deepracer<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14055648895627587, 'fraction_numerical': 0.0752776297996148, 'mean_word_length': 4.450971632789814, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 30, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '228421', 'license': 'MIT-0', 'n_tokens_mistral': 11064, 'n_tokens_neox': 10065, 'n_words': 1257, 'path': 'Advanced workshops/reInvent2019-400/logs_training_evaluation.ipynb', 'repo_name': 'seantbrady/aws-deepracer-workshops', 'size': '24403'}