<jupyter_start><jupyter_text>if you wish to set which cores to useaffinity_mask = {4, 5, 7} affinity_mask = {6, 7, 9} affinity_mask = {0, 1, 3} affinity_mask = {2, 3, 5} affinity_mask = {0, 2, 4, 6} pid = 0os.sched_setaffinity(pid, affinity_mask) print("CPU affinity mask is modified to %s for process id 0" % affinity_mask)   DEFAULT 'CarRacing-v3' environment values continuos action = (steering_angle, throttle, brake)ACT = [[0, 0, 0], [-0.4, 0, 0], [0.4, 0, 0], [0, 0.6, 0], [0, 0, 0.8]] discrete actions: center_steering and no gas/brake, steer left, steer right, accel, brake       --> actually a good choice, because car_dynamics softens the action's diff for gas and steeringREWARDS  reward given each step: step taken, distance to centerline, normalized speed [0-1], normalized steer angle [0-1] reward given on new tile touched: %proportional of advance, %advance/steps_taken reward given at episode end: all tiles touched (track finished), patience or off-raod exceeded, out of bounds, max_steps exceeded reward for obstacles:  obstacle hit (each step), obstacle collided (episode end)GYM_REWARD = [ -0.1, 0.0, 0.0, 0.0,   10.0, 0.0,     0,  -0, -100, -0,     -0, -0 ]STD_REWARD = [ -0.1, 0.0, 0.0, 0.0,    1.0, 0.0,   100, -20, -100, -50,    -0, -0 ]CONT_REWARD =[-0.11, 0.1, 0.0, 0.0,    1.0, 0.0,   100, -20, -100, -50,    -5, -100 ] see docu for RETURN computation details DEFAULT Environment Parameters (not related to RL Algorithm!)game_color = 1            State (frame) color option: 0 = RGB, 1 = Grayscale, 2 = Green onlyindicators = True         show or not bottom Info Panelframes_per_state = 4      stacked (rolling history) Frames on each state [1-inf], latest observation always on first Frameskip_frames = 3           number of consecutive Frames to skip between history saves [0-4]discre = ACT              Action discretization function, format [[steer0, throtle0, brake0], [steer1, ...], ...]. None for continuoususe_track = 1             number of times to use the same Track, [1-100]. More than 20 high risk of overfitting!!episodes_per_track = 1    number of evenly distributed starting points on each track [1-20]. Every time you call reset(), the env automatically starts at the next pointtr_complexity = 12        generated Track geometric Complexity, [6-20]tr_width = 45             relative Track Width, [30-50]patience = 2.0            max time in secs without Progress, [0.5-20]off_track = 1.0           max time in secs Driving on Grass, [0.0-5]f_reward = CONT_REWARD    Reward Funtion coefficients, refer to Docu for detailsnum_obstacles = 5         Obstacle objects placed on track [0-10]end_on_contact = False    Stop Episode on contact with obstacle, not recommended for starting-phase of trainingobst_location = 0         array pre-setting obstacle Location, in %track. Negative value means tracks's left-hand side. 0 for random locationoily_patch = False        use all obstacles as Low-friction road (oily patch)verbose = 2<jupyter_code>## Choose one agent, see Docu for description
#agent='CarRacing-v0'
#agent='CarRacing-v1'
agent='CarRacing-v3'

# Stop training when the model reaches the reward threshold
callback_on_best = StopTrainingOnRewardThreshold(reward_threshold = 170, verbose=1)

seed = 2000
## SIMULATION param  
## Changing these makes world models incompatible!!
game_color = 2
indicators = True
fpst = 4
skip = 3
actions = [[0, 0, 0], [-0.4, 0, 0], [0.4, 0, 0], [0, 0.6, 0], [0, 0, 0.8]]  #this is ACT

obst_loc = [6, -12, 25, -50, 75, -37, 62, -87, 95, -29]  #track percentage, negative for obstacle to the left-hand side

## Loading drive_pretained model

import pickle
root = 'ppo_cnn_gym-mod_'
file = root+'c{:d}_f{:d}_s{:d}_{}_a{:d}'.format(game_color,fpst,skip,indicators,len(actions))

model = PPO2.load(file)
## This model param
use = 6       # number of times to use same track [1,100]
ept = 10      # different starting points on same track [1,20]
patience = 1.0
track_complexity = 12
#REWARD2 = [-0.05, 0.1, 0.0, 0.0,   2.0, 0.0,   100, -20, -100, -50,   -5, -100]

if agent=='CarRacing-v3': 
    env = gym.make(agent, seed=seed, 
        game_color=game_color,
        indicators=indicators,
        frames_per_state=fpst,
        skip_frames=skip,   
#        discre=actions,          #passing custom actions
        use_track = use,       
        episodes_per_track = ept,  
        tr_complexity = track_complexity, 
        tr_width = 45,
        patience = patience,
        off_track = patience,
        end_on_contact = True,     #learning to avoid obstacles the-hard-way
        oily_patch = False,
        num_obstacles = 5,         #some obstacles
        obst_location = obst_loc,  #passing fixed obstacle location
#        f_reward = REWARD2,        #passing a custom reward function
        verbose = 2 )            
else: 
    env = gym.make(agent)

env = DummyVecEnv([lambda: env])
## Training on obstacles
model.set_env(env)
batch_size = 256
updates = 700
model.learn(total_timesteps = updates*batch_size, log_interval=1) #, callback=eval_callback)
#Save last updated model

file = root+'c{:d}_f{:d}_s{:d}_{}_a{:d}__u{:d}_e{:d}_p{}_bs{:d}'.format(
    game_color,fpst,skip,indicators,len(actions),use,ept,patience,batch_size)

model.save(file, cloudpickle=True)
param_list=model.get_parameter_list()

env.close()
## This model param #2
use = 6       # number of times to use same track [1,100]
ept = 10      # different starting points on same track [1,20]
patience = 1.0
track_complexity = 12
#REWARD2 = [-0.05, 0.1, 0.0, 0.0,   2.0, 0.0,   100, -20, -100, -50,   -5, -100]
seed = 25000

if agent=='CarRacing-v3': 
    env2 = gym.make(agent, seed=seed, 
        game_color=game_color,
        indicators=indicators,
        frames_per_state=fpst,
        skip_frames=skip,   
#        discre=actions,          #passing custom actions
        use_track = use,       
        episodes_per_track = ept,  
        tr_complexity = track_complexity, 
        tr_width = 45,
        patience = patience,
        off_track = patience,
        end_on_contact = False,    # CHANGED 
        oily_patch = False,
        num_obstacles = 5,         #some obstacles
        obst_location = 0,         #using random obstacle location
#        f_reward = REWARD2,        #passing a custom reward function
        verbose = 3 )            
else: 
    env2 = gym.make(agent)

env2 = DummyVecEnv([lambda: env2])
## Training on obstacles
model.set_env(env2)
#batch_size = 384
updates = 1500
## Separate evaluation env
test_freq = 100      #policy updates until evaluation
test_episodes_per_track = 5   #number of starting points on test_track
eval_log = './evals/'

env_test = gym.make(agent, seed=int(3.14*seed), 
        game_color=game_color,
        indicators=indicators,
        frames_per_state=fpst,
        skip_frames=skip,   
#        discre=actions,            #passing custom actions
        use_track = 1,           #change test track after 1 ept round
        episodes_per_track = test_episodes_per_track,  
        tr_complexity = 12,      #test on a medium complexity track
        tr_width = 45,
        patience = 2.0,
        off_track = 2.0,
        end_on_contact = False,
        oily_patch = False,
        num_obstacles = 5,
        obst_location = obst_loc)  #passing fixed obstacle location

env_test = DummyVecEnv([lambda: env_test])

eval_callback = EvalCallback(env_test, callback_on_new_best=callback_on_best,  #None,
                             n_eval_episodes=test_episodes_per_track*3, eval_freq=test_freq*batch_size,
                             best_model_save_path=eval_log, log_path=eval_log, deterministic=True, 
                             render = False)

model.learn(total_timesteps = updates*batch_size, log_interval=1, callback=eval_callback)
#Save last updated model

#file = root+'c{:d}_f{:d}_s{:d}_{}_a{:d}__u{:d}_e{:d}_p{}_bs{:d}'.format(
#    game_color,fpst,skip,indicators,len(actions),use,ept,patience,batch_size)

model.save(file+'_II', cloudpickle=True)
param_list=model.get_parameter_list()

env2.close()
env_test.close()
## Enjoy last trained policy

if agent=='CarRacing-v3':  #create an independent test environment, almost everything in std/random definition
    env3 = gym.make(agent, seed=None, 
        game_color=game_color,
        indicators = True,
        frames_per_state=fpst,
        skip_frames=skip,   
#        discre=actions,
        use_track = 2,       
        episodes_per_track = 1,  
        patience = 5.0,
        off_track = 3.0    )
else:
    env3 = gym.make(agent)

env3 = DummyVecEnv([lambda: env3])
obs = env3.reset()
print(obs.shape)        

done = False
pasos = 0
_states=None

while not done: # and pasos<1500:
    action, _states = model.predict(obs, deterministic=True)
    obs, reward, done, info = env3.step(action)
    env3.render()
    pasos+=1
    
env3.close()
print()
print(reward, done, pasos) #, info)
## Enjoy best eval_policy

obs = env3.reset()
print(obs.shape)        

## Load bestmodel from eval
#if not isinstance(model_test, PPO2):
model_test = PPO2.load(eval_log+'best_model', env3)

done = False
pasos = 0
_states=None

while not done: # and pasos<1500:
    action, _states = model_test.predict(obs, deterministic=True)
    obs, reward, done, info = env3.step(action)
    env3.render()
    pasos+=1
    
env3.close()
print()
print(reward, done, pasos)
print(action, _states)
model_test.save(file+'_evalbest', cloudpickle=True)
env2.close()
env3.close()
env_test.close()
print(action, _states)
obs.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11206806007772292, 'fraction_numerical': 0.04190736267198824, 'mean_word_length': 3.129228100607112, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '917', 'license': 'MIT', 'n_tokens_mistral': 3470, 'n_tokens_neox': 3196, 'n_words': 920, 'path': 'examples/Train_ppo_cnn+eval_contact-(pretrained).ipynb', 'repo_name': 'pleslabay/CarRacing-mod', 'size': '9521'}