<commit_before># -*- encoding: utf-8 -*-
import gym
import time
import ctypes
import tempfile
import utils.logger
import multiprocessing
import tensorflow as tf
import numpy as np

from utils import checkpoint_utils
from utils.decorators import only_on_train
from utils.hogupdatemv import apply_grads_mom_rmsprop, apply_grads_adam, apply_grads_adamax
from contextlib import contextmanager
from multiprocessing import Process


CHECKPOINT_INTERVAL = 100000
ONE_LIFE_GAMES = [
    #Atari
    'Bowling-v0',
    'Boxing-v0',
    'Carnival-v0',
    'DoubleDunk-v0',
    'Enduro-v0',
    'FishingDerby-v0',
    'Freeway-v0',
    'IceHockey-v0',
    'JourneyEscape-v0',
    'Pong-v0',
    'PrivateEye-v0',
    'Skiing-v0',
    'Tennis-v0',
    #Classic Control
    'CartPole-v0',
    'CartPole-v1',
    'Pendulum-v0',
    'MountainCar-v0',
    'Acrobot-v1',
    'MountainCarContinuous-v0',
    'Pendulum-v0',
    #Box2D
    'LunarLander-v2',
    'LunarLanderContinuous-v2',
    'BipedalWalker-v2',
    'BipedalWalkerHardcore-v2',
    'CarRacing-v0',
    #MuJoCo
    'InvertedPendulum-v1',
    'IvertedDoublePendulum-v1',
    'Reacher-v1',
    'HalfCheetah-v1',
    'Swimmer-v1',
    'Hopper-v1',
    'Walker2d-v1',
    'Ant-v1',
    'Humanoid-v1',
    'HumanoidStandup-v1',
]
 
logger = utils.logger.getLogger('actor_learner')


class ActorLearner(Process):
    
    def __init__(self, args):
        super(ActorLearner, self).__init__()
       
        self.summ_base_dir = args.summ_base_dir
        
        self.local_step = 0
        self.global_step = args.global_step
        self.local_episode = 0
        self.last_saving_step = 0

        self.saver = None
        self.actor_id = args.actor_id
        self.alg_type = args.alg_type
        self.use_monitor = args.use_monitor
        self.max_local_steps = args.max_local_steps
        self.optimizer_type = args.opt_type
        self.optimizer_mode = args.opt_mode
        self.num_actions = args.num_actions
        self.initial_lr = args.initial_lr
        self.lr_annealing_steps = args.lr_annealing_steps
        self.num_actor_learners = args.num_actor_learners
        self.is_train = args.is_train
        self.input_shape = args.input_shape
        self.reward_clip_val = args.reward_clip_val
        self.q_update_interval = args.q_update_interval
        self.restore_checkpoint = args.restore_checkpoint
        self.random_seed = args.random_seed
        
        # Shared mem vars
        self.learning_vars = args.learning_vars
            
        if self.optimizer_mode == 'local':
            if self.optimizer_type == 'rmsprop':
                self.opt_st = np.ones(size, dtype=ctypes.c_float)
            else:
                self.opt_st = np.zeros(size, dtype=ctypes.c_float)
        elif self.optimizer_mode == 'shared':
                self.opt_st = args.opt_state

        # rmsprop/momentum
        self.alpha = args.momentum
        # adam
        self.b1 = args.b1
        self.b2 = args.b2
        self.e = args.e
        
        if args.env == 'GYM':
            from environments.atari_environment import AtariEnvironment
            self.emulator = AtariEnvironment(
                args.game,
                self.random_seed,
                args.visualize,
                use_rgb=args.use_rgb,
                frame_skip=args.frame_skip,
                agent_history_length=args.history_length,
                max_episode_steps=args.max_episode_steps,
                single_life_episodes=args.single_life_episodes,
            )
        elif args.env == 'ALE':
            from environments.emulator import Emulator
            self.emulator = Emulator(
                args.rom_path, 
                args.game, 
                args.visualize, 
                self.actor_id,
                self.random_seed,
                args.single_life_episodes)
        else:
            raise Exception('Invalid environment `{}`'.format(args.env))
            
        self.grads_update_steps = args.grads_update_steps
        self.max_global_steps = args.max_global_steps
        self.gamma = args.gamma

        self.rescale_rewards = args.rescale_rewards
        self.max_achieved_reward = -float('inf')
        if self.rescale_rewards:
            self.thread_max_reward = 1.0

        # Barrier to synchronize all actors after initialization is done
        self.barrier = args.barrier
        self.game = args.game

        # Initizlize Tensorboard summaries
        self.summary_ph, self.update_ops, self.summary_ops = self.setup_summaries()
        self.summary_op = tf.summary.merge_all()


    def reset_hidden_state(self):
        """
        Override in subclass if needed
        """
        pass


    def is_master(self):
        return self.actor_id == 0


    def test(self, num_episodes=100):
        """
        Run test monitor for `num_episodes`
        """
        target_params = self.session.run(self.target_network.params)
        self.assign_vars(self.local_network, target_params)

        rewards = list()
        for episode in range(num_episodes):
            s = self.emulator.get_initial_state()
            self.reset_hidden_state()
            total_episode_reward = 0
            episode_over = False

            while not episode_over:
                a = self.choose_next_action(s)[0]
                s, reward, episode_over = self.emulator.next(a)

                total_episode_reward += reward

            else:
                rewards.append(total_episode_reward)
                logger.info("EPISODE {0} -- REWARD: {1}, RUNNING AVG: {2:.0f}±{3:.0f}, BEST: {4}".format(
                    episode,
                    total_episode_reward,
                    np.array(rewards).mean(),
                    2*np.array(rewards).std(),
                    max(rewards),
                ))


    def synchronize_workers(self):
        if self.is_master():
            # Initialize network parameters
            g_step = checkpoint_utils.restore_vars(self.saver, self.session, self.game, self.alg_type, self.max_local_steps, self.restore_checkpoint)<commit_msg>remove target network assignment from test method<commit_after># -*- encoding: utf-8 -*-
import gym
import time
import ctypes
import tempfile
import utils.logger
import multiprocessing
import tensorflow as tf
import numpy as np

from utils import checkpoint_utils
from utils.decorators import only_on_train
from utils.hogupdatemv import apply_grads_mom_rmsprop, apply_grads_adam, apply_grads_adamax
from contextlib import contextmanager
from multiprocessing import Process


CHECKPOINT_INTERVAL = 100000
ONE_LIFE_GAMES = [
    #Atari
    'Bowling-v0',
    'Boxing-v0',
    'Carnival-v0',
    'DoubleDunk-v0',
    'Enduro-v0',
    'FishingDerby-v0',
    'Freeway-v0',
    'IceHockey-v0',
    'JourneyEscape-v0',
    'Pong-v0',
    'PrivateEye-v0',
    'Skiing-v0',
    'Tennis-v0',
    #Classic Control
    'CartPole-v0',
    'CartPole-v1',
    'Pendulum-v0',
    'MountainCar-v0',
    'Acrobot-v1',
    'MountainCarContinuous-v0',
    'Pendulum-v0',
    #Box2D
    'LunarLander-v2',
    'LunarLanderContinuous-v2',
    'BipedalWalker-v2',
    'BipedalWalkerHardcore-v2',
    'CarRacing-v0',
    #MuJoCo
    'InvertedPendulum-v1',
    'IvertedDoublePendulum-v1',
    'Reacher-v1',
    'HalfCheetah-v1',
    'Swimmer-v1',
    'Hopper-v1',
    'Walker2d-v1',
    'Ant-v1',
    'Humanoid-v1',
    'HumanoidStandup-v1',
]
 
logger = utils.logger.getLogger('actor_learner')


class ActorLearner(Process):
    
    def __init__(self, args):
        super(ActorLearner, self).__init__()
       
        self.summ_base_dir = args.summ_base_dir
        
        self.local_step = 0
        self.global_step = args.global_step
        self.local_episode = 0
        self.last_saving_step = 0

        self.saver = None
        self.actor_id = args.actor_id
        self.alg_type = args.alg_type
        self.use_monitor = args.use_monitor
        self.max_local_steps = args.max_local_steps
        self.optimizer_type = args.opt_type
        self.optimizer_mode = args.opt_mode
        self.num_actions = args.num_actions
        self.initial_lr = args.initial_lr
        self.lr_annealing_steps = args.lr_annealing_steps
        self.num_actor_learners = args.num_actor_learners
        self.is_train = args.is_train
        self.input_shape = args.input_shape
        self.reward_clip_val = args.reward_clip_val
        self.q_update_interval = args.q_update_interval
        self.restore_checkpoint = args.restore_checkpoint
        self.random_seed = args.random_seed
        
        # Shared mem vars
        self.learning_vars = args.learning_vars
            
        if self.optimizer_mode == 'local':
            if self.optimizer_type == 'rmsprop':
                self.opt_st = np.ones(size, dtype=ctypes.c_float)
            else:
                self.opt_st = np.zeros(size, dtype=ctypes.c_float)
        elif self.optimizer_mode == 'shared':
                self.opt_st = args.opt_state

        # rmsprop/momentum
        self.alpha = args.momentum
        # adam
        self.b1 = args.b1
        self.b2 = args.b2
        self.e = args.e
        
        if args.env == 'GYM':
            from environments.atari_environment import AtariEnvironment
            self.emulator = AtariEnvironment(
                args.game,
                self.random_seed,
                args.visualize,
                use_rgb=args.use_rgb,
                frame_skip=args.frame_skip,
                agent_history_length=args.history_length,
                max_episode_steps=args.max_episode_steps,
                single_life_episodes=args.single_life_episodes,
            )
        elif args.env == 'ALE':
            from environments.emulator import Emulator
            self.emulator = Emulator(
                args.rom_path, 
                args.game, 
                args.visualize, 
                self.actor_id,
                self.random_seed,
                args.single_life_episodes)
        else:
            raise Exception('Invalid environment `{}`'.format(args.env))
            
        self.grads_update_steps = args.grads_update_steps
        self.max_global_steps = args.max_global_steps
        self.gamma = args.gamma

        self.rescale_rewards = args.rescale_rewards
        self.max_achieved_reward = -float('inf')
        if self.rescale_rewards:
            self.thread_max_reward = 1.0

        # Barrier to synchronize all actors after initialization is done
        self.barrier = args.barrier
        self.game = args.game

        # Initizlize Tensorboard summaries
        self.summary_ph, self.update_ops, self.summary_ops = self.setup_summaries()
        self.summary_op = tf.summary.merge_all()


    def reset_hidden_state(self):
        """
        Override in subclass if needed
        """
        pass


    def is_master(self):
        return self.actor_id == 0


    def test(self, num_episodes=100):
        """
        Run test monitor for `num_episodes`
        """
        rewards = list()
        for episode in range(num_episodes):
            s = self.emulator.get_initial_state()
            self.reset_hidden_state()
            total_episode_reward = 0
            episode_over = False

            while not episode_over:
                a = self.choose_next_action(s)[0]
                s, reward, episode_over = self.emulator.next(a)

                total_episode_reward += reward

            else:
                rewards.append(total_episode_reward)
                logger.info("EPISODE {0} -- REWARD: {1}, RUNNING AVG: {2:.0f}±{3:.0f}, BEST: {4}".format(
                    episode,
                    total_episode_reward,
                    np.array(rewards).mean(),
                    2*np.array(rewards).std(),
                    max(rewards),
                ))


    def synchronize_workers(self):
        if self.is_master():
            # Initialize network parameters
            g_step = checkpoint_utils.restore_vars(self.saver, self.session, self.game, self.alg_type, self.max_local_steps, self.restore_checkpoint)
{'fraction_non_alphanumeric': 0.09283468475922554, 'fraction_numerical': 0.011087208340228364, 'mean_word_length': 2.011961126339397, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2331298', 'n_tokens_mistral': 4028, 'n_tokens_neox': 3837, 'n_words': 653}