<commit_before>from __future__ import division

import numpy as np
import theano

from sft.agent.DeepQAgentGpu import ReplayBuffer, DeepQAgentGpu


class PropReplayBuffer(object):
	def __init__(self, size, view_size, action_hist_size, pct_pos):
		self.size = size
		self.v = np.zeros((size, 1, view_size.w, view_size.h), dtype=theano.config.floatX)
		self.ah = np.zeros((size, 1, action_hist_size.w, action_hist_size.h), dtype=theano.config.floatX)
		self.a = np.zeros((size, 1), dtype=np.int32)
		self.v2 = np.zeros((size, 1, view_size.w, view_size.h), dtype=theano.config.floatX)
		self.ah2 = np.zeros((size, 1, action_hist_size.w, action_hist_size.h), dtype=theano.config.floatX)
		self.r = np.zeros((size, 1), dtype=theano.config.floatX)
		self.t = np.zeros((size, 1), dtype=np.int32)
		# for prop rpl
		max_len_pos = int(np.round(pct_pos * size, 0))
		self.pct_pos = pct_pos
		self.top_pos = 0
		self.len_pos = 0
		self.start_oth = max_len_pos
		self.top_oth = max_len_pos
		self.len_oth = 0

	def add_to_pos(self, v, ah, a, v2, ah2, r, t):
		# write to current pos index
		self.v[self.top_pos] = v
		self.ah[self.top_pos] = ah
		self.a[self.top_pos] = a
		self.v2[self.top_pos] = v2
		self.ah2[self.top_pos] = ah2
		self.r[self.top_pos] = r
		self.t[self.top_pos] = t
		self.top_pos += 1
		if self.len_pos < self.start_oth:
			self.len_pos += 1
		if self.top_pos == self.start_oth:
			self.top_pos = 0

	def add_to_oth(self, v, ah, a, v2, ah2, r, t):
		# write to current other index
		self.v[self.top_oth] = v
		self.ah[self.top_oth] = ah
		self.a[self.top_oth] = a
		self.v2[self.top_oth] = v2
		self.ah2[self.top_oth] = ah2
		self.r[self.top_oth] = r
		self.t[self.top_oth] = t
		self.top_oth += 1
		if self.len_oth < self.size - self.start_oth:
			self.len_oth += 1
		if self.top_oth == self.size:
			self.top_oth = self.start_oth

	def _get_batch_indices(self, start, curr_len, batch_size):
		if curr_len > 0:
			return np.random.randint(start, start + curr_len, batch_size if curr_len > batch_size else curr_len)
		else:
			return None

	def draw_batch(self, batch_size):
		batch_size_pos = int(np.round(self.pct_pos * batch_size, 0))
		batch_size_oth = batch_size - batch_size_pos
		indices_pos = self._get_batch_indices(0, self.len_pos, batch_size_pos)
		indices_oth = self._get_batch_indices(self.start_oth, self.len_oth, batch_size_oth)
		if indices_pos is None:
			indices = indices_oth
		elif indices_oth is None:
			indices = indices_pos
		else:
			indices = np.append(indices_pos, indices_oth)
		return self.v[indices], self.ah[indices], self.a[indices], self.v2[indices], self.ah2[indices], \
			   self.r[indices], self.t[indices]


class DeepQAgentGpuPropReplay(DeepQAgentGpu):
	# actions: possible actions
	# epsilon: epsilon-greedy strategy
	# epsilon: discount function for epsilon
	# batch size: size of minibatches for experience replay (see doi:10.1038/nature14236)
	# buffer size: size of experience pool from which minibatches are randomly sampled
	# start_learn: after how many experiences (buffer size) we start learning based on experiences
	# learn_steps: every learn_steps steps the model gets updated
	DEF_POS_PORTION = 0.7
	def __init__(self, logger, actions, batch_size, buffer_size, start_learn, learn_interval, view_size,
				 action_hist, model, pos_portion=DEF_POS_PORTION):
		self.prop_rpl_buffer = PropReplayBuffer(buffer_size, view_size, action_hist.get_size(), pos_portion)
		self.last_exps = []
		super(DeepQAgentGpuPropReplay, self).__init__(logger, actions, batch_size, buffer_size, start_learn, learn_interval, view_size,
				 action_hist, model)

	def incorporate_reward(self, old_state, action, new_state, reward):
		""" incorporates reward, states, action into replay list and updates the parameters of model """
		self.logger.log_parameter("reward", reward)
		old_view = old_state.view
		old_actions = self.action_hist.get_history(old_state.actions)
		is_terminal = new_state is None
		if not is_terminal:
			new_view = new_state.view
			new_actions = self.action_hist.get_history(new_state.actions)
		else:
			new_view = np.zeros(old_view.shape, dtype=theano.config.floatX)
			new_actions = np.zeros(old_actions.shape, dtype=theano.config.floatX)
		terminal = 1 if is_terminal else 0
		exp_new = (old_view, old_actions, action, new_view, new_actions, reward, terminal)

		self.last_exps.append(exp_new)

		len_all_rpl = self.prop_rpl_buffer.len_pos + self.prop_rpl_buffer.len_oth

		if len_all_rpl >= self.start_learn and self.learn_steps % self.learn_interval == 0:<commit_msg>Set default proportions in proportional replay agent to 0.5<commit_after>from __future__ import division

import numpy as np
import theano

from sft.agent.DeepQAgentGpu import ReplayBuffer, DeepQAgentGpu


class PropReplayBuffer(object):
	def __init__(self, size, view_size, action_hist_size, pct_pos):
		self.size = size
		self.v = np.zeros((size, 1, view_size.w, view_size.h), dtype=theano.config.floatX)
		self.ah = np.zeros((size, 1, action_hist_size.w, action_hist_size.h), dtype=theano.config.floatX)
		self.a = np.zeros((size, 1), dtype=np.int32)
		self.v2 = np.zeros((size, 1, view_size.w, view_size.h), dtype=theano.config.floatX)
		self.ah2 = np.zeros((size, 1, action_hist_size.w, action_hist_size.h), dtype=theano.config.floatX)
		self.r = np.zeros((size, 1), dtype=theano.config.floatX)
		self.t = np.zeros((size, 1), dtype=np.int32)
		# for prop rpl
		max_len_pos = int(np.round(pct_pos * size, 0))
		self.pct_pos = pct_pos
		self.top_pos = 0
		self.len_pos = 0
		self.start_oth = max_len_pos
		self.top_oth = max_len_pos
		self.len_oth = 0

	def add_to_pos(self, v, ah, a, v2, ah2, r, t):
		# write to current pos index
		self.v[self.top_pos] = v
		self.ah[self.top_pos] = ah
		self.a[self.top_pos] = a
		self.v2[self.top_pos] = v2
		self.ah2[self.top_pos] = ah2
		self.r[self.top_pos] = r
		self.t[self.top_pos] = t
		self.top_pos += 1
		if self.len_pos < self.start_oth:
			self.len_pos += 1
		if self.top_pos == self.start_oth:
			self.top_pos = 0

	def add_to_oth(self, v, ah, a, v2, ah2, r, t):
		# write to current other index
		self.v[self.top_oth] = v
		self.ah[self.top_oth] = ah
		self.a[self.top_oth] = a
		self.v2[self.top_oth] = v2
		self.ah2[self.top_oth] = ah2
		self.r[self.top_oth] = r
		self.t[self.top_oth] = t
		self.top_oth += 1
		if self.len_oth < self.size - self.start_oth:
			self.len_oth += 1
		if self.top_oth == self.size:
			self.top_oth = self.start_oth

	def _get_batch_indices(self, start, curr_len, batch_size):
		if curr_len > 0:
			return np.random.randint(start, start + curr_len, batch_size if curr_len > batch_size else curr_len)
		else:
			return None

	def draw_batch(self, batch_size):
		batch_size_pos = int(np.round(self.pct_pos * batch_size, 0))
		batch_size_oth = batch_size - batch_size_pos
		indices_pos = self._get_batch_indices(0, self.len_pos, batch_size_pos)
		indices_oth = self._get_batch_indices(self.start_oth, self.len_oth, batch_size_oth)
		if indices_pos is None:
			indices = indices_oth
		elif indices_oth is None:
			indices = indices_pos
		else:
			indices = np.append(indices_pos, indices_oth)
		return self.v[indices], self.ah[indices], self.a[indices], self.v2[indices], self.ah2[indices], \
			   self.r[indices], self.t[indices]


class DeepQAgentGpuPropReplay(DeepQAgentGpu):
	# actions: possible actions
	# epsilon: epsilon-greedy strategy
	# epsilon: discount function for epsilon
	# batch size: size of minibatches for experience replay (see doi:10.1038/nature14236)
	# buffer size: size of experience pool from which minibatches are randomly sampled
	# start_learn: after how many experiences (buffer size) we start learning based on experiences
	# learn_steps: every learn_steps steps the model gets updated
	DEF_POS_PORTION = 0.5
	def __init__(self, logger, actions, batch_size, buffer_size, start_learn, learn_interval, view_size,
				 action_hist, model, pos_portion=DEF_POS_PORTION):
		self.prop_rpl_buffer = PropReplayBuffer(buffer_size, view_size, action_hist.get_size(), pos_portion)
		self.last_exps = []
		super(DeepQAgentGpuPropReplay, self).__init__(logger, actions, batch_size, buffer_size, start_learn, learn_interval, view_size,
				 action_hist, model)

	def incorporate_reward(self, old_state, action, new_state, reward):
		""" incorporates reward, states, action into replay list and updates the parameters of model """
		self.logger.log_parameter("reward", reward)
		old_view = old_state.view
		old_actions = self.action_hist.get_history(old_state.actions)
		is_terminal = new_state is None
		if not is_terminal:
			new_view = new_state.view
			new_actions = self.action_hist.get_history(new_state.actions)
		else:
			new_view = np.zeros(old_view.shape, dtype=theano.config.floatX)
			new_actions = np.zeros(old_actions.shape, dtype=theano.config.floatX)
		terminal = 1 if is_terminal else 0
		exp_new = (old_view, old_actions, action, new_view, new_actions, reward, terminal)

		self.last_exps.append(exp_new)

		len_all_rpl = self.prop_rpl_buffer.len_pos + self.prop_rpl_buffer.len_oth

		if len_all_rpl >= self.start_learn and self.learn_steps % self.learn_interval == 0:
{'fraction_non_alphanumeric': 0.11158657011191574, 'fraction_numerical': 0.01228878648233487, 'mean_word_length': 4.5647130647130645, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '112421', 'n_tokens_mistral': 3731, 'n_tokens_neox': 3424, 'n_words': 763}