<commit_before>import tensorflow as tf
import numpy as np
import random
from collections import deque
from models.dqn import DQN
from torch import optim
import torch
from torch.autograd import Variable
import torch.nn.functional as F
from config import opt

# TODO put in params
# Hyper Parameters:
GAMMA = 0.99  # decay rate of past observations
OBSERVE = 32  # timesteps to observe before training
REPLAY_MEMORY_SIZE = 10000  # number of previous transitions to remember
BATCH_SIZE = 32  # size of minibatch
# FINAL_EPSILON = 0
# INITIAL_EPSILON = 0.1
# or alternative:
FINAL_EPSILON = 0.0001  # final value of epsilon
INITIAL_EPSILON = 0.1  # starting value of epsilon
UPDATE_TIME = 100
EXPLORE = 100000.  # frames over which to anneal epsilon


class DQNTargetAgent:
    def __init__(self):
        self.replay_memory = deque()
        self.time_step = 0
        self.actions = opt.actions
        self.epsilon = INITIAL_EPSILON
        self.policynetwork = DQN()
        self.targetnetwork = DQN()
        self.optimizer = optim.Adam(self.policynetwork.parameters(), 0.01)


        if opt.cuda:
            self.policynetwork = self.policynetwork.cuda()
            self.targetnetwork = self.targetnetwork.cuda()

    def initialise(self):
        self.policynetwork = DQN()

    def load_policynetwork(self, old_modal):
        self.policynetwork.load_state_dict(old_modal)

    def train_policynetwork(self):
        minibatch = random.sample(self.replay_memory, BATCH_SIZE)

        batch_state, batch_action, batch_reward, batch_next_state, batch_terminal = zip(*minibatch)
        batch_state = torch.cat(batch_state)
        batch_next_state = torch.cat(batch_next_state)
        batch_next_state.volatile = True

        batch_action = Variable(torch.LongTensor(list(batch_action)).unsqueeze(1))
        batch_reward = Variable(torch.FloatTensor(list(batch_reward)))

        if opt.cuda:
            batch_action = batch_action.cuda()
            batch_reward = batch_reward.cuda()
            batch_next_state = batch_next_state.cuda()

        current_q_values = self.policynetwork(batch_state).gather(1, batch_action)
        max_next_q_values = self.targetnetwork(batch_next_state).max(1)[0]
        expected_q_values = batch_reward + (GAMMA * max_next_q_values)
        # Undo volatility introduced above
        expected_q_values = Variable(expected_q_values.data)

        if opt.cuda:
            expected_q_values = expected_q_values.cuda()
        loss = F.mse_loss(current_q_values, expected_q_values)

        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()


    def update(self, current_state, action, reward, next_state, terminal):
        self.replay_memory.append(
            (current_state, action, reward, next_state, terminal))
        if len(self.replay_memory) > REPLAY_MEMORY_SIZE:
            self.replay_memory.popleft()
        global OBSERVE
        if self.time_step > OBSERVE:
            self.train_policynetwork()

        self.time_step += 1

    def get_action(self, state):
        action = 0
        if random.random() <= self.epsilon:
            action = random.randrange(self.actions)
        else:
            qvalue = self.policynetwork(state)
            action = np.argmax(qvalue.data[0])
        # change epsilon
        if self.epsilon > FINAL_EPSILON and self.time_step > OBSERVE:
            self.epsilon -= (INITIAL_EPSILON - FINAL_EPSILON) / EXPLORE

        return action

    def finish_episode(self, episode):
        # Update target network every 10 epoch
        if episode % 10 == 0:
            self.targetnetwork.load_state_dict(self.policynetwork.state_dict())
<commit_msg>Unsqueeze + use huber loss<commit_after>import tensorflow as tf
import numpy as np
import random
from collections import deque
from models.dqn import DQN
from torch import optim
import torch
from torch.autograd import Variable
import torch.nn.functional as F
from config import opt

# TODO put in params
# Hyper Parameters:
GAMMA = 0.99  # decay rate of past observations
OBSERVE = 32  # timesteps to observe before training
REPLAY_MEMORY_SIZE = 10000  # number of previous transitions to remember
BATCH_SIZE = 32  # size of minibatch
# FINAL_EPSILON = 0
# INITIAL_EPSILON = 0.1
# or alternative:
FINAL_EPSILON = 0.0001  # final value of epsilon
INITIAL_EPSILON = 0.1  # starting value of epsilon
UPDATE_TIME = 100
EXPLORE = 100000.  # frames over which to anneal epsilon


class DQNTargetAgent:
    def __init__(self):
        self.replay_memory = deque()
        self.time_step = 0
        self.actions = opt.actions
        self.epsilon = INITIAL_EPSILON
        self.policynetwork = DQN()
        self.targetnetwork = DQN()
        self.optimizer = optim.Adam(self.policynetwork.parameters(), 0.01)


        if opt.cuda:
            self.policynetwork = self.policynetwork.cuda()
            self.targetnetwork = self.targetnetwork.cuda()

    def initialise(self):
        self.policynetwork = DQN()

    def load_policynetwork(self, old_modal):
        self.policynetwork.load_state_dict(old_modal)

    def train_policynetwork(self):
        minibatch = random.sample(self.replay_memory, BATCH_SIZE)

        batch_state, batch_action, batch_reward, batch_next_state, batch_terminal = zip(*minibatch)
        batch_state = torch.cat(batch_state)
        batch_next_state = torch.cat(batch_next_state)
        batch_next_state.volatile = True

        batch_action = Variable(torch.LongTensor(list(batch_action)).unsqueeze(1))
        batch_reward = Variable(torch.FloatTensor(list(batch_reward)))

        if opt.cuda:
            batch_action = batch_action.cuda()
            batch_reward = batch_reward.cuda()
            batch_next_state = batch_next_state.cuda()

        current_q_values = self.policynetwork(batch_state).gather(1, batch_action)
        max_next_q_values = self.targetnetwork(batch_next_state).max(1)[0]
        expected_q_values = batch_reward + (GAMMA * max_next_q_values)
        # Undo volatility introduced above
        expected_q_values = Variable(expected_q_values.data).unsqueeze(1)

        if opt.cuda:
            expected_q_values = expected_q_values.cuda()
        loss = F.smooth_l1_loss(current_q_values, expected_q_values)

        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()


    def update(self, current_state, action, reward, next_state, terminal):
        self.replay_memory.append(
            (current_state, action, reward, next_state, terminal))
        if len(self.replay_memory) > REPLAY_MEMORY_SIZE:
            self.replay_memory.popleft()
        global OBSERVE
        if self.time_step > OBSERVE:
            self.train_policynetwork()

        self.time_step += 1

    def get_action(self, state):
        action = 0
        if random.random() <= self.epsilon:
            action = random.randrange(self.actions)
        else:
            qvalue = self.policynetwork(state)
            action = np.argmax(qvalue.data[0])
        # change epsilon
        if self.epsilon > FINAL_EPSILON and self.time_step > OBSERVE:
            self.epsilon -= (INITIAL_EPSILON - FINAL_EPSILON) / EXPLORE

        return action

    def finish_episode(self, episode):
        # Update target network every 10 epoch
        if episode % 10 == 0:
            self.targetnetwork.load_state_dict(self.policynetwork.state_dict())

{'fraction_non_alphanumeric': 0.08256632376827287, 'fraction_numerical': 0.012994044396318355, 'mean_word_length': 2.868586387434555, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4322906', 'n_tokens_mistral': 2478, 'n_tokens_neox': 2327, 'n_words': 507}