<commit_before>import random
from capstone.policy import EGreedyPolicy
from capstone.policy import RandomPolicy


class Sarsa(object):

    def __init__(self, env, policy=EGreedyPolicy(epsilon=0.1), qf={}, alpha=0.1,
                 gamma=0.99, n_episodes=1000):
        self.env = env
        self.policy = policy
        self.qf = qf
        self.alpha = alpha
        self.gamma = gamma
        self.n_episodes = n_episodes

    def init(self):
        '''Initializes the q-value if unvisited'''
        state = self.env.cur_state()
        actions = self.env.actions(state)
        for action in actions:
            if (state, action) not in self.qf:
                self.qf[(state, action)] = random.random() - 0.5

    def learn(self):
        for episode in range(1, self.n_episodes + 1):
            print('Episode {}'.format(episode))
            self.env.reset()
            step = 1
            self.init()
            action = self.policy.action(self.env, qf=self.qf)
            while not self.env.is_terminal():
                print('Step {}'.format(step))
                state = self.env.cur_state()
                reward, next_state = self.env.do_action(action)
                next_action = None
                next_state_next_action_value = None
                actions = self.env.actions(self.env.cur_state())
                if not actions:
                    next_state_next_action_value = 0
                else:
                    self.init()
                    next_action = self.policy.action(self.env, qf=self.qf)
                    next_state_next_action_value = self.qf[(next_state, next_action)]
                update_value = reward + (self.gamma * next_state_next_action_value) - self.qf[(state, action)]
                self.qf[(state, action)] += self.alpha * update_value
                action = next_action
                step += 1<commit_msg>Rename update_value to td_error<commit_after>import random
from capstone.policy import EGreedyPolicy
from capstone.policy import RandomPolicy


class Sarsa(object):

    def __init__(self, env, policy=EGreedyPolicy(epsilon=0.1), qf={}, alpha=0.1,
                 gamma=0.99, n_episodes=1000):
        self.env = env
        self.policy = policy
        self.qf = qf
        self.alpha = alpha
        self.gamma = gamma
        self.n_episodes = n_episodes

    def init(self):
        '''Initializes the q-value if unvisited'''
        state = self.env.cur_state()
        actions = self.env.actions(state)
        for action in actions:
            if (state, action) not in self.qf:
                self.qf[(state, action)] = random.random() - 0.5

    def learn(self):
        for episode in range(1, self.n_episodes + 1):
            print('Episode {}'.format(episode))
            self.env.reset()
            step = 1
            self.init()
            action = self.policy.action(self.env, qf=self.qf)
            while not self.env.is_terminal():
                print('Step {}'.format(step))
                state = self.env.cur_state()
                reward, next_state = self.env.do_action(action)
                next_action = None
                next_state_next_action_value = None
                actions = self.env.actions(self.env.cur_state())
                if not actions:
                    next_state_next_action_value = 0
                else:
                    self.init()
                    next_action = self.policy.action(self.env, qf=self.qf)
                    next_state_next_action_value = self.qf[(next_state, next_action)]
                td_error = reward + (self.gamma * next_state_next_action_value) - self.qf[(state, action)]
                self.qf[(state, action)] += self.alpha * td_error
                action = next_action
                step += 1
{'fraction_non_alphanumeric': 0.10501319261213721, 'fraction_numerical': 0.00949868073878628, 'mean_word_length': 1.7352092352092352, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2239659', 'n_tokens_mistral': 1161, 'n_tokens_neox': 1090, 'n_words': 236}