<commit_before>import random
from capstone.environment import Environment
from capstone.game import TicTacToe
from capstone.mdp import GameMDP
from capstone.player import AlphaBeta, RandPlayer
from capstone.util import ZobristHashing


class TabularQLearning(object):

    def __init__(self, env, policy=RandPlayer(), alpha=0.1, gamma=0.99, n_episodes=1000):
        self.env = env
        self.policy = RandPlayer()
        self.alpha = alpha
        self.gamma = gamma
        self.n_episodes = n_episodes
        self._table = {}

    def max_q_value(self, state, actions):
        if not actions:
            return 0
        best_value = -100000
        for next_action in actions:
            temp_value = self._table.get((state, next_action), random.random() - 0.5)
            if temp_value > best_value:
                best_value = temp_value
        return best_value

    def learn(self):
        import random
        for episode in range(self.n_episodes):
            print('Episode {}'.format(episode))
            self.env.reset()
            step = 0
            while not self.env.is_terminal():
                print('Step {}'.format(step))
                state = self.env.cur_state()
                action = random.choice(self.env.actions())
                reward, next_state = self.env.do_action(action)
                max_q_value = self.max_q_value(next_state, self.env.actions())
                q_value = self._table.get((state, action), 0.1)
                update_value = reward + (self.gamma * max_q_value) - q_value
                self._table[(state, action)] = q_value + (self.alpha * update_value)
                step += 1
        print('Results:')
        for key, value in self._table.iteritems():
            print(key)
            print(value)
            print('*' * 60)

game = TicTacToe(
    'X-O'
    'XO-'
    '-XO'
)
ab = AlphaBeta()
mdp = GameMDP(game, ab, 1)
env = Environment(mdp)
td0 = TabularQLearning(env)
td0.learn()<commit_msg>Rename _table to table<commit_after>import random
from capstone.environment import Environment
from capstone.game import TicTacToe
from capstone.mdp import GameMDP
from capstone.player import AlphaBeta, RandPlayer
from capstone.util import ZobristHashing


class TabularQLearning(object):

    def __init__(self, env, policy=RandPlayer(), alpha=0.1, gamma=0.99, n_episodes=1000):
        self.env = env
        self.policy = RandPlayer()
        self.alpha = alpha
        self.gamma = gamma
        self.n_episodes = n_episodes
        self.table = {}

    def max_q_value(self, state, actions):
        if not actions:
            return 0
        best_value = -100000
        for next_action in actions:
            temp_value = self.table.get((state, next_action), random.random() - 0.5)
            if temp_value > best_value:
                best_value = temp_value
        return best_value

    def learn(self):
        import random
        for episode in range(self.n_episodes):
            print('Episode {}'.format(episode))
            self.env.reset()
            step = 0
            while not self.env.is_terminal():
                print('Step {}'.format(step))
                state = self.env.cur_state()
                action = random.choice(self.env.actions())
                reward, next_state = self.env.do_action(action)
                max_q_value = self.max_q_value(next_state, self.env.actions())
                q_value = self.table.get((state, action), 0.1)
                update_value = reward + (self.gamma * max_q_value) - q_value
                self.table[(state, action)] = q_value + (self.alpha * update_value)
                step += 1
        print('Results:')
        for key, value in self.table.iteritems():
            print(key)
            print(value)
            print('*' * 60)

game = TicTacToe(
    'X-O'
    'XO-'
    '-XO'
)
ab = AlphaBeta()
mdp = GameMDP(game, ab, 1)
env = Environment(mdp)
td0 = TabularQLearning(env)
td0.learn()
{'fraction_non_alphanumeric': 0.10791729702471003, 'fraction_numerical': 0.01361573373676248, 'mean_word_length': 2.0751937984496123, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4987920', 'n_tokens_mistral': 1276, 'n_tokens_neox': 1203, 'n_words': 282}