<jupyter_start><jupyter_text>Network Helper<jupyter_code>import operator
import random
from functools import reduce


class BaseGameSpec(object):
    def __init__(self):
        """Abstract base class for the specification for running/training on a game.

        Examples:
            spec = TicTacToeGameSpec()
            result = spec.play_game(func_a, func_b)
        """
        raise NotImplementedError('This is an abstract base class')

    def new_board(self):
        raise NotImplementedError()

    def apply_move(self, board_state, move, side):
        raise NotImplementedError()

    def available_moves(self, board_state):
        raise NotImplementedError()

    def has_winner(self, board_state):
        raise NotImplementedError()

    def evaluate(self, board_state):
        """An evaluation function for this game, gives an estimate of how good the board position is for the plus player.
        There is no specific range for the values returned, they just need to be relative to each other.

        Args:
            board_state (tuple): State of the board

        Returns:
            number
        """
        raise NotImplementedError()

    def board_dimensions(self):
        """Returns the dimensions of the board for this game

        Returns:
            tuple of ints: one int for each dimension of the board, this will normally be 2 ints
        """
        raise NotImplementedError()

    def board_squares(self):
        """The number of squares on the board. This can be used for the number of input nodes to a network.

        Returns:
            int
        """
        return reduce(operator.mul, self.board_dimensions(), 1)

    def outputs(self):
        """The number of moves that could be made in this kind of game, weather or not they are legal. For most games
        this will be every single square on the board, but for connect 4 this is different. If we wanted to do chess in
        the future this method may need to get a bit more complicated.

        Returns:
            int
        """
        return self.board_squares()

    def flat_move_to_tuple(self, move_index):
        """If board is 2d then we return a tuple for where we moved to.
        e.g if the board is a 3x3 size and our move_index was 6 then
        this method will return (2, 0)

        Args:
            move_index (int): The index of the square we moved to

        Returns:
            tuple or int: For where we moved in board coordinates
        """
        if len(self.board_dimensions()) == 1:
            return move_index

        board_x = self.board_dimensions()[0]
        return int(move_index / board_x), move_index % board_x

    def tuple_move_to_flat(self, tuple_move):
        """Does the inverse operation to flat_move_to_tuple

        Args:
            tuple_move (tuple):

        Returns:
            int :
        """
        if len(self.board_dimensions()) == 1:
            return tuple_move[0]
        else:
            return tuple_move[0] * self.board_dimensions()[0] + tuple_move[1]
	
	
    def show_board(self,c):
        pc = {0:' ', -1:'x',1:'o'}
        print(" %s | %s | %s" % (pc[c[0][0]],pc[c[0][1]],pc[c[0][2]]))
        print("---+---+---")
        print(" %s | %s | %s" % (pc[c[1][0]],pc[c[1][1]],pc[c[1][2]]))
        print("---+---+---")
        print(" %s | %s | %s" % (pc[c[2][0]],pc[c[2][1]],pc[c[2][2]]))
        print()

    def play_game(self, plus_player_func, minus_player_func, log=False, board_state=None):
        """Run a single game of until the end, using the provided function args to determine the moves for each
        player.

        Args:
            plus_player_func ((board_state(3 by 3 tuple of int), side(int)) -> move((int, int))): Function that takes the
                current board_state and side this player is playing, and returns the move the player wants to play.
            minus_player_func ((board_state(3 by 3 tuple of int), side(int)) -> move((int, int))): Function that takes the
                current board_state and side this player is playing, and returns the move the player wants to play.
            log (bool): If True progress is logged to console, defaults to False
            board_state: Optionally have the game start from this position, rather than from a new board

        Returns:
            int: 1 if the plus_player_func won, -1 if the minus_player_func won and 0 for a draw
        """
		
        board_state = board_state or self.new_board()
        player_turn = 1
		
        while True:
            _available_moves = list(self.available_moves(board_state))

            if len(_available_moves) == 0:
                # draw
                if log:
                    print("no moves left, game ended a draw")
                return 0.,{'illegal_move':0,'draw':1}
            if player_turn > 0:
                move = plus_player_func(board_state, 1)
            else:
                move = minus_player_func(board_state, -1)

            if move not in _available_moves:
                #print("illegal move ", move)
                # if a player makes an invalid move the other player wins
                if log:
                    print("illegal move ", move)
                return -player_turn,{'illegal_move':1,'draw':0}
				

            board_state = self.apply_move(board_state, move, player_turn)
            if log:
                #print(board_state)
                self.show_board(board_state)

            winner = self.has_winner(board_state)
            if winner != 0:
                if log:
                    print("we have a winner, side: %s" % player_turn)
                return winner , {'illegal_move':0,'draw':0}
            player_turn = -player_turn

    def get_random_player_func(self):
        """Return a function that makes moves for the current game by choosing a move randomly
        NOTE: this move returns the function that makes the random move so should be used like so:
        Examples:
            self.play_game(self.get_random_player_func(), self.get_random_player_func())

        Returns:
            board_state, side (int) -> move : function that plays this game by making random moves
        """
        #print("Random Move Called")
        return lambda board_state, side: random.choice(list(self.available_moves(board_state)))

"""
Full code for running a game of tic-tac-toe on a 3 by 3 board.
Two players take turns making moves on squares of the board, the first to get 3 in a row, including diagonals, wins. If
there are no valid moves left to make the game ends a draw.

The main method to use here is play_game which simulates a game to the end using the function args it takes to determine
where each player plays.
The board is represented by a 3 x 3 tuple of ints. A 0 means no player has played in a space, 1 means player one has
played there, -1 means the seconds player has played there. The apply_move method can be used to return a copy of a
given state with a given move applied. This can be useful for doing min-max or monte carlo sampling.
"""
import itertools
import random


from techniques.min_max import evaluate


def _new_board():
    """Return a emprty tic-tac-toe board we can use for simulating a game.

    Returns:
        3x3 tuple of ints
    """
    return ((0, 0, 0),
            (0, 0, 0),
            (0, 0, 0))


def apply_move(board_state, move, side):
    """Returns a copy of the given board_state with the desired move applied.

    Args:
        board_state (3x3 tuple of int): The given board_state we want to apply the move to.
        move (int, int): The position we want to make the move in.
        side (int): The side we are making this move for, 1 for the first player, -1 for the second player.

    Returns:
        (3x3 tuple of int): A copy of the board_state with the given move applied for the given side.
    """
    move_x, move_y = move

    def get_tuples():
        for x in range(3):
            if move_x == x:
                temp = list(board_state[x])
                temp[move_y] = side
                yield tuple(temp)
            else:
                yield board_state[x]

    return tuple(get_tuples())


def available_moves(board_state):
    """Get all legal moves for the current board_state. For Tic-tac-toe that is all positions that do not currently have
    pieces played.

    Args:
        board_state: The board_state we want to check for valid moves.

    Returns:
        Generator of (int, int): All the valid moves that can be played in this position.
    """
    for x, y in itertools.product(range(3), range(3)):
        if board_state[x][y] == 0:
            yield (x, y)


def _has_3_in_a_line(line):
    return all(x == -1 for x in line) | all(x == 1 for x in line)


def has_winner(board_state):
    """Determine if a player has won on the given board_state.

    Args:
        board_state (3x3 tuple of int): The current board_state we want to evaluate.

    Returns:
        int: 1 if player one has won, -1 if player 2 has won, otherwise 0.
    """
    # check rows
    for x in range(3):
        if _has_3_in_a_line(board_state[x]):
            return board_state[x][0]
    # check columns
    for y in range(3):
        if _has_3_in_a_line([i[y] for i in board_state]):
            return board_state[0][y]

    # check diagonals
    if _has_3_in_a_line([board_state[i][i] for i in range(3)]):
        return board_state[0][0]
    if _has_3_in_a_line([board_state[2 - i][i] for i in range(3)]):
        return board_state[0][2]

    return 0  # no one has won, return 0 for a draw


def play_game(plus_player_func, minus_player_func, log=False):
    """Run a single game of tic-tac-toe until the end, using the provided function args to determine the moves for each
    player.

    Args:
        plus_player_func ((board_state(3 by 3 tuple of int), side(int)) -> move((int, int))): Function that takes the
            current board_state and side this player is playing, and returns the move the player wants to play.
        minus_player_func ((board_state(3 by 3 tuple of int), side(int)) -> move((int, int))): Function that takes the
            current board_state and side this player is playing, and returns the move the player wants to play.
        log (bool): If True progress is logged to console, defaults to False

    Returns:
        int: 1 if the plus_player_func won, -1 if the minus_player_func won and 0 for a draw
    """
    #print("Inside tic game_play")

    board_state = _new_board()
    player_turn = 1

    while True:
        _available_moves = list(available_moves(board_state))

        if len(_available_moves) == 0:
            # draw
            if log:
                print("no moves left, game ended a draw")
            return 0.
        if player_turn > 0:
            move = plus_player_func(board_state, 1)
        else:
            move = minus_player_func(board_state, -1)

        if move not in _available_moves:
            # if a player makes an invalid move the other player wins
            if log:
                print("illegal move ", move)
            return -player_turn

        board_state = apply_move(board_state, move, player_turn)
        if log:
            print(board_state)

        winner = has_winner(board_state)
        if winner != 0:
            if log:
                print("we have a winner, side: %s" % player_turn)
            return winner
        player_turn = -player_turn


def random_player(board_state, _):
    """A player func that can be used in the play_game method. Given a board state it chooses a move randomly from the
    valid moves in the current state.

    Args:
        board_state (3x3 tuple of int): The current state of the board
        _: the side this player is playing, not used in this function because we are simply choosing the moves randomly

    Returns:
        (int, int): the move we want to play on the current board
    """
    moves = list(available_moves(board_state))
    return random.choice(moves)


class TicTacToeGameSpec(BaseGameSpec):
    def __init__(self):
        self.available_moves = available_moves
        self.has_winner = has_winner
        self.new_board = _new_board
        self.apply_move = apply_move
        self.evaluate = evaluate

    def board_dimensions(self):
        return 3, 3



import operator
import pickle
from functools import reduce
import math

import numpy as np
import tensorflow as tf


def create_network(input_nodes, hidden_nodes, output_nodes=None, output_softmax=True):
    """Create a network with relu activations at each layer

    Args:
        output_nodes: (int): Number of output nodes, if None then number of input nodes is used
        input_nodes (int or tuple(int)): The size of the board this network will work on. The output layer will also be
            this size if not specified. Can be an int if 1d or a tuple of ints for a 2d+ dim board
        hidden_nodes ([int]): The number of hidden nodes in each hidden layer
        output_softmax (bool): If True softmax is used in the final layer, otherwise just use the activation with no
            non-linearity function

    Returns:
        (input_layer, output_layer, [variables]) : The final item in the tuple is a list containing all the parameters,
            wieghts and biases used in this network
    """
    output_nodes = output_nodes or input_nodes

    variables = []

    with tf.name_scope('network'):
        if isinstance(input_nodes, tuple):
            input_layer = tf.placeholder("float", (None,) + input_nodes)
            flat_size = reduce(operator.mul, input_nodes, 1)
            current_layer = tf.reshape(input_layer, (-1, flat_size))
        else:
            input_layer = tf.placeholder("float", (None, input_nodes))
            current_layer = input_layer

        for hidden_nodes in hidden_nodes:
            last_layer_nodes = int(current_layer.get_shape()[-1])
            hidden_weights = tf.Variable(
                tf.truncated_normal((last_layer_nodes, hidden_nodes), stddev=1. / np.sqrt(last_layer_nodes)),
                name='weights')
            hidden_bias = tf.Variable(tf.constant(0.01, shape=(hidden_nodes,)), name='biases')

            variables.append(hidden_weights)
            variables.append(hidden_bias)

            current_layer = tf.nn.relu(
                tf.matmul(current_layer, hidden_weights) + hidden_bias)

        if isinstance(output_nodes, tuple):
            output_nodes = reduce(operator.mul, input_nodes, 1)

        # for some reason having output std divided by np.sqrt(output_nodes) massively outperforms np.sqrt(hidden_nodes)
        output_weights = tf.Variable(
            tf.truncated_normal((hidden_nodes, output_nodes), stddev=1. / np.sqrt(output_nodes)), name="output_weights")
        output_bias = tf.Variable(tf.constant(0.01, shape=(output_nodes,)), name="output_bias")

        variables.append(output_weights)
        variables.append(output_bias)

        output_layer = tf.matmul(current_layer, output_weights) + output_bias
        if output_softmax:
            output_layer = tf.nn.softmax(output_layer)

    return input_layer, output_layer, variables


def save_network(session, tf_variables, file_path):
    """Save the given set of variables to the given file using the given session

    Args:
        session (tf.Session): session within which the variables has been initialised
        tf_variables (list of tf.Variable): list of variables which will be saved to the file
        file_path (str): path of the file we want to save to.
    """
    variable_values = session.run(tf_variables)
    with open(file_path, mode='wb') as f:
        pickle.dump(variable_values, f)


def load_network(session, tf_variables, file_path):
    """Load the given set of variables from the given file using the given session

    Args:
        session (tf.Session): session within which the variables has been initialised
        tf_variables (list of tf.Variable): list of variables which will set up with the values saved to the file. List
            order matters, in must be the exact same order as was used to save and all of the same shape.
        file_path (str): path of the file we want to load from.
    """
    with open(file_path, mode='rb') as f:
        variable_values = pickle.load(f)

    try:
        if len(variable_values) != len(tf_variables):
            raise ValueError("Network in file had different structure, variables in file: %s variables in memeory: %s"
                             % (len(variable_values), len(tf_variables)))
        for value, tf_variable in zip(variable_values, tf_variables):
            session.run(tf_variable.assign(value))
    except ValueError as ex:
        # TODO: maybe raise custom exception
        raise ValueError("""Tried to load network file %s with different architecture from the in memory network.
Error was %s
Either delete the network file to train a new network from scratch or change the in memory network to match that dimensions of the one in the file""" % (file_path, ex))


def invert_board_state(board_state):
    """Returns the board state inverted, so all 1 are replaced with -1 and visa-versa

    Args:
        board_state (tuple of tuple of ints): The board we want to invert

    Returns:
        (tuple of tuple of ints) The board state for the other player
    """
    return tuple(tuple(-board_state[j][i] for i in range(len(board_state[0]))) for j in range(len(board_state)))


def get_stochastic_network_move(session, input_layer, output_layer, board_state, side,
                                valid_only=False, game_spec=None):
    #print ("valid_only:",valid_only)
    """Choose a move for the given board_state using a stocastic policy. A move is selected using the values from the
     output_layer as a categorical probability distribution to select a single move

    Args:
        session (tf.Session): Session used to run this network
        input_layer (tf.Placeholder): Placeholder to the network used to feed in the board_state
        output_layer (tf.Tensor): Tensor that will output the probabilities of the moves, we expect this to be of
            dimesensions (None, board_squares) and the sum of values across the board_squares to be 1.
        board_state: The board_state we want to get the move for.
        side: The side that is making the move.

    Returns:
        (np.array) It's shape is (board_squares), and it is a 1 hot encoding for the move the network has chosen.
    """
    np_board_state = np.array(board_state)
    if side == -1:
        np_board_state = -np_board_state

    np_board_state = np_board_state.reshape(1, *input_layer.get_shape().as_list()[1:])
    probability_of_actions = session.run(output_layer,
                                         feed_dict={input_layer: np_board_state})[0]

    if valid_only:
        available_moves = list(game_spec.available_moves(board_state))
        
        
        #print("available_moves:",available_moves)
        
        if len(available_moves) == 1:
            move = np.zeros(game_spec.board_squares())
            np.put(move, game_spec.tuple_move_to_flat(available_moves[0]), 1)
            return move
        available_moves_flat = [game_spec.tuple_move_to_flat(x) for x in available_moves]
        
        print("available_moves_flat:",available_moves_flat)
        for i in range(game_spec.board_squares()):
            if probability_of_actions[i]<0:
                print("probability less than zero")
            if i not in available_moves_flat:
                probability_of_actions[i] = 0.
        
        print("probability of actions:",probability_of_actions)
        prob_mag = sum(probability_of_actions)
        if prob_mag !=0:
            probability_of_actions /= sum(probability_of_actions)
        print("probability of actions:",probability_of_actions)

    try:
        move = np.random.multinomial(1, probability_of_actions)
    except ValueError:
        # sometimes because of rounding errors we end up with probability_of_actions summing to greater than 1.
        # so need to reduce slightly to be a valid value
        move = np.random.multinomial(1, probability_of_actions / (1. + 1e-6))
    
    print("probability of actions:%s Move:%s"%(probability_of_actions,move))
    return move


def get_deterministic_network_move(session, input_layer, output_layer, board_state, side, valid_only=False,
                                   game_spec=None):
    """Choose a move for the given board_state using a deterministic policy. A move is selected using the values from
    the output_layer and selecting the move with the highest score.

    Args:
        session (tf.Session): Session used to run this network
        input_layer (tf.Placeholder): Placeholder to the network used to feed in the board_state
        output_layer (tf.Tensor): Tensor that will output the probabilities of the moves, we expect this to be of
            dimesensions (None, board_squares).
        board_state: The board_state we want to get the move for.
        side: The side that is making the move.

    Returns:
        (np.array) It's shape is (board_squares), and it is a 1 hot encoding for the move the network has chosen.
    """
    np_board_state = np.array(board_state)
    np_board_state = np_board_state.reshape(1, *input_layer.get_shape().as_list()[1:])
    if side == -1:
        np_board_state = -np_board_state

    probability_of_actions = session.run(output_layer,
                                         feed_dict={input_layer: np_board_state})[0]

    if valid_only:
        available_moves = game_spec.available_moves(board_state)
        available_moves_flat = [game_spec.tuple_move_to_flat(x) for x in available_moves]
        for i in range(game_spec.board_squares()):
            if i not in available_moves_flat:
                probability_of_actions[i] = 0

    move = np.argmax(probability_of_actions)
    one_hot = np.zeros(len(probability_of_actions))
    one_hot[move] = 1.
    return one_hot

import collections
import os
import random

import numpy as np
import tensorflow as tf

#from common.network_helpers import load_network, get_stochastic_network_move,get_deterministic_network_move, save_network


def train_policy_gradients(game_spec,
                           create_network,
                           network_file_path,
                           save_network_file_path=None,
                           opponent_func=None,
                           number_of_games=10000,
                           print_results_every=1000,
                           learn_rate=1e-4,
                           batch_size=100,
                           randomize_first_player=True,
						   plot_stats = True):
    """Train a network using policy gradients

    Args:
        save_network_file_path (str): Optionally specifiy a path to use for saving the network, if unset then
            the network_file_path param is used.
        opponent_func (board_state, side) -> move: Function for the opponent, if unset we use an opponent playing
            randomly
        randomize_first_player (bool): If True we alternate between being the first and second player
        game_spec (games.base_game_spec.BaseGameSpec): The game we are playing
        create_network (->(input_layer : tf.placeholder, output_layer : tf.placeholder, variables : [tf.Variable])):
            Method that creates the network we will train.
        network_file_path (str): path to the file with weights we want to load for this network
        number_of_games (int): number of games to play before stopping
        print_results_every (int): Prints results to std out every x games, also saves the network
        learn_rate (float):
        batch_size (int):

    Returns:
        (variables used in the final network : list, win rate: float)
    """
	
    save_network_file_path = save_network_file_path or network_file_path
    opponent_func = opponent_func or game_spec.get_random_player_func()
    reward_placeholder = tf.placeholder("float", shape=(None,))
    actual_move_placeholder = tf.placeholder("float", shape=(None, game_spec.outputs()))

    input_layer, output_layer, variables = create_network()

    policy_gradient = tf.log(
        tf.reduce_sum(tf.multiply(actual_move_placeholder, output_layer), axis=1)) * reward_placeholder
    train_step = tf.train.AdamOptimizer(learn_rate).minimize(-policy_gradient)

    with tf.Session() as session:
        session.run(tf.global_variables_initializer())

        if network_file_path and os.path.isfile(network_file_path):
            print("loading pre-existing network")
            load_network(session, variables, network_file_path)

        mini_batch_board_states, mini_batch_moves, mini_batch_rewards = [], [], []
        results = collections.deque(maxlen=print_results_every)
        mini_batch_illegal_moves = 0
        illegal_move = 0
        draw_count = 0
        #for plotting
        episode_win = []
        episode_wrong_moves = []
        episode_count = []
		
        def make_training_move(board_state, side):
            mini_batch_board_states.append(np.ravel(board_state) * side)
            move = get_deterministic_network_move(session, input_layer, output_layer, board_state, side,valid_only = True,game_spec = game_spec)
            mini_batch_moves.append(move)
            #print("Training Move Called")
            return game_spec.flat_move_to_tuple(move.argmax())
		
        for episode_number in range(1, number_of_games):
            # randomize if going first or second
            if (not randomize_first_player) or bool(random.getrandbits(1)):
                reward,info = game_spec.play_game(make_training_move, opponent_func, log=False)
            else:
                reward,info = game_spec.play_game(opponent_func, make_training_move, log=False)
                reward = -reward
            #print(reward) 
            #print(type(info))
            #print(info)
            results.append(reward)
            mini_batch_illegal_moves+=info['illegal_move']
            draw_count+=info['draw']

            # we scale here so winning quickly is better winning slowly and loosing slowly better than loosing quick
            last_game_length = len(mini_batch_board_states) - len(mini_batch_rewards)

            reward /= float(last_game_length)

            mini_batch_rewards += ([reward] * last_game_length)

            if episode_number % batch_size == 0:
                normalized_rewards = mini_batch_rewards - np.mean(mini_batch_rewards)

                rewards_std = np.std(normalized_rewards)
                print("rewards_std:",rewards_std)
                if rewards_std > 0.2:
                    normalized_rewards /= rewards_std
                else:
                    normalized_rewards /= 0.2
                    print("warning: got mini batch std of 0.")

                np_mini_batch_board_states = np.array(mini_batch_board_states) \
                    .reshape(len(mini_batch_rewards), *input_layer.get_shape().as_list()[1:])

                session.run(train_step, feed_dict={input_layer: np_mini_batch_board_states,
                                                   reward_placeholder: normalized_rewards,
                                                   actual_move_placeholder: mini_batch_moves})

                # clear batches
                del mini_batch_board_states[:]
                del mini_batch_moves[:]
                del mini_batch_rewards[:]
				

            if episode_number % print_results_every == 0:
                import matplotlib.pyplot as plt
                episode_win.append(_win_rate(print_results_every, results)) 
                episode_wrong_moves.append(mini_batch_illegal_moves/print_results_every) 
                episode_count.append(episode_number)
                
                # red dashes, blue squares and green triangles
                plt.plot(episode_count, episode_win, 'r', episode_count, episode_wrong_moves, 'g')
                plt.show()	   

                print("episode: %s win_rate: %s illegal_moves: %s draw_count: %s" % (episode_number, _win_rate(print_results_every, results),mini_batch_illegal_moves,draw_count))
                mini_batch_illegal_moves = 0
                draw_count = 0
                if network_file_path:
                    save_network(session, variables, save_network_file_path)
                    #print("file saved")

        if network_file_path:
            save_network(session, variables, save_network_file_path)

    return variables, _win_rate(print_results_every, results)


def _win_rate(print_results_every, results):
    #print("Random Move Called")
    return 0.5 + sum(results) / (print_results_every * 2.)
"""
Builds and trains a neural network that uses policy gradients to learn to play Tic-Tac-Toe.

The input to the network is a vector with a number for each space on the board. If the space has one of the networks
pieces then the input vector has the value 1. -1 for the opponents space and 0 for no piece.

The output of the network is a also of the size of the board with each number learning the probability that a move in
that space is the best move.

The network plays successive games randomly alternating between going first and second against an opponent that makes
moves by randomly selecting a free space. The neural network does NOT initially have any way of knowing what is or is not
a valid move, so initially it must learn the rules of the game.

I have trained this version with success at 3x3 tic tac toe until it has a success rate in the region of 75% this maybe
as good as it can do, because 3x3 tic-tac-toe is a theoretical draw, so the random opponent will often get lucky and
force a draw.	
"""
import functools



BATCH_SIZE = 100  # every how many games sto do a parameter update?
LEARN_RATE = 1e-4
PRINT_RESULTS_EVERY_X = 1000  # every how many games to print the results
NETWORK_FILE_PATH = 'network_500000.p'  # path to save the network to
Save_Network_file_path = 'new_test_20000_valid_only_second_iteration.p'
NUMBER_OF_GAMES_TO_RUN = 20000

# to play a different game change this to another spec, e.g TicTacToeXGameSpec or ConnectXGameSpec, to get these to run
# well may require tuning the hyper parameters a bit
game_spec = TicTacToeGameSpec()

create_network_func = functools.partial(create_network, game_spec.board_squares(), (100, 100, 100))

train_policy_gradients(game_spec, create_network_func, NETWORK_FILE_PATH,save_network_file_path=Save_Network_file_path,
                       number_of_games=NUMBER_OF_GAMES_TO_RUN,
                       batch_size=BATCH_SIZE,
                       learn_rate=LEARN_RATE,
                       print_results_every=PRINT_RESULTS_EVERY_X)<jupyter_output>loading pre-existing network
rewards_std: 0.215447776476
rewards_std: 0.212772850273
rewards_std: 0.168688427781
warning: got mini batch std of 0.
rewards_std: 0.184956709462
warning: got mini batch std of 0.
rewards_std: 0.19458777993
warning: got mini batch std of 0.
rewards_std: 0.196220804622
warning: got mini batch std of 0.
rewards_std: 0.193973820299
warning: got mini batch std of 0.
rewards_std: 0.177772218072
warning: got mini batch std of 0.
rewards_std: 0.237881675355
rewards_std: 0.266415495216
{'fraction_non_alphanumeric': 0.07540366328419172, 'fraction_numerical': 0.012221584019401367, 'mean_word_length': 2.5399299672427427, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '460200', 'license': 'MIT', 'n_tokens_mistral': 9218, 'n_tokens_neox': 8716, 'n_words': 3059, 'path': 'Untitled.ipynb', 'repo_name': 'alphagamatoe/AlphaToe', 'size': '31338'}