<jupyter_start><jupyter_text>ML-Brawlhalla: Trial 3  Import PackagesThe environment module is not included in this code, so that results cannot be abused.<jupyter_code>from time import sleep
import numpy as np
import tensorflow as tf
from tensorflow import keras

from paiutils import neural_network as nn
from paiutils import reinforcement as rl
from paiutils import reinforcement_agents as ra

from environment import BrawlhallaFollow2 as Brawlhalla<jupyter_output><empty_output><jupyter_text>Create Environment<jupyter_code>stack_size = 1
time_size = 3
env = Brawlhalla(stack_size=stack_size)
max_steps = 5000<jupyter_output><empty_output><jupyter_text>Create Models<jupyter_code>x0 = keras.layers.Input((time_size, 64 * 64 + 11, stack_size))
frame_input = keras.layers.Lambda(
    lambda x: tf.reshape(tf.transpose(x[:,:,:-11,:], [0, 3, 2, 1]), [-1, 64, 64, stack_size * time_size])
)(x0)
other_input = keras.layers.Lambda(
    lambda x: tf.reshape(x[:,:,-11:,:], [-1, 11 * stack_size * time_size])
)(x0)

x = nn.conv2d(32, 5, 2)(frame_input)
x = nn.conv2d(64, 3, 2)(x)
x = nn.conv2d(128, 3, 2)(x)
x = nn.conv2d(256, 3, 2)(x)
x = nn.conv2d(512, 3, 2)(x)
x = keras.layers.Flatten()(x)
x = keras.layers.Concatenate()([x, other_input])
x = nn.dense(128)(x)
outputs = keras.layers.Dense(9, activation='softmax')(x)
model = keras.models.Model(inputs=x0, outputs=outputs)
model.compile(optimizer=keras.optimizers.Adam(.0005), loss='mse')
model.summary()<jupyter_output>Model: "functional_1"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_1 (InputLayer)            [(None, 3, 4107, 1)] 0                                            
__________________________________________________________________________________________________
lambda (Lambda)                 (None, 64, 64, 3)    0           input_1[0][0]                    
__________________________________________________________________________________________________
conv2d (Conv2D)                 (None, 32, 32, 32)   2432        lambda[0][0]                     
__________________________________________________________________________________________________
batch_normalization (BatchNorma (None, 32, 32, 32)   128         conv2d[0][0]          [...]<jupyter_text>Create the Agent<jupyter_code>def create_memory(shape, dtype):
    if shape == (None, time_size, 64 * 64 + 11, stack_size):
        void_state = np.append(np.zeros(64 * 64), np.ones(11))
        void_state = np.expand_dims(void_state, axis=-1)
        return rl.ETDMemory(time_size, void_state)
    return rl.Memory()

agent = rl.PGAgent(model, .99, create_memory=create_memory)<jupyter_output><empty_output><jupyter_text>Train the Agent<jupyter_code>save_dir = 'saves/'

sleep(2)
agent.set_playing_data(memorizing=True, verbose=True)
env.play_episodes(agent, 50, max_steps, random=True,
                  verbose=True, episode_verbose=False)
agent.save(save_dir, note='PG_random')

agent.set_playing_data(
    training=True, memorizing=True, batch_size=64,
    mini_batch=12800, epochs=1, repeat=1,
    entropy_coef=0, verbose=True
)
num_episodes = 10
for ndx in range(10):
    print(f'Save Loop: {ndx}')
    result = env.play_episodes(
        agent, num_episodes, max_steps,
        verbose=True, episode_verbose=False
    )
    agent.save(save_dir, note=f'PG_{ndx}_{result}')<jupyter_output>Time: 12:57:16 - Episode: 1 - Steps: 415 - Total Reward: 0.09500000000001241 - Best Total Reward: 0.09500000000001241 - Average Total Reward: 0.09500000000001241 - Memory Size: 415
Time: 12:57:34 - Episode: 2 - Steps: 311 - Total Reward: -0.21099999999999186 - Best Total Reward: 0.09500000000001241 - Average Total Reward: -0.057999999999989726 - Memory Size: 726
Time: 12:57:56 - Episode: 3 - Steps: 457 - Total Reward: -1.5269999999999908 - Best Total Reward: 0.09500000000001241 - Average Total Reward: -0.5476666666666568 - Memory Size: 1183
Time: 12:58:26 - Episode: 4 - Steps: 677 - Total Reward: 2.9430000000000063 - Best Total Reward: 2.9430000000000063 - Average Total Reward: 0.325000000000009 - Memory Size: 1860
Time: 12:58:40 - Episode: 5 - Steps: 277 - Total Reward: -0.8869999999999951 - Best Total Reward: 2.9430000000000063 - Average Total Reward: 0.08260000000000818 - Memory Size: 2137
Time: 12:58:50 - Episode: 6 - Steps: 159 - Total Reward: -2.639 - Best Total Reward: 2.9430000[...]<jupyter_text>Test the Agent<jupyter_code>path = 'saves/trial3'
agent.load(path, load_data=False)
sleep(2)
agent.set_playing_data(training=False, memorizing=False)
env.play_episodes(agent, 20, 10000, verbose=True,
                  episode_verbose=False)<jupyter_output>Time: 19:15:27 - Episode: 1 - Steps: 627 - Total Reward: 4.6929999999999765 - Best Total Reward: 4.6929999999999765 - Average Total Reward: 4.6929999999999765 - Memory Size: 0
Time: 19:15:40 - Episode: 2 - Steps: 163 - Total Reward: -0.6129999999999955 - Best Total Reward: 4.6929999999999765 - Average Total Reward: 2.0399999999999903 - Memory Size: 0
Time: 19:16:10 - Episode: 3 - Steps: 587 - Total Reward: 2.5930000000000093 - Best Total Reward: 4.6929999999999765 - Average Total Reward: 2.22433333333333 - Memory Size: 0
Time: 19:16:23 - Episode: 4 - Steps: 181 - Total Reward: -1.570999999999997 - Best Total Reward: 4.6929999999999765 - Average Total Reward: 1.275499999999998 - Memory Size: 0
Time: 19:16:38 - Episode: 5 - Steps: 258 - Total Reward: -2.3579999999999997 - Best Total Reward: 4.6929999999999765 - Average Total Reward: 0.5487999999999985 - Memory Size: 0
Time: 19:16:45 - Episode: 6 - Steps: 59 - Total Reward: 4.091000000000001 - Best Total Reward: 4.6929999999999765 - Avera[...]
{'fraction_non_alphanumeric': 0.1296707046579082, 'fraction_numerical': 0.1460501620883808, 'mean_word_length': 4.2668463611859835, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 32, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '625794', 'license': 'MIT', 'n_tokens_mistral': 2557, 'n_tokens_neox': 1883, 'n_words': 481, 'path': 'trial3.ipynb', 'repo_name': 'Tiger767/ML-Brawlhalla', 'size': '5861'}