<jupyter_start><jupyter_text>Remove outer game area and normalize to 0-1<jupyter_code>def preprocess(image):
    image = image / 255
    image = image[34:194:3,0:160:3,0:3]
    return image
def create_Q_model(learning_rate=0.001):
    model = Sequential()

    model.add(Conv2D(16, (3,3), padding='same', activation='relu', input_shape=(54, 54, 3)))
    model.add(MaxPooling2D())
    model.add(Conv2D(24, (3,3), padding='same', activation='relu'))
    model.add(MaxPooling2D())
    model.add(Conv2D(32, (3,3), padding='same', activation='relu'))
    model.add(MaxPooling2D())
    model.add(Flatten())
    model.add(Dense(128, activation='relu'))
    model.add(Dense(128, activation='relu'))
    model.add(Dense(ACTION_SIZE, activation='linear'))

    optimizer = Adam(lr=learning_rate)
    model.compile(loss='mse', optimizer=optimizer)

    return model
class Memory():
    def __init__(self, max_size):
        self.buffer = deque(maxlen=max_size)

    def add(self, experience):
        self.buffer.append(experience)
    
    def size(self):
        return len(self.buffer)

    def sample(self, batch_size):
        idx = np.random.choice(
            np.arange(len(self.buffer)),
            size=batch_size,
            replace=False
        )
        return [self.buffer[ii] for ii in idx]
model = create_Q_model()
memory = Memory(max_size = MEMORY_SIZE)
f = open("log.txt", "w")
explore_rate = EXPLORE_INIT
frame = 0

for ep in range(0, EPISODES):
    env.reset()
    state, reward, done, _ = env.step(env.action_space.sample())
    state = preprocess(state)
    state= np.expand_dims(state, axis=0)
    frame += 1
    
    total_reward = 0
    
    explore_rate = EXPLORE_INIT - (EXPLORE_INIT - EXPLORE_FINAL) * (ep + 1) / EPISODES
    
    while True:
        if np.random.rand() > explore_rate:
            action = np.argmax(model.predict(state)[0])
        else:
            action = env.action_space.sample()
        
        next_state, reward, done, _ = env.step(env.action_space.sample())
        next_state = preprocess(next_state)
        next_state = np.expand_dims(next_state, axis=0)
        frame += 1
        state = next_state
        
#         env.render()
        
        total_reward += reward
        
        memory.add((state, action, reward, done, next_state))
        
        if memory.size() >= MEMORT_START_SIZE:
            minibatch = memory.sample(BATCH_SIZE)
            
            inputs = np.zeros((BATCH_SIZE, 54, 54, 3))
            targets = np.zeros((BATCH_SIZE, ACTION_SIZE))
            
            for i, (state_b, action_b, reward_b, done_b, next_state_b) in enumerate(minibatch):
                inputs[i:i+1] = state_b[0]
                if done_b:
                    target = reward_b
                else:
                    target = reward_b + GAMMA * np.amax(model.predict(next_state_b))
                targets[i] = model.predict(next_state_b)[0]
                targets[i][action_b] = target
                
            model.fit(inputs, targets, epochs=1, verbose=0)
        
        if done:
            log_message = "Episode {}, Total Reward {}, Explore Rate {}".format(ep + 1, total_reward, explore_rate)
            f.write(log_message + "\n")
            f.flush()
            print(log_message)
            break
            
    if ep % 5 == 0:
        model.save('model.h5')

f.close()
print("!")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12988539523949455, 'fraction_numerical': 0.021745518660005878, 'mean_word_length': 2.2857142857142856, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '509030', 'license': 'MIT', 'n_tokens_mistral': 1176, 'n_tokens_neox': 1120, 'n_words': 214, 'path': 'pong-dqn.ipynb', 'repo_name': 'neungkl/flappy-bird-DQN', 'size': '3403'}