<jupyter_start><jupyter_text>Define DQN Network<jupyter_code>import keras
import tensorflow.compat.v1 as tf
from tensorflow.compat.v1.keras import models, Sequential, layers


def QNetwork(X_state, name):
    
    with tf.variable_scope(name, reuse = tf.AUTO_REUSE) as scope:
    
        hid1 = tf.layers.dense(X_state, 16, activation = 'relu')
        hid2 = tf.layers.dense(hid1, 12 , activation= 'relu')
        outputs = tf.layers.dense(hid2, 10, activation = 'softmax')

    trainable_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 
                                        scope = scope.name)
    trainable_vars_by_name = {var.name[len(scope.name):]: var for var in trainable_vars}
    return outputs, trainable_vars_by_name
    
    
X_state = tf.placeholder(tf.float32, shape = (None, 1), name = 'X')

online_q_values, online_vars = QNetwork(X_state, name= 'q_networks/online')
target_q_values, target_vars = QNetwork(X_state, name= 'q_networks/target')

copy_ops = [target_var.assign(online_vars[var_name]) for var_name, target_var in target_vars.items()] 
copy_online_to_target = tf.group(*copy_ops)

X_action = tf.placeholder(tf.int32, shape = [None])
q_value = tf.reduce_sum(target_q_values * tf.one_hot(X_action, 10), 
                       axis = 1, keep_dims = True)

y = tf.placeholder(tf.float32, shape=[None, 1])
error = tf.abs(y - q_value)
clipped_error = tf.clip_by_value(error, 0.0, 1.0)
linear_error = 2 * (error - clipped_error)
loss = tf.reduce_mean(tf.square(clipped_error) + linear_error)


learning_rate = 0.001
momentum = 0.95
global_step = tf.Variable(0, trainable=False, name='global_step')
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum, use_nesterov=True)
training_op = optimizer.minimize(loss, global_step=global_step)
init = tf.global_variables_initializer()
saver = tf.train.Saver()
from collections import deque
replay_memory_size = 500000
replay_memory = deque([], maxlen=replay_memory_size)

def sample_memories(batch_size):
    indices = np.random.permutation(len(replay_memory))[:batch_size]
    cols = [[], [], [], [], []] # state, action, reward, next_state, continue
    for idx in indices:
        memory = replay_memory[idx]
        for col, value in zip(cols, memory):
            col.append(value)
    cols = [np.array(col) for col in cols]
    return (cols[0], cols[1], cols[2].reshape(-1, 1), cols[3],cols[4].reshape(-1, 1))


eps_min = 0.1
eps_max = 1.0
eps_decay_steps = 2000000
def epsilon_greedy(q_values, step):
    epsilon = max(eps_min, eps_max - (eps_max-eps_min) * step/eps_decay_steps)
    if np.random.rand() < epsilon:
        return np.random.randint(n_outputs) # random action
    else:
        return np.argmax(q_values) # optimal action
    
    
n_steps = 4000000 # total number of training steps
training_start = 10000 # start training after 10,000 game iterations
training_interval = 4 # run a training step every 4 game iterations
save_steps = 1000 # save the model every 1,000 training steps
copy_steps = 10000 # copy online DQN to target DQN every 10,000 training steps
discount_rate = 0.99
skip_start = 90 # Skip the start of every game (it's just waiting time).
batch_size = 50
iteration = 0 # game iterations
checkpoint_path = "./my_dqn.ckpt"
done = True # env needs to be reset
tennis = pytennis(fps = 50)

with tf.Session() as sess:
    if os.path.isfile(checkpoint_path + ".index"):
        saver.restore(sess, checkpoint_path)
    else:
        init.run()
        copy_online_to_target.run()
    while True:
        step = global_step.eval()
        if step >= n_steps:
            break
        iteration += 1
        if done: # game over, start again
            obs = tennis.reset()
            for skip in range(skip_start): # skip the start of each game
                obs, reward, done, info = env.step(0)
            state = preprocess_observation(obs)
            
            
        # Online DQN evaluates what to do
        q_values = online_q_values.eval(feed_dict={X_state: [state]})
        action = epsilon_greedy(q_values, step)
        
        # Online DQN plays
        obs, reward, done, info = env.step(action)
        next_state = preprocess_observation(obs)
        
        
        # Let's memorize what just happened
        replay_memory.append((state, action, reward, next_state, 1.0 - done))
        state = next_state
        
        if iteration < training_start or iteration % training_interval != 0:
            continue # only train after warmup period and at regular intervals
        
        # Sample memories and use the target DQN to produce the target Q-Value
        X_state_val, X_action_val, rewards, X_next_state_val, continues = (sample_memories(batch_size))
        next_q_values = target_q_values.eval(feed_dict={X_state: X_next_state_val})
        max_next_q_values = np.max(next_q_values, axis=1, keepdims=True)
        y_val = rewards + continues * discount_rate * max_next_q_values
        
        # Train the online DQN
        training_op.run(feed_dict={X_state: X_state_val,X_action: X_action_val, y: y_val})
        
        # Regularly copy the online DQN to the target DQN
        if step % copy_steps == 0:
            copy_online_to_target.run()
        # And save regularly
        if step % save_steps == 0:
            saver.save(sess, checkpoint_path)
        
        #show the game
        tennis.render()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10090556274256145, 'fraction_numerical': 0.021622620587691738, 'mean_word_length': 3.075301204819277, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '428273', 'license': 'Apache-2.0', 'n_tokens_mistral': 1873, 'n_tokens_neox': 1759, 'n_words': 441, 'path': 'pytennis/Environment-Original.ipynb', 'repo_name': 'elishatofunmi/ReinEnv', 'size': '5411'}