<jupyter_start><jupyter_text>연습<jupyter_code>import numpy as np
import tensorflow as tf

# set parameters ###############################################################
initial_burning_epoch = 0
epoch = 5
lr_rate = 1e-2
copy_period = 5
epsilon_0 = 1e-5
# set parameters ###############################################################

# state
states = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
N_STATES = len(states)

# action
actions = [0, 1, 2, 3] 
N_ACTIONS = len(actions)

# transition probabilities
P = np.empty((N_STATES, N_ACTIONS, N_STATES))

#                0   1   2   3   4   5   6   7   8   9  10
P[ 0, 0, :] = [ .9,  0,  0,  0, .1,  0,  0,  0,  0,  0,  0]
P[ 0, 1, :] = [ .1, .8,  0,  0, .1,  0,  0,  0,  0,  0,  0]
P[ 0, 2, :] = [ .9, .1,  0,  0,  0,  0,  0,  0,  0,  0,  0]
P[ 0, 3, :] = [ .1, .1,  0,  0, .8,  0,  0,  0,  0,  0,  0]

#                0   1   2   3   4   5   6   7   8   9  10
P[ 1, 0, :] = [ .8, .2,  0,  0,  0,  0,  0,  0,  0,  0,  0]
P[ 1, 1, :] = [  0, .2, .8,  0,  0,  0,  0,  0,  0,  0,  0]
P[ 1, 2, :] = [ .1, .8, .1,  0,  0,  0,  0,  0,  0,  0,  0]
P[ 1, 3, :] = [ .1, .8, .1,  0,  0,  0,  0,  0,  0,  0,  0]

#                0   1   2   3   4   5   6   7   8   9  10
P[ 2, 0, :] = [  0, .8, .1,  0,  0, .1,  0,  0,  0,  0,  0]
P[ 2, 1, :] = [  0,  0, .1, .8,  0, .1,  0,  0,  0,  0,  0]
P[ 2, 2, :] = [  0, .1, .8, .1,  0,  0,  0,  0,  0,  0,  0]
P[ 2, 3, :] = [  0, .1,  0, .1,  0, .8,  0,  0,  0,  0,  0]

#                0   1   2   3   4   5   6   7   8   9  10
P[ 3, 0, :] = [  0,  0,  0,  1,  0,  0,  0,  0,  0,  0,  0]
P[ 3, 1, :] = [  0,  0,  0,  1,  0,  0,  0,  0,  0,  0,  0]
P[ 3, 2, :] = [  0,  0,  0,  1,  0,  0,  0,  0,  0,  0,  0]
P[ 3, 3, :] = [  0,  0,  0,  1,  0,  0,  0,  0,  0,  0,  0]

#                0   1   2   3   4   5   6   7   8   9  10
P[ 4, 0, :] = [ .1,  0,  0,  0, .8,  0,  0, .1,  0,  0,  0]
P[ 4, 1, :] = [ .1,  0,  0,  0, .8,  0,  0, .1,  0,  0,  0]
P[ 4, 2, :] = [ .8,  0,  0,  0, .2,  0,  0,  0,  0,  0,  0]
P[ 4, 3, :] = [  0,  0,  0,  0, .2,  0,  0, .8,  0,  0,  0]

#                0   1   2   3   4   5   6   7   8   9  10
P[ 5, 0, :] = [  0,  0, .1,  0,  0, .8,  0,  0,  0, .1,  0]
P[ 5, 1, :] = [  0,  0, .1,  0,  0,  0, .8,  0,  0, .1,  0]
P[ 5, 2, :] = [  0,  0, .8,  0,  0, .1, .1,  0,  0,  0,  0]
P[ 5, 3, :] = [  0,  0,  0,  0,  0, .1, .1,  0,  0, .8,  0]

#                0   1   2   3   4   5   6   7   8   9  10
P[ 6, 0, :] = [  0,  0,  0,  0,  0,  0,  1,  0,  0,  0,  0]
P[ 6, 1, :] = [  0,  0,  0,  0,  0,  0,  1,  0,  0,  0,  0]
P[ 6, 2, :] = [  0,  0,  0,  0,  0,  0,  1,  0,  0,  0,  0]
P[ 6, 3, :] = [  0,  0,  0,  0,  0,  0,  1,  0,  0,  0,  0]

#                0   1   2   3   4   5   6   7   8   9  10
P[ 7, 0, :] = [  0,  0,  0,  0, .1,  0,  0, .9,  0,  0,  0]
P[ 7, 1, :] = [  0,  0,  0,  0, .1,  0,  0, .1, .8,  0,  0]
P[ 7, 2, :] = [  0,  0,  0,  0, .8,  0,  0, .1, .1,  0,  0]
P[ 7, 3, :] = [  0,  0,  0,  0,  0,  0,  0, .9, .1,  0,  0]

#                0   1   2   3   4   5   6   7   8   9  10
P[ 8, 0, :] = [  0,  0,  0,  0,  0,  0,  0, .8, .2,  0,  0]
P[ 8, 1, :] = [  0,  0,  0,  0,  0,  0,  0,  0, .2, .8,  0]
P[ 8, 2, :] = [  0,  0,  0,  0,  0,  0,  0, .1, .8, .1,  0]
P[ 8, 3, :] = [  0,  0,  0,  0,  0,  0,  0, .1, .8, .1,  0]

#                0   1   2   3   4   5   6   7   8   9  10
P[ 9, 0, :] = [  0,  0,  0,  0,  0, .1,  0,  0, .8, .1,  0]
P[ 9, 1, :] = [  0,  0,  0,  0,  0, .1,  0,  0,  0, .1, .8]
P[ 9, 2, :] = [  0,  0,  0,  0,  0, .8,  0,  0, .1,  0, .1]
P[ 9, 3, :] = [  0,  0,  0,  0,  0,  0,  0,  0, .1, .8, .1]

#                0   1   2   3   4   5   6   7   8   9  10
P[10, 0, :] = [  0,  0,  0,  0,  0,  0, .1,  0,  0, .8, .1]
P[10, 1, :] = [  0,  0,  0,  0,  0,  0, .1,  0,  0,  0, .9]
P[10, 2, :] = [  0,  0,  0,  0,  0,  0, .8,  0,  0, .1, .1]
P[10, 3, :] = [  0,  0,  0,  0,  0,  0,  0,  0,  0, .1, .9]

# rewards
R = -0.02 * np.ones((N_STATES, N_ACTIONS)) 
R[3,:] = 1.
R[6,:] = -1.
    
# discount factor
gamma = 0.99

# policy
if 0: 
    # bad policy 
    policy = np.empty((N_STATES, N_ACTIONS))
    policy[0,:] = [0,1,0,0]
    policy[1,:] = [0,1,0,0]
    policy[2,:] = [0,1,0,0]
    policy[3,:] = [0,1,0,0]
    policy[4,:] = [0,0,0,1]
    policy[5,:] = [0,1,0,0]
    policy[6,:] = [0,1,0,0]
    policy[7,:] = [0,1,0,0]
    policy[8,:] = [0,1,0,0]
    policy[9,:] = [0,0,1,0]
    policy[10,:] = [0,0,1,0]
elif 0: 
    # random policy
    policy = 0.25*np.ones((N_STATES, N_ACTIONS))
elif 0: 
    # optimal policy 
    policy = np.empty((N_STATES, N_ACTIONS))
    policy[0,:] = [0,1,0,0]
    policy[1,:] = [0,1,0,0]
    policy[2,:] = [0,1,0,0]
    policy[3,:] = [0,1,0,0]
    policy[4,:] = [0,0,1,0]
    policy[5,:] = [0,0,1,0]
    policy[6,:] = [0,0,1,0]
    policy[7,:] = [0,0,1,0]
    policy[8,:] = [1,0,0,0]
    policy[9,:] = [1,0,0,0]
    policy[10,:] = [1,0,0,0]
elif 1: 
    # optimal policy + noise 
    # we use optimal policy with probability 1/(1+ep)
    # we use random policy with probability ep/(1+ep)
    ep = 0.1
    policy = np.empty((N_STATES, N_ACTIONS))
    policy[0,:] = [0,1,0,0]
    policy[1,:] = [0,1,0,0]
    policy[2,:] = [0,1,0,0]
    policy[3,:] = [0,1,0,0]
    policy[4,:] = [0,0,1,0]
    policy[5,:] = [0,0,1,0]
    policy[6,:] = [0,0,1,0]
    policy[7,:] = [0,0,1,0]
    policy[8,:] = [1,0,0,0]
    policy[9,:] = [1,0,0,0]
    policy[10,:] = [1,0,0,0]
    policy = policy + (ep/4)*np.ones((N_STATES, N_ACTIONS))
    policy = policy / np.sum(policy, axis=1).reshape((N_STATES,1))
" TD "
# https://www.tensorflow.org/api_docs/python/tf/placeholder
state = tf.placeholder(tf.int32, shape=[None], name='state')
action = tf.placeholder(tf.int32, shape=[None], name='action')
reward = tf.placeholder(tf.float32, shape=[None], name='reward')
next_state = tf.placeholder(tf.int32, shape=[None], name='next_state')
next_action = tf.placeholder(tf.int32, shape=[None], name='next_action')
done_holder = tf.placeholder(tf.bool, shape=[None], name='done_holder')

#어떤 구조를 만들어주기 위해 쓰는 것. 그 구조를 활용할때는 feed_dict로 각각의 부분에 해당하는 값을 대응시키는 것이 필요
#https://gdyoon.tistory.com/5

# https://www.tensorflow.org/api_docs/python/tf/one_hot
# one_hot(x,y) x의 내용 개수 만큼 row를 만들어주고, y값 만큼 column을 만들어주고, 그리고 x의 내용 번째 값을 1, 아닌 것을 0으로
# one_hot([2,0,1],3) = [[0,0,1],[1,0,0],[0,1,0]]
state_one_hot = tf.one_hot(state, N_STATES)
action_one_hot = tf.one_hot(action, N_ACTIONS)
next_state_one_hot = tf.one_hot(next_state, N_STATES)
next_action_one_hot = tf.one_hot(next_action, N_ACTIONS)

# https://www.tensorflow.org/api_docs/python/tf/concat
#데이터 합치기
#t1= [[1,0,0],[0,1,0],[0,0,1]]
#t2=[[2,0,0],[0,2,0],[0,0,2]]
#concat([t1,t2],0)=[[1,0,0],[0,1,0],[0,0,1],[2,0,0],[0,2,0],[0,0,2]]
#concat([t1,t2],1)=[[1,0,0,2,0,0],[0,1,0,0,2,0],[0,0,1,0,0,2]]
state_action_one_hot = tf.concat([state_one_hot, action_one_hot], 1)
next_state_action_one_hot = tf.concat([next_state_one_hot, next_action_one_hot], 1)
# Previously
# Q = np.zeros((N_STATES, N_ACTIONS))
# Q[3, :] = 1.
# Q[6, :] = -1.

# state s: one hot encode
# action a: one hot encode
# Q[s,a]: [s as one hot encode, a as one hot encode] @ W 

with tf.variable_scope('main_net') as scope:
    W = tf.get_variable(name='W', \
                        shape=[N_STATES + N_ACTIONS, 1], \
                        dtype=tf.float32, \
                        initializer=tf.random_uniform_initializer(-1.0, 1.0))

with tf.variable_scope('target_net') as scope:
    W_target = tf.get_variable(name='W_target', \
                        shape=[N_STATES + N_ACTIONS, 1], \
                        dtype=tf.float32, \
                        initializer=tf.random_uniform_initializer(-1.0, 1.0))
    
# variable copy
from_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'main_net')
# print(from_vars)
# [<tf.Variable 'main_net/W:0' shape=(15, 1) dtype=float32_ref>]

to_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'target_net')
# print(to_vars)
# [<tf.Variable 'target_net/W_target:0' shape=(15, 1) dtype=float32_ref>]

copy_ops = [to_vars[i].assign(from_vars[i]) for i in range(len(from_vars))]

# TD

# Previously for V
# V(S_t)\leftarrow V(S_t)+\alpha(R_{t+1}+\gamma V(S_{t+1})-V(S_t))
# V[s] += alpha * (R[s,a] + gamma * V[s1] - V[s])
# Q[s,a] += alpha * (R[s,a] + gamma * Q[s1,a1] - Q[s,a])

# TD target and TD error
TD_target_not_done = reward \
                                    + gamma * tf.reduce_max(tf.matmul(next_state_action_one_hot, W_target), \
                                                            axis=1, \
                                                            keepdims=True)
TD_target_done = reward # this reward is actually reward + gamma * final_reward. 마지막엔 tf.reduce이하의 내용이 1 혹은 -1이 나와야하는데 그렇게 안나오기 떄문에 아예 따로 값을 해주는 걸로

Q = tf.matmul(state_action_one_hot, W)

TD_error_not_done = TD_target_not_done - Q
TD_error_done = TD_target_done - Q

error = tf.to_float(done_holder) * TD_error_done \
            + (1-tf.to_float(done_holder)) * TD_error_not_done  #done holder에 트루 폴스 줘서 error값을 done 값에 따라 다르게 주기 위해서 식 이렇게 쓴 것

loss = tf.reduce_mean(tf.square(error))
opt = tf.train.GradientDescentOptimizer(learning_rate=lr_rate)
train_ops = opt.minimize(loss, \
                         var_list=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, \
                                                    'main_net')) #여기에 var list로 제한을 시킨 것 : 거기에 있는 var만 바꿔서 트레이닝을 해라.
" Session "
with tf.Session() as sess:
    tf.global_variables_initializer().run()
    
    " Initial Burning"

    gradient_update_number = 0

    for this_epoch in range(initial_burning_epoch):
        
        print("================================================", this_epoch )
        
        epsilon = epsilon_0 / (1 + this_epoch)

        # indicate game is not over yet
        done = False

        # choose initial state randomly, not from 3 or 6
        s = np.random.choice([0, 1, 2, 4, 5, 7, 8, 9, 10]) 

        # random action selection
        a = np.random.choice(actions)

        while not done:
            
            if gradient_update_number % copy_period == 0:
                sess.run(copy_ops)
                
            # choose next state using transition probabilities
            s1 = np.random.choice(states, p=P[s, a, :])
            print('next state :', s1)

            # random action selection
            a1 = np.random.choice(actions)
            
            if s1 == 3:
                # if game is over, 
                # ready to break while loop by letting done = True
                done = True
                
                # train
                # reward =  R[s,a] + gamma*1
                feed_dict = {state: [s], action: [a], reward: [R[s,a] + gamma*1], next_state: [s1], next_action: [a1], done_holder: [done]}
                sess.run(train_ops, feed_dict=feed_dict)

                gradient_update_number += 1
                
            elif s1 == 6:
                # if game is over, 
                # ready to break while loop by letting done = True
                done = True
                
                # train
                # reward =  R[s,a] - gamma*1
                feed_dict = {state: [s], action: [a], reward: [R[s,a] - gamma*1], next_state: [s1], next_action: [a1], done_holder: [done]}
                sess.run(train_ops, feed_dict=feed_dict)

                gradient_update_number += 1
                               
            else:
                # if game is not over, continue playing game
                
                # train
                # reward =  R[s,a]
                feed_dict = {state: [s], action: [a], reward: [R[s,a]], next_state: [s1], next_action: [a1], done_holder: [done]}
                sess.run(train_ops, feed_dict=feed_dict)

  #              gradient_update_number += 1
                
                s = s1
                a = a1
                
    " Train "
                
    gradient_update_number = 0

    for this_epoch in range(epoch):
        
        print("================================================", this_epoch )
        
        epsilon = epsilon_0 / (1 + this_epoch)

        # indicate game is not over yet
        done = False

        # choose initial state randomly, not from 3 or 6
        s = np.random.choice([0, 1, 2, 4, 5, 7, 8, 9, 10]) 

        # this is policy update part of algorithm 
        # epsilon-greedy policy update
        # choose action using udated epsilon-greedy policy
        # if this part is replaced and if we choose action using current policy
        # this whole algorithm becomes TD, not SARSA, SARSA(lambda) nor Q-Learning
        if np.random.uniform(0.,1.) < epsilon:
            # random action selection
            a = np.random.choice(actions)
        else:
            # greedy action selection
            a = np.argmax(Q[s, :])

        while not done:
            
            if gradient_update_number % copy_period == 0:
                sess.run(copy_ops)
                
            # choose next state using transition probabilities
            s1 = np.random.choice(states, p=P[s, a, :])
            print('next state :', s1)

            # this is policy update part of algorithm 
            # epsilon-greedy policy update
            # choose action using udated epsilon-greedy policy
            # if this part is replaced and if we choose action using current policy
            # this whole algorithm becomes TD, not SARSA, SARSA(lambda) nor Q-Learning
            if np.random.uniform(0.,1.) < epsilon:
                # random action selection
                a1 = np.random.choice(actions)
            else:
                # greedy action selection
                a1 = np.argmax(Q[s1, :])
            
            if s1 == 3:
                # if game is over, 
                # ready to break while loop by letting done = True
                done = True
                
                # train
                # reward =  R[s,a] + gamma*1
                feed_dict = {state: [s], action: [a], reward: [R[s,a] + gamma*1], next_state: [s1], next_action: [a1], done_holder: [done]}
                sess.run(train_ops, feed_dict=feed_dict)

                gradient_update_number += 1
                
            elif s1 == 6:
                # if game is over, 
                # ready to break while loop by letting done = True
                done = True
                
                # train
                # reward =  R[s,a] - gamma*1
                feed_dict = {state: [s], action: [a], reward: [R[s,a] - gamma*1], next_state: [s1], next_action: [a1], done_holder: [done]}
                sess.run(train_ops, feed_dict=feed_dict)

                gradient_update_number += 1
                               
            else:
                # if game is not over, continue playing game
                
                # train
                # reward =  R[s,a]
                feed_dict = {state: [s], action: [a], reward: [R[s,a]], next_state: [s1], next_action: [a1], done_holder: [done]}
                sess.run(train_ops, feed_dict=feed_dict)

   #             gradient_update_number += 1
                
                s = s1
                a = a1
            
        print ()

    Q_final = np.empty((N_STATES,N_ACTIONS))
    for s in range(N_STATES):
        for a in range(N_ACTIONS):
            feed_dict = {state: [s], action: [a]}
            Q_now = sess.run(Q, feed_dict=feed_dict)
            Q_final[s,a] = Q_now[0][0]
    print(Q_final)<jupyter_output>================================================ 0
next state : 0
next state : 0
next state : 0
next state : 1
next state : 0
next state : 1
next state : 1
next state : 1
next state : 0
next state : 4
next state : 4
next state : 4
next state : 7
next state : 7
next state : 4
next state : 0
next state : 4
next state : 0
next state : 0
next state : 0
next state : 0
next state : 4
next state : 0
next state : 4
next state : 4
next state : 4
next state : 4
next state : 7
next state : 7
next state : 7
next state : 4
next state : 4
next state : 0
next state : 0
next state : 4
next state : 4
next state : 0
next state : 4
next state : 4
next state : 4
next state : 7
next state : 7
next state : 7
next state : 7
next state : 8
next state : 7
next state : 7
next state : 4
next state : 7
next state : 7
next state : 7
next state : 4
next state : 4
next state : 0
next state : 4
next state : 4
next state : 4
next state : 4
next state : 0
next state : 4
next state : 4
next state : 4
next state : 7
next[...]
{'fraction_non_alphanumeric': 0.17758547138765313, 'fraction_numerical': 0.07325248339325979, 'mean_word_length': 1.7469032474054236, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 6, 'https://': 4, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 150, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '315941', 'license': 'MIT', 'n_tokens_mistral': 7328, 'n_tokens_neox': 6967, 'n_words': 1881, 'path': '19-1/Machine-Learning-3/Test2 for HW4.ipynb', 'repo_name': 'neulbo-187/graduate-course', 'size': '16409'}