<jupyter_start><jupyter_text>Cab-Driver Agent<jupyter_code>! git clone https://github.com/alokproc/deep-reinforcement-learning1
cd/content/deep-reinforcement-learning1

ls
# Importing libraries
import numpy as np
import random
import math
from collections import deque
import collections
import pickle

# for building DQN model
from keras import layers
from keras import Sequential
from keras.layers import Dense, Activation, Flatten
from keras.optimizers import Adam

# for plotting graphs
import matplotlib.pyplot as plt

# Import the environment
from Env import CabDriver
env = CabDriver()<jupyter_output><empty_output><jupyter_text>Defining Time Matrix<jupyter_code># Loading the time matrix provided
Time_matrix = np.load("TM.npy")<jupyter_output><empty_output><jupyter_text>Tracking the state-action pairs for checking convergence<jupyter_code>def encode_track_state(state):
    return ('-'.join(str(e) for e in state))
def encode_track_action(action):
    return ('-'.join(str(e) for e in action))
# Tracking q-values for all possible moves from position 4 at the start of the episode
def initialise_tracking_states(episodes):
    sample_q_values = [('4-0-0', '1-2'), ('4-0-0', '2-1'), ('4-0-0', '1-3'), ('4-0-0', '3-1'), ('4-0-0', '1-4'), ('4-0-0', '4-1'), ('4-0-0', '1-5'), ('4-0-0', '5-1'), ('4-0-0', '2-3'), 
                       ('4-0-0', '3-2'), ('4-0-0', '2-4'), ('4-0-0', '4-2'), ('4-0-0', '2-5'), ('4-0-0', '5-2'), ('4-0-0', '3-4'), ('4-0-0', '4-3'), ('4-0-0', '3-5'), ('4-0-0', '5-3'), 
                       ('4-0-0', '4-5'), ('4-0-0', '5-4'), ('4-0-0', '0-0')]    
    for q_value in sample_q_values:
        state = q_value[0]
        action = q_value[1]
        States_track[state][action] = []
# Function to save q-value for tracking states-action pair
def save_tracking_states(curr_state, curr_action, curr_q_value):
    for state in States_track.keys():
        if state == curr_state:
            for action in States_track[state].keys():
                if action == curr_action:
                    States_track[state][action].append(curr_q_value)
#Defining a function to save the Q-dictionary as a pickle file
def save_obj(obj, name ):
    with open(name + '.pkl', 'wb') as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)<jupyter_output><empty_output><jupyter_text>Agent Class- If the memory size is greater than mini-batch size, you randomly sample experiences from memory as per the mini-batch size and do the following:- Initialise your input and output batch for training the model- Calculate the target Q value for each sample: reward + gamma*max(Q(s'a,))- Get Q(s', a) values from the last trained model- Update the input batch as your encoded state and output batch as your Q-values- Then fit your DQN model using the updated input and output batch.<jupyter_code>class DQNAgent:
    def __init__(self, state_size, action_size):
        # Define size of state and action
        self.state_size = state_size
        self.action_size = action_size

        # Write here: Specify you hyper parameters for the DQN
        self.discount_factor = 0.99
        self.learning_rate =  0.001      
        self.epsilon = 1.0
        self.epsilon_decay = 0.999
        self.epsilon_min = 0.01
        
        self.batch_size = 32  
        self.train_start = 50
        # create replay memory using deque
        self.memory = deque(maxlen=2000)

       # create main model and target model
        #print ("Main NN Model : ")
        self.model = self.build_model()
        print ("\nTarget NN Model : ")
        self.target_model = self.build_model()

        
            
    # approximate Q function using Neural Network
    def build_model(self):
        model = Sequential()
        # Write your code here: Add layers to your neural nets       
        # input layer - input dimension would be state size
        model.add(Dense(32, input_dim=self.state_size, activation='relu', kernel_initializer='he_uniform'))
        
        # layer - 2
        model.add(Dense(32, activation='relu', kernel_initializer='he_uniform'))
        
        # layer - 3
        model.add(Dense(32, activation='relu', kernel_initializer='he_uniform'))
        
        # layer - 4
        model.add(Dense(32, activation='relu', kernel_initializer='he_uniform'))

        # the output layer: output size is total number of possible actions     
        model.add(Dense(self.action_size, activation='relu', kernel_initializer='he_uniform'))
        model.compile(loss='mse',optimizer=Adam(lr=self.learning_rate))
        model.summary
        return model

    def update_target_model(self):
        # update the target Q-value network to current Q-value network after training for a episode. 
        # this means that weights an biases of target Q-value network will become same as current Q-value network.
        self.target_model.set_weights(self.model.get_weights())


    def get_action(self, state, possible_actions):      
        if np.random.rand() <= self.epsilon:
            # explore: choose a random action from all possible actions
            # possible actions list would be given by environment
            choice = random.choice(possible_actions)
        else:
            # choose the action with the highest q(s, a)
            q_values = self.model.predict(state)
            possible_action_q_values = list(q_values[0][index] for index in possible_actions)
            max_q_value = np.max(possible_action_q_values)
            choice = np.where(q_values[0] == max_q_value)
            choice = choice[0][0]
            
        return choice
        
    def get_q_values(self, state):
        return self.model.predict(state)[0]
    
    def append_sample(self, state, action, reward, next_state, done):
        # append the tuple (s, a, r, s', done) to memory (replay buffer) after every action
        self.memory.append((state, action, reward, next_state, done))
    
        # decay in ε after we generate each sample from the environment
        if self.epsilon > self.epsilon_min:
            self.epsilon *= self.epsilon_decay    
    
    def train_model(self):
        # don't start the model training until memory queue is of certain size
        if len(self.memory) < self.train_start:
            return
        
        # sample batch from the memory
        batch_size = min(self.batch_size, len(self.memory))
        # pick samples randomly from replay memory (with batch_size) and train the network
        mini_batch = random.sample(self.memory, self.batch_size)

        # initialise your input and output batch for training the model
        update_input = np.zeros((self.batch_size, self.state_size))
        update_target = np.zeros((self.batch_size, self.state_size)) 
        actions, rewards, terminal_states = [], [], []

        for i in range(self.batch_size):

            update_input[i] = mini_batch[i][0]
            actions.append(mini_batch[i][1])
            rewards.append(mini_batch[i][2])
            update_target[i] = mini_batch[i][3]
            terminal_states.append(mini_batch[i][4])

        # predict the target from earlier model
        target = self.model.predict(update_input)

        # get the target for the Q-network
        target_qval = self.target_model.predict(update_target)  

        for i in range(self.batch_size):
            if terminal_states[i]:
                target[i][actions[i]] = rewards[i]
            else: # non-terminal state
                target[i][actions[i]] = rewards[i] + self.discount_factor * np.max(target_qval[i])

        # fit your model and track the loss values
        self.model.fit(update_input, target, batch_size=self.batch_size, epochs=1, verbose=0)
    
    def save(self, name):
        self.model.save_weights(name)
        
    def get_model_weights(self):
        return self.model.get_weights()
Episodes = 2000<jupyter_output><empty_output><jupyter_text>DQN block<jupyter_code>import pylab
scores, episodes = [], []
state = env.state_init
state_encoded = env.state_encod_arch1(state)
state_size = len(state_encoded)
action_size = len(env.action_space)
agent = DQNAgent(state_size, action_size)
States_track = collections.defaultdict(dict)
initialise_tracking_states(Episodes)

for episode in range(Episodes):

    # Write code here
    # Call the environment
    # Call all the initialised variables of the environment
    terminal_state = False
    score = 0
    reward = 0
    total_days = 1
    previous_day = 0
    current_day = 0
    env.reset()
    

    #Call the DQN agent
    action_space, state_space, state = env.reset()
    track_state = state
    
    while not terminal_state:
        
        # Write your code here
        # 1. Pick epsilon-greedy action from possible actions for the current state
        # 2. Evaluate your reward and next state
        # 3. Append the experience to the memory
        # 4. Train the model by calling function agent.train_model
        # 5. Keep a track of rewards, Q-values, loss
        
        state_encoded = env.state_encod_arch1(state)
        state_encoded = np.reshape(state_encoded, [1, state_size])
        
        # get possible list of actions from the environment
        possible_actions, action_list = env.requests(state)
        
        # pick epsilon-greedy action from possible actions for the current state
        # possible actions is given by the environment
        action = agent.get_action(state_encoded, possible_actions)
        
        # evaluate your reward and next state
        reward = env.reward_func(state, env.action_space[action], Time_matrix)
        next_state = env.next_state_func(state, env.action_space[action], Time_matrix)
        next_state_encoded = env.state_encod_arch1(next_state)
        next_state_encoded = np.reshape(next_state_encoded, [1, state_size])
        
        # append the experience to the memory
        agent.append_sample(state_encoded, action, reward, next_state_encoded, terminal_state)
        
        # train the model by calling function agent.train_model
        agent.train_model()
        
        # keep track of rewards
        score += reward
        previous_day = state[2]
        state = next_state
        current_day = state[2]
        if previous_day != current_day:
            total_days = total_days + 1
            
        if total_days > 30:
            terminal_state = True
            
        # once you reach end of 30 days, target model is updated, scores and episode saved
        if terminal_state == True:
            agent.update_target_model()
            scores.append(score)
            episodes.append(episode)  
    
    # this is track q-value convergence
    # once the episode is complete, q-values will be predicted for all possible actions for the initial state
    # this way is much easier to track because other states would be difficult to track as it is heavily dependant on the location and action taken
    track_state_encode = encode_track_state(track_state)
    track_state_encoded = env.state_encod_arch1(track_state)
    track_state_encoded = np.reshape(track_state_encoded, [1, state_size])
    track_q_values = agent.get_q_values(track_state_encoded)
    
    for index in range(len(env.action_space)):
        track_action_encode = encode_track_action(env.action_space[index])
        save_tracking_states(track_state_encode, track_action_encode, track_q_values[index])
    
    #save the graph, model (h5, pickle file)
    if (episode + 1) % 200 == 0:
        print("Episode:", episode + 1, "  score:", score, "  memory length:", len(agent.memory), "  epsilon:", agent.epsilon)
        pylab.plot(episodes, scores)
        pylab.savefig("cardriver_dqn_" + str(score) + ".png")
        agent.save("cardriver_dqn_" + str(score) + ".h5")
        #agent.save_model_graph(str(score))
        # this is the only one required for assignment. this will create pickle file
        save_obj(agent.get_model_weights(), "cardriver_dqn_" + str(score))<jupyter_output><empty_output><jupyter_text>Tracking Convergence<jupyter_code># convergence with scores
plt.plot(episodes, scores)

# State-action q-value graph
values = States_track['4-0-0']['4-3']
plt.plot(values)

# State-action q-value graph
values = States_track['4-0-0']['4-2']
plt.plot(values)<jupyter_output><empty_output><jupyter_text>Epsilon-decay sample function Try building a similar epsilon-decay function for your model.<jupyter_code>time = np.arange(0,10000)
epsilon = []
for i in range(0,10000):
    epsilon.append(0 + (1 - 0) * np.exp(-0.0009*i))
plt.plot(time, epsilon)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10203919701658336, 'fraction_numerical': 0.017694199793699912, 'mean_word_length': 2.8721966205837175, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '68205', 'license': 'CNRI-Python', 'n_tokens_mistral': 3913, 'n_tokens_neox': 3777, 'n_words': 1062, 'path': 'RLF_0221A_Saminathan_Renganayagi_DQN_Agent_Arch1_.ipynb', 'repo_name': 'renganaya/Reinforcement-Learning-cab-Driver', 'size': '12603'}