<jupyter_start><jupyter_text>Import the necessary package<jupyter_code>import make_env as world
import numpy as np
from keras.models import Sequential
from keras.layers import Dense
import random<jupyter_output><empty_output><jupyter_text>load the environment<jupyter_code>env = world.make_env('simple')
env.discrete_action_input = True
for i in range(100):
    val =  env.render(mode='Test')
    val = env.step(action_n=[np.random.choice([0,1,2,3,4])])<jupyter_output><empty_output><jupyter_text>Create the model<jupyter_code>agent_move = Sequential()
agent_move.add(Dense(64,activation='relu', input_dim = 5))
agent_move.add(Dense(50,activation='relu'))
agent_move.add(Dense(32,activation='relu'))
agent_move.add(Dense(24,activation='relu'))
agent_move.add(Dense(16,activation='relu'))
agent_move.add(Dense(10,activation='relu'))
agent_move.add(Dense(8, activation='relu'))
agent_move.add(Dense(4,activation='relu'))
agent_move.add(Dense(1,activation='linear'))
agent_move.summary()
agent_move.compile(loss='mean_squared_error', optimizer='Adam')<jupyter_output><empty_output><jupyter_text>Introduce target network<jupyter_code>agent_target = Sequential()
agent_target.add(Dense(64,activation='relu', input_dim = 5))
agent_target.add(Dense(50,activation='relu'))
agent_target.add(Dense(32,activation='relu'))
agent_target.add(Dense(24,activation='relu'))
agent_target.add(Dense(16,activation='relu'))
agent_target.add(Dense(10,activation='relu'))
agent_target.add(Dense(8, activation='relu'))
agent_target.add(Dense(4,activation='relu'))
agent_target.add(Dense(1,activation='linear'))
agent_target.summary()
agent_move.save_weights('./random_weight')
episode = 1000
max_episode_length = 400
discount_factor = 0.9
for i in range(276,episode):
    
    #replace the experience replay 
    if ((i+1) % 25) == 0:
        episode_experience = episode_experience[-2000:]
        
        #replace old network with new network
        agent_move.save_weights('./weight')
        
        agent_target.load_weights('./weight')
        
        agent_move.load_weights('./random_weight')
    
    if i == 0:
        episode_experience = []
    
    
    epsilon = 1 / (np.sqrt(i+10))
    
    S = env.reset()
    S = S[0]
    
    A = np.random.choice([0,1,2,3,4])

    for j in range(max_episode_length):
        
        observe = env.step(action_n=[A])
        
        # Reward clip so that it values from 1 to -1
        R = observe[1][0] 
        
        if R < -1.5:
            R = -1
        else :
            R = (1.5 + R) / (1.5)
        
        S_new = observe[0][0]
        
        Test = np.array([[S_new[0],S_new[1],S_new[2],S_new[3], 0],
                         [S_new[0],S_new[1],S_new[2],S_new[3], 1],
                         [S_new[0],S_new[1],S_new[2],S_new[3], 2],
                         [S_new[0],S_new[1],S_new[2],S_new[3], 3],
                         [S_new[0],S_new[1],S_new[2],S_new[3], 4]])
        
        #value of all possible action from target network
        all_action_value = agent_move.predict(Test)
        
        #chose action which is maximum value 
        A_new = np.argmax(all_action_value)
        
        
        #epsilon - greedy 
        rand = np.random.uniform([0])
        
        if rand[0] > epsilon:
            A_new = A_new
        else :
            A_new = np.random.choice([0,1,2,3,4])
        
        
        Target = agent_target.predict(Test)[A_new][0]
        
        
        episode_experience.append([S[0],S[1],S[2],S[3],A,R + discount_factor * Target])
            
        S = S_new
        A = A_new
        
        if (j % (200)) == 0 :
            env.reset()
            
            S = env.reset()
            S = S[0]
    
            A = np.random.choice([0,1,2,3,4])
    
        if (i == 0 and j > 65) or (i > 0) :
            minibatch = random.sample(episode_experience,64)
    
            Test = np.array(minibatch)
        
            agent_move.fit(Test[:,0:5], Test[:,5], epochs=3)
    
import time

S = env.reset()
S = S[0]
    
A = np.random.choice([0,1,2,3,4])

epsilon = 0.0
agent_target.load_weights('./weight')
for _ in range(100):
    time.sleep(.5)
    observe = env.step(action_n=[A])
        
    chek =  env.render(mode='Test')
        
    # Reward clip so that it values from 1 to -1
    R = observe[1][0] 
        
    if R < -1.5:
        R = -1
    else :
        R = (1.5 + R) / (1.5)
        
    S_new = observe[0][0]
        
    Test = np.array([[S_new[0],S_new[1],S_new[2],S_new[3], 0],
                    [S_new[0],S_new[1],S_new[2],S_new[3], 1],
                    [S_new[0],S_new[1],S_new[2],S_new[3], 2],
                    [S_new[0],S_new[1],S_new[2],S_new[3], 3],
                    [S_new[0],S_new[1],S_new[2],S_new[3], 4]])
    
   
    #value of all possible action from target network
    all_action_value = agent_target.predict(Test)
        
    #chose action which is maximum value 
    A_new = np.argmax(all_action_value)
    
    rand = np.random.uniform([0])
    if rand[0] > epsilon:
            A_new = A_new
    else :
            A_new = np.random.choice([0,1,2,3,4])
    
    S = S_new
    A = A_new
        
env2 = world.make_env('simple_adversary')
env2.render("test")
agent2 = env2.agents
env2.render('Test')
env.step(action_n=[3])
for i in range(100):
    if (i%10) == 0:
        print(i,'1')
    
    if ((i+1) % 10) == 0:
        print(i, '2')
env.reset()
i
episode = [[1,2],[2,4],[3,1],[4,2]]
time.sleep(3)
random.sample(episode,3)
i<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15617631053872666, 'fraction_numerical': 0.040812624705242155, 'mean_word_length': 2.4854614412136535, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '68625', 'license': 'MIT', 'n_tokens_mistral': 2159, 'n_tokens_neox': 2046, 'n_words': 332, 'path': '.ipynb_checkpoints/project-Copy3-checkpoint.ipynb', 'repo_name': 'HemantharajM/Reinforcement-Learning-Intro-to-expect', 'size': '5513'}