<jupyter_start><jupyter_text>For example in Humanoid-v2 with 20 roll outs we have20 sets of training data. Each ahave 17 actions and 376 observations.The would means the input of the model would be 376 and the output would be 17.<jupyter_code># load the gym enviornment
env = gym.make(model)
print('observation space:', env.observation_space)
print('action space:', env.action_space)
# check the observation space dimentionality of the enviornment
spc = env.observation_space
print(type(spc.shape))
print(env.observation_space.shape[0])
"""
Prepare the data, including:
    normalization and save the normalization parameters
"""

num_rollout = 20

x_i = loaded["observations"]
y_i = np.squeeze(loaded["actions"],axis=1)
num_examples = x_i.shape[0]

num_ins = x_i.shape[1]
num_outs = y_i.shape[1]

print ("X RAW shape:", x_i.shape, "Y RAW shape:",y_i.shape)


bat = num_examples / num_rollout
# print(bat)
bat = int(bat)

x = np.array([], dtype=np.float16).reshape(0,num_ins * 2 )
y = np.array([], dtype=np.float16).reshape(0,num_outs)


for idx, i in enumerate(range(0,num_examples,bat)):
    start = i
    end = i+bat
    xpreK = x_i[start:end-1,:]
    xK = x_i[start+1:end,:]
    x = np.concatenate( (x,np.concatenate((xK, xK-xpreK ) , axis = 1) ), axis = 0)
    y = np.concatenate( (y , y_i[start+1:end,:]), axis = 0  )


print ("After Adding \"obs_k - obs_(k-1)\". X shape:", x.shape, "Y shape:",y.shape)

num_examples = x.shape[0]


# normalize the values
def normalize(x,mean=None,std=None):
    if (mean is None):
        mean = np.mean(x,axis=0)
        std = np.std(x,axis=0)
        normalized = (x-mean)/std
        return normalized, mean, std
    else:
        return (x-mean)/std

def unnormalize(x, mean, std):
    unnorm = x * std + mean
    return unnorm

x_norm, x_mean, x_std = normalize(x)
y_norm, y_mean, y_std = normalize(y)

print ("shape of the normlization results for x_norm {}, x_mean {} and x_std {}".format(
    x_norm.shape, x_mean.shape, x_std.shape))
print ("shape of the normlization results for y_norm {}, y_mean {} and y_std {}".format(
    y_norm.shape, y_mean.shape, y_std.shape))

# remove the nans from the normalized vector
x_mask = (x_std!=0)
y_mask = (y_std!=0)
t = np.tile(x_mask,(num_examples,1))
x_norm = x_norm[t].reshape([num_examples,-1])
t = np.tile(y_mask,(num_examples,1))
y_norm = y_norm[t].reshape([num_examples,-1])
print ("The shapes after removing the dimenstions with **std = 0**, x_norm shape {}, y_norm shape {}".format(
    x_norm.shape, y_norm.shape))

y_un = unnormalize(y_norm,y_mean,y_std)
print("error after undoing the normalization" ,np.sum(np.abs(y_un - y)))

print ("Is there any Nan in training date?" ,np.isnan(x_norm).any(),np.isnan(y_norm).any(),
       np.isinf(x_norm).any(),np.isinf(y_norm).any())
# save the normalization paramters
toSave = dict()
toSave["x_mean"] = x_mean
toSave["y_mean"] = y_mean
toSave["x_std"] = x_std
toSave["y_std"] = y_std
pickle.dump(toSave,open( "/home/yousof/courses/cs294_DeepRL/hw1/expert_data/"+model+"_normalization.pkl", "wb" ) )
# lets make a dense network to learn frome the training data as easy as possible

model_in_size = x_norm.shape[1]
model_out_size = y_norm.shape[1]

x_place = tf.placeholder(tf.float16,shape=(None, model_in_size),name="input")
y_place = tf.placeholder(tf.float16,shape=(None, model_out_size),name="label")

mean_init = 0
std_init = 0.001
regularizer = tf.contrib.layers.l2_regularizer(scale=0.1)

dens = tf.layers.dense(x_place,512,
                       activation=tf.nn.tanh,
                       kernel_initializer=tf.truncated_normal_initializer(stddev=std_init),
                       name = "dense1",
                       kernel_regularizer=regularizer)
# dens = tf.layers.dense(dens,1024,
#                        activation=tf.nn.sigmoid,
#                        kernel_initializer=tf.truncated_normal_initializer(stddev=std_init),
#                        name = "dense2",
#                        kernel_regularizer=regularizer)
# dens = tf.layers.dense(dens,128,
#                        activation=tf.nn.tanh,
#                        kernel_initializer=tf.truncated_normal_initializer(stddev=std_init),
#                        name = "dense3",
#                        kernel_regularizer=regularizer)
# dens = tf.layers.dense(dens,128,
#                        activation=tf.nn.tanh,
#                        kernel_initializer=tf.truncated_normal_initializer(stddev=std_init),
#                        name = "dense4",
#                        kernel_regularizer=regularizer)
pred = tf.layers.dense(dens,model_out_size,
                       activation=tf.nn.tanh,
                       kernel_initializer=tf.truncated_normal_initializer(stddev=std_init),
                       name = "pred",
                       kernel_regularizer=regularizer)

loss = tf.losses.mean_squared_error(y_place , pred)
optmizer_op = tf.train.RMSPropOptimizer(learning_rate=0.001)
train_op = optmizer_op.minimize(loss)

saver = tf.train.Saver()
def get_batches(batch_size,x,y):
    x_size = x.shape[1]
    y_size = y.shape[1]
    total_number = x.shape[0]
    xy = np.concatenate((x,y),axis=1)
    np.random.shuffle(xy)
    for batch_i in range(0,total_number,batch_size):
        start = batch_i
        end = batch_i+ batch_size
        end = min (end, total_number)
        x_out = xy[start:end,0:x_size]
        y_out = xy[start:end,x_size:]
        #print(x_out.shape,y_out.shape)
        yield x_out,y_out 
num_epoch = 5000
batch_size = 1024*16

#merged = tf.summary.merge_all()
writer = tf.summary.FileWriter(logfile, tf.get_default_graph())

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    for epoch in range(num_epoch):
#         i_batch = 0
#         for batch_x, batch_y in get_batches(batch_size,x_norm,y_norm):
#             i_batch += 1
#             out = sess.run([loss ,train_op], feed_dict = {x_place: batch_x , y_place: batch_y})
#             print("Epoch: {} batch: {} loss: {}"
#                   .format(epoch,i_batch,out[0]))
        out = sess.run([loss ,train_op], feed_dict = {x_place: x_norm , y_place: y_norm})
        print("Epoch: {} loss: {}"
              .format(epoch,out[0]))
#         print("===================================================")
    save_path = saver.save(sess, "models/"+model+".ckpt")
tf.reset_default_graph()
model_path = "models/" + model
render = True


def prepareInput(obs, preObs, size):
    obs =obs.reshape([1,size])
    preObs = preObs.reshape([1,size])
    model_in = np.concatenate( (obs,obs-preObs), axis = 1 )
    model_in = normalize(model_in,mean = x_mean, std = x_std)
    model_in = model_in[x_mask.reshape([1,size*2])]
    model_in = np.expand_dims(model_in, axis=0)
    return model_in

def prepareOutput(modelOut):
    return unnormalize(modelOut,y_mean,y_std)

with tf.Session() as sess:
    # load the model
    saver = tf.train.import_meta_graph(model_path + '.ckpt.meta')
    saver.restore(sess,model_path+".ckpt")
    graph = tf.get_default_graph()
    policy_out = graph.get_tensor_by_name('pred/Tanh:0')#I_want_to_train_only_these/fc8/BiasAdd:0
    x_place = graph.get_tensor_by_name('input:0')
    
    def run_policy(policy_in):
        return sess.run(policy_out, feed_dict = {x_place: policy_in})
    
    env = gym.make(model)
    
    obs_len = env.observation_space.shape[0]
    obsPre = np.zeros((1,obs_len),dtype=np.float16)
    
    max_steps = 1000
    num_rollouts = 2
    returns = []
    observations = []
    actions = []
    
    for i in range(num_rollouts):
        obs = env.reset()
        observations.append(obs)
        done = False
        totalr = 0
        steps = 0
        
        model_in = prepareInput(obs,obsPre,obs_len)
        obsPre = obs
        preAction = run_policy(model_in)
        action = prepareOutput(preAction)
        obs, r, done, _ = env.step(action)
        while True: # Not done
            model_in = prepareInput(obs,obsPre,obs_len)
            obsPre = obs
            preAction = run_policy(model_in)
            action = prepareOutput(preAction)
            obs, r, done, _ = env.step(action)
            totalr += r
            steps += 1
            if render:
                env.render()
            if steps % 100 == 0: 
#                 print("",end="\r")
                print("%i/%i"%(steps, max_steps),end="\r")
            if steps >= max_steps:
                break
        returns.append(totalr)
        num_rollouts += 1
    print('returns', returns)
    print('mean return', np.mean(returns))
    print('std of return', np.std(returns))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1373096151610278, 'fraction_numerical': 0.015463318218811766, 'mean_word_length': 2.9064486830154403, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '209948', 'license': 'MIT', 'n_tokens_mistral': 3137, 'n_tokens_neox': 2984, 'n_words': 588, 'path': 'hw1/Test and Learn.ipynb', 'repo_name': 'yosoufe/cs294_DeepRL', 'size': '8601'}