<jupyter_start><jupyter_text>This uses the open AI gym baseline with some slight modifications. Mostly copied code from herehttps://github.com/openai/baselines/tree/master/baselines/ppo1. Looks like the version I got from pip and the version currently on master don't quite sync up. (For example there is no tf_util.save_state fcn so we save manually<jupyter_code>from baselines.common.cmd_util import make_mujoco_env, mujoco_arg_parser
from baselines.common import tf_util as U
import tensorflow as tf
from baselines import logger
import os
import sys
from baselines.ppo1 import mlp_policy, pposgd_simple

import gym
import gym_ucsb_robolab
import policies.mlp_relu_policy as mlp_relu_policy

import numpy as np
from mpl_toolkits import mplot3d
%matplotlib ipympl
import matplotlib.pyplot as plt
import itertools

#Needed for saving 
import errno, datetime, time, inspect

def train(env_id, num_timesteps, seed=0):

    U.make_session(num_cpu=16).__enter__()
    
    def policy_fn(name, ob_space, ac_space):
        #return mlp_policy.MlpPolicy(name=name, ob_space=ob_space, ac_space=ac_space, hid_size=64, num_hid_layers=64)
        return mlp_relu_policy.ReluMlpPolicy(name=name, ob_space=ob_space, ac_space=ac_space, hid_size=64, num_hid_layers=4)

    env = gym.make(env_id)
    pi = pposgd_simple.learn(env, policy_fn,
            max_timesteps=num_timesteps,
            timesteps_per_actorbatch=2048,
            clip_param=0.2, entcoeff=0.0,
            optim_epochs=10, optim_stepsize=3e-4, optim_batchsize=64,
            gamma=0.99, lam=0.95, schedule='linear',
        )
    env.close()
   
    return pi



def save_results(filename, description = None):
    """ 
    description: saves the results of a run of the second cell (the one that calls train) in this notebook
    
    """

    save_dir = "data/" + filename + "/"
    os.makedirs(save_dir)
    
    if description is None:
        description  = input("please enter a description of the run")
        
    datetime_str = str(datetime.datetime.today())
    datetime_str = datetime_str.replace(" ", "_")
    
    runtime_str = str(datetime.timedelta(seconds = runtime))
    
    readme = open(save_dir + "README.txt", "w+")
    readme.write("datetime: " + datetime_str + "\n\n")
    readme.write("enviroment: " + env_name + "\n\n")
    readme.write("description: " + description + "\n\n")
    readme.write("time_elapsed: " + runtime_str + "\n\n")
    readme.write("num_timesteps: " + str(num_timesteps) + "\n\n")
    readme.write("seed: " + str(seed) + "\n\n")
    readme.close()

    # TODO add code snippets that correspond to the run
    # TODO somehow store the tensorboard logs here after the fact
    
    saver = tf.train.Saver()
    saver.save(tf.get_default_session(), save_dir + filename)
    
    os.rename("./tmp_logs/", save_dir + "tensorboard")
   
   
#env_name = "Acrobot-v1"
env_name = "InvertedPendulum-v2"
#env_name = 'InvertedPendulumPyBulletEnv-v0'
#env_name = "su_cartpole_et-v0"
#env_name = "InvertedDoublePendulum-v2"

# comment one of these lines to switch between loading weights or training them from scratch
load_pretrained_network = True
#load_pretrained_network = False


if load_pretrained_network: #load the weights
    save_name = 'invertedpendulum_3layer'
    
    pi = train(env_name, num_timesteps=1, seed=0)
    # TODO eventually need to switch to .load_variables() instead of U.load_state() but this didn't work by default for me
    U.load_state(os.getcwd() + '/data/'+ save_name + '/' + save_name)
    
else: #run the RL algorithm
    num_timesteps = 2e6
    seed = 0
    
    print("training")
    
    start_time = time.time()
    
    logger.configure(dir = "./tmp_logs", format_strs=["tensorboard"] )
    with tf.device("/cpu:0"):    
        pi= train(env_name, num_timesteps=num_timesteps, seed=seed)

    runtime = time.time() - start_time


# Plays out a trained policy

#env = make_mujoco_env(env_name,seed=0)
env = gym.make(env_name)
ob = env.reset()     

while True:
    action = pi.act(stochastic=False, ob=ob)[0]
    ob, _, done, _ =  env.step(action)
    #if reward == 1:
    #    print("balanced")
    env.render()
    if done:
        ob = env.reset()
        
#U.save_state("./saved/5mil_flat")
all_weights = pi.get_variables()
fc1 = all_weights[4]
fc1_weights = fc1.value()
fc1_weights.eval() # only positive weights contribute anything

#input_iter = itertools.combinations_with_replacement(range(-10,11),4)
#input_data = np.array([np.array(x) for x in input_iter],dtype='float32')
#output_data = np.array([pi.act(0, x)[0] for x in input_data],dtype='float32')

input_iter = itertools.combinations_with_replacement(range(-10,11),2)
input_data = np.array([np.concatenate((np.zeros(2), np.array(x))) for x in input_iter],dtype='float32')
output_data = np.array([pi.act(0, x)[0] for x in input_data],dtype='float32')
from sklearn import datasets, linear_model
from sklearn.metrics import mean_squared_error, r2_score
regr = linear_model.LinearRegression()

regr.fit(input_data,output_data.T.flatten())
#regr.fit(input_data,output_data)
#regr.fit(index, output_data.T.flatten())

lin_predict = regr.predict(input_data)
#lin_predict = regr.predict(index)

print("coefs are", regr.coef_)
print("mean sqared error:", mean_squared_error(lin_predict, output_data))
plt.plot(output_data)
plt.figure()
plt.plot(lin_predict)
input_data = np.array([[x, 0, 0, 0] for x in range(-10,11)])
output_data = np.array([pi.act(0, x)[0] for x in input_data])
plt.plot(input_data[:,0], output_data)
plt.figure()

input_data = np.array([[0, x, 0, 0] for x in range(-10,11)])
output_data = np.array([pi.act(0, x)[0] for x in input_data])
plt.plot(input_data[:,1], output_data)
plt.figure()

input_data = np.array([[0, 0, x, 0] for x in range(-10,11)])
output_data = np.array([pi.act(0, x)[0] for x in input_data])
plt.plot(input_data[:,2], output_data)
plt.figure()

input_data = np.array([[0, 0, 0, x] for x in range(-10,11)])
output_data = np.array([pi.act(0, x)[0] for x in input_data])
plt.plot(input_data[:,3], output_data)
plt.figure()

input_data = np.array([[x, x, x, x] for x in range(-10,11)])
output_data = np.array([pi.act(0, x)[0] for x in input_data])
plt.plot(input_data[:,0], output_data)
x = input_data[:,2]
y = input_data[:,3]
z= output_data.flatten()
z2 = lin_predict.flatten()
ax = plt.axes(projection='3d')
ax.plot_trisurf(x, y, z, cmap='viridis', edgecolor='none');

ax2 = plt.axes(projection='3d')
ax2.plot_trisurf(x, y, z2, cmap='viridis', edgecolor='none');
import scipy.io

all_weights = pi.get_variables()

# I'll fix this later..
kernel0 = all_weights[13].value().eval()
bias0   = all_weights[14].value().eval()

kernel1 = all_weights[15].value().eval()
bias1   = all_weights[16].value().eval()

kernel2 = all_weights[17].value().eval()
bias2   = all_weights[18].value().eval()

name_dict = {"kernel0": kernel0, "kernel1":kernel1, "kernel2":kernel2, "bias0":bias0, "bias1":bias1, "bias2":bias2}
scipy.io.savemat("savemat_test", name_dict)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13863506567675615, 'fraction_numerical': 0.023272415762421474, 'mean_word_length': 4.384319754035357, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '273643', 'license': 'MIT', 'n_tokens_mistral': 2637, 'n_tokens_neox': 2567, 'n_words': 574, 'path': 'cartpole/cartpole_ppo1.ipynb', 'repo_name': 'mmolnar0/sgillen_research', 'size': '7004'}