# Path: osim/http/client.py
# class Client(object):
#     """
#     Gym client to interface with gym_http_server
#     """
#     def __init__(self, remote_base):
#         self.remote_base = remote_base
#         self.session = requests.Session()
#         self.session.headers.update({'Content-type': 'application/json'})
#         self.instance_id = None
# 
#     def _parse_server_error_or_raise_for_status(self, resp):
#         j = {}
#         try:
#             j = resp.json()
#         except:
#             # Most likely json parse failed because of network error, not server error (server
#             # sends its errors in json). Don't let parse exception go up, but rather raise default
#             # error.
#             resp.raise_for_status()
#         if resp.status_code != 200 and "message" in j:  # descriptive message from server side
#             raise ServerError(message=j["message"], status_code=resp.status_code)
#         resp.raise_for_status()
#         return j
# 
#     def _post_request(self, route, data):
#         url = urlparse.urljoin(self.remote_base, route)
#         logger.info("POST {}\n{}".format(url, json.dumps(data)))
#         resp = self.session.post(urlparse.urljoin(self.remote_base, route),
#                             data=json.dumps(data))
#         return self._parse_server_error_or_raise_for_status(resp)
# 
#     def _get_request(self, route):
#         url = urlparse.urljoin(self.remote_base, route)
#         logger.info("GET {}".format(url))
#         resp = self.session.get(url)
#         return self._parse_server_error_or_raise_for_status(resp)
# 
#     def env_create(self, token, env_id = "Run"):
#         route = '/v1/envs/'
#         data = {'env_id': env_id,
#                 'token': token,
#                 'version': pkg_resources.get_distribution("osim-rl").version }
#         try:
#             resp = self._post_request(route, data)
#         except ServerError as e:
#             sys.exit(e.message)
#         self.instance_id = resp['instance_id']
#         self.env_monitor_start("tmp", force=True)
#         return self.env_reset()
# 
#     def env_reset(self):
#         route = '/v1/envs/{}/reset/'.format(self.instance_id)
#         resp = self._post_request(route, None)
#         observation = resp['observation']
#         return observation
# 
#     def env_step(self, action, render=False):
#         route = '/v1/envs/{}/step/'.format(self.instance_id)
#         data = {'action': action, 'render': render}
#         resp = self._post_request(route, data)
#         observation = resp['observation']
#         reward = resp['reward']
#         done = resp['done']
#         info = resp['info']
#         return [observation, reward, done, info]
# 
#     def env_monitor_start(self, directory,
#                               force=False, resume=False, video_callable=False):
#         route = '/v1/envs/{}/monitor/start/'.format(self.instance_id)
#         data = {'directory': directory,
#                 'force': force,
#                 'resume': resume,
#                 'video_callable': video_callable}
#         self._post_request(route, data)
# 
#     def submit(self):
#         route = '/v1/envs/{}/monitor/close/'.format(self.instance_id)
#         result = self._post_request(route, None)
#         if result['reward']:
#             print("Your total reward from this submission: %f" % result['reward'])
#         else:
#             print("There was an error in your submission. Please contact administrators.")
#         route = '/v1/envs/{}/close/'.format(self.instance_id)
#         self.env_close()
# 
#     def env_close(self):
#         route = '/v1/envs/{}/close/'.format(self.instance_id)
#         self._post_request(route, None)


import numpy as np
import sys
import numpy as np
import argparse
import math
from keras.models import Sequential, Model
from keras.layers import Dense, Activation, Flatten, Input, concatenate
from keras.optimizers import Adam
from rl.agents import DDPGAgent
from rl.memory import SequentialMemory
from rl.random import OrnsteinUhlenbeckProcess
from osim.env import *
from osim.http.client import Client
from keras.optimizers import RMSprop
x = Dense(1)(x)
x = Activation('linear')(x)
critic = Model(inputs=[action_input, observation_input], outputs=x)
print(critic.summary())

# Set up the agent for training
memory = SequentialMemory(limit=100000, window_length=1)
random_process = OrnsteinUhlenbeckProcess(theta=.15, mu=0., sigma=.2, size=env.get_action_space_size())
agent = DDPGAgent(nb_actions=nb_actions, actor=actor, critic=critic, critic_action_input=action_input,
                  memory=memory, nb_steps_warmup_critic=100, nb_steps_warmup_actor=100,
                  random_process=random_process, gamma=.99, target_model_update=1e-3,
                  delta_clip=1.)
# agent = ContinuousDQNAgent(nb_actions=env.noutput, V_model=V_model, L_model=L_model, mu_model=mu_model,
#                            memory=memory, nb_steps_warmup=1000, random_process=random_process,
#                            gamma=.99, target_model_update=0.1)
agent.compile(Adam(lr=.001, clipnorm=1.), metrics=['mae'])

# Okay, now it's time to learn something! We visualize the training here for show, but this
# slows down training quite a lot. You can always safely abort the training prematurely using
# Ctrl + C.
if args.train:
    agent.fit(env, nb_steps=nallsteps, visualize=False, verbose=1, nb_max_episode_steps=env.time_limit, log_interval=10000)
    # After training is done, we save the final weights.
    agent.save_weights(args.model, overwrite=True)

# If TEST and TOKEN, submit to crowdAI
if not args.train and args.token:
    agent.load_weights(args.model)
    # Settings
    remote_base = 'http://grader.crowdai.org:1729'

# Path: osim/http/client.py
# class Client(object):
#     """
#     Gym client to interface with gym_http_server
#     """
#     def __init__(self, remote_base):
#         self.remote_base = remote_base
#         self.session = requests.Session()
#         self.session.headers.update({'Content-type': 'application/json'})
#         self.instance_id = None
# 
#     def _parse_server_error_or_raise_for_status(self, resp):
#         j = {}
#         try:
#             j = resp.json()
#         except:
#             # Most likely json parse failed because of network error, not server error (server
#             # sends its errors in json). Don't let parse exception go up, but rather raise default
#             # error.
#             resp.raise_for_status()
#         if resp.status_code != 200 and "message" in j:  # descriptive message from server side
#             raise ServerError(message=j["message"], status_code=resp.status_code)
#         resp.raise_for_status()
#         return j
# 
#     def _post_request(self, route, data):
#         url = urlparse.urljoin(self.remote_base, route)
#         logger.info("POST {}\n{}".format(url, json.dumps(data)))
#         resp = self.session.post(urlparse.urljoin(self.remote_base, route),
#                             data=json.dumps(data))
#         return self._parse_server_error_or_raise_for_status(resp)
# 
#     def _get_request(self, route):
#         url = urlparse.urljoin(self.remote_base, route)
#         logger.info("GET {}".format(url))
#         resp = self.session.get(url)
#         return self._parse_server_error_or_raise_for_status(resp)
# 
#     def env_create(self, token, env_id = "Run"):
#         route = '/v1/envs/'
#         data = {'env_id': env_id,
#                 'token': token,
#                 'version': pkg_resources.get_distribution("osim-rl").version }
#         try:
#             resp = self._post_request(route, data)
#         except ServerError as e:
#             sys.exit(e.message)
#         self.instance_id = resp['instance_id']
#         self.env_monitor_start("tmp", force=True)
#         return self.env_reset()
# 
#     def env_reset(self):
#         route = '/v1/envs/{}/reset/'.format(self.instance_id)
#         resp = self._post_request(route, None)
#         observation = resp['observation']
#         return observation
# 
#     def env_step(self, action, render=False):
#         route = '/v1/envs/{}/step/'.format(self.instance_id)
#         data = {'action': action, 'render': render}
#         resp = self._post_request(route, data)
#         observation = resp['observation']
#         reward = resp['reward']
#         done = resp['done']
#         info = resp['info']
#         return [observation, reward, done, info]
# 
#     def env_monitor_start(self, directory,
#                               force=False, resume=False, video_callable=False):
#         route = '/v1/envs/{}/monitor/start/'.format(self.instance_id)
#         data = {'directory': directory,
#                 'force': force,
#                 'resume': resume,
#                 'video_callable': video_callable}
#         self._post_request(route, data)
# 
#     def submit(self):
#         route = '/v1/envs/{}/monitor/close/'.format(self.instance_id)
#         result = self._post_request(route, None)
#         if result['reward']:
#             print("Your total reward from this submission: %f" % result['reward'])
#         else:
#             print("There was an error in your submission. Please contact administrators.")
#         route = '/v1/envs/{}/close/'.format(self.instance_id)
#         self.env_close()
# 
#     def env_close(self):
#         route = '/v1/envs/{}/close/'.format(self.instance_id)
#         self._post_request(route, None)


# Path: examples/under-construction/ddpg.keras-rl.py
import numpy as np
import sys
import numpy as np
import argparse
import math
from keras.models import Sequential, Model
from keras.layers import Dense, Activation, Flatten, Input, concatenate
from keras.optimizers import Adam
from rl.agents import DDPGAgent
from rl.memory import SequentialMemory
from rl.random import OrnsteinUhlenbeckProcess
from osim.env import *
from osim.http.client import Client
from keras.optimizers import RMSprop

x = Dense(1)(x)
x = Activation('linear')(x)
critic = Model(inputs=[action_input, observation_input], outputs=x)
print(critic.summary())

# Set up the agent for training
memory = SequentialMemory(limit=100000, window_length=1)
random_process = OrnsteinUhlenbeckProcess(theta=.15, mu=0., sigma=.2, size=env.get_action_space_size())
agent = DDPGAgent(nb_actions=nb_actions, actor=actor, critic=critic, critic_action_input=action_input,
                  memory=memory, nb_steps_warmup_critic=100, nb_steps_warmup_actor=100,
                  random_process=random_process, gamma=.99, target_model_update=1e-3,
                  delta_clip=1.)
# agent = ContinuousDQNAgent(nb_actions=env.noutput, V_model=V_model, L_model=L_model, mu_model=mu_model,
#                            memory=memory, nb_steps_warmup=1000, random_process=random_process,
#                            gamma=.99, target_model_update=0.1)
agent.compile(Adam(lr=.001, clipnorm=1.), metrics=['mae'])

# Okay, now it's time to learn something! We visualize the training here for show, but this
# slows down training quite a lot. You can always safely abort the training prematurely using
# Ctrl + C.
if args.train:
    agent.fit(env, nb_steps=nallsteps, visualize=False, verbose=1, nb_max_episode_steps=env.time_limit, log_interval=10000)
    # After training is done, we save the final weights.
    agent.save_weights(args.model, overwrite=True)

# If TEST and TOKEN, submit to crowdAI
if not args.train and args.token:
    agent.load_weights(args.model)
    # Settings
    remote_base = 'http://grader.crowdai.org:1729'

    client = Client(remote_base)