<jupyter_start><jupyter_text>OpenAI Gym ~ HER applicable problems (Acrobot-v1)- *Wheeler task definition ( task wrapper, State decoder settings, NeuralNetwork, ReplayBuffer, .. )*  Import generics<jupyter_code>import os, time

os.chdir("..")

import numpy as np
import toml, gym

import torch
from torch.multiprocessing import Queue, Process<jupyter_output><empty_output><jupyter_text>Prepare statistic helpers<jupyter_code>import matplotlib.pyplot as plt
%matplotlib inline

def print_stats(scores, title):
    fig = plt.figure()
    ax = fig.add_subplot(111)
    plt.plot(np.arange(1, len(scores)+1), scores)
    plt.ylabel('Rewards')
    plt.xlabel(title)
    plt.show()<jupyter_output><empty_output><jupyter_text>Load task configs ~ this should be adopted offline for particular task<jupyter_code>CFG = toml.loads(open('cfg.toml').read())
GYM_CFG = toml.loads(open('gym.toml').read())

torch.set_default_tensor_type(CFG['tensor'])

CFG['task'] = "Acrobot-v1"
GYM_CFG['her_state_size'] = 1
GYM_CFG['her_state_features'] = 4
GYM_CFG['max_n_episode'] = 1000<jupyter_output><empty_output><jupyter_text>Import wheeler environment and particular utils we want to use ~ general ones ( shared across tasks )<jupyter_code>from utils.task import Task
from utils.taskinfo import *

from utils.rbf import *
from utils.normalizer import *

from utils.taskmgr import *
from utils.replay import *

from utils.fastmem import Memory

from utils.curiosity import *

from agent.zer0bot import agent_launch<jupyter_output><empty_output><jupyter_text>Define Task wrapper ~ when is goal met, how to step ( update rewards function, .. ), when / how to reset<jupyter_code># https://github.com/openai/gym/blob/master/gym/envs/classic_control/acrobot.py
# ._terminal() + _get_ob() ; we add this to state, to be able to easier calculate distance from target
def transform(s):
    return np.hstack([
        np.arccos(s[0]),
        np.arccos(s[2]),
        s,
        ])

def goal_select(total_after, n_step):
    if total_after <= n_step + 1: # only last n_state reminds
        return 0
    if random.randint(0, 2):
        return random.randint(1, n_step)
    return random.randint(1, total_after - 1 - n_step)

def fun_reward(s, gs, cfg):
    return -1 * int(-np.cos(s[0]) - np.cos(s[1]+s[0]) < gs[0])

def sample_goal(goal, cfg):
    state_size = len(goal) // cfg['history_count']
    g = goal[-state_size:]
    return [-np.cos(g[0]) - np.cos(g[1]+g[0])]
class GymTask(Task):
    def step_ex(self, action, test = False):
        action, a = self.softmax_policy(action, test)
        state, reward, done, _ = self.env.step(self.bot_id, self.objective_id, a)

        state = transform(state)

        if test: state = [state]
        else: reward = fun_reward(state, self.goal(0), self.cfg)
            
        return action, state, reward, done, True

    def reset(self, seed = None, test = False):
        state = super().reset(seed)
        state = transform(state[0])
        return [state.reshape(-1)]

    def goal_met(self, states, rewards, n_steps):
        return sum(rewards) > -250.

    def goal(self, ind = 0):
        return np.array([[1.]])
    
    def update_goal(self, _, goals, states, n_goals, n_states, updates):
        for i, (g, s, n_g, n, u) in enumerate(zip(goals, states, n_goals, n_states, updates)):
            if u:
                g = states[i + goal_select(len(states) - i, self.cfg['n_step'])]
                g = sample_goal(g, self.cfg)
                n_g = g
            
            yield (
                fun_reward(s, g, self.cfg),
                g, s,
                n_g, n,
            )<jupyter_output><empty_output><jupyter_text>Generic proxy for creating our Task ( multiprocess environments purpose mainly ) - but can also add wrapping function approx values ( action value to tanh, sigmoid, .. ) - this not works well with PPO now<jupyter_code>class GymInfo(TaskInfo):
    def __init__(self, env, replaybuf, factory, Mgr, args):
        super().__init__(
                len(transform(env.reset())), 2, -2, +2,
                CFG,
                replaybuf,
                factory, Mgr, args)

    def new(self, cfg, bot_id, objective_id):
        return GymTask(cfg,
                self.env,
                objective_id, bot_id,
                self.action_low, self.action_high)

    @staticmethod
    def factory(ind): # bare metal task creation
        global CFG
        print("created %i-th task"%ind)
        return gym.make(CFG['task'])<jupyter_output><empty_output><jupyter_text>Implement callback for testing policy ~ per X training rounds, we want to test it ~ enable visuals if you want<jupyter_code>def context_callback(count, print_every):
    context_callback.z = 0
    context_callback.scores = []
    context_callback.total_scores = []
    def callback(task, agent, scores):
        context_callback.z += 1
        context_callback.total_scores += scores
        if 0 == context_callback.z % print_every:
            print("\nTEST review : ", np.mean(context_callback.scores[-print_every:]))
        # we can save scores to main queue, and avarage them, or we can ..
        # run testing w/ visuals :
        for _ in range(count):
            done, states, rewards = task.test_policy(agent)
            rewards = sum(rewards)
            context_callback.scores.append(rewards)            
            if not done:
                return None

        print("\n")
        print("="*80)
        print("training over", context_callback.z * GYM_CFG['n_simulations'] * GYM_CFG['mcts_rounds'])
        print("="*80)
        
        print_stats(context_callback.scores, "all-out::testing phase")

        return context_callback.total_scores
    return callback<jupyter_output><empty_output><jupyter_text>Prepare neural network which we will be using<jupyter_code>from models import ddpg_model, noisy_model, state_action_model

def CriticNN(state_size, action_size, wrap_value, cfg):
    return state_action_model.Critic(state_size, action_size, wrap_value, cfg, fcs1_units=256, fc2_units=128)

def ActorNN(state_size, action_size, wrap_action, cfg):
    return noisy_model.Actor(state_size, action_size, wrap_action, cfg, hiddens=[128, 64])
    return noisy_model.Actor(state_size, action_size, wrap_action, cfg, hiddens=[64])
    return noisy_model.Actor(state_size, action_size, wrap_action, cfg, hiddens=[400, 300])<jupyter_output><empty_output><jupyter_text>Select encoders<jupyter_code>from utils.encoders import *
from utils.rnn import *#GRUEncoder

def encoderstack(env):
    state_size = len(transform(env.reset()))
    norm = GlobalNormalizer(GYM_CFG, state_size)
    return norm
    experience = GRUEncoder(GYM_CFG, norm.total_size())#GRU#LSTM
    encoder_norm = StackedEncoder(GYM_CFG, state_size, norm, experience)
    encoder_norm.share_memory()
    return encoder_norm<jupyter_output><empty_output><jupyter_text>Cook Task : replay buffer ( fast / prio-gae-rnn ) + task manager ( local / remote / unity )<jupyter_code>def taskfactory(env):
#    return GymInfo(env, Memory, GymInfo.factory, LocalTaskManager, ())
    return GymInfo(env, ReplayBuffer, GymInfo.factory, LocalTaskManager, ())
    return GymInfo(env, ReplayBuffer, GymInfo.factory, RemoteTaskManager, (LocalTaskManager, 1 + GYM_CFG['n_simulations']))<jupyter_output><empty_output><jupyter_text>Glue it all together ~ select buffer, encoders, agents, ... and RUN!!<jupyter_code># well this does not fit to encoders, just temporary; encoders are in fact TODO to refactor and properly rethink!
class GoalEncoder(IEncoder):
    def __init__(self, cfg, size):
        super().__init__({ 'history_count' : 1, 'history_features' : 0 })
        self.size = size
        self.net = torch.nn.Linear(size, cfg['her_state_features'], bias=False)
    def out_size(self):
        assert False, "should not be called"
    def forward(self, goals, _):
        return self.net(goals)

def main():
    print(GYM_CFG)

    env = gym.make(CFG['task'])
    
    encoder = encoderstack(env)
    task_factory = taskfactory(env)
    task = task_factory.new(GYM_CFG, 0, -1)
    
    callback = context_callback(10, 30)
    def callback_task(agent, stop_q):
        return callback(task, agent, stop_q)

    stop_q = Queue()
    agent_launch(0, GYM_CFG, task_factory, encoder, ActorNN, CriticNN, stop_q, callback_task, GoalEncoder(GYM_CFG, 1))
    
    scores = stop_q.get()
    print("FINISHED!")
    print_stats(scores, "learning algorithm")

if '__main__' == __name__:
    main()<jupyter_output>{'ddpg': True, 'gae': False, 'n_step': 3, 'batch_size': 128, 'select_count': 10, 'discount_rate': 0.99, 'n_simulations': 1, 'send_exp_delta': 20, 'critic_learn_delta': 20, 'fast_exp_epochs': 15, 'full_replay_count': 10, 'lr_actor': 0.0003, 'lr_critic': 0.0001, 'mcts_rounds': 1, 'max_n_episode': 1000, 'max_ep_draw_count': 40, 'detached_actor': False, 'freeze_delta': 2, 'freeze_count': 4, 'history_count': 4, 'rnn_n_layers': 2, 'full_rnn_out': False, 'history_features': 256, 'her_state_size': 1, 'her_state_features': 4, 'postpone_exploring_while_learning': 2, 'ppo_eps': 0.2, 'loss_min': 1000, 'gae_tau': 0.95, 'replay_size': 10000, 'her_max_ratio': 3, 'replay_reanalyze': True, 'replay_cleaning': False, 'prob_treshold': 3.0, 'replay_alpha': 0.7, 'replay_beta_iters': 10000, 'replay_beta_base': 0.3, 'replay_beta_top': 0.9, 'pg_mean': True, 'action_repeat': 1, 'good_reach': 1, 'advantages_enabled': True, 'advantages_boost': False, 'normalize_advantages': False, 'attention_enabled': False, 'att[...]
{'fraction_non_alphanumeric': 0.12498684902682798, 'fraction_numerical': 0.01641241451867438, 'mean_word_length': 3.687376725838264, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 45, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '250293', 'license': 'MIT', 'n_tokens_mistral': 3383, 'n_tokens_neox': 3191, 'n_words': 764, 'path': 'envs/her.ipynb', 'repo_name': 'rezer0dai/wheeler', 'size': '9505'}