<jupyter_start><jupyter_text>Python Notebook to interact with gym-battery and battery-agentThis python notebook is a working document to interact with and test the environment and the agent.Note: In order for this to work, gym-battery needs to be installed as a package, using pip install -e gym-battery from wherever gym-battery exists.The ipython notebook should exist in battery dispatch by default and should be ableto access those resources so it does not necessarily need to be build/installed using pip.<jupyter_code>import gym
import gym_battery 
import numpy as np
import pandas as pd
env = gym.make('gym_battery:battery-v0', **{'N_actions':5})
env.set_standard_system()
import pickle
simple_load = pd.read_pickle("simple_load_1d.pkl")
#simple_load = pd.read_clipboard()
simple_load.value.plot()
env.load.initialize(simple_load)
env.fit_load_to_space()
# Show the possible action mapping the agent can take
env.action_mapping
print(env.observation_space.low)
print("to")
print(env.observation_space.high)
# Set how to structure the environment. 'count_days' will generate the a single day as an episode. THe number of days
# given indicates how many differnet days to use.
# This needs to be changed so that it generates LONGER episodes, not DIFFERENT episodes, but this hasn't been done yet.
env.episode_type = 'count_days'
env.run_N_episodes = 1
# Get the do-nothing value for taking no action
def dict_key_by_val(d, val):
    for k in d.keys():
        if d[k] == val:
            return k
    raise ValueError("value not found in dictionary")
    
act0 = dict_key_by_val(env.action_mapping, 0)
act0
''' Set up the agent and the discretizer.'''
from batterydispatch.agent.agents import MonteCarloAgent
from batterydispatch.agent.discretizers import Box_Discretizer

from batterydispatch.agent.policies import do_nothing
agent = MonteCarloAgent()
agent.set_policy(do_nothing, {'do_nothing_action': act0})

# Note, you can change the size of the state sapce by changing the number of buckets, below
agent.set_discretizer(Box_Discretizer(env.observation_space, N=[6, 4, 12, 12]))
agent.actions = env.action_space
agent.learning_rate = 0.05 # used for the updates of the Q estimates
agent.subtype = 'on-policy' # Setup the MC agent for off-policy learning

global eps
eps=0
agent.discretizer.buckets<jupyter_output><empty_output><jupyter_text>Plot the day of data that we will be trying to learn from<jupyter_code>done = False
state = env.reset()
i = 0
while not done:
    i+=1
    _,reward,done, details = env.step(act0)

from matplotlib import pyplot as plt
plt.plot(env.grid_flow.net_flow)
try:
    print(list(env.grid_flow.start_date)[0])
except:
    pass
print(i)
print(reward)
default_reward = reward
plt.show()

# We then initialize the agent state-action estimates, based on the original billing period.
# We also give the do_nothing action a small bonus of 100, in order to prevent the agent from arbitrarily taking action.
agent.initialize_state_actions(new_default=default_reward,
                              do_nothing_action = act0,
                              do_nothing_bonus = 100)
agent.policy_args<jupyter_output><empty_output><jupyter_text>Set up the function to run the episodes, and run episodes until convergence.<jupyter_code>from batterydispatch.agent.functions import log_history, run_episode
# We then set the final parameters guiding the episodes: The agents proclivity for random actions, 
# the number of episodes without a policy change before we can say we've converge.
agent.set_greedy_policy(eta=0.125)
agent.patience = 10000
agent.name
agent.learning_rate = 0.075



from IPython.display import clear_output

for iteration in range(30):
    notes = 'Take2:Run of a Monte Carlo On Policy agent on Simple Day with seeds, run for 10,000 episodes: Seed {}'.format(iteration)

    agent.set_greedy_policy(eta=0.1)
    starting_learning_rate = 0.075
    agent.patience_counter = 0
    agent.initialize_state_actions(new_default=default_reward,
                              do_nothing_action = act0,
                              do_nothing_bonus = 100)    
    
    agent.set_seed(iteration)
    env.set_seed(iteration)
    
    i=30

    eps=0
    history = []
    while eps < 10001:
        i+=1
        eps+= 1

        if i>30:
            i=0
            clear_output()
            print(notes)
        print(eps, end=" | ")
        run_episode.run_episodes(env, agent, eps, history, default_reward, random_charge = False, run_type="once")
        agent.learning_rate = starting_learning_rate * np.exp(-0.0002*eps)
    
    agent.set_greedy_policy(eta=0)
    reward = run_episode.run_episodes(env, agent, eps, history, default_reward, random_charge=False, run_type='once')
    
    log_history.save_results(env, agent, history, reward, scenario = notes, agent_name=agent.name, notes='Iteration {}'.format(iteration))
    
    
agent.set_greedy_policy(0)
run_episode.run_episodes(env, agent, eps, history, default_reward, random_charge=False, run_type='once')

default_reward
log_history.save_results(env, agent, history, reward, scenario = notes, agent_name=agent.name, notes='Iteration {}'.format(iteration))

from matplotlib import pyplot as plt
plt.plot(np.exp(-0.0002*np.arange(0,10000))*0.075)
plt.show()
agent.history
Qs = pd.DataFrame.from_dict(agent.S_A_values, orient='index')
Qs.to_clipboard()
counts = pd.DataFrame.from_dict(agent.S_A_frequency, orient='index')
counts.to_clipboard

print(f"The agent converged after {eps} episodes")<jupyter_output><empty_output><jupyter_text>Agent has taken between 10 and 30 minutes, and between 700 and 2262 episodes, to converge on day 1. Optimal policy:Current reward of -397414.125 / -406791.825, 5600.0 / 6000.0, patience=21For 2 days, agent took 5 hours 8 minutes, and converged after 21200 episodes.  Then we allow the agent to take entirely greedy actions and run the algorithm to see how much the agent learned.<jupyter_code>agent.set_greedy_policy(eta=0)
    
state = env.reset(random_charge=False)
done = False
while not done:
    action = agent.get_action(state, list(env.action_mapping.keys()), 0.25)
    #print(state)
    #action = int(input("action:"))

    #print(action)
    state, reward, done, details = env.step(action)

try:
    new_demand = max(env.grid_flow.net_flow)
    orig_demand = max(env.grid_flow.load)
except AttributeError:
    new_demand = "???"
    orig_demand = "???"
    
    env.grid_flow['final_reward'] = reward
    env.grid_flow['original_reward'] = default_reward


print(f"Current reward of {reward} / {default_reward}, {new_demand} / {orig_demand}, patience={agent.patience_counter}")



DF = save_results(scenario='Day1_load', agent_name='DynaQ', notes="ran the DynaQ agent again on the Day1 data, for a second (same agent)")
pd.to_datetime(DF.saved_timestamp)
sum(DF.index.duplicated())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09293843011380216, 'fraction_numerical': 0.022176831047563468, 'mean_word_length': 4.351288056206089, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '79622', 'license': 'Apache-2.0', 'n_tokens_mistral': 2298, 'n_tokens_neox': 2151, 'n_words': 642, 'path': 'ipynb to run experiments/MC On Policy on Simple Day.ipynb', 'repo_name': 'danbolinson/BatteryAgent', 'size': '6854'}