<jupyter_start><jupyter_text>Exercise 2.5Design and conduct an experiment to demonstrate thediculties that sample-average methods have for nonstationary problems. Use a modifiedversion of the 10-armed testbed in which all the q*(a) start out equal and then takeindependent random walks (say by adding a normally distributed increment with mean 0and standard deviation 0.01 to all the q*(a) on each step). Prepare plots like Figure 2.2for an action-value method using sample averages, incrementally computed, and anotheraction-value method using a constant step-size parameter, alpha = 0.1. Use epsilon = 0.1 andlonger runs, say of 10,000 steps. Summary- Create a bandit with 10 arms;- Instantiate q*(a) = 0 for all a, then take random walks for each a (normal distribution, mean=0, deviation=0.01);- Use the sample average technique and a bandit with alpha = 0.1;- Create all the tests with epsilon = 1;<jupyter_code># Import and instantiate all the data
import numpy as np
import matplotlib.pyplot as plt

n_bandits = 1000
n_steps = 1000

R = [0 for _ in range(10)]
 
print("n_steps:", n_steps)
print("n_bandits:", n_bandits)
print("\nrandom:", random)
print("R:", R)
# Create the bandit object
class Bandit(object):
  def __init__(self, k=10, epsilon=0.1, alpha=None):
    # Initialize all the variables
    self.k = k
    self.epsilon = epsilon

    self.Q = [0 for _ in range(self.k)]
    self.N = [0 for _ in range(self.k)]

    if alpha is None:
      self.alpha = lambda a: 1/self.N[a]
    else:
      self.alpha = alpha
    
    self.random = random
  
  def _choose_action(self):
    # If random number < epsilon -> select random action
    if np.random.uniform(0, 1) < self.epsilon:
      action = np.random.randint(self.k)
    # Otherwise -> select the 'best' action (according to our estimations)
    else:
      action = np.argmax(self.Q)
    
    # Return the chosen action
    return action

  def step(self):
    # Choose an action and update the Q value
    a = self._choose_action()

    self.N[a] += 1
    self.Q[a] += self.alpha(a) * (R[a] - self.Q[a])

    return R[a]
# Create the bandits
sample_avg_bandits = [Bandit() for _ in range(n_bandits)]
constant_bandits = [Bandit(alpha=lambda a: 0.1) for _ in range(n_bandits)]

x = [i for i in range(n_steps)]

# Get the rewards
sample_avg_r = [0 for _ in range(n_steps)]
constant_r = [0 for _ in range(n_steps)]

# Run with the number of steps
for i in range(n_steps):
  # Run sample average bandits
  for bandit in sample_avg_bandits:
    sample_avg_r[i] += bandit.step()
  
  # Run all constant alpha bandits
  for bandit in constant_bandits:
    constant_r[i] += bandit.step()

  # Get the average of all the bandits
  sample_avg_r[i] /= n_bandits
  constant_r[i] /= n_bandits

  # Take random walks
  if random:
    for action in range(5):
      R[action] += np.random.normal(0, 0.01) 

# Plot the results
plt.plot(x, sample_avg_r)
plt.plot(x, constant_r)
plt.legend(["Sample Average", "Constant Step Size"])
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10294604435617345, 'fraction_numerical': 0.019529956967891428, 'mean_word_length': 3.649230769230769, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '197707', 'license': 'MIT', 'n_tokens_mistral': 1052, 'n_tokens_neox': 992, 'n_words': 357, 'path': 'RL Book Exercises/Exercise_2_5.ipynb', 'repo_name': 'ProfessorQu/Reinforcement-Learning', 'size': '3021'}