<jupyter_start><jupyter_text>Example of using Bandits from GenRL  Setup<jupyter_code>!git clone https://github.com/SforAiDl/genrl.git
!pip install -e genrl
import matplotlib.pyplot as plt
import torch<jupyter_output><empty_output><jupyter_text>Epsilon Greedy Policy on a Bernoulli Bandit<jupyter_code>from genrl.agents import EpsGreedyMABAgent, BernoulliMAB
from genrl.trainers import MABTrainer

bandit = BernoulliMAB(arms=50, context_type="int")  
agent = EpsGreedyMABAgent(bandit, eps=0.1)
trainer = MABTrainer(agent, bandit)             
results = trainer.train(2000)

plt.plot(results["cumulative_regrets"])<jupyter_output>Started at 27-08-20 18:48:38
Training EpsGreedyMABAgent on BernoulliMAB for 2000 timesteps
timestep                  regret/regret             reward/reward             regret/cumulative_regret  reward/cumulative_reward  regret/regret_moving_avg  reward/reward_moving_avg  
100                       0                         1                         14                        86                        0.14                      0.86                      
200                       0                         1                         19                        181                       0.05                      0.95                      
300                       0                         1                         29                        271                       0.1                       0.9                       
400                       0                         1                         34                        366                       0.05                      0.95                [...]<jupyter_text>Linear Posterior Policy with the Covertype Dataset based Bandit<jupyter_code>from genrl.agents import LinearPosteriorAgent
from genrl.trainers import DCBTrainer
from genrl.utils.data_bandits import CovertypeDataBandit

bandit = CovertypeDataBandit(download=True)
agent = LinearPosteriorAgent(bandit)
trainer = DCBTrainer(agent, bandit)
results = trainer.train(10000)

plt.plot(results["cumulative_regrets"])<jupyter_output>Downloading https://archive.ics.uci.edu/ml/machine-learning-databases/covtype/covtype.data.gz to /content/data/Covertype/covtype.data.gz

Started at 25-08-20 18:24:09
Training LinearPosteriorAgent on CovertypeDataBandit for 10000 timesteps
timestep                  regret/regret             reward/reward             regret/cumulative_regret  reward/cumulative_reward  regret/regret_moving_avg  reward/reward_moving_avg  
100                       0                         1                         86                        14                        0.86                      0.14                      
200                       1                         0                         168                       32                        0.84                      0.16                      
300                       1                         0                         252                       48                        0.84                      0.16                      
400                       1 [...]<jupyter_text>Implementing a new Multi-Armed Bandit by extending `MABAgent` base class<jupyter_code>from genrl.agents import MABAgent
from genrl.agents import EpsGreedyMABAgent, BernoulliMAB

class ReinforcementComparison(MABAgent):
    def __init__(self, bandit, alpha, beta):
        super(ReinforcementComparison, self).__init__(bandit)
        self.alpha = alpha
        self.beta = beta
        self._pi = torch.zeros(self._bandit.arms)
        self._r = torch.zeros(self._bandit.arms)

    def select_action(self, context):
        p = torch.softmax(self._pi, 0)
        a = torch.distributions.Categorical(p).sample()
        return a.item()

    def update_params(self, context, action, reward):
        self._pi[action] += self.beta * (reward - torch.mean(self._r))
        self._r[action] = self._r[action] * (1 - self.alpha) + self._r[action] * self.alpha

bandit = BernoulliMAB(arms=50, context_type="int")
agent = ReinforcementComparison(bandit, 0.3, 0.3)
trainer = MABTrainer(agent, bandit)
results = trainer.train(2000)

plt.plot(results["cumulative_regrets"])<jupyter_output>Started at 25-08-20 18:26:48
Training ReinforcementComparison on BernoulliMAB for 2000 timesteps
timestep                  regret/regret             reward/reward             regret/cumulative_regret  reward/cumulative_reward  regret/regret_moving_avg  reward/reward_moving_avg  
100                       0                         1                         44                        56                        0.44                      0.56                      
200                       0                         1                         78                        122                       0.34                      0.66                      
300                       0                         1                         111                       189                       0.33                      0.67                      
400                       1                         0                         151                       249                       0.4                       0.6           [...]<jupyter_text>Implementing a new agent by extending the `DCBAgent` base class<jupyter_code>from genrl.agents import DCBAgent, BernoulliMAB
from genrl.trainers import DCBTrainer
from genrl.agents.bandits.contextual.common import NeuralBanditModel, TransitionDB

class NeuralAgent(DCBAgent):
    def __init__(self, bandit, **kwargs):
        super(NeuralAgent, self).__init__(bandit, **kwargs)
        self.model = (
            NeuralBanditModel(
                context_dim=self.context_dim,
                n_actions=self.n_actions,
                hidden_dims=kwargs.get("hidden_dims", [64]),
                **kwargs
            )
            .to(torch.float)
            .to(self.device)
        )
        self.db = TransitionDB(self.device)
        self.t = 0

    def select_action(self, context):
        self.t += 1
        if self.t < self.n_actions * self.init_pulls:
            return torch.tensor(
                self.t % self.n_actions, device=self.device, dtype=torch.int
            ).view(1)

        results = self.model(context)
        action = torch.argmax(results["pred_rewards"]).to(torch.int).view(1)
        return action

    def update_db(self, context, action, reward):
        self.db.add(context, action, reward)

    def update_param(self, action, batch_size=512, train_epochs=20):
        self.model.train_model(self.db, train_epochs, batch_size)

bandit = BernoulliMAB(arms=10, context_type="tensor")
agent = NeuralAgent(bandit)
trainer = DCBTrainer(agent, bandit)
results = trainer.train(5000)

plt.plot(results["cumulative_regrets"])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.07210401891252956, 'fraction_numerical': 0.03930260047281324, 'mean_word_length': 1.3684394681595522, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 18, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 70, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '409439', 'license': 'MIT', 'n_tokens_mistral': 2018, 'n_tokens_neox': 1771, 'n_words': 376, 'path': 'examples/Bandit_demo.ipynb', 'repo_name': 'sisaman/genrl', 'size': '6768'}