<jupyter_start><jupyter_text>MAB experimentIn this notebook we will simulate a multi armed bandit experiment, using different approaches to solve it<jupyter_code>import os
import sys
sys.path.append('/Users/fci02/Documents/GitHub/decisioning-analysis/test_and_learn/production_modules/')

import numpy as np
import pandas as pd

from scipy.stats import f_oneway
from tqdm import tqdm

from contextual_mab.experiments.framework import MABFramework
from contextual_mab.experiments.framework import run_experiment
from contextual_mab.data_utils.data_generation  import generate_experimental_dataset

import matplotlib.pyplot as plt
import seaborn as sns<jupyter_output><empty_output><jupyter_text>Experimental data generation<jupyter_code>n_1 = 5000
noise_scale_1=5.

cw_1 = [.05,.03,.01]

ct_prm_1 = {'X1':{'loc':0.,'scale':1.},
          'X2':{'loc':10.,'scale':5.},
          'X3':{'loc':-.3,'scale':.5}}

ord_prm_1 = {'O1':{'start':1,'stop':11,'weights':[.1]*10},
           'O2':{'start':1,'stop':4,'weights':[.3,.4,.3]}}

catg_prm_1 = {'C1':{'levels':['Yes','No'],'weights':[.4,.6]},
            'C2':{'levels':['Green','Yellow'],'weights':[.2,.8]},
            'C3':{'levels':['A','B','C'],'weights':[.4,.1,.5]}}

weights_1 = [[-0.85427315,  1.16572722,  0.8890073,  -0.57988417, 0.15193386, 
              -0.28800726, -0.06606457,  0.36732494, -0.03666541, -0.61067511],
             [ 0.46719077, -1.64435559,  0.69791627, -0.14981489,  0.26023682,  
               0.67528998, 1.52150038,  1.05417964,  0.37329345,  0.79700709],
             [ 1.62075116, 0.15865047, -0.85728784,  1.23667642, -0.58707557,
               0.05713119, -0.47387454,  0.51293855, -0.55820087, -2.14815787]]
n_2 = 5000
cw_2 = [.02,.07,.03]

ct_prm_2 = {'X1':{'loc':5.,'scale':1.},
            'X2':{'loc':1.,'scale':5.},
            'X3':{'loc':1.3,'scale':.5}}

ord_prm_2 = {'O1':{'start':1,'stop':11,'weights':[.2,.2,.05,.05,.05,.05,.1,.1,.1,.1]},
             'O2':{'start':1,'stop':4,'weights':[.1,.6,.3]}}

catg_prm_2 = {'C1':{'levels':['Yes','No'],'weights':[.1,.9]},
              'C2':{'levels':['Green','Yellow'],'weights':[.5,.5]},
              'C3':{'levels':['A','B','C'],'weights':[.6,.2,.2]}}

noise_scale_2=5.

weights_2 = [[ 0.2249653,   0.54732847,  0.76620536,  0.7461608,  -0.76568111,
              -0.13241893, -1.82046231, -0.47742618, -0.96465132, -0.68848216],
             [-0.04717597,  1.49105257, -0.6332578,  -1.03206255, -1.30374031,
              -0.48575409, 0.01466847, 0.54927814,  0.72014772,  0.42807199],
             [-0.56907754, 0.04875765,  0.89346343,  0.62619356, -2.19116666,
              1.70168624, 0.34768686,  0.26208243,  1.27787397, -2.07476064]]
n_3 = 5000
cw_3 = [.01,.03,.06]

ct_prm_3 = {'X1':{'loc':-2.,'scale':1.},
            'X2':{'loc':-20.,'scale':5.},
            'X3':{'loc':11.3,'scale':.5}}

ord_prm_3 = {'O1':{'start':1,'stop':11,'weights':[.5,.0,.0,.0,.05,.05,.1,.1,.1,.1]},
             'O2':{'start':1,'stop':4,'weights':[.5,.1,.4]}}

catg_prm_3 = {'C1':{'levels':['Yes','No'],'weights':[.5,.5]},
              'C2':{'levels':['Green','Yellow'],'weights':[.8,.2]},
              'C3':{'levels':['A','B','C'],'weights':[.35,.35,.3]}}

noise_scale_3=5.

weights_3 = [[ 0.2249653,   0.54732847,  0.76620536,  0.7461608,  -0.76568111, 
              -0.13241893, -1.82046231, -0.47742618, -0.96465132, -0.68848216],
             [-0.04717597,  1.49105257, -0.6332578,  -1.03206255, -1.30374031,
              -0.48575409, 0.01466847,  0.54927814,  0.72014772,  0.42807199],
             [-0.56907754,  0.04875765,  0.89346343,  0.62619356, -2.19116666,
              1.70168624, 0.34768686,  0.26208243,  1.27787397, -2.07476064]]
seed = 0
experiment_data = generate_experimental_dataset([n_1,n_2,n_3],
                                                [cw_1,cw_2,cw_3],
                                                [ct_prm_1,ct_prm_2,ct_prm_3],
                                                [ord_prm_1,ord_prm_2,ord_prm_3],
                                                [catg_prm_1,catg_prm_2,catg_prm_3],
                                                [noise_scale_1,noise_scale_2,noise_scale_3],
                                                [weights_1,weights_2,weights_3],
                                                seed,
                                                output_info=True)<jupyter_output>Group 1<jupyter_text>Single run  ABC greedy<jupyter_code>batch_size = 2000

run_experiment(experiment_data, batch_size, MABFramework,{'strategy':'static-one-fits-all','n_actions':3,'static_min_steps':2})<jupyter_output>100%|██████████| 7/7 [00:00<00:00, 70.39it/s]<jupyter_text>Thompson Sampling<jupyter_code>run_experiment(experiment_data, batch_size, MABFramework,{'strategy':'dynamic-one-fits-all','n_actions':3,'alphas':[1.,1.,1.],'betas':[1.,1.,1.]})<jupyter_output>100%|██████████| 7/7 [00:00<00:00, 85.74it/s]<jupyter_text>Contextual Thompson Sampling Bayesian Logistic<jupyter_code>from contextual_mab.predictive_models.bayesian_logistic import BayesianLogisticRegression
action_cols = [column for column in experiment_data.columns if 'action' in column]
predictors = pd.get_dummies(experiment_data.drop(columns=action_cols),drop_first=True).columns.tolist()
predictors
run_experiment(experiment_data, batch_size, MABFramework,{'strategy':'contextual-one-fits-one','n_actions':3,
                                                          'modelling_approach':BayesianLogisticRegression,
                                                          'modelling_approach_pms':{'n_samples':500,'n_chains':2,
                                                                                    'predictors':predictors.copy(),
                                                                                    'tune':1000,
                                                                                    'check_prog':False}})<jupyter_output>0%|          | 0/7 [00:00<?, ?it/s]Auto-assigning NUTS sampler...
Initializing NUTS using adapt_diag...
Multiprocess sampling (2 chains in 2 jobs)
NUTS: [C3_C, C3_B, C2_Yellow, C1_Yes, O2, O1, X3, X2, X1, Intercept]
Auto-assigning NUTS sampler...
Initializing NUTS using adapt_diag...
Multiprocess sampling (2 chains in 2 jobs)
NUTS: [C3_C, C3_B, C2_Yellow, C1_Yes, O2, O1, X3, X2, X1, Intercept]
Auto-assigning NUTS sampler...
Initializing NUTS using adapt_diag...
Multiprocess sampling (2 chains in 2 jobs)
NUTS: [C3_C, C3_B, C2_Yellow, C1_Yes, O2, O1, X3, X2, X1, Intercept]
 29%|██▊       | 2/7 [00:42<01:47, 21.47s/it]Auto-assigning NUTS sampler...
Initializing NUTS using adapt_diag...
Multiprocess sampling (2 chains in 2 jobs)
NUTS: [C3_C, C3_B, C2_Yellow, C1_Yes, O2, O1, X3, X2, X1, Intercept]
Auto-assigning NUTS sampler...
Initializing NUTS using adapt_diag...
Multiprocess sampling (2 chains in 2 jobs)
NUTS: [C3_C, C3_B, C2_Yellow, C1_Yes, O2, O1, X3, X2, X1, Intercept]
Auto-assignin[...]<jupyter_text>Bootstrap Oracles<jupyter_code>from contextual_mab.predictive_models.oracles import BootstrapOracle
from sklearn.linear_model import LogisticRegression
logistic_params = {'solver':'lbfgs',
                   'max_iter':500,
                   'random_state':0}
run_experiment(experiment_data, batch_size, MABFramework,{'strategy':'contextual-one-fits-one','n_actions':3,
                                                          'modelling_approach':BootstrapOracle,
                                                          'modelling_approach_pms':{'n_bootstrap':1000,
                                                                                    'learner_class':LogisticRegression,
                                                                                    'learner_class_params':logistic_params,
                                                                                    'check_prog':False}})<jupyter_output>100%|██████████| 7/7 [05:25<00:00, 56.02s/it]<jupyter_text>Running many experiments<jupyter_code>batch_size = 2000
abc_approval_rates = []
for seed in range(10):
    experiment_data = generate_experimental_dataset([n_1,n_2,n_3],
                                                [cw_1,cw_2,cw_3],
                                                [ct_prm_1,ct_prm_2,ct_prm_3],
                                                [ord_prm_1,ord_prm_2,ord_prm_3],
                                                [catg_prm_1,catg_prm_2,catg_prm_3],
                                                [noise_scale_1,noise_scale_2,noise_scale_3],
                                                [weights_1,weights_2,weights_3],
                                                seed)
    abc_approval_rates.append(run_experiment(experiment_data, batch_size, MABFramework,{'strategy':'static-one-fits-all','n_actions':3,'static_min_steps':2}))
ts_approval_rates = []
for seed in range(10):
    experiment_data = generate_experimental_dataset([n_1,n_2,n_3],
                                                [cw_1,cw_2,cw_3],
                                                [ct_prm_1,ct_prm_2,ct_prm_3],
                                                [ord_prm_1,ord_prm_2,ord_prm_3],
                                                [catg_prm_1,catg_prm_2,catg_prm_3],
                                                [noise_scale_1,noise_scale_2,noise_scale_3],
                                                [weights_1,weights_2,weights_3],
                                                seed)
    ts_approval_rates.append(run_experiment(experiment_data, batch_size, MABFramework,{'strategy':'dynamic-one-fits-all','n_actions':3,'alphas':[1.,1.,1.],'betas':[1.,1.,1.]}))
from statsmodels.stats.weightstats import ztest
ztest(abc_approval_rates,ts_approval_rates)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15652263163296726, 'fraction_numerical': 0.12562865647131274, 'mean_word_length': 2.1031847133757964, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 75, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '490664', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 4074, 'n_tokens_neox': 3442, 'n_words': 453, 'path': 'examples/Contextual_MAB_experiment.ipynb', 'repo_name': 'Fracappo87/RL', 'size': '9743'}