<jupyter_start><jupyter_text>Upper Confidence Bound Algorithm<jupyter_code>#Initialization
N = len(df.index)       # number of iterations
d = 4                   # number of possible messages
Qt_a = 0
Nt_a = np.zeros(d) #number of times action a has been selected prior to T
                            #If Nt(a) = 0, then a is considered to be a maximizing action.
c = 5                       #a number greater than = that controls the degree of exploration

sum_rewards = np.zeros(d) #cumulative sum of reward by selecting message

#loop through no of rounds 
hist_t = [] #holds the natural log
hist_achieved_rewards = [] #holds the history of the cumulative rewards chosen
hist_best_possible_rewards = [] #holds the history of cumulative optimal rewards
hist_random_choice_rewards = [] #holds the history of actions selected by random

#t = time
for t in range(0,N):
    UCB_Values = np.zeros(d) #array holding the ucb values. we pick the max
    max_upper_bnd = 0
    ad = 0
    
    action_selected = 0
    for a in range(0, d):
        if (Nt_a[a] > 0):
            ln_t = math.log(t) #natural log of t
            hist_t.append(ln_t) #to plot natural log of t

            #calculate the UCB
            Qt_a = sum_rewards[a]/Nt_a[a]
            ucb_value = Qt_a + c*(ln_t/Nt_a[a]) 
            UCB_Values[a] = ucb_value

        #if this equals zero, choose as the maximum. Cant divide by negative     
        elif (Nt_a[a] == 0):
            UCB_Values[a] = 1e500 #make large value
        
    #select the max value
    action_selected = np.argmax(UCB_Values)
    #print("selected {0}".format(action_selected))
    
    #Update Values
    Nt_a[action_selected] += 1
    reward = df.values[t, action_selected+1]
    sum_rewards[action_selected] += reward
    
    #for analysis
    r_ = df.values[t,[1,2,3,4]] #get all rewards for time t to a vector
    r_best = r_[np.argmax(r_)]      #select the best action
    
    pick_random = random.randrange(d)
    r_random = r_[pick_random] #np.random.choice(r_) #select reward for random action
    if len(hist_achieved_rewards)>0:
        hist_achieved_rewards.append(hist_achieved_rewards[-1]+reward)
        hist_best_possible_rewards.append(hist_best_possible_rewards[-1]+r_best)
        hist_random_choice_rewards.append(hist_random_choice_rewards[-1]+r_random)
    else:
        hist_achieved_rewards.append(reward)
        hist_best_possible_rewards.append(r_best)
        hist_random_choice_rewards.append(r_random)
plt.bar(['m1','m2','m3','m4'],Nt_a)
#plt.bar([0,1,2,3,4,5,6,7,8,9],Nt_a)
plt.plot(hist_best_possible_rewards)
plt.title("Rewards Achieved")
plt.plot(hist_achieved_rewards)
plt.title("Rewards Achieved")
hist_achieved_rewards[-1]
plt.plot(hist_random_choice_rewards)
plt.title("Rewards Achieved")
hist_random_choice_rewards[-1]
Nt_a
Nt_a
plt.plot(hist_t)
plt.title("Natural Log of t")
Nt_a
q = [1,10,3]
np.random.choice(q)
Nt_a
total_cum_rewards = []
sum_rewards = np.zeros(d)

sum_rewards[2] = 10
total_cum_rewards.append(sum_rewards)

sum_rewards[1] = 15
total_cum_rewards.append(sum_rewards)

total_cum_rewards
r = [1,2,3]
r[-1]
r_ = df.values[2,[1,2,3,4]]
r_[np.argmax(r_)]
 df.values[2,[1,2,3,4]]
r_ = df.values[2,[1,2,3,4]]
r_ = np.random.choice(r_)
np.random.choice(r_)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11212675198049969, 'fraction_numerical': 0.021633150517976842, 'mean_word_length': 3.47275204359673, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '339572', 'license': 'Apache-2.0', 'n_tokens_mistral': 1271, 'n_tokens_neox': 1213, 'n_words': 275, 'path': 'ReinforcementLearning/Message Selection Algorithm.ipynb', 'repo_name': 'handsonmachinelearning/Tutorials', 'size': '3282'}