<jupyter_start><jupyter_text>Functions for creating a dog<jupyter_code>WALKING_TIME = 0
EATING_TIME = 0
PLAYING_TIME = 0

food_consumption_rate = 1.0 / (30 * 3600)
affection_consumption_rate = 1.0 / (50 * 3600)
walking_fat_converge_rate = 0.2
walking_affection_converge_rate = 0.4
playing_fat_converge_rate = 0.1
playing_affection_converge_rate = 0.20
eating_food_increase = 0.6
eating_fat_increase = 0.25
def transform_to_dec(value, dec=10):
    return np.floor(dec*value)/dec

def apply_decreasing_rate(value: float, rate: float) -> float:
    """
    Apply a decreasing rate to a value
    :param value: current value
    :param rate: per second
    :return: updated value
    """
    return value - (60 * rate)

def converge(value: float, target: float, ratio: float) -> float:
    diff: float = (target - value) * ratio
    return value + diff

def update_walking(dog_table_row):
    update_fat = converge(dog_table_row.iloc[0]['fat'], 0.0, walking_fat_converge_rate)
    update_affection = converge(dog_table_row.iloc[0]['affection'], 1.0, walking_affection_converge_rate)
    return (update_fat, update_affection)

def update_feeding(dog_table_row):
    update_food = min(dog_table_row.iloc[0]['food'] + eating_food_increase, 1.0)
    update_fat = min(dog_table_row.iloc[0]['fat'] + eating_fat_increase, 1.0)
    return (update_food, update_fat)

def update_playing(dog_table_row):
    update_fat = converge(dog_table_row.iloc[0]['fat'], 0.0, playing_fat_converge_rate)
    update_affection = converge(dog_table_row.iloc[0]['affection'], 1.0, playing_affection_converge_rate)
    return (update_fat, update_affection)

def update_food(dog_table_row):
    return transform_to_dec(max(
        0.0,
        apply_decreasing_rate(dog_table_row.iloc[0]['food'], food_consumption_rate)
    ))

def update_affection(dog_table_row):
    return transform_to_dec(max(
        0.0,
        apply_decreasing_rate(dog_table_row.iloc[0]['affection'], affection_consumption_rate)
    ))

def update_happiness(dog_table_row) -> float:  # between 0 and 1
    return min(
        dog_table_row.iloc[0]['food'],
        1.0 - dog_table_row.iloc[0]['fat'],
        dog_table_row.iloc[0]['affection']
    )

def update_alive(dog_table_row):
    return dog_table_row.iloc[0]['alive'] & (dog_table_row.iloc[0]['happiness'] > 0.0)

def new_state_after_1_minute(dog_table, action_taken='NO ACTION'):
    
    dog_table_row = dog_table.tail(1)
    new_state_dog_id = dog_table_row.iloc[0]['dog_id']
    
    new_state_food = update_food(dog_table_row)
    new_state_fat = dog_table_row.iloc[0]['fat']
    new_state_affection = update_affection(dog_table_row)
    
    new_state_last_action_taken = dog_table_row.iloc[0]['last_action_taken']
    new_state_can_action_be_taken = dog_table_row.iloc[0]['can_action_be_taken']
    new_state_minutes_since_last_action = dog_table_row.iloc[0]['minutes_since_last_action']
    
    new_state_minute = dog_table_row.iloc[0]['minute'] + 1
    new_state_minutes_since_last_action = dog_table_row.iloc[0]['minutes_since_last_action'] + 1
    
    new_state_happiness = update_happiness(dog_table_row)
    new_state_alive = update_alive(dog_table_row)
    
    if ((action_taken != 'NO ACTION') & (dog_table_row.iloc[0]['can_action_be_taken'])):
        new_state_last_action_taken = action_taken
        new_state_can_action_be_taken = False
        new_state_minutes_since_last_action = 0
    
    if ((dog_table_row.iloc[0]['last_action_taken'] == 'WALKING') &
        (dog_table_row.iloc[0]['minutes_since_last_action'] == WALKING_TIME)):
        new_state_fat, new_state_affection = update_walking(dog_table_row)
        new_state_last_action_taken = 'NO ACTION'
        new_state_can_action_be_taken = True
        new_state_minutes_since_last_action = 0        

    if ((dog_table_row.iloc[0]['last_action_taken'] == 'FEEDING') &
        (dog_table_row.iloc[0]['minutes_since_last_action'] == EATING_TIME)):
        new_state_food, new_state_fat = update_feeding(dog_table_row)
        new_state_last_action_taken = 'NO ACTION'
        new_state_can_action_be_taken = True
        new_state_minutes_since_last_action = 0

    if ((dog_table_row.iloc[0]['last_action_taken'] == 'PLAYING') &
        (dog_table_row.iloc[0]['minutes_since_last_action'] == PLAYING_TIME)):
        new_state_fat, new_state_affection = update_playing(dog_table_row)
        new_state_last_action_taken = 'NO ACTION'
        new_state_can_action_be_taken = True
        new_state_minutes_since_last_action = 0
        
    dog_table = dog_table.append({
        'dog_id': new_state_dog_id,
        'minute': new_state_minute,
        'happiness': new_state_happiness,
        'food': new_state_food,
        'fat': new_state_fat,
        'affection': new_state_affection,
        'alive': new_state_alive,
        'can_action_be_taken': new_state_can_action_be_taken, 
        'last_action_taken': new_state_last_action_taken,
        'minutes_since_last_action': new_state_minutes_since_last_action
        }, ignore_index=True)
    
    return dog_table<jupyter_output><empty_output><jupyter_text>Functions for RL  Initialize dog<jupyter_code>def init_dog_table():

    dog_table = pd.DataFrame(columns=[
        'dog_id', 'minute',
        'happiness', 'food', 'fat', 'affection',
        'alive', 'can_action_be_taken', 'last_action_taken', 'minutes_since_last_action'
    ])

    #dog_table = dog_table.append({
    #    'dog_id': 1, 'minute': 0,
    #    'happiness': 0,
    #    'food': transform_to_dec(1), 
    #    'fat': transform_to_dec(0), 
    #    'affection': transform_to_dec(1),
    #    'alive': True, 'can_action_be_taken': True, 'last_action_taken': 'NO ACTION', 'minutes_since_last_action': 0
    #    }, ignore_index=True)

    dog_table = dog_table.append({
        'dog_id': 1, 'minute': 0,
        'happiness': 0,
        'food': transform_to_dec(random.random()), 
        'fat': transform_to_dec(random.random()), 
        'affection': transform_to_dec(random.random()),
        'alive': True, 'can_action_be_taken': True, 'last_action_taken': 'NO ACTION', 'minutes_since_last_action': 0
        }, ignore_index=True)
    
    dog_table['happiness'] = update_happiness(dog_table)
    
    return dog_table<jupyter_output><empty_output><jupyter_text>Choose action<jupyter_code># epsilon = 0.3

def choose_action(state, Q, action_space, epsilon):
    state_q = get_state_q(state,0)
    if np.random.uniform(0, 1) < epsilon:
        action = random.choices(range(len(action_space)), k = 1)[0]
    else:
        action = np.argmax(Q[state_q, :])
    return action<jupyter_output><empty_output><jupyter_text>Run an episode<jupyter_code>def create_episode(Q, max_steps, epsilon):
    
    dog_table = init_dog_table()
    state = dog_table.tail(1)

    t = 0
    while ((state.iloc[0]['alive']) & (t < max_steps)):
        t += 1
        action = choose_action(
            state = state,
            Q = Q,
            action_space = action_space,
            epsilon = epsilon
        )
        dog_table = new_state_after_1_minute(dog_table, action_taken = action_space[action])
        state = dog_table.tail(1)

    return dog_table<jupyter_output><empty_output><jupyter_text>Update Q function<jupyter_code>def get_state_q(current_state, t):
    return int('{:01d}{:01d}{:01d}'.format(
        min(9,int(current_state.iloc[t]['food']*10)),
        min(9,int(current_state.iloc[t]['fat']*10)),
        min(9,int(current_state.iloc[t]['affection']*10))
    ))
# Function to learn the Q-value
def update(Q, state_t, reward_t, action_t, state_t_1, gamma, alpha):
    
    Q[state_t, action_t] = Q[state_t, action_t] + alpha * (reward_t + gamma * max(Q[state_t_1,]) - Q[state_t, action_t])
    
    return Q<jupyter_output><empty_output><jupyter_text>Define reward<jupyter_code>def define_reward(state, cnt_limit = 20):
    
    dog_table_next_hour = state.copy()
    cnt = 0
    
    while ((dog_table_next_hour.tail(1).iloc[0]['alive']) & (cnt < cnt_limit)):
        cnt += 1
        dog_table_next_hour = new_state_after_1_minute(dog_table_next_hour, action_taken='NO ACTION')

    average_happiness = dog_table_next_hour['happiness'].sum()/cnt_limit
    
    return average_happiness<jupyter_output><empty_output><jupyter_text>Plots<jupyter_code>def intermediate_plots(len_steps_plot, all_overall_reward_t):
    
    # Create some mock data
    fig, ax1 = plt.subplots()

    color = 'tab:red'
    ax1.set_xlabel('episodes')
    ax1.set_ylabel('nber steps', color=color)
    ax1.plot(range(len(len_steps_plot)), len_steps_plot, color=color)
    ax1.tick_params(axis='y', labelcolor=color)

    ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis

    color = 'tab:blue'
    ax2.set_ylabel('average reward', color=color)  # we already handled the x-label with ax1
    ax2.plot(range(len(all_overall_reward_t)), all_overall_reward_t, color=color)
    ax2.tick_params(axis='y', labelcolor=color)

    fig.tight_layout()  # otherwise the right y-label is slightly clipped
    plt.show()
def display_analyse_Q(Q):
    
    analyse_Q = pd.DataFrame(columns=[
        'state_id', 'food_id', 'fat_id', 'affection_id',
        'no_action', 'walking', 'feeding', 'playing',
    ])

    for i in trange(Q.shape[0]):
        analyse_Q = analyse_Q.append({
            'state_id': int(i),
            'food_id': int(np.floor(i/100)),
            'fat_id': int(np.floor(i/10 % 10)),
            'affection_id': int(np.floor(i % 10)),
            'no_action': Q[i,0],
            'walking': Q[i,1],
            'feeding': Q[i,2],
            'playing': Q[i,3]
            }, ignore_index=True)

    analyse_Q.groupby('food_id').agg({
        'no_action': 'mean',
        'walking': 'mean',
        'feeding': 'mean',
        'playing': 'mean'
    }).plot()

    analyse_Q.groupby('fat_id').agg({
        'no_action': 'mean',
        'walking': 'mean',
        'feeding': 'mean',
        'playing': 'mean'
    }).plot()

    analyse_Q.groupby('affection_id').agg({
        'no_action': 'mean',
        'walking': 'mean',
        'feeding': 'mean',
        'playing': 'mean'
    }).plot()<jupyter_output><empty_output><jupyter_text>This is where things are done<jupyter_code>observation_space_n = 10*10*10
action_space = ['NO ACTION', 'WALKING', 'FEEDING', 'PLAYING']
action_space_n = len(action_space)
Q = np.full(
    shape=(observation_space_n, action_space_n),
    fill_value=3.25
)
Q_previous = Q.copy()
all_overall_reward = []
all_overall_reward_plot = []
all_overall_reward_t = []
all_overall_reward_t_plot = []
len_steps = []
len_steps_plot = []
nber_steps = 0
max_steps = 1000
epsilon_initial = 0.5
alpha_initial = 1
gamma = 1
convergence = False
# Starting with Q-learning
#for episode in trange(total_episodes):
while not convergence:
    
    epsilon = 1
    alpha = 1
    # epsilon = epsilon_initial / np.ceil( (len(all_overall_reward)+1) / 10000)
    # alpha = alpha_initial  / np.ceil( (len(all_overall_reward)+1) / 10000)
    # Something like alpha = k/(k+t) can work well
    
    dog_table = create_episode(
        Q = Q, 
        max_steps = max_steps,
        epsilon = epsilon)
    # print(dog_table)

    for t in range(len(dog_table)-1):

        state_t = get_state_q(dog_table, t)

        action_t_name = dog_table.iloc[t]['last_action_taken'] if (
            dog_table.iloc[t]['minutes_since_last_action'] == 0
        ) else 'NO ACTION'
        action_t = action_space.index(action_t_name)

        reward_t = (sum(dog_table.loc[dog_table['minute'] >= t]['happiness'])/(len(dog_table)-t))
        
        state_t_1 = get_state_q(dog_table, t + 1)

        #print(state_t, action_t_name, action_t, reward_t, state_t_1)
        #print(Q[state_t, action_t])
        Q = update(Q, state_t, reward_t, action_t, state_t_1,
                   gamma = 1,
                   alpha = alpha)
        #print(Q[state_t, action_t])
    
    overall_reward = sum(dog_table['happiness'])
    nber_steps += len(dog_table)-1
    len_steps.append(len(dog_table)-1)
    all_overall_reward.append(overall_reward)
    all_overall_reward_t.append(overall_reward/len(dog_table))
    
    # print(episode, t, overall_reward, overall_reward/t)
    if len(all_overall_reward) % 1000 == 0:
        
        all_overall_reward_plot.append(np.mean(all_overall_reward[-100:]))
        all_overall_reward_t_plot.append(np.mean(all_overall_reward_t[-100:]))
        len_steps_plot.append(np.mean(len_steps[-100:]))
        distance_q_matrices = np.linalg.norm(Q_previous - Q)
        convergence = distance_q_matrices < 0.0001
        Q_previous = Q.copy()
        
        print('Iter. {:7d} - Steps {:8d} - Eps. {:0.6f} - Alp. {:0.6f} - Dist. {:06.5f} - Total R. {:06.3f} - Avg R. {:0.3f}'.format(
            len(all_overall_reward),
            nber_steps,
            epsilon, alpha,
            distance_q_matrices,
            np.mean(all_overall_reward[-100:]),
            np.mean(all_overall_reward_t[-100:])
        ))
        intermediate_plots(len_steps_plot, all_overall_reward_t_plot)

    if len(all_overall_reward) % 10000 == 0:
        display_analyse_Q(Q)

for i in range(Q.shape[1]):
    print(Q[:,1])
dog_table = create_episode(
        Q = Q,
        max_steps = max_steps,
        epsilon = epsilon_initial / np.ceil( (len(all_overall_reward)+1) / 1000))
dog_table
sum(dog_table['happiness'])
display_analyse_Q(Q)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1278358208955224, 'fraction_numerical': 0.019402985074626865, 'mean_word_length': 3.567484662576687, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '393640', 'license': 'MIT', 'n_tokens_mistral': 5199, 'n_tokens_neox': 4887, 'n_words': 766, 'path': 'notebooks/first_tests/2.3-yp-first_try.ipynb', 'repo_name': 'Blitzy29/doggo', 'size': '13400'}