<jupyter_start><jupyter_text>Create dataset<jupyter_code>patient = HIVPatient(clipping=False,logscale=False)
FQI_buffer = Buffer(50000)

for j in tqdm(range(30)):
    s = patient.reset(mode="unhealthy")
    for i in range(200):
        a = np.random.choice(4)
        s_, r, d, _ = patient.step(a)
        FQI_buffer.append(s,a,r,s_,d)
        s = s_<jupyter_output>100%|██████████| 30/30 [08:28<00:00, 16.96s/it]<jupyter_text>FQI<jupyter_code>from sklearn.ensemble import RandomForestRegressor
from typing import Optional,List, Callable
from sklearn.base import BaseEstimator

def estimator_factory(*args, **kwargs):
    return RandomForestRegressor(*args, **kwargs)


def update(memory : Buffer,
    gamma : float = 0.98,
    estimator : Optional[BaseEstimator] = None,
    estimator_factory: Callable = estimator_factory):

    states, actions, rewards, next_state, done  = memory.get()
    actions = np.expand_dims(actions,axis=1)
    target = np.expand_dims(rewards,axis=1)
    #target = rewards

    if estimator is not None:
        q_values = np.zeros((len(rewards),4))
        for a in range(4):
            actions_ = a*np.ones((len(rewards),1))
            X = np.concatenate((next_state,actions_),axis=1)
            q_values[:,a] = estimator.predict(X)
            
        #print(f"q_values --> {q_values}")
        qmax = np.expand_dims(np.max(q_values,axis=1),axis=1)
        #print(f"Shape de qmax --> {qmax.shape}, Shape de target --> {target.shape}, Shape de done --> {done.shape}")
        target += gamma*qmax * (1 - np.expand_dims(done,axis=1))
        #print(f"Shape de target --> {target.shape}")

    if estimator is None:
        estimator = estimator_factory()


    #print(f"States shape --> {states.shape}, actions shape --> {actions.shape},target{target.shape}")
    data = np.concatenate((states,actions),axis=1)
    #print(f"Data shape --> {data.shape}")
    estimator.fit(data,target)


    return estimator<jupyter_output><empty_output><jupyter_text>Premier entraînement !<jupyter_code>estimator = None
for _ in tqdm(range(100)):

    estimator = update(FQI_buffer,estimator=estimator)<jupyter_output>0%|          | 0/100 [00:00<?, ?it/s]/tmp/ipykernel_1097/1251894869.py:38: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  estimator.fit(data,target)
  1%|          | 1/100 [00:03<06:22,  3.86s/it]/tmp/ipykernel_1097/1251894869.py:38: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  estimator.fit(data,target)
  2%|▏         | 2/100 [00:08<06:44,  4.12s/it]/tmp/ipykernel_1097/1251894869.py:38: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().
  estimator.fit(data,target)
  3%|▎         | 3/100 [00:13<07:14,  4.48s/it]/tmp/ipykernel_1097/1251894869.py:38: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n[...]<jupyter_text>Deuxième entraînement<jupyter_code>for j in tqdm(range(30)):

    s = patient.reset(mode="unhealthy")

    for step in range(200):
        
        if np.random.random()<0.15:
            action = np.random.choice(4)
        
        else:
            if estimator is not None:
                greedy = np.zeros((4,2))
                for a in range(4):
                    #print(f"States shape --> {np.expand_dims(s.T,axis=1).shape}, actions shape --> {np.expand_dims(np.array([a]),axis=1).shape}")
                    sta = np.expand_dims(s,axis=1)
                    act = np.expand_dims(np.array([a]),axis=1)
                    X = np.concatenate((sta,act),axis=0).T
                    #print((f"shape of X --> {X.shape}"))
                    q = estimator.predict(X)
                    #print(f'Q -->{q}')
                    greedy[a,0] , greedy[a,1] = a , q

                #print(greedy)
                action = greedy[np.argmax(greedy[:,0]),0]
            
            else:
                action = np.random.choice(4)


        s_, r, d, _ = patient.step(int(action))
        FQI_buffer.append(s,a,r,s_,d)
        s = s_


while _ in tqdm(range(100)):
    estimator = update(FQI_buffer,estimator=estimator)<jupyter_output>100%|██████████| 100/100 [00:00<00:00, 56096.08it/s]<jupyter_text>Affichons les résultats<jupyter_code>s = patient.reset(mode="unhealthy")
T1,T2,T1_,T2_,V,E = np.zeros(200), np.zeros(200), np.zeros(200), np.zeros(200), np.zeros(200), np.zeros(200)
actions = []
for step in tqdm(range(200)):
    T1[step], T2[step], T1_[step], T2_[step], V[step], E[step] = s 

    greedy = np.zeros((4,2))
        
    for a in range(4):
                    #print(f"States shape --> {np.expand_dims(s.T,axis=1).shape}, actions shape --> {np.expand_dims(np.array([a]),axis=1).shape}")
        sta = np.expand_dims(s,axis=1)
        act = np.expand_dims(np.array([a]),axis=1)
        X = np.concatenate((sta,act),axis=0).T
                    #print((f"shape of X --> {X.shape}"))
        q = estimator.predict(X)
                    #print(f'Q -->{q}')
        greedy[a,0] , greedy[a,1] = a , q

                #print(greedy)
    action = greedy[np.argmax(greedy[:,0]),0]
    actions += [action]
            


    s_, r, d, _ = patient.step(int(action))
    FQI_buffer.append(s,a,r,s_,d)
    s = s_<jupyter_output>100%|██████████| 200/200 [00:59<00:00,  3.37it/s]<jupyter_text>Post traitement<jupyter_code>actions
plt.plot(np.log10(E))
plt.plot(np.log10(T1))
FQI_buffer.get()
np.array(FQI_buffer.actions).shape
_, _, rewards, _, _  = FQI_buffer.get()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15943035776311218, 'fraction_numerical': 0.04984369572768323, 'mean_word_length': 2.904406779661017, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '4191', 'license': 'MIT', 'n_tokens_mistral': 2313, 'n_tokens_neox': 2104, 'n_words': 393, 'path': 'Paul/Paul.ipynb', 'repo_name': 'Fiona55/hiv_experiment', 'size': '5758'}