<jupyter_start><jupyter_text>Humanoid v2  Expert performance:returns [10399.465466297353, 10337.698440078071, 10338.03251510115, 10363.347077257335, 10520.142508634532, 10437.992393534745, 10375.590303129682, 10427.51001606666, 10265.334918376444, 10383.854722941793, 10383.34055343508, 10392.749227673174, 10386.273548840532, 10377.634438183406, 10445.100629123428, 10468.288190410423, 10431.12064455722, 10522.290767465352, 10412.97541826066, 10371.666146306461] mean return 10402.020396283677 std of return 58.8302440319096<jupyter_code>pickle_in = open("expert_data/Humanoid-v2.pkl","rb")
humanoid_expert = pickle.load(pickle_in)
humanoid_expert.keys()
observations = humanoid_expert['observations']
actions = humanoid_expert['actions']<jupyter_output><empty_output><jupyter_text>Define model<jupyter_code>observations.shape, actions.shape
actions = actions.reshape(-1, 17)
NUM_TRAIN = len(observations)

observations_tensor = torch.from_numpy(observations).float()
actions_tensor = torch.from_numpy(actions).float()
dataset = torch.utils.data.TensorDataset(observations_tensor, actions_tensor)
loader_train = torch.utils.data.DataLoader(dataset, batch_size=128, 
                          sampler=torch.utils.data.sampler.SubsetRandomSampler(range(NUM_TRAIN)))
def training_loop(loader_train, model, optimizer, epochs=1, print_every=1):
    criterion = nn.MSELoss()
    for e in range(epochs):
        for t, (x, y) in enumerate(loader_train):
            model.train()
            scores = model(x)
            loss = criterion(scores, y)
            
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            
        if e % print_every == 0:
            print('Epoch %d, loss = %.4f' % (e, loss.item()))
    print('Epoch %d, loss = %.4f' % (e, loss.item()))
def init_weights(m):
    if type(m) == nn.Linear:
        torch.nn.init.xavier_uniform_(m.weight)
        m.bias.data.fill_(0.01)

model = nn.Sequential(
    nn.Linear(376, 30),
    nn.ReLU(),
    nn.Linear(30, 15),
    nn.ReLU(),
    nn.Linear(15, 17)
)

model.apply(init_weights)

optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
training_loop(loader_train, model, optimizer, epochs=30, print_every=5)
model(observations_tensor[1])
torch.save(model, open("policies/humanoid_3l_fc.md", 'wb'))
with open("policies/humanoid_3l_fc.pkl", 'wb') as f: 
    pickle.dump(model, f, pickle.HIGHEST_PROTOCOL)<jupyter_output><empty_output><jupyter_text>Model performance:returns [226.90052892567294, 212.3321194289658, 217.80195366576146, 204.62086051618942, 182.3108776594606, 236.57988388099307, 191.89164222695177, 215.00037708242786, 181.21875479360577, 236.3635765411728, 217.53778560875043, 221.21379086735246, 205.28428884513102, 334.3184507917499, 207.5064707678762, 219.45422690228023, 256.09821927536933, 204.7539900068401, 206.02273922400704, 207.93419224419713] mean return 219.25723646273778 std of return 31.578925930927426   Reacher v2  Expert performance:returns [-4.198410075394117, -1.3188410506994994, -5.7354880391280965, -3.06706735463171, -3.620617467815022, -4.786380708719675, -1.3852393193543773, -3.3516123491724876, -1.2547292814536883, -6.505562306648722, -5.98983699591411, -3.526049717732696, -2.4442175691811974, -4.521031261323542, -6.025657346033555, -3.1225293997079513, -4.643144853035947, -6.123864040766139, -2.9946819433519587, -3.53563183633491] mean return -3.90752964581997 std of return 1.5917489917071845<jupyter_code>pickle_in = open("expert_data/Reacher-v2.pkl","rb")
reacher_expert = pickle.load(pickle_in)

observations = reacher_expert['observations']

actions = reacher_expert['actions']

observations.shape, actions.shape
actions = actions.reshape(-1, 2)
NUM_TRAIN = len(observations)

observations_tensor = torch.from_numpy(observations).float()
actions_tensor = torch.from_numpy(actions).float()
dataset = torch.utils.data.TensorDataset(observations_tensor, actions_tensor)
loader_train = torch.utils.data.DataLoader(dataset, batch_size=128, 
                          sampler=torch.utils.data.sampler.SubsetRandomSampler(range(NUM_TRAIN)))
model = nn.Sequential(
    nn.Linear(11, 30),
    nn.ReLU(),
    nn.Linear(30, 15),
    nn.ReLU(),
    nn.Linear(15, 2)
)

model.apply(init_weights)

optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
training_loop(loader_train, model, optimizer, epochs=45, print_every=5)
model(observations_tensor[1])
actions_tensor[1]
torch.save(model, open("policies/reacher_3l_fc.md", 'wb'))<jupyter_output><empty_output><jupyter_text>Model performance:returns [-14.307916560565811, -9.077888321317364, -13.208763647245032, -9.222286514236384, -13.518732555193447, -15.503348259528812, -7.9961926282811016, -11.179946908777184, -13.191927016184627, -12.024706431236597, -6.32562905638001, -7.256840410833916, -6.73461669557026, -8.210166523930585, -5.142029045800414, -13.92841154171759, -11.264429763848174, -9.91746506852077, -13.744988381990433, -4.8021436262973936] mean return -10.327921447872793  std of return 3.217880999896169   More demonstrations  Stats for 100 demonstrations:mean return -3.700508867917537 std of return 1.7148592869182697<jupyter_code>pickle_in = open("expert_data/Reacher-v2.pkl","rb")
reacher_expert = pickle.load(pickle_in)
observations = reacher_expert['observations']
actions = reacher_expert['actions']
observations.shape, actions.shape
actions = actions.reshape(-1, 2)<jupyter_output><empty_output><jupyter_text>Train on all data<jupyter_code>NUM_TRAIN = len(observations)

observations_tensor = torch.from_numpy(observations).float()
actions_tensor = torch.from_numpy(actions).float()
dataset = torch.utils.data.TensorDataset(observations_tensor, actions_tensor)
loader_train = torch.utils.data.DataLoader(dataset, batch_size=128, 
                          sampler=torch.utils.data.sampler.SubsetRandomSampler(range(NUM_TRAIN)))
## Change ReLU on arctang expReLU

model = nn.Sequential(
    nn.Linear(11, 30),
    nn.ReLU(),
    nn.Linear(30, 15),
    nn.ReLU(),
    nn.Linear(15, 2)
)

model.apply(init_weights)

optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
training_loop(loader_train, model, optimizer, epochs=45, print_every=5)
model(observations_tensor[1])
actions_tensor[1]
torch.save(model, open("policies/reacher_3l_fc_100demos.md", 'wb'))<jupyter_output><empty_output><jupyter_text>mean return -6.852096058204267 std of return 3.469663395745245<jupyter_code>!python run_model.py policies/reacher_3l_fc_100demos.md Reacher-v2 --num_rollouts 100<jupyter_output>loading and building expert policy
loaded and built
/home/raznem/anaconda3/lib/python3.7/site-packages/gym/envs/registration.py:14: PkgResourcesDeprecationWarning: Parameters to load are deprecated.  Call .resolve and .require separately.
  result = entry_point.load(False)
running build_ext
[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.[0m
[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.[0m
iter 0
iter 1
iter 2
iter 3
iter 4
iter 5
iter 6
iter 7
iter 8
iter 9
iter 10
iter 11
iter 12
iter 13
iter 14
iter 15
iter 16
iter 17
iter 18
iter 19
iter 20
iter 21
iter 22
iter 23
iter 24
iter 25
iter 26
iter 27
iter 28
iter 29
iter 30
iter 31
iter 32
iter 33
iter 34
iter 35
iter 36
iter 37
iter 38
iter 39
iter 40
iter 41
iter 42
iter 43
iter 44
iter 45
iter 46
iter 47
iter 48
iter 49
iter 50
iter 51
iter 52
iter 53
iter 54
iter 55
iter 56
iter 57
iter 58
iter 59
iter 60
iter 61
iter 6[...]
{'fraction_non_alphanumeric': 0.11742021276595745, 'fraction_numerical': 0.2327127659574468, 'mean_word_length': 5.231151615575808, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '612796', 'license': 'MIT', 'n_tokens_mistral': 3969, 'n_tokens_neox': 2866, 'n_words': 524, 'path': 'hw1/Behavioral Cloning.ipynb', 'repo_name': 'raznem/RL_cs294', 'size': '7520'}