<jupyter_start><jupyter_text>CRAR<jupyter_code>import math
import torch
import torch.nn as nn
from torch.optim import Adam
import gym
from utils import ReplayBuffer
import random
import numpy as np
from IPython.display import clear_output
import matplotlib.pyplot as plt
%matplotlib inline
USE_CUDA = torch.cuda.is_available()
device = 'cuda' if USE_CUDA else 'cpu'
device = torch.device(device)
def plot(frame_idx, rewards, losses):
    clear_output(True)
    plt.figure(figsize=(20,5))
    plt.subplot(131)
    plt.title('frame %s. reward: %s' % (frame_idx, np.mean(rewards[-10:])))
    plt.plot(rewards)
    plt.subplot(132)
    plt.title('loss')
    plt.plot(losses)
    plt.show()<jupyter_output><empty_output><jupyter_text>CartPole<jupyter_code>env_id = "CartPole-v0"
env = gym.make(env_id)
from model import SimpleQNetwork
model = SimpleQNetwork(env.observation_space.shape, env.action_space.n, device).to(device)
optimizer = Adam(model.parameters())
replay_buffer = ReplayBuffer(1000)
epsilon_start = 1.0
epsilon_final = 0.01
epsilon_decay = 500

epsilon_by_frame = lambda frame_idx: epsilon_final + (epsilon_start - epsilon_final) * math.exp(-1. * frame_idx / epsilon_decay)
plt.plot([epsilon_by_frame(i) for i in range(10000)])
from loss import q_td_loss_optim
compute_td_loss = q_td_loss_optim
num_frames = 10000
batch_size = 32
gamma      = 0.99

losses = []
all_rewards = []
episode_reward = 0

state = env.reset()

for frame_idx in range(1, num_frames + 1):
    epsilon = epsilon_by_frame(frame_idx)
    action = model.act(state, epsilon)
    
    next_state, reward, done, _ = env.step(action)
    replay_buffer.push(state, action, reward, next_state, done)
    
    state = next_state
    episode_reward += reward
    
    if done:
        state = env.reset()
        all_rewards.append(episode_reward)
        episode_reward = 0
        
    if len(replay_buffer) > batch_size:
        loss = compute_td_loss(batch_size, gamma, model, optimizer, replay_buffer, device)
        losses.append(loss.item())
        
    if frame_idx % 200 == 0:
        plot(frame_idx, all_rewards, losses)<jupyter_output><empty_output><jupyter_text>Pong<jupyter_code>from wrappers import make_atari, wrap_deepmind, wrap_pytorch
env_id = "PongNoFrameskip-v4"
env    = make_atari(env_id)
env    = wrap_deepmind(env)
env    = wrap_pytorch(env)
from model import QNetwork
model = QNetwork(env.observation_space.shape, env.action_space.n, device).to(device)
optimizer = Adam(model.parameters(), lr=0.00001)
replay_initial = 1000
replay_buffer = ReplayBuffer(100000)
epsilon_start = 1.0
epsilon_final = 0.01
epsilon_decay = 100000

epsilon_by_frame = lambda frame_idx: epsilon_final + (epsilon_start - epsilon_final) * math.exp(-1. * frame_idx / epsilon_decay)
from loss import q_td_loss_optim
compute_td_loss = q_td_loss_optim
num_frames = 1400000
batch_size = 32
gamma      = 0.99

losses = []
all_rewards = []
episode_reward = 0

state = env.reset()
for frame_idx in range(1, num_frames + 1):
    epsilon = epsilon_by_frame(frame_idx)
    action = model.act(state, epsilon)
    
    next_state, reward, done, _ = env.step(action)
    replay_buffer.push(state, action, reward, next_state, done)
    
    state = next_state
    episode_reward += reward
    
    if done:
        state = env.reset()
        all_rewards.append(episode_reward)
        episode_reward = 0
        
    if len(replay_buffer) > replay_initial:
        loss = compute_td_loss(batch_size, gamma, model, optimizer, replay_buffer, device)
        losses.append(loss.data.item())
        
    if frame_idx % 10000 == 0:
        plot(frame_idx, all_rewards, losses)
torch.save(model.state_dict(), 'saved_weights/qnetwork.pt')
model.load_state_dict(torch.load('saved_weights/qnetwork.pt'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10547184773988898, 'fraction_numerical': 0.026169706582077717, 'mean_word_length': 4.183561643835616, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '467490', 'license': 'MIT', 'n_tokens_mistral': 1423, 'n_tokens_neox': 1344, 'n_words': 277, 'path': 'experiments/CRAR_QNet.ipynb', 'repo_name': 'aayn/yarl', 'size': '3783'}