<commit_before>from Tkinter import *
import numpy as np

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.autograd as autograd
from torch.autograd import Variable

master = Tk()
goal = 0
var_goal = StringVar()

def force_reinforce(var, reward):
    if var.creator.reward is torch.autograd.stochastic_function._NOT_PROVIDED:
        var.creator.reward = reward
    else:
        var.creator.reward += reward

class Policy(nn.Module):
    def __init__(self):
        super(Policy, self).__init__()
        self.fc1 = nn.Linear(2, 2)
        self.fc2 = nn.Linear(2, 2)

        self.sm = Variable(torch.Tensor([0,0])).unsqueeze(0) # short memory
        self.lm = Variable(torch.Tensor([0,0])).unsqueeze(0) # long memory
        self.saved_actions = []
        self.rewards = []

    def forward(self, x):
        self.sm = 0.5*self.sm + 0.5*x
        signal = torch.abs(self.sm - self.lm)
        print(signal)

        y = F.relu(self.fc1(signal))
        action_scores = self.fc2(signal)

        self.lm = 0.99*self.lm + 0.01*x

        return F.softmax(action_scores)

model = Policy()
optimizer = optim.Adam(model.parameters(), lr=1e-2)

def select_action(state):
    probs = model(Variable(state))
    print(probs)
    action = probs.multinomial()
    return action


def rand_learn():
    indice = np.random.choice(range(len(model.rewards)))
    action = model.saved_actions[indice]
    reward = model.rewards[indice]
    force_reinforce(action,reward)
    optimizer.zero_grad()
    autograd.backward([action], [None], retain_variables=True)
    optimizer.step()


def learn():
    action = model.saved_actions[-1]
    reward = model.rewards[-1]
    force_reinforce(action,reward)
    optimizer.zero_grad()<commit_msg>clean net structure<commit_after>from Tkinter import *
import numpy as np

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.autograd as autograd
from torch.autograd import Variable

master = Tk()
goal = 0
var_goal = StringVar()

def force_reinforce(var, reward):
    if var.creator.reward is torch.autograd.stochastic_function._NOT_PROVIDED:
        var.creator.reward = reward
    else:
        var.creator.reward += reward

class Policy(nn.Module):
    def __init__(self):
        super(Policy, self).__init__()
        self.fc1 = nn.Linear(2, 2)

        self.sm = Variable(torch.Tensor([0,0])).unsqueeze(0) # short memory
        self.lm = Variable(torch.Tensor([0,0])).unsqueeze(0) # long memory
        self.saved_actions = []
        self.rewards = []

    def forward(self, x):
        self.sm = 0.5*self.sm + 0.5*x
        signal = torch.abs(self.sm - self.lm)
        print(signal)

        action_scores = self.fc1(signal)

        self.lm = 0.99*self.lm + 0.01*x

        return F.softmax(action_scores)

model = Policy()
optimizer = optim.Adam(model.parameters(), lr=1e-2)

def select_action(state):
    probs = model(Variable(state))
    print(probs)
    action = probs.multinomial()
    return action


def rand_learn():
    indice = np.random.choice(range(len(model.rewards)))
    action = model.saved_actions[indice]
    reward = model.rewards[indice]
    force_reinforce(action,reward)
    optimizer.zero_grad()
    autograd.backward([action], [None], retain_variables=True)
    optimizer.step()


def learn():
    action = model.saved_actions[-1]
    reward = model.rewards[-1]
    force_reinforce(action,reward)
    optimizer.zero_grad()
{'fraction_non_alphanumeric': 0.12492853058890795, 'fraction_numerical': 0.015437392795883362, 'mean_word_length': 3.1164705882352943, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '173105', 'n_tokens_mistral': 1243, 'n_tokens_neox': 1191, 'n_words': 244}