<commit_before>import sampling, hyper, gaussian
import torch, random, sys
from torch.autograd import Variable
from torch import nn, optim
from tqdm import trange
from tensorboardX import SummaryWriter

import matplotlib.pyplot as plt
import util

torch.manual_seed(1)

"""
Simple experiment: learn the identity function from one tensor to another
"""
w = SummaryWriter()

BATCH = 32
SHAPE = (8, )
CUDA = True

nzs = 8 # hyper.prod(SHAPE)*6

model = gaussian.ParamASHLayer(SHAPE, SHAPE, k=nzs) #
# model.initialize(SHAPE, batch_size=64,iterations=100, lr=0.05)

if CUDA:
    model.cuda()

#x = Variable(torch.rand((BATCH, ) + SHAPE))
#print('--- x', x)

#y = model(x)
#print('--- y', y)

N = 5000

criterion = nn.MSELoss()
optimizer = optim.Adam(model.parameters(), lr=0.01)

plt.figure(figsize=(5,5))
util.makedirs('./spread/')

for i in trange(N):

    x = torch.rand((BATCH,) + SHAPE)
    if CUDA:
        x = x.cuda()
    x = Variable(x)

    optimizer.zero_grad()

    y = model(x)
    loss = criterion(y, x) # compute the loss
    loss.backward()        # compute the gradients
    optimizer.step()

    w.add_scalar('identity/plain', torch.sqrt(loss).data[0], i)

    # if(i != 0 and i % (N/25) == 0):
    means, sigmas, values = model.hyper(x)
    #     print(sigmas, sigmas.grad)
    #     print(values, values.grad)
    #     # print(list(model.parameters()))
    #
    # print('LOSS', torch.sqrt(loss))

    if i % 50 == 0:
        plt.clf()
        util.plot(means, sigmas, values)
        plt.xlim((-1, 8))
        plt.ylim((-1, 8))
        plt.axis('square')
        plt.savefig('./spread/means{:04}.png'.format(i))

for i in range(20):
    x = Variable(torch.rand((1,) + SHAPE))
    y = model(x)

    print('diff', torch.abs(x - y).unsqueeze(0))

    print('********')<commit_msg>CUDA for tests post training<commit_after>import sampling, hyper, gaussian
import torch, random, sys
from torch.autograd import Variable
from torch import nn, optim
from tqdm import trange
from tensorboardX import SummaryWriter

import matplotlib.pyplot as plt
import util

torch.manual_seed(1)

"""
Simple experiment: learn the identity function from one tensor to another
"""
w = SummaryWriter()

BATCH = 32
SHAPE = (8, )
CUDA = True

nzs = 8 # hyper.prod(SHAPE)*6

model = gaussian.ParamASHLayer(SHAPE, SHAPE, k=nzs) #
# model.initialize(SHAPE, batch_size=64,iterations=100, lr=0.05)

if CUDA:
    model.cuda()

#x = Variable(torch.rand((BATCH, ) + SHAPE))
#print('--- x', x)

#y = model(x)
#print('--- y', y)

N = 5000

criterion = nn.MSELoss()
optimizer = optim.Adam(model.parameters(), lr=0.01)

plt.figure(figsize=(5,5))
util.makedirs('./spread/')

for i in trange(N):

    x = torch.rand((BATCH,) + SHAPE)
    if CUDA:
        x = x.cuda()
    x = Variable(x)

    optimizer.zero_grad()

    y = model(x)
    loss = criterion(y, x) # compute the loss
    loss.backward()        # compute the gradients
    optimizer.step()

    w.add_scalar('identity/plain', torch.sqrt(loss).data[0], i)

    # if(i != 0 and i % (N/25) == 0):
    means, sigmas, values = model.hyper(x)
    #     print(sigmas, sigmas.grad)
    #     print(values, values.grad)
    #     # print(list(model.parameters()))
    #
    # print('LOSS', torch.sqrt(loss))

    if i % 50 == 0:
        plt.clf()
        util.plot(means, sigmas, values)
        plt.xlim((-1, 8))
        plt.ylim((-1, 8))
        plt.axis('square')
        plt.savefig('./spread/means{:04}.png'.format(i))

for i in range(20):
    x = torch.rand((1,) + SHAPE)
    if CUDA:
        x = x.cuda()
    x = Variable(x)

    y = model(x)

    print('diff', torch.abs(x - y).unsqueeze(0))

    print('********')
{'fraction_non_alphanumeric': 0.16885964912280702, 'fraction_numerical': 0.022478070175438597, 'mean_word_length': 2.9879781420765026, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1034866', 'n_tokens_mistral': 1435, 'n_tokens_neox': 1345, 'n_words': 325}