<commit_before>import os
import numpy as np

from shutil import copyfile

import torch
import torch.nn as nn
from torch.autograd import Variable

from pymongo import MongoClient

import word2vec as W2V
from word2vec import vocabulary_size
from word2vec import WindowBatchLoader

from sklearn.metrics import f1_score

from tempfile import gettempdir

import argparse


# CNN Model (2 conv layer)
class CNN(nn.Module):
    def __init__(self, emb_size, win_size, hidden_size):
        super(CNN, self).__init__()
        self.win_size = win_size
        self.emb_size = emb_size
        self.emb = nn.Embedding(vocabulary_size, emb_size)
        self.layer1 = nn.Sequential(
            nn.Conv2d(1, 64, kernel_size=emb_size, padding=2),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.MaxPool2d(2))
        self.layer2 = nn.Sequential(
            nn.Conv2d(64, 32, kernel_size=emb_size, padding=2),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(2))
        self.layer3 = nn.Sequential(
            nn.Linear(160, hidden_size),
            nn.BatchNorm1d(hidden_size),
            nn.Sigmoid())
        self.fc = nn.Linear(hidden_size, vocabulary_size)
        self.sf = nn.Softmax()

    def forward(self, x):
        out = self.emb(x).unsqueeze(1)
        out = self.layer1(out)
        out = self.layer2(out)
        out = out.view(out.size(0), -1)
        out = self.layer3(out)
        out = self.fc(out)
        out = self.sf(out)
        return out


class MLP(nn.Module):

    def __init__(self, emb_size, win_size, hidden_size):
        super(MLP, self).__init__()
        self.win_size = win_size
        self.emb_size = emb_size
        self.emb = nn.Embedding(vocabulary_size, emb_size)
        self.layer1 = nn.Sequential(
            nn.Linear(2 * win_size * emb_size, hidden_size),
            nn.BatchNorm1d(hidden_size),
            nn.Sigmoid())
        self.layer2 = nn.Sequential(
            nn.Linear(hidden_size, hidden_size // 2),
            nn.BatchNorm1d(hidden_size // 2),
            nn.Sigmoid())
        self.layer3 = nn.Sequential(
            nn.Linear(hidden_size // 2, hidden_size),
            nn.BatchNorm1d(hidden_size),
            nn.Sigmoid())
        self.fc = nn.Linear(hidden_size, vocabulary_size)
        self.sf = nn.Softmax()

    def forward(self, x):
        out = self.emb(x)
        out = out.view(out.size(0), -1)
        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.fc(out)
        out = self.sf(out)
        return out


def device(device_str):
    return int(device_str[-1])


def train(model, train_loader, test_loader):

    # Hyper Parameters
    num_epochs = args.num_epochs

    # Loss and Optimizer
    optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=1e-4)

    criterion = nn.CrossEntropyLoss()
    train_loss = 0
    best_loss = np.inf
    start_epoch = 0

    # optionally resume from a checkpoint
    if args.resume:
        if os.path.isfile(args.resume):
            print("=> loading checkpoint '%s'" % args.resume)
            checkpoint = torch.load(args.resume)
            start_epoch = checkpoint['epoch']
            best_loss = checkpoint['best_loss']
            model.load_state_dict(checkpoint['state_dict'])
            optimizer.load_state_dict(checkpoint['optimizer'])
            print("=> loaded checkpoint '%s' (epoch %s)" %
                  (args.resume, checkpoint['epoch'] + 1))
        else:
            print("=> no checkpoint found at '%s'" % args.resume)

    for epoch in range(start_epoch, num_epochs):

        for step, (batch_inputs, batch_labels) in enumerate(train_loader):

            inp = torch.from_numpy(batch_inputs).long()
            lbl = torch.from_numpy(batch_labels).long().view(-1)

            if use_cuda:
                with torch.cuda.device(device(args.device)):
                    inp.cuda()
                    lbl.cuda()
                    model.cuda()

            x = Variable(inp)
            y = Variable(lbl)

            model.train()
            optimizer.zero_grad()
            y_hat = model(x)

            loss = criterion(y_hat, y)

            train_loss += loss.data[0]
            loss.backward()
            optimizer.step()

            # loss = p_pos.log() + p_neg.log()

            if (step + 1) % args.steps_per_stats == 0:
                test_loss = 0
                f1 = 0
                for i, (batch_inputs, batch_labels) in enumerate(test_loader):

                    inp = torch.from_numpy(batch_inputs).long()
                    lbl = torch.from_numpy(batch_labels).long().view(-1)

                    if use_cuda:
                        with torch.cuda.device(device(args.device)):
                            inp.cuda()
                            lbl.cuda()
                            model.cuda()
<commit_msg>Move parameters buffers and tensors to GPU if use_cuda==True.<commit_after>import os
import numpy as np

from shutil import copyfile

import torch
import torch.nn as nn
from torch.autograd import Variable

from pymongo import MongoClient

import word2vec as W2V
from word2vec import vocabulary_size
from word2vec import WindowBatchLoader

from sklearn.metrics import f1_score

from tempfile import gettempdir

import argparse


# CNN Model (2 conv layer)
class CNN(nn.Module):
    def __init__(self, emb_size, win_size, hidden_size):
        super(CNN, self).__init__()
        self.win_size = win_size
        self.emb_size = emb_size
        self.emb = nn.Embedding(vocabulary_size, emb_size)
        self.layer1 = nn.Sequential(
            nn.Conv2d(1, 64, kernel_size=emb_size, padding=2),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.MaxPool2d(2))
        self.layer2 = nn.Sequential(
            nn.Conv2d(64, 32, kernel_size=emb_size, padding=2),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(2))
        self.layer3 = nn.Sequential(
            nn.Linear(160, hidden_size),
            nn.BatchNorm1d(hidden_size),
            nn.Sigmoid())
        self.fc = nn.Linear(hidden_size, vocabulary_size)
        self.sf = nn.Softmax()

    def forward(self, x):
        out = self.emb(x).unsqueeze(1)
        out = self.layer1(out)
        out = self.layer2(out)
        out = out.view(out.size(0), -1)
        out = self.layer3(out)
        out = self.fc(out)
        out = self.sf(out)
        return out


class MLP(nn.Module):

    def __init__(self, emb_size, win_size, hidden_size):
        super(MLP, self).__init__()
        self.win_size = win_size
        self.emb_size = emb_size
        self.emb = nn.Embedding(vocabulary_size, emb_size)
        self.layer1 = nn.Sequential(
            nn.Linear(2 * win_size * emb_size, hidden_size),
            nn.BatchNorm1d(hidden_size),
            nn.Sigmoid())
        self.layer2 = nn.Sequential(
            nn.Linear(hidden_size, hidden_size // 2),
            nn.BatchNorm1d(hidden_size // 2),
            nn.Sigmoid())
        self.layer3 = nn.Sequential(
            nn.Linear(hidden_size // 2, hidden_size),
            nn.BatchNorm1d(hidden_size),
            nn.Sigmoid())
        self.fc = nn.Linear(hidden_size, vocabulary_size)
        self.sf = nn.Softmax()

    def forward(self, x):
        out = self.emb(x)
        out = out.view(out.size(0), -1)
        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.fc(out)
        out = self.sf(out)
        return out


def device(device_str):
    return int(device_str[-1])


def train(model, train_loader, test_loader):

    # Hyper Parameters
    num_epochs = args.num_epochs

    # Loss and Optimizer
    optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=1e-4)

    criterion = nn.CrossEntropyLoss()
    train_loss = 0
    best_loss = np.inf
    start_epoch = 0

    # optionally resume from a checkpoint
    if args.resume:
        if os.path.isfile(args.resume):
            print("=> loading checkpoint '%s'" % args.resume)
            checkpoint = torch.load(args.resume)
            start_epoch = checkpoint['epoch']
            best_loss = checkpoint['best_loss']
            model.load_state_dict(checkpoint['state_dict'])
            optimizer.load_state_dict(checkpoint['optimizer'])
            print("=> loaded checkpoint '%s' (epoch %s)" %
                  (args.resume, checkpoint['epoch'] + 1))
        else:
            print("=> no checkpoint found at '%s'" % args.resume)

    if use_cuda:
        with torch.cuda.device(device(args.device)):
            model.cuda()

    for epoch in range(start_epoch, num_epochs):

        for step, (batch_inputs, batch_labels) in enumerate(train_loader):

            inp = torch.from_numpy(batch_inputs).long()
            lbl = torch.from_numpy(batch_labels).long().view(-1)

            if use_cuda:
                with torch.cuda.device(device(args.device)):
                    inp = inp.cuda()
                    lbl = lbl.cuda()

            x = Variable(inp)
            y = Variable(lbl)

            model.train()
            optimizer.zero_grad()
            y_hat = model(x)

            loss = criterion(y_hat, y)

            train_loss += loss.data[0]
            loss.backward()
            optimizer.step()

            # loss = p_pos.log() + p_neg.log()

            if (step + 1) % args.steps_per_stats == 0:
                test_loss = 0
                f1 = 0
                for i, (batch_inputs, batch_labels) in enumerate(test_loader):

                    inp = torch.from_numpy(batch_inputs).long()
                    lbl = torch.from_numpy(batch_labels).long().view(-1)

                    if use_cuda:
                        with torch.cuda.device(device(args.device)):
                            inp = inp.cuda()
                            lbl = lbl.cuda()

{'fraction_non_alphanumeric': 0.1119718309859155, 'fraction_numerical': 0.01488933601609658, 'mean_word_length': 1.8847939640162508, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '739281', 'n_tokens_mistral': 3261, 'n_tokens_neox': 3127, 'n_words': 641}