<commit_before>import itertools
from collections import namedtuple

import numpy as np
import theano
from theano import tensor as T


_Weights = namedtuple('_Weights', ['w0', 'w1', 'v'])


class _FactorizationMachine(object):
    """Base class for factorization machines.

    Warning: This class should not be used directly. Use derived classes
    instead (FactorizationMachineClassifier and FactorizationMachineRegressor).
    """
    def __init__(self,
                 feature_count,
                 classifier=False,
                 k = 8,
                 stdev = 0.1):
        self.classifier = classifier
        d = feature_count

        # *** Symbolic variables ***
        X = T.matrix()
        y = T.vector()
        beta_w1 = T.scalar()
        beta_v = T.scalar()

        # *** Model parameters ***
        # bias term (intercept)
        w0_init = np.zeros(1)
        self.w0 = theano.shared(w0_init, allow_downcast=True)
        # first order coefficients
        w1_init = np.zeros(d)
        self.w1 = theano.shared(w1_init, allow_downcast=True)
        # interaction factors
        v_init = stdev * np.random.randn(k, d)
        self.v = theano.shared(v_init, allow_downcast=True)

        # *** The Model ***
        # The formula for pairwise interactions is from the bottom left
        # of page 997 of Rendle 2010, "Factorization Machines."
        # This version scales linearly in k and d, as opposed to O(d^2).
        interactions = 0.5 * T.sum((T.dot(X, T.transpose(self.v)) ** 2) - \
                                   T.dot(X ** 2, T.transpose(self.v ** 2)), axis=1)
        y_hat = T.addbroadcast(self.w0,0) + T.dot(X, self.w1) + interactions
        if self.classifier:
            y_hat = T.nnet.sigmoid(y_hat)

        # *** Loss Function ***
        if self.classifier:
            error = T.mean(T.nnet.binary_crossentropy(y_hat, y))
        else:
            error = T.mean((y - y_hat)**2)
        # regularization
        L2 = beta_w1 * T.mean(self.w1 ** 2) + beta_v * T.mean(self.v ** 2)
        loss = error + L2

        # *** Learning ***
        updates = []
        params = [self.w0, self.w1, self.v]
        grads = T.grad(cost=loss, wrt=params)
        # rmsprop TODO: adam
        lr, rho, epsilon = 0.001, 0.9, 1e-6
        for p, g in zip(params, grads):
            acc = theano.shared(p.get_value() * 0.)
            acc_new = rho * acc + (1 - rho) * g ** 2
            gradient_scaling = T.sqrt(acc_new + epsilon)
            g = g / gradient_scaling
            updates.append((acc, acc_new))
            updates.append((p, p - lr * g))

        self.theano_train = theano.function(
            inputs=[X, y, beta_w1, beta_v], outputs=loss, updates=updates, allow_input_downcast=True)

        self.theano_cost = theano.function(
            inputs=[X, y, beta_w1, beta_v], outputs=loss, allow_input_downcast=True)

        # *** Prediction ***
        self.theano_predict = theano.function(
            inputs=[X], outputs=y_hat, allow_input_downcast=True)


    def get_weights(self):
        """Returns a _Weights namedtuple"""
        return _Weights(*(w.get_value() for w in (self.w0, self.w1, self.v)))


    def set_weights(self, weights):
        """Sets weights from a _Weights namedtuple"""
        self.w0.set_value(weights.w0)
        self.w1.set_value(weights.w1)
        self.v.set_value(weights.v)


    def fit(self, X, y,
            batch_size=32,
            nb_epoch=10,
            shuffle=True,
            verbose=False,
            beta_w1=0.0,
            beta_v=0.0):
        """Learns the weights of a factorization machine with mini-batch gradient
        descent. The weights that minimize the loss function (across epochs) are
        retained."""
        # TODO: support for validation
        n = X.shape[0]
        if batch_size > n:
            batch_size = n
        min_loss = float('inf')
        min_loss_weights = self.get_weights()
        for i in range(nb_epoch):
            if shuffle:
                indices = np.arange(n)
                np.random.shuffle(indices)
                X, y = X[indices], y[indices]
            for start in itertools.count(0, batch_size):
                if start >= n:
                    break
                stop = min(start + batch_size, n)
                self.theano_train(X[start:stop], y[start:stop], beta_w1, beta_v)
            current_loss = self.theano_cost(X, y, beta_w1, beta_v)
            if current_loss < min_loss:
                min_loss = current_loss
                min_loss_weights = self.get_weights()<commit_msg>bigger batches<commit_after>import itertools
from collections import namedtuple

import numpy as np
import theano
from theano import tensor as T


_Weights = namedtuple('_Weights', ['w0', 'w1', 'v'])


class _FactorizationMachine(object):
    """Base class for factorization machines.

    Warning: This class should not be used directly. Use derived classes
    instead (FactorizationMachineClassifier and FactorizationMachineRegressor).
    """
    def __init__(self,
                 feature_count,
                 classifier=False,
                 k = 8,
                 stdev = 0.1):
        self.classifier = classifier
        d = feature_count

        # *** Symbolic variables ***
        X = T.matrix()
        y = T.vector()
        beta_w1 = T.scalar()
        beta_v = T.scalar()

        # *** Model parameters ***
        # bias term (intercept)
        w0_init = np.zeros(1)
        self.w0 = theano.shared(w0_init, allow_downcast=True)
        # first order coefficients
        w1_init = np.zeros(d)
        self.w1 = theano.shared(w1_init, allow_downcast=True)
        # interaction factors
        v_init = stdev * np.random.randn(k, d)
        self.v = theano.shared(v_init, allow_downcast=True)

        # *** The Model ***
        # The formula for pairwise interactions is from the bottom left
        # of page 997 of Rendle 2010, "Factorization Machines."
        # This version scales linearly in k and d, as opposed to O(d^2).
        interactions = 0.5 * T.sum((T.dot(X, T.transpose(self.v)) ** 2) - \
                                   T.dot(X ** 2, T.transpose(self.v ** 2)), axis=1)
        y_hat = T.addbroadcast(self.w0,0) + T.dot(X, self.w1) + interactions
        if self.classifier:
            y_hat = T.nnet.sigmoid(y_hat)

        # *** Loss Function ***
        if self.classifier:
            error = T.mean(T.nnet.binary_crossentropy(y_hat, y))
        else:
            error = T.mean((y - y_hat)**2)
        # regularization
        L2 = beta_w1 * T.mean(self.w1 ** 2) + beta_v * T.mean(self.v ** 2)
        loss = error + L2

        # *** Learning ***
        updates = []
        params = [self.w0, self.w1, self.v]
        grads = T.grad(cost=loss, wrt=params)
        # rmsprop TODO: adam
        lr, rho, epsilon = 0.001, 0.9, 1e-6
        for p, g in zip(params, grads):
            acc = theano.shared(p.get_value() * 0.)
            acc_new = rho * acc + (1 - rho) * g ** 2
            gradient_scaling = T.sqrt(acc_new + epsilon)
            g = g / gradient_scaling
            updates.append((acc, acc_new))
            updates.append((p, p - lr * g))

        self.theano_train = theano.function(
            inputs=[X, y, beta_w1, beta_v], outputs=loss, updates=updates, allow_input_downcast=True)

        self.theano_cost = theano.function(
            inputs=[X, y, beta_w1, beta_v], outputs=loss, allow_input_downcast=True)

        # *** Prediction ***
        self.theano_predict = theano.function(
            inputs=[X], outputs=y_hat, allow_input_downcast=True)


    def get_weights(self):
        """Returns a _Weights namedtuple"""
        return _Weights(*(w.get_value() for w in (self.w0, self.w1, self.v)))


    def set_weights(self, weights):
        """Sets weights from a _Weights namedtuple"""
        self.w0.set_value(weights.w0)
        self.w1.set_value(weights.w1)
        self.v.set_value(weights.v)


    def fit(self, X, y,
            batch_size=50,
            nb_epoch=10,
            shuffle=True,
            verbose=False,
            beta_w1=0.0,
            beta_v=0.0):
        """Learns the weights of a factorization machine with mini-batch gradient
        descent. The weights that minimize the loss function (across epochs) are
        retained."""
        # TODO: support for validation
        n = X.shape[0]
        if batch_size > n:
            batch_size = n
        min_loss = float('inf')
        min_loss_weights = self.get_weights()
        for i in range(nb_epoch):
            if shuffle:
                indices = np.arange(n)
                np.random.shuffle(indices)
                X, y = X[indices], y[indices]
            for start in itertools.count(0, batch_size):
                if start >= n:
                    break
                stop = min(start + batch_size, n)
                self.theano_train(X[start:stop], y[start:stop], beta_w1, beta_v)
            current_loss = self.theano_cost(X, y, beta_w1, beta_v)
            if current_loss < min_loss:
                min_loss = current_loss
                min_loss_weights = self.get_weights()
{'fraction_non_alphanumeric': 0.11447296559257236, 'fraction_numerical': 0.015510649918077553, 'mean_word_length': 1.9029803424223208, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '445382', 'n_tokens_mistral': 2978, 'n_tokens_neox': 2860, 'n_words': 702}