<commit_before>import theano
import theano.tensor as T
from lasagne import init
from lasagne.layers import Conv2DLayer, TransposedConv2DLayer, ConcatLayer, SliceLayer, DropoutLayer
from base import Model
from ..lasagne_extensions.layers import (SampleLayer, MultinomialLogDensityLayer,
                                       GaussianLogDensityLayer, StandardNormalLogDensityLayer, BernoulliLogDensityLayer,
                                       InputLayer, DenseLayer, DimshuffleLayer, ElemwiseSumLayer, ReshapeLayer,
                                       NonlinearityLayer, BatchNormLayer, get_all_params, get_output)
from ..lasagne_extensions.objectives import categorical_crossentropy, categorical_accuracy
from ..lasagne_extensions.nonlinearities import rectify, softplus, sigmoid, softmax
from ..lasagne_extensions.updates import total_norm_constraint
from ..lasagne_extensions.updates import adam
from parmesan.distributions import log_normal
from theano.tensor.shared_randomstreams import RandomStreams
import numpy as np


class ConvSDGMSSL(Model):
    """
    The :class:'SDGMSSL' class represents the implementation of the model described in the
    Auxiliary Generative Models article on Arxiv.org.
    """

    def __init__(self, input_size, n_a, n_z, n_y, qa_hid, qz_hid, qy_hid, px_hid, pa_hid, model_logger, nonlinearity=rectify,
                 n_mi_features=0, dropout_prob=0.0, px_nonlinearity=None, x_dist='bernoulli', batchnorm=False, seed=1234,
                 conv_output_size=512):

        super(ConvSDGMSSL, self).__init__(input_size**2, qz_hid + px_hid, n_a + n_z, nonlinearity, model_logger)
        self.x_dist = x_dist
        self.n_y = n_y
        self.input_size = input_size
        self.n_mi_features = n_mi_features
        self.n_a = n_a
        self.n_z = n_z
        self.batchnorm = batchnorm
        self._srng = RandomStreams(seed)


        # Decide Glorot initializaiton of weights.
        init_w = 1e-3
        hid_w = ""
        if nonlinearity == rectify or nonlinearity == softplus:
            hid_w = "relu"

        # Define symbolic variables for theano functions.
        self.sym_beta = T.scalar('beta')  # scaling constant beta
        self.sym_x_l = T.matrix('x')  # labeled inputs
        self.sym_t_l = T.matrix('t')  # labeled targets
        self.sym_x_u = T.matrix('x')  # unlabeled inputs
        self.sym_bs_l = T.iscalar('bs_l')  # number of labeled data
        self.sym_samples = T.iscalar('samples')  # MC samples
        self.sym_z = T.matrix('z')  # latent variable z
        self.sym_a = T.matrix('a')  # auxiliary variable a

        # Assist methods for collecting the layers
        def dense_layer(layer_in, n, dist_w=init.GlorotNormal, dist_b=init.Normal):
            dense = DenseLayer(layer_in, n, dist_w(hid_w), dist_b(init_w), None)
            if batchnorm:
                dense = BatchNormLayer(dense)
            if dropout_prob != 0.0:
                dense = DropoutLayer(dense, dropout_prob)
            return NonlinearityLayer(dense, self.transf)

        def stochastic_layer(layer_in, n, samples, nonlin=None):
            mu = DenseLayer(layer_in, n, init.Normal(init_w), init.Normal(init_w), nonlin)
            logvar = DenseLayer(layer_in, n, init.Normal(init_w), init.Normal(init_w), nonlin)
            return SampleLayer(mu, logvar, eq_samples=samples, iw_samples=1), mu, logvar


        # Return the number of elements in a tensor, ignoring the first (batch size)
        # axis
        def num_elems(tensor):
            try:
                num_elems = 1
                for val in tensor.output_shape[1:]:
                    num_elems *= val
                return num_elems
            except:
                return -2

        #
        # Functions that define the convolutional and deconvolutional sections of the 
        # networks (they are opposites of one another)
        #

        def conv_net(input_layer):
            if self.n_mi_features != 0:
                conv_input = SliceLayer(input_layer, indices=slice(0,input_layer.shape[1] - self.n_mi_features))
                mi_input = SliceLayer(input_layer, indices=slice(input_layer.shape[1]-2,None))
            else:
                conv_input = input_layer
                mi_input = None

            conv_input = ReshapeLayer(conv_input, (-1, 1, self.input_size, self.input_size))

            conv_layer_output_shapes = []
            output = Conv2DLayer(conv_input, 64, 5, stride=2, pad='same')
            conv_layer_output_shapes.append(output.output_shape[2])
            output = Conv2DLayer(output, 128, 5, stride=2, pad='same')
            conv_layer_output_shapes.append(output.output_shape[2])
            output = ReshapeLayer(output, (-1, num_elems(output)))
            if mi_input is not None:
                output = ConcatLayer([output, mi_input], axis=1)
            output = BatchNormLayer(DenseLayer(output, conv_output_size))
            return output, conv_layer_output_shapes

        def deconv_net(input_layer, conv_layer_output_shapes):
            output = BatchNormLayer(DenseLayer(input_layer, 128*7*7 + self.n_mi_features))
            if self.n_mi_features != 0:
                deconv_input = SliceLayer(output, indices=slice(0,128*7*7))
                mi_features = SliceLayer(output, indices=slice(128*7*7, 128*7*7 + self.n_mi_features))

            else:<commit_msg>Fixing bug in implementation of MI features<commit_after>import theano
import theano.tensor as T
from lasagne import init
from lasagne.layers import Conv2DLayer, TransposedConv2DLayer, ConcatLayer, SliceLayer, DropoutLayer
from base import Model
from ..lasagne_extensions.layers import (SampleLayer, MultinomialLogDensityLayer,
                                       GaussianLogDensityLayer, StandardNormalLogDensityLayer, BernoulliLogDensityLayer,
                                       InputLayer, DenseLayer, DimshuffleLayer, ElemwiseSumLayer, ReshapeLayer,
                                       NonlinearityLayer, BatchNormLayer, get_all_params, get_output)
from ..lasagne_extensions.objectives import categorical_crossentropy, categorical_accuracy
from ..lasagne_extensions.nonlinearities import rectify, softplus, sigmoid, softmax
from ..lasagne_extensions.updates import total_norm_constraint
from ..lasagne_extensions.updates import adam
from parmesan.distributions import log_normal
from theano.tensor.shared_randomstreams import RandomStreams
import numpy as np


class ConvSDGMSSL(Model):
    """
    The :class:'SDGMSSL' class represents the implementation of the model described in the
    Auxiliary Generative Models article on Arxiv.org.
    """

    def __init__(self, input_size, n_a, n_z, n_y, qa_hid, qz_hid, qy_hid, px_hid, pa_hid, model_logger, nonlinearity=rectify,
                 n_mi_features=0, dropout_prob=0.0, px_nonlinearity=None, x_dist='bernoulli', batchnorm=False, seed=1234,
                 conv_output_size=512):

        super(ConvSDGMSSL, self).__init__(input_size**2, qz_hid + px_hid, n_a + n_z, nonlinearity, model_logger)
        self.x_dist = x_dist
        self.n_y = n_y
        self.input_size = input_size
        self.n_mi_features = n_mi_features
        self.n_a = n_a
        self.n_z = n_z
        self.batchnorm = batchnorm
        self._srng = RandomStreams(seed)


        # Decide Glorot initializaiton of weights.
        init_w = 1e-3
        hid_w = ""
        if nonlinearity == rectify or nonlinearity == softplus:
            hid_w = "relu"

        # Define symbolic variables for theano functions.
        self.sym_beta = T.scalar('beta')  # scaling constant beta
        self.sym_x_l = T.matrix('x')  # labeled inputs
        self.sym_t_l = T.matrix('t')  # labeled targets
        self.sym_x_u = T.matrix('x')  # unlabeled inputs
        self.sym_bs_l = T.iscalar('bs_l')  # number of labeled data
        self.sym_samples = T.iscalar('samples')  # MC samples
        self.sym_z = T.matrix('z')  # latent variable z
        self.sym_a = T.matrix('a')  # auxiliary variable a

        # Assist methods for collecting the layers
        def dense_layer(layer_in, n, dist_w=init.GlorotNormal, dist_b=init.Normal):
            dense = DenseLayer(layer_in, n, dist_w(hid_w), dist_b(init_w), None)
            if batchnorm:
                dense = BatchNormLayer(dense)
            if dropout_prob != 0.0:
                dense = DropoutLayer(dense, dropout_prob)
            return NonlinearityLayer(dense, self.transf)

        def stochastic_layer(layer_in, n, samples, nonlin=None):
            mu = DenseLayer(layer_in, n, init.Normal(init_w), init.Normal(init_w), nonlin)
            logvar = DenseLayer(layer_in, n, init.Normal(init_w), init.Normal(init_w), nonlin)
            return SampleLayer(mu, logvar, eq_samples=samples, iw_samples=1), mu, logvar


        # Return the number of elements in a tensor, ignoring the first (batch size)
        # axis
        def num_elems(tensor):
            try:
                num_elems = 1
                for val in tensor.output_shape[1:]:
                    num_elems *= val
                return num_elems
            except:
                return -2

        #
        # Functions that define the convolutional and deconvolutional sections of the 
        # networks (they are opposites of one another)
        #

        def conv_net(input_layer):
            if self.n_mi_features != 0:
                conv_input = SliceLayer(input_layer, indices=slice(0,input_layer.shape[1] - self.n_mi_features))
                mi_input = SliceLayer(input_layer, indices=slice(input_layer.shape[1]-self.n_mi_features, None))
            else:
                conv_input = input_layer
                mi_input = None

            conv_input = ReshapeLayer(conv_input, (-1, 1, self.input_size, self.input_size))

            conv_layer_output_shapes = []
            output = Conv2DLayer(conv_input, 64, 5, stride=2, pad='same')
            conv_layer_output_shapes.append(output.output_shape[2])
            output = Conv2DLayer(output, 128, 5, stride=2, pad='same')
            conv_layer_output_shapes.append(output.output_shape[2])
            output = ReshapeLayer(output, (-1, num_elems(output)))
            if mi_input is not None:
                output = ConcatLayer([output, mi_input], axis=1)
            output = BatchNormLayer(DenseLayer(output, conv_output_size))
            return output, conv_layer_output_shapes

        def deconv_net(input_layer, conv_layer_output_shapes):
            output = BatchNormLayer(DenseLayer(input_layer, 128*7*7 + self.n_mi_features))
            if self.n_mi_features != 0:
                deconv_input = SliceLayer(output, indices=slice(0,128*7*7))
                mi_features = SliceLayer(output, indices=slice(128*7*7, 128*7*7 + self.n_mi_features))

            else:
{'fraction_non_alphanumeric': 0.08589127218934911, 'fraction_numerical': 0.011926775147928994, 'mean_word_length': 2.557053600789214, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6583060', 'n_tokens_mistral': 3424, 'n_tokens_neox': 3287, 'n_words': 748}