<commit_before>import theano.tensor as T
from theano.tensor.shared_randomstreams import RandomStreams
from theano import function
from theano import Param

from logistic_sgd import LogisticRegression  # , load_data
from mlp import HiddenLayer
from .dA import dA
from linear_regression_sgd import LinearRegression


class SdA(object):
    '''Stacked denoising auto-encoder class (SdA)

    A stacked denoising autoencoder model is obtained by stacking several
    dAs. The hidden layer of the dA at layer `i` becomes the input of
    the dA at layer `i+1`. The first layer dA gets as input the input of
    the SdA, and the hidden layer of the last dA represents the output.
    Note that after pretraining, the SdA is dealt with as a normal MLP,
    the dAs are only used to initialize the weights.
    '''

    def __init__(
        self,
        np_rng,
        hidden_layers_sizes,
        theano_rng=None,
        n_ins=None,
        n_outs=None,
        corruption_levels=None,
        method='logistic_regression',
        scoring='mean_absolute_error',
    ):
        self.tanh_layers = []
        self.dA_layers = []
        self.params = []
        self.n_layers = len(hidden_layers_sizes)
        assert self.n_layers > 0
        if not theano_rng:
            theano_rng = RandomStreams(np_rng.randint(2 ** 30))
        # allocate symbolic variables for the data
        self.x = T.matrix('x')
        # end-snippet-1
        # The SdA is an MLP, for which all weights of intermediate layers
        # are shared with a different denoising autoencoders
        # We will first construct the SdA as a deep multilayer perceptron,
        # and when constructing each sigmoidal layer we also construct a
        # denoising autoencoder that shares weights with that layer
        # During pretraining we will train these autoencoders (which will
        # lead to chainging the weights of the MLP as well)
        # During finetunining we will finish training the SdA by doing
        # stochastich gradient descent on the MLP

        # start-snippet-2
        for i in range(self.n_layers):
            # construct the sigmoidal layer
            # the size of the input is either the number of hidden units of
            # the layer below or the input size if we are on the first layer
            # the input to this layer is either the activation of the hidden
            # layer below or the input of the SdA if you are on the first
            # layer
            if i == 0:
                input_size = n_ins
                layer_input = self.x
            else:
                input_size = hidden_layers_sizes[i - 1]
                layer_input = self.tanh_layers[-1].output
            tanh_layer = HiddenLayer(
                np_rng=np_rng,
                input_dat=layer_input,
                n_in=input_size,
                n_out=hidden_layers_sizes[i],
                activation=T.tanh
                )
            # add the layer to our list of layers
            self.tanh_layers.append(tanh_layer)
            # its arguably a philosophical question...
            # but we are going to only declare that the parameters of the
            # tanh_layers are parameters of the StackedDAA
            # the visible biases in the dA are parameters of those
            # dA, but not the SdA
            self.params.extend(tanh_layer.params)

            # Construct a denoising autoencoder that shared weights with this
            # layer
            dA_layer = dA(
                np_rng=np_rng,
                theano_rng=theano_rng,
                input_dat=layer_input,
                n_visible=input_size,
                n_hidden=hidden_layers_sizes[i],
                W=tanh_layer.W,
                bhid=tanh_layer.b
            )
            self.dA_layers.append(dA_layer)
        if n_outs is not None:
            if method == 'logistic_regression':
                # We now need to add a logistic layer on top of the MLP
                self.top_layer = LogisticRegression(
                    input_dat=self.tanh_layers[-1].output,
                    n_in=hidden_layers_sizes[-1],
                    n_out=n_outs
                )

                self.params.extend(self.top_layer.params)
                # the labels are presented as 1D vector of [int] labels
                self.y = T.ivector('y')
                # construct a function that implements one step of
                # finetunining

                # compute the cost for second phase of training,
                # defined as the negative log likelihood
                self.finetune_cost = (
                    self.top_layer.negative_log_likelihood(self.y))
                # compute the gradients with respect to the model
                # parameters
                # symbolic variable that points to the number of
                # errors made on the
                # minibatch given by self.x and self.y
                self.errors = self.top_layer.errors(self.y)
            elif method == 'linear_regression':
                # the method for generalised regression goes here
                self.top_layer = LinearRegression(
                    input_dat=self.tanh_layers[-1].output,
                    n_in=hidden_layers_sizes[-1],
                    n_out=n_outs
                )

                self.params.extend(self.top_layer.params)
                # the response are presented as matix
                self.y = T.matrix('y')
                # construct a function that implements one step of
                # finetunining

                # compute the cost for second phase of training,
                self.finetune_cost = (
                    self.top_layer.errors(self.y, scoring=scoring))
                self.errors = self.finetune_cost
            else:
                raise SystemExit('Give correct method lr or gr!')
        self.y_pred = self.top_layer.y_pred

    def get_final_hidden_layer(self, input_dat):
        '''
        Given input, gets the final output.
        '''
        assert len(self.dA_layers) > 0
        h_values = []
        h_values.append(input_dat)
        for da in self.dA_layers:
            h_values.append(da.get_hidden_values(h_values[-1]))<commit_msg>start to work on SdA<commit_after>import theano.tensor as T
from theano.tensor.shared_randomstreams import RandomStreams
from theano import function
from theano import Param

from mlp import HiddenLayer
from .dA import dA


class SdA(object):
    '''Stacked denoising auto-encoder class (SdA)

    A stacked denoising autoencoder model is obtained by stacking several
    dAs. The hidden layer of the dA at layer `i` becomes the input of
    the dA at layer `i+1`. The first layer dA gets as input the input of
    the SdA, and the hidden layer of the last dA represents the output.
    Note that after pretraining, the SdA is dealt with as a normal MLP,
    the dAs are only used to initialize the weights.
    '''

    def __init__(
        self,
        np_rng,
        hidden_layers_sizes,
        theano_rng=None,
        n_ins=None,
        n_outs=None,
        corruption_levels=None,
    ):
        self.tanh_layers = []
        self.dA_layers = []
        self.params = []
        self.n_layers = len(hidden_layers_sizes)
        assert self.n_layers > 0
        if not theano_rng:
            theano_rng = RandomStreams(np_rng.randint(2 ** 30))
        # allocate symbolic variables for the data
        self.x = T.matrix('x')
        # end-snippet-1
        # The SdA is an MLP, for which all weights of intermediate layers
        # are shared with a different denoising autoencoders
        # We will first construct the SdA as a deep multilayer perceptron,
        # and when constructing each sigmoidal layer we also construct a
        # denoising autoencoder that shares weights with that layer
        # During pretraining we will train these autoencoders (which will
        # lead to chainging the weights of the MLP as well)
        # During finetunining we will finish training the SdA by doing
        # stochastich gradient descent on the MLP

        # start-snippet-2
        for i in range(self.n_layers):
            # construct the sigmoidal layer
            # the size of the input is either the number of hidden units of
            # the layer below or the input size if we are on the first layer
            # the input to this layer is either the activation of the hidden
            # layer below or the input of the SdA if you are on the first
            # layer
            if i == 0:
                input_size = n_ins
                layer_input = self.x
            else:
                input_size = hidden_layers_sizes[i - 1]
                layer_input = self.tanh_layers[-1].output
            tanh_layer = HiddenLayer(
                np_rng=np_rng,
                input_dat=layer_input,
                n_in=input_size,
                n_out=hidden_layers_sizes[i],
                activation=T.tanh
                )
            # add the layer to our list of layers
            self.tanh_layers.append(tanh_layer)
            # its arguably a philosophical question...
            # but we are going to only declare that the parameters of the
            # tanh_layers are parameters of the StackedDAA
            # the visible biases in the dA are parameters of those
            # dA, but not the SdA
            self.params.extend(tanh_layer.params)

            # Construct a denoising autoencoder that shared weights with this
            # layer
            dA_layer = dA(
                np_rng=np_rng,
                theano_rng=theano_rng,
                input_dat=layer_input,
                n_visible=input_size,
                n_hidden=hidden_layers_sizes[i],
                W=tanh_layer.W,
                bhid=tanh_layer.b
            )
            self.dA_layers.append(dA_layer)
        self.y_pred = self.top_layer.y_pred

    def get_final_hidden_layer(self, input_dat):
        '''
        Given input, gets the final output.
        '''
        assert len(self.dA_layers) > 0
        h_values = []
        h_values.append(input_dat)
        for da in self.dA_layers:
            h_values.append(da.get_hidden_values(h_values[-1]))
{'fraction_non_alphanumeric': 0.05709545188366192, 'fraction_numerical': 0.002830372828420847, 'mean_word_length': 1.7709572742022714, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6225830', 'n_tokens_mistral': 2911, 'n_tokens_neox': 2777, 'n_words': 945}