<commit_before>"""
@file
@brief Taken from `Tutorial on logistic regression <http://deeplearning.net/tutorial/logreg.html>`_

This tutorial introduces logistic regression using Theano and stochastic
gradient descent. Logistic regression is a probabilistic, linear classifier. It is parametrized
by a weight matrix :math:`W` and a bias vector :math:`b`. Classification is
done by projecting data points onto a set of hyperplanes, the distance to
which is used to determine a class membership probability.
Mathematically, this can be written as:

.. math::

      P(Y=i|x, W,b) &= f_i(W x + b) \\\\
                    &= \\frac {e^{W_i x + b_i}} {\\sum_j e^{W_j x + b_j}}

:math:`f_i`` is the activation function attached to the neuron *i*
(see `rectifier <https://en.wikipedia.org/wiki/Rectifier_(neural_networks)>`_).
The output of the model or prediction is then done by taking the argmax of
the vector whose i'th element is :math:`P(Y=i|x)`.

.. math::

    y_{pred} = \\arg \\max_i P(Y=i|x,W,b)


This tutorial presents a stochastic gradient descent optimization method
suitable for large datasets. Example:

::

    from src.ensae_teaching_cs.examples.theano_logreg import sgd_optimization_mnist, predict
    from pyensae.datasource import download_data

    dataset = "mnist.pkl.gz"
    if not os.path.exists(dataset):
        download_data(dataset, website ="http://deeplearning.net/data/mnist/")

    model = "log_reg_theano.bin"
    sgd_optimization_mnist(dataset=dataset, saved_model=model, n_epochs=2, fLOG=print)
    pred = predict(model, dataset, 10)

"""
import gzip
import os
import sys
import timeit
import pickle

import numpy

from pyquickhelper.loghelper import noLOG


class TheanoLogisticRegression(object):
    """
    Multi-class Logistic Regression Class

    The logistic regression is fully described by a weight matrix :math:`W`
    and bias vector :math:`b`. Classification is done by projecting data
    points onto a set of hyperplanes, the distance to which is used to
    determine a class membership probability.
    """

    def __init__(self, input, n_in, n_out):
        """ Initialize the parameters of the logistic regression

        :type input: theano.tensor.TensorType
        :param input: symbolic variable that describes the input of the
                      architecture (one minibatch)

        :type n_in: int
        :param n_in: number of input units, the dimension of the space in
                     which the datapoints lie

        :type n_out: int
        :param n_out: number of output units, the dimension of the space in
                      which the labels lie

        """
        # Delayed import as theano is not maintained anymore.
        import theano
        import theano.tensor as T

        self._theano = theano
        self._T = T

        # start-snippet-1
        # initialize with 0 the weights W as a matrix of shape (n_in, n_out)
        self.W = theano.shared(
            value=numpy.zeros(
                (n_in, n_out),
                dtype=theano.config.floatX
            ),
            name='W',
            borrow=True
        )
        # initialize the biases b as a vector of n_out 0s
        self.b = theano.shared(
            value=numpy.zeros(
                (n_out,),
                dtype=theano.config.floatX
            ),
            name='b',
            borrow=True
        )

        # symbolic expression for computing the matrix of class-membership
        # probabilities
        # Where:
        # W is a matrix where column-k represent the separation hyperplane for
        # class-k
        # x is a matrix where row-j  represents input training sample-j
        # b is a vector where element-k represent the free parameter of
        # hyperplane-k
        self.p_y_given_x = self._T.nnet.softmax(
            self._T.dot(input, self.W) + self.b)

        # symbolic description of how to compute prediction as class whose
        # probability is maximal
        self.y_pred = self._T.argmax(self.p_y_given_x, axis=1)
        # end-snippet-1

        # parameters of the model
        self.params = [self.W, self.b]

        # keep track of model input
        self.input = input

    def __getstate__(self):
        """
        Avoids pickling modules (_T, _theano).
        """
        return {k: v for k, v in self.__dict__ if k in {'W', 'b', 'p_y_given_x', 'params', 'input', 'y_pred'}}

    def __setstate__(self, d):<commit_msg>fix bug (items)<commit_after>"""
@file
@brief Taken from `Tutorial on logistic regression <http://deeplearning.net/tutorial/logreg.html>`_

This tutorial introduces logistic regression using Theano and stochastic
gradient descent. Logistic regression is a probabilistic, linear classifier. It is parametrized
by a weight matrix :math:`W` and a bias vector :math:`b`. Classification is
done by projecting data points onto a set of hyperplanes, the distance to
which is used to determine a class membership probability.
Mathematically, this can be written as:

.. math::

      P(Y=i|x, W,b) &= f_i(W x + b) \\\\
                    &= \\frac {e^{W_i x + b_i}} {\\sum_j e^{W_j x + b_j}}

:math:`f_i`` is the activation function attached to the neuron *i*
(see `rectifier <https://en.wikipedia.org/wiki/Rectifier_(neural_networks)>`_).
The output of the model or prediction is then done by taking the argmax of
the vector whose i'th element is :math:`P(Y=i|x)`.

.. math::

    y_{pred} = \\arg \\max_i P(Y=i|x,W,b)


This tutorial presents a stochastic gradient descent optimization method
suitable for large datasets. Example:

::

    from src.ensae_teaching_cs.examples.theano_logreg import sgd_optimization_mnist, predict
    from pyensae.datasource import download_data

    dataset = "mnist.pkl.gz"
    if not os.path.exists(dataset):
        download_data(dataset, website ="http://deeplearning.net/data/mnist/")

    model = "log_reg_theano.bin"
    sgd_optimization_mnist(dataset=dataset, saved_model=model, n_epochs=2, fLOG=print)
    pred = predict(model, dataset, 10)

"""
import gzip
import os
import sys
import timeit
import pickle

import numpy

from pyquickhelper.loghelper import noLOG


class TheanoLogisticRegression(object):
    """
    Multi-class Logistic Regression Class

    The logistic regression is fully described by a weight matrix :math:`W`
    and bias vector :math:`b`. Classification is done by projecting data
    points onto a set of hyperplanes, the distance to which is used to
    determine a class membership probability.
    """

    def __init__(self, input, n_in, n_out):
        """ Initialize the parameters of the logistic regression

        :type input: theano.tensor.TensorType
        :param input: symbolic variable that describes the input of the
                      architecture (one minibatch)

        :type n_in: int
        :param n_in: number of input units, the dimension of the space in
                     which the datapoints lie

        :type n_out: int
        :param n_out: number of output units, the dimension of the space in
                      which the labels lie

        """
        # Delayed import as theano is not maintained anymore.
        import theano
        import theano.tensor as T

        self._theano = theano
        self._T = T

        # start-snippet-1
        # initialize with 0 the weights W as a matrix of shape (n_in, n_out)
        self.W = theano.shared(
            value=numpy.zeros(
                (n_in, n_out),
                dtype=theano.config.floatX
            ),
            name='W',
            borrow=True
        )
        # initialize the biases b as a vector of n_out 0s
        self.b = theano.shared(
            value=numpy.zeros(
                (n_out,),
                dtype=theano.config.floatX
            ),
            name='b',
            borrow=True
        )

        # symbolic expression for computing the matrix of class-membership
        # probabilities
        # Where:
        # W is a matrix where column-k represent the separation hyperplane for
        # class-k
        # x is a matrix where row-j  represents input training sample-j
        # b is a vector where element-k represent the free parameter of
        # hyperplane-k
        self.p_y_given_x = self._T.nnet.softmax(
            self._T.dot(input, self.W) + self.b)

        # symbolic description of how to compute prediction as class whose
        # probability is maximal
        self.y_pred = self._T.argmax(self.p_y_given_x, axis=1)
        # end-snippet-1

        # parameters of the model
        self.params = [self.W, self.b]

        # keep track of model input
        self.input = input

    def __getstate__(self):
        """
        Avoids pickling modules (_T, _theano).
        """
        return {k: v for k, v in self.__dict__.items() if k in {'W', 'b', 'p_y_given_x', 'params', 'input', 'y_pred'}}

    def __setstate__(self, d):
{'fraction_non_alphanumeric': 0.08660975061783868, 'fraction_numerical': 0.001797348910357223, 'mean_word_length': 2.398091603053435, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5649573', 'n_tokens_mistral': 2685, 'n_tokens_neox': 2549, 'n_words': 915}