<commit_before>#!/usr/bin/env python3

"""
A Deep Neural Network (multilayer Perceptron) sklearn-style estimator abstract
class.

Similar to sklearn.neural_network.MLPClassifier, but using TensorFlow.
"""

from abc import ABCMeta, abstractmethod
import logging
import re
from warnings import warn

import numpy as np
import scipy.sparse as sp

from sklearn.base import BaseEstimator
from sklearn.utils import (check_array, check_random_state, check_X_y,
                           DataConversionWarning)

from sklearn.utils.validation import NotFittedError

import tensorflow as tf
from tensorflow.python.framework.ops import Graph
from tensorflow.python.framework import random_seed as tf_random_seed

from muffnn.core import TFPicklingBase, affine


_LOGGER = logging.getLogger(__name__)


class MLPBaseEstimator(TFPicklingBase, BaseEstimator, metaclass=ABCMeta):
    """Base class for multilayer perceptron models

    Notes
    -----
    There is currently no dropout for sparse input layers.
    """

    def _transform_targets(self, y):
        # This can be overridden to, e.g., map label names to indices when
        # fitting a classifier.
        return y

    def fit(self, X, y):
        """Fit the model.

        Parameters
        ----------
        X : numpy array or sparse matrix of shape [n_samples, n_features]
            Training data
        y : numpy array of shape [n_samples, n_targets]
            Target values

        Returns
        -------
        self : returns an instance of self.
        """
        _LOGGER.info("Fitting %s", re.sub(r"\s+", r" ", repr(self)))

        # Mark the model as not fitted (i.e., not fully initialized based on
        # the data).
        self._is_fitted = False

        # Call partial fit, which will initialize and then train the model.
        return self.partial_fit(X, y)

    def _fit_targets(self, y):
        # This can be overwritten to set instance variables that pertain to the
        # targets (e.g., an array of class labels).
        pass

    def partial_fit(self, X, y, **kwargs):
        """Fit the model on a batch of training data.

        Parameters
        ----------
        X : numpy array or sparse matrix of shape [n_samples, n_features]
            Training data
        y : numpy array of shape [n_samples, n_targets]
            Target values

        Returns
        -------
        self : returns an instance of self.
        """

        X, y = self._check_inputs(X, y)

        random_state = check_random_state(self.random_state)
        assert self.batch_size > 0, "batch_size <= 0"

        # Initialize the model if it hasn't been already by a previous call.
        if self._is_fitted:
            y = self._transform_targets(y)
        else:
            self._fit_targets(y, **kwargs)
            y = self._transform_targets(y)

            self.is_sparse_ = sp.issparse(X)
            self.input_layer_sz_ = X.shape[1]

            # Instantiate the graph.  TensorFlow seems easier to use by just
            # adding to the default graph, and as_default lets you temporarily
            # set a graph to be treated as the default graph.
            self.graph_ = Graph()
            with self.graph_.as_default():
                tf_random_seed.set_random_seed(
                    random_state.randint(0, 10000000))

                tf.get_variable_scope().set_initializer(
                    tf.uniform_unit_scaling_initializer(self.init_scale))

                self._build_tf_graph()

                # Train model parameters.
                self._session.run(tf.initialize_all_variables())

            # Set an attributed to mark this as at least partially fitted.
            self._is_fitted = True

        # Train the model with the given data.
        with self.graph_.as_default():
            n_examples = X.shape[0]
            start_idx = 0
            epoch = 0
            indices = np.arange(n_examples)
            random_state.shuffle(indices)

            while True:
                batch_ind = indices[start_idx:start_idx + self.batch_size]
                feed_dict = self._make_feed_dict(X[batch_ind],
                                                 y[batch_ind])
                obj_val, _ = self._session.run(
                    [self._obj_func, self._train_step], feed_dict=feed_dict)
                _LOGGER.info("objective: %.4f, epoch: %d, idx: %d",
                             obj_val, epoch, start_idx)
                start_idx += self.batch_size
                if start_idx > n_examples - self.batch_size:
                    start_idx = 0
                    epoch += 1
                    if epoch >= self.n_epochs:
                        break
                    random_state.shuffle(indices)

        return self

    def _check_inputs(self, X, y):
        # Check that the input X is an array or sparse matrix.
        # Convert to CSR if it's in another sparse format.
        X, y = check_X_y(X, y, accept_sparse='csr', multi_output=True)

        if y.ndim == 2 and y.shape[1] == 1:
            # Following
            # https://github.com/scikit-learn/scikit-learn/blob/51a765a/sklearn/ensemble/forest.py#L223,
            # issue a warning if an Nx1 array was provided.
            warn("A column-vector y was passed when a 1d array was"
                 " expected. Please change the shape of y to "
                 "(n_samples,), for example using ravel().",
                 DataConversionWarning, stacklevel=2)
            y = y[:, 0]
        return X, y

    def __getstate__(self):
        # Handles TF persistence
        state = super().__getstate__()

        # Add attributes of this estimator
        state.update(dict(activation=self.activation,
                          batch_size=self.batch_size,
                          dropout=self.dropout,
                          hidden_units=self.hidden_units,
                          init_scale=self.init_scale,
                          random_state=self.random_state,
                          n_epochs=self.n_epochs,
                          ))

        # Add fitted attributes if the model has been fitted.
        if self._is_fitted:
            state['input_layer_sz_'] = self.input_layer_sz_
            state['is_sparse_'] = self.is_sparse_

        return state

    @abstractmethod
    def _init_model_output(self, t):
        pass

    @abstractmethod
    def _init_model_objective_fn(self, t):
        pass

    def _set_up_graph(self):
        """Initialize TF objects (needed before fitting or restoring)."""

        # A placeholder to control dropout for training vs. prediction.
        self._dropout = \
            tf.placeholder(dtype=np.float32, shape=(), name="dropout")

        # Input layers.
        if self.is_sparse_:
            self._input_indices = \
                tf.placeholder(np.int64, [None, 2], "input_indices")
            self._input_values = \
                tf.placeholder(np.float32, [None], "input_values")
            self._input_shape = \
                tf.placeholder(np.int64, [2], "input_shape")
            # t will be the current layer as we build up the graph below.
            t = tf.SparseTensor(self._input_indices, self._input_values,
                                self._input_shape)
        else:
            self._input_values = \
                tf.placeholder(np.float32, [None, self.input_layer_sz_],
                               "input_values")
            t = self._input_values

        # Hidden layers.
        for i, layer_sz in enumerate(self.hidden_units):
            if self.is_sparse_ and i == 0:
                t = affine(t, layer_sz, input_size=self.input_layer_sz_,
                           scope='layer_%d' % i, sparse_input=True)
            else:
                t = tf.nn.dropout(t, keep_prob=self._dropout)
                t = affine(t, layer_sz, scope='layer_%d' % i)
            t = t if self.activation is None else self.activation(t)

        # The output layer and objective function depend on the model
        # (e.g., classification vs regression).
        t = self._init_model_output(t)
        self._init_model_objective_fn(t)

        self._train_step = tf.train.AdamOptimizer().minimize(self._obj_func)

    def _make_feed_dict(self, X, y=None):
        # Make the dictionary mapping tensor placeholders to input data.

        if self.is_sparse_:
            # TF's sparse matrix is initialized by DoK data.
            X = X.todok()
            # Make sure the input is a 2-d array.
            indices = np.array(list(X.keys()) if X.nnz > 0
                               else np.zeros((0, 2)))
            values = np.array(list(X.values()))

            feed_dict = {
                self._input_indices: indices,
                self._input_values: values,
                self._input_shape: X.shape
            }
        else:
            feed_dict = {
                self._input_values: X
            }

        if y is None:
            # If y is None, then we are doing prediction and should fix
            # dropout.
            feed_dict[self._dropout] = 1.0
        else:
            feed_dict[self.input_targets_] = y
            feed_dict[self._dropout] = \
                self.dropout if self.dropout is not None else 1.0

        return feed_dict

    def _compute_output(self, X):
        """Get the outputs of the network, for use in prediction methods."""

        if not self._is_fitted:
            raise NotFittedError("Call fit before prediction")

        X = check_array(X, accept_sparse=['csr', 'dok', 'lil', 'csc', 'coo'])

        if self.is_sparse_:
            # For sparse input, make the input a CSR matrix since it can be
            # indexed by row.
            X = X.tocsr() if sp.issparse(X) else sp.csr_matrix(X)
        elif sp.issparse(X):
            # Convert sparse input to dense.
            X = X.todense().A

        # Make predictions in batches.
        pred_batches = []
        start_idx = 0
        n_examples = X.shape[0]
        with self.graph_.as_default():
            while start_idx < n_examples:
                X_batch = \
                    X[start_idx:min(start_idx + self.batch_size, n_examples)]
                feed_dict = self._make_feed_dict(X_batch)
                start_idx += self.batch_size
                pred_batches.append(
                    self._session.run(self.output_layer_, feed_dict=feed_dict))
        y_pred = np.concatenate(pred_batches)
        return y_pred

    @abstractmethod
    def predict(self, X):
        pass
<commit_msg>added monitor kwarg to MLP<commit_after>#!/usr/bin/env python3

"""
A Deep Neural Network (multilayer Perceptron) sklearn-style estimator abstract
class.

Similar to sklearn.neural_network.MLPClassifier, but using TensorFlow.
"""

from abc import ABCMeta, abstractmethod
import logging
import re
from warnings import warn

import numpy as np
import scipy.sparse as sp

from sklearn.base import BaseEstimator
from sklearn.utils import (check_array, check_random_state, check_X_y,
                           DataConversionWarning)

from sklearn.utils.validation import NotFittedError

import tensorflow as tf
from tensorflow.python.framework.ops import Graph
from tensorflow.python.framework import random_seed as tf_random_seed

from muffnn.core import TFPicklingBase, affine


_LOGGER = logging.getLogger(__name__)


class MLPBaseEstimator(TFPicklingBase, BaseEstimator, metaclass=ABCMeta):
    """Base class for multilayer perceptron models

    Notes
    -----
    There is currently no dropout for sparse input layers.
    """

    def _transform_targets(self, y):
        # This can be overridden to, e.g., map label names to indices when
        # fitting a classifier.
        return y

    def fit(self, X, y, monitor=None):
        """Fit the model.

        Parameters
        ----------
        X : numpy array or sparse matrix of shape [n_samples, n_features]
            Training data
        y : numpy array of shape [n_samples, n_targets]
            Target values
        monitor : callable, optional
            The monitor is called after each iteration with the current
            iteration, a reference to the estimator, and a dictionary with
            {'loss': loss_value} representing the loss calculated by the
            objective function at this iteration.
            If the callable returns True the fitting procedure is stopped.
            The monitor can be used for various things such as computing
            held-out estimates, early stopping, model introspection,
            and snapshoting.

        Returns
        -------
        self : returns an instance of self.
        """
        _LOGGER.info("Fitting %s", re.sub(r"\s+", r" ", repr(self)))

        # Mark the model as not fitted (i.e., not fully initialized based on
        # the data).
        self._is_fitted = False

        # Call partial fit, which will initialize and then train the model.
        return self.partial_fit(X, y, monitor=monitor)

    def _fit_targets(self, y):
        # This can be overwritten to set instance variables that pertain to the
        # targets (e.g., an array of class labels).
        pass

    def partial_fit(self, X, y, monitor=None, **kwargs):
        """Fit the model on a batch of training data.

        Parameters
        ----------
        X : numpy array or sparse matrix of shape [n_samples, n_features]
            Training data
        y : numpy array of shape [n_samples, n_targets]
            Target values
        monitor : callable, optional
            The monitor is called after each iteration with the current
            iteration, a reference to the estimator, and a dictionary with
            {'loss': loss_value} representing the loss calculated by the
            objective function at this iteration.
            If the callable returns True the fitting procedure is stopped.
            The monitor can be used for various things such as computing
            held-out estimates, early stopping, model introspection,
            and snapshoting.

        Returns
        -------
        self : returns an instance of self.
        """

        X, y = self._check_inputs(X, y)

        random_state = check_random_state(self.random_state)
        assert self.batch_size > 0, "batch_size <= 0"

        # Initialize the model if it hasn't been already by a previous call.
        if self._is_fitted:
            y = self._transform_targets(y)
        else:
            self._fit_targets(y, **kwargs)
            y = self._transform_targets(y)

            self.is_sparse_ = sp.issparse(X)
            self.input_layer_sz_ = X.shape[1]

            # Instantiate the graph.  TensorFlow seems easier to use by just
            # adding to the default graph, and as_default lets you temporarily
            # set a graph to be treated as the default graph.
            self.graph_ = Graph()
            with self.graph_.as_default():
                tf_random_seed.set_random_seed(
                    random_state.randint(0, 10000000))

                tf.get_variable_scope().set_initializer(
                    tf.uniform_unit_scaling_initializer(self.init_scale))

                self._build_tf_graph()

                # Train model parameters.
                self._session.run(tf.initialize_all_variables())

            # Set an attributed to mark this as at least partially fitted.
            self._is_fitted = True

        # Train the model with the given data.
        with self.graph_.as_default():
            n_examples = X.shape[0]
            indices = np.arange(n_examples)
            random_state.shuffle(indices)

            for epoch in range(self.n_epochs):
                random_state.shuffle(indices)
                for start_idx in range(0, n_examples, self.batch_size):
                    batch_ind = indices[start_idx:start_idx + self.batch_size]
                    feed_dict = self._make_feed_dict(X[batch_ind],
                                                     y[batch_ind])
                    obj_val, _ = self._session.run(
                        [self._obj_func, self._train_step],
                        feed_dict=feed_dict)
                    _LOGGER.debug("objective: %.4f, epoch: %d, idx: %d",
                                  obj_val, epoch, start_idx)

                _LOGGER.info("objective: %.4f, epoch: %d, idx: %d",
                             obj_val, epoch, start_idx)

                if monitor:
                    stop_early = monitor(epoch, self, {'loss': obj_val})
                    if stop_early:
                        _LOGGER.info(
                            "stopping early due to monitor function.")
                        return self

        return self

    def _check_inputs(self, X, y):
        # Check that the input X is an array or sparse matrix.
        # Convert to CSR if it's in another sparse format.
        X, y = check_X_y(X, y, accept_sparse='csr', multi_output=True)

        if y.ndim == 2 and y.shape[1] == 1:
            # Following
            # https://github.com/scikit-learn/scikit-learn/blob/51a765a/sklearn/ensemble/forest.py#L223,
            # issue a warning if an Nx1 array was provided.
            warn("A column-vector y was passed when a 1d array was"
                 " expected. Please change the shape of y to "
                 "(n_samples,), for example using ravel().",
                 DataConversionWarning, stacklevel=2)
            y = y[:, 0]
        return X, y

    def __getstate__(self):
        # Handles TF persistence
        state = super().__getstate__()

        # Add attributes of this estimator
        state.update(dict(activation=self.activation,
                          batch_size=self.batch_size,
                          dropout=self.dropout,
                          hidden_units=self.hidden_units,
                          init_scale=self.init_scale,
                          random_state=self.random_state,
                          n_epochs=self.n_epochs,
                          ))

        # Add fitted attributes if the model has been fitted.
        if self._is_fitted:
            state['input_layer_sz_'] = self.input_layer_sz_
            state['is_sparse_'] = self.is_sparse_

        return state

    @abstractmethod
    def _init_model_output(self, t):
        pass

    @abstractmethod
    def _init_model_objective_fn(self, t):
        pass

    def _set_up_graph(self):
        """Initialize TF objects (needed before fitting or restoring)."""

        # A placeholder to control dropout for training vs. prediction.
        self._dropout = \
            tf.placeholder(dtype=np.float32, shape=(), name="dropout")

        # Input layers.
        if self.is_sparse_:
            self._input_indices = \
                tf.placeholder(np.int64, [None, 2], "input_indices")
            self._input_values = \
                tf.placeholder(np.float32, [None], "input_values")
            self._input_shape = \
                tf.placeholder(np.int64, [2], "input_shape")
            # t will be the current layer as we build up the graph below.
            t = tf.SparseTensor(self._input_indices, self._input_values,
                                self._input_shape)
        else:
            self._input_values = \
                tf.placeholder(np.float32, [None, self.input_layer_sz_],
                               "input_values")
            t = self._input_values

        # Hidden layers.
        for i, layer_sz in enumerate(self.hidden_units):
            if self.is_sparse_ and i == 0:
                t = affine(t, layer_sz, input_size=self.input_layer_sz_,
                           scope='layer_%d' % i, sparse_input=True)
            else:
                t = tf.nn.dropout(t, keep_prob=self._dropout)
                t = affine(t, layer_sz, scope='layer_%d' % i)
            t = t if self.activation is None else self.activation(t)

        # The output layer and objective function depend on the model
        # (e.g., classification vs regression).
        t = self._init_model_output(t)
        self._init_model_objective_fn(t)

        self._train_step = tf.train.AdamOptimizer().minimize(self._obj_func)

    def _make_feed_dict(self, X, y=None):
        # Make the dictionary mapping tensor placeholders to input data.

        if self.is_sparse_:
            # TF's sparse matrix is initialized by DoK data.
            X = X.todok()
            # Make sure the input is a 2-d array.
            indices = np.array(list(X.keys()) if X.nnz > 0
                               else np.zeros((0, 2)))
            values = np.array(list(X.values()))

            feed_dict = {
                self._input_indices: indices,
                self._input_values: values,
                self._input_shape: X.shape
            }
        else:
            feed_dict = {
                self._input_values: X
            }

        if y is None:
            # If y is None, then we are doing prediction and should fix
            # dropout.
            feed_dict[self._dropout] = 1.0
        else:
            feed_dict[self.input_targets_] = y
            feed_dict[self._dropout] = \
                self.dropout if self.dropout is not None else 1.0

        return feed_dict

    def _compute_output(self, X):
        """Get the outputs of the network, for use in prediction methods."""

        if not self._is_fitted:
            raise NotFittedError("Call fit before prediction")

        X = check_array(X, accept_sparse=['csr', 'dok', 'lil', 'csc', 'coo'])

        if self.is_sparse_:
            # For sparse input, make the input a CSR matrix since it can be
            # indexed by row.
            X = X.tocsr() if sp.issparse(X) else sp.csr_matrix(X)
        elif sp.issparse(X):
            # Convert sparse input to dense.
            X = X.todense().A

        # Make predictions in batches.
        pred_batches = []
        start_idx = 0
        n_examples = X.shape[0]
        with self.graph_.as_default():
            while start_idx < n_examples:
                X_batch = \
                    X[start_idx:min(start_idx + self.batch_size, n_examples)]
                feed_dict = self._make_feed_dict(X_batch)
                start_idx += self.batch_size
                pred_batches.append(
                    self._session.run(self.output_layer_, feed_dict=feed_dict))
        y_pred = np.concatenate(pred_batches)
        return y_pred

    @abstractmethod
    def predict(self, X):
        pass

{'fraction_non_alphanumeric': 0.08329633247491341, 'fraction_numerical': 0.004972915371636622, 'mean_word_length': 1.7360301263362488, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7442622', 'n_tokens_mistral': 6462, 'n_tokens_neox': 6107, 'n_words': 1812}