<commit_before>"""ML-ENSEMBLE

:author: <NAME>
:copyright: 2017
:licence: MIT

Cross-validation jobs for an :class:`Evaluator` instance.
"""

from .estimation import (fit_trans,
                         _slice_array)

from ..externals.joblib import delayed
from ..utils import pickle_load
from ..utils.exceptions import FitFailedWarning
from ..externals.sklearn.base import clone

import os
import warnings

try:
    from time import perf_counter as time
except ImportError:
    from time import time


class Evaluation(object):

    """Evaluation engine.

    Run a job for an :class:`Evaluator` instance.

    Parameters
    ----------
    evaluator : :class:`Evaluator`
        Evaluator instance to run job for.
    """

    def __init__(self, evaluator):
        self.evaluator = evaluator

    def preprocess(self, parallel, X, y, dir):
        """Fit preprocessing pipelines.

        Fit all preprocessing pipelines in parallel and store as a
        ``preprocessing_`` attribute on the :class:`Evaluator`.

        Parameters
        ----------
        parallel : :class:`joblib.Parallel`
            The instance to use for parallel fitting.

        X : array-like of shape [n_samples, n_features]
            Training set to use for estimation. Can be memmaped.

        y : array-like of shape [n_samples, ]
            labels for estimation. Can be memmaped.

        dir : directory of cache to dump fitted transformers before assembly.
        """
        preprocessing = _expand_instance_list(self.evaluator.preprocessing,
                                              self.evaluator.indexer)

        parallel(delayed(fit_trans)(dir=dir,
                                    case=case,
                                    inst=instance_list,
                                    X=X,
                                    y=y,
                                    idx=tri,
                                    name=None)
                 for case, tri, _, instance_list in preprocessing)

        self.evaluator.preprocessing_ = \
            [(tup[0], pickle_load(os.path.join(dir, '%s__t' % tup[0])))
             for tup in preprocessing]

    def evaluate(self, parallel, X, y, dir):
        """cross-validation of estimators.

        Parameters
        ----------
        parallel : :class:`joblib.Parallel`
            The instance to use for parallel fitting.

        X : array-like of shape [n_samples, n_features]
            Training set to use for estimation. Can be memmaped.

        y : array-like of shape [n_samples, ]
            labels for estimation. Can be memmaped.

        dir : str
            directory of cache to dump fitted transformers before assembly.
        """
        preprocessing = dict(getattr(self.evaluator, 'preprocessing_', []))
        estimators = _expand_instance_list(self.evaluator.estimators,
                                           self.evaluator.indexer)

        scores = parallel(delayed(fit_score)(
                case=case,
                tr_list=preprocessing[case] if case in preprocessing else [],
                est_name=est_name,
                est=est,
                params=(i, params),
                X=X,
                y=y,
                idx=(tri, tei),
                scorer=self.evaluator.scorer,
                error_score=self.evaluator.error_score)
                          for case, tri, tei, est_list in estimators
                          for est_name, est in est_list
                          for i, params in
                          enumerate(
                                  self.evaluator.params[_name(case, est_name)]
                                    ))
        self.evaluator.scores_ = scores


###############################################################################
def _name(case, est_name):
    """Get correct param_dict name."""
    if case is not None:
        return case.split('__')[0], est_name.split('__')[0]
    else:
        return est_name.split('__')[0]


def fit_score(case, tr_list, est_name, est, params, X, y, idx, scorer,
              error_score):
    """Wrapper around fit function to determine how to handle exceptions."""

    if error_score is None:
        # If fit or scoring fails, we raise errors.
        return _fit_score(case, tr_list, est_name, est, params, X, y, idx,
                          scorer, error_score)

    else:
        # Otherwise, we issue a warning and set an error score.
        try:
            return _fit_score(case, tr_list, est_name, est, params, X, y, idx,
                              scorer, error_score)
        except Exception as exception:

            warnings.warn("Cross validation failed. Setting error score {}"
                          ".".format(error_score), FitFailedWarning)

            return case, est_name, params[0], error_score, error_score, 0


def _fit_score(case, tr_list, est_name, est, params, X, y, idx, scorer,
               error_score):
    """Fit an estimator and generate scores for train and test set."""
    est = clone(est).set_params(**params[1])

    # Prepare training set
    xtrain, ytrain, _ = _slice_array(X, y, idx[0])

    for tr_name, tr in tr_list:
        xtrain = tr.transform(xtrain)
<commit_msg>rebase for blend transformer<commit_after>"""ML-ENSEMBLE

:author: <NAME>
:copyright: 2017
:licence: MIT

Cross-validation jobs for an :class:`Evaluator` instance.
"""

from .estimation import (fit_trans,
                         _slice_array)

from ..externals.joblib import delayed
from ..utils import pickle_load
from ..utils.exceptions import FitFailedWarning
from ..externals.sklearn.base import clone

import os
import warnings

try:
    from time import perf_counter as time
except ImportError:
    from time import time


class Evaluation(object):

    """Evaluation engine.

    Run a job for an :class:`Evaluator` instance.

    Parameters
    ----------
    evaluator : :class:`Evaluator`
        Evaluator instance to run job for.
    """

    def __init__(self, evaluator):
        self.evaluator = evaluator

    def preprocess(self, parallel, X, y, dir):
        """Fit preprocessing pipelines.

        Fit all preprocessing pipelines in parallel and store as a
        ``preprocessing_`` attribute on the :class:`Evaluator`.

        Parameters
        ----------
        parallel : :class:`joblib.Parallel`
            The instance to use for parallel fitting.

        X : array-like of shape [n_samples, n_features]
            Training set to use for estimation. Can be memmaped.

        y : array-like of shape [n_samples, ]
            labels for estimation. Can be memmaped.

        dir : directory of cache to dump fitted transformers before assembly.
        """
        preprocessing = _expand_instance_list(self.evaluator.preprocessing,
                                              self.evaluator.indexer)

        parallel(delayed(fit_trans)(dir=dir,
                                    case=case,
                                    inst=instance_list,
                                    X=X,
                                    y=y,
                                    idx=tri,
                                    name=None)
                 for case, tri, _, instance_list in preprocessing)

        self.evaluator.preprocessing_ = \
            [(tup[0], pickle_load(os.path.join(dir, '%s__t' % tup[0])))
             for tup in preprocessing]

    def evaluate(self, parallel, X, y, dir):
        """cross-validation of estimators.

        Parameters
        ----------
        parallel : :class:`joblib.Parallel`
            The instance to use for parallel fitting.

        X : array-like of shape [n_samples, n_features]
            Training set to use for estimation. Can be memmaped.

        y : array-like of shape [n_samples, ]
            labels for estimation. Can be memmaped.

        dir : str
            directory of cache to dump fitted transformers before assembly.
        """
        preprocessing = dict(getattr(self.evaluator, 'preprocessing_', []))
        estimators = _expand_instance_list(self.evaluator.estimators,
                                           self.evaluator.indexer)

        scores = parallel(delayed(fit_score)(
                case=case,
                tr_list=preprocessing[case] if case in preprocessing else [],
                est_name=est_name,
                est=est,
                params=(i, params),
                X=X,
                y=y,
                idx=(tri, tei),
                scorer=self.evaluator.scorer,
                error_score=self.evaluator.error_score)
                          for case, tri, tei, est_list in estimators
                          for est_name, est in est_list
                          for i, params in
                          enumerate(
                                  self.evaluator.params[_name(case, est_name)]
                                    ))
        self.evaluator.scores_ = scores


###############################################################################
def _name(case, est_name):
    """Get correct param_dict name."""
    if case is not None:
        return case.split('__')[0], est_name.split('__')[0]
    else:
        return est_name.split('__')[0]


def fit_score(case, tr_list, est_name, est, params, X, y, idx, scorer,
              error_score):
    """Wrapper around fit function to determine how to handle exceptions."""

    if error_score is None:
        # If fit or scoring fails, we raise errors.
        return _fit_score(case, tr_list, est_name, est, params, X, y, idx,
                          scorer, error_score)

    else:
        # Otherwise, we issue a warning and set an error score.
        try:
            return _fit_score(case, tr_list, est_name, est, params, X, y, idx,
                              scorer, error_score)
        except Exception as exception:

            warnings.warn("Cross validation failed. Setting error score {}"
                          ".".format(error_score), FitFailedWarning)

            return case, est_name, params[0], error_score, error_score, 0


def _fit_score(case, tr_list, est_name, est, params, X, y, idx, scorer,
               error_score):
    """Fit an estimator and generate scores for train and test set."""
    est = clone(est).set_params(**params[1])

    # Prepare training set
    xtrain, ytrain, _ = _slice_array(X, y, idx[0])

    for tr_name, tr in tr_list:
        xtrain = tr.transform(xtrain)

    # We might have to rebase the training labels since a BlendEnsemble would
    # make xtrain. Since Blend is sequential, we can discard the first 'n'
    # observation until the dimensions match
    rebase = ytrain.shape[0] - xtrain.shape[0]
    ytrain = ytrain[rebase:]

{'fraction_non_alphanumeric': 0.10334883720930232, 'fraction_numerical': 0.0026046511627906975, 'mean_word_length': 1.7687355137780068, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3717151', 'n_tokens_mistral': 2925, 'n_tokens_neox': 2809, 'n_words': 869}