<commit_before>"""
The :mod:`sklearn.pipeline` module implements utilities to build a composite
estimator, as a chain of transforms and estimators.
"""
# Author: <NAME>
#         <NAME>
#         <NAME>
#         <NAME>
#         <NAME>
# License: BSD

from collections import defaultdict
from abc import ABCMeta, abstractmethod

import numpy as np
from scipy import sparse

from .base import clone, BaseEstimator, TransformerMixin
from .externals.joblib import Parallel, delayed, Memory
from .externals import six
from .utils import tosequence
from .utils.metaestimators import if_delegate_has_method
from .utils import Bunch

__all__ = ['Pipeline', 'FeatureUnion']


class _BasePipeline(six.with_metaclass(ABCMeta, BaseEstimator)):
    """Handles parameter management for classifiers composed of named steps.
    """

    @abstractmethod
    def __init__(self):
        pass

    def _replace_step(self, steps_attr, name, new_val):
        # assumes `name` is a valid step name
        new_steps = getattr(self, steps_attr)[:]
        for i, (step_name, _) in enumerate(new_steps):
            if step_name == name:
                new_steps[i] = (name, new_val)
                break
        setattr(self, steps_attr, new_steps)

    def _get_params(self, steps_attr, deep=True):
        out = super(_BasePipeline, self).get_params(deep=False)
        if not deep:
            return out
        steps = getattr(self, steps_attr)
        out.update(steps)
        for name, estimator in steps:
            if estimator is None:
                continue
            for key, value in six.iteritems(estimator.get_params(deep=True)):
                out['%s__%s' % (name, key)] = value
        return out

    def _set_params(self, steps_attr, **params):
        # Ensure strict ordering of parameter setting:
        # 1. All steps
        if steps_attr in params:
            setattr(self, steps_attr, params.pop(steps_attr))
        # 2. Step replacement
        step_names, _ = zip(*getattr(self, steps_attr))
        for name in list(six.iterkeys(params)):
            if '__' not in name and name in step_names:
                self._replace_step(steps_attr, name, params.pop(name))
        # 3. Step parameters and other initilisation arguments
        super(_BasePipeline, self).set_params(**params)
        return self

    def _validate_names(self, names):
        if len(set(names)) != len(names):
            raise ValueError('Names provided are not unique: '
                             '{0!r}'.format(list(names)))
        invalid_names = set(names).intersection(self.get_params(deep=False))
        if invalid_names:
            raise ValueError('Step names conflict with constructor arguments: '
                             '{0!r}'.format(sorted(invalid_names)))
        invalid_names = [name for name in names if '__' in name]
        if invalid_names:
            raise ValueError('Step names must not contain __: got '
                             '{0!r}'.format(invalid_names))


class Pipeline(_BasePipeline):
    """Pipeline of transforms with a final estimator.

    Sequentially apply a list of transforms and a final estimator.
    Intermediate steps of the pipeline must be 'transforms', that is, they
    must implement fit and transform methods.
    The final estimator only needs to implement fit.
    The transformers in the pipeline can be cached using ```memory`` argument.

    The purpose of the pipeline is to assemble several steps that can be
    cross-validated together while setting different parameters.
    For this, it enables setting parameters of the various steps using their
    names and the parameter name separated by a '__', as in the example below.
    A step's estimator may be replaced entirely by setting the parameter
    with its name to another estimator, or a transformer removed by setting
    to None.

    Read more in the :ref:`User Guide <pipeline>`.

    Parameters
    ----------
    steps : list
        List of (name, transform) tuples (implementing fit/transform) that are
        chained, in the order in which they are chained, with the last object
        an estimator.

    memory : Instance of joblib.Memory or string, optional (default=None)
        Used to caching the fitted transformers of the transformer of the
        pipeline. By default, no cache is performed.
        If a string is given, it is the path to the caching directory.
        Enabling caching triggers a clone of the transformers before fitting.
        Therefore, the transformer instance given to the pipeline cannot be
        inspected directly. Use the attribute ``named_steps`` or ``steps``
        to inspect estimators within the pipeline.
        Caching the transformers is advantageous when fitting is time
        consuming.


    Attributes
    ----------
    named_steps : bunch object, a dictionary with attribute access
        Read-only attribute to access any step parameter by user given name.
        Keys are step names and values are steps parameters.

    Examples<commit_msg>[MRG+1] DOC remove repetition in the Pipeline/memory doc (#8669)<commit_after>"""
The :mod:`sklearn.pipeline` module implements utilities to build a composite
estimator, as a chain of transforms and estimators.
"""
# Author: <NAME>
#         <NAME>
#         <NAME>
#         <NAME>
#         <NAME>
# License: BSD

from collections import defaultdict
from abc import ABCMeta, abstractmethod

import numpy as np
from scipy import sparse

from .base import clone, BaseEstimator, TransformerMixin
from .externals.joblib import Parallel, delayed, Memory
from .externals import six
from .utils import tosequence
from .utils.metaestimators import if_delegate_has_method
from .utils import Bunch

__all__ = ['Pipeline', 'FeatureUnion']


class _BasePipeline(six.with_metaclass(ABCMeta, BaseEstimator)):
    """Handles parameter management for classifiers composed of named steps.
    """

    @abstractmethod
    def __init__(self):
        pass

    def _replace_step(self, steps_attr, name, new_val):
        # assumes `name` is a valid step name
        new_steps = getattr(self, steps_attr)[:]
        for i, (step_name, _) in enumerate(new_steps):
            if step_name == name:
                new_steps[i] = (name, new_val)
                break
        setattr(self, steps_attr, new_steps)

    def _get_params(self, steps_attr, deep=True):
        out = super(_BasePipeline, self).get_params(deep=False)
        if not deep:
            return out
        steps = getattr(self, steps_attr)
        out.update(steps)
        for name, estimator in steps:
            if estimator is None:
                continue
            for key, value in six.iteritems(estimator.get_params(deep=True)):
                out['%s__%s' % (name, key)] = value
        return out

    def _set_params(self, steps_attr, **params):
        # Ensure strict ordering of parameter setting:
        # 1. All steps
        if steps_attr in params:
            setattr(self, steps_attr, params.pop(steps_attr))
        # 2. Step replacement
        step_names, _ = zip(*getattr(self, steps_attr))
        for name in list(six.iterkeys(params)):
            if '__' not in name and name in step_names:
                self._replace_step(steps_attr, name, params.pop(name))
        # 3. Step parameters and other initilisation arguments
        super(_BasePipeline, self).set_params(**params)
        return self

    def _validate_names(self, names):
        if len(set(names)) != len(names):
            raise ValueError('Names provided are not unique: '
                             '{0!r}'.format(list(names)))
        invalid_names = set(names).intersection(self.get_params(deep=False))
        if invalid_names:
            raise ValueError('Step names conflict with constructor arguments: '
                             '{0!r}'.format(sorted(invalid_names)))
        invalid_names = [name for name in names if '__' in name]
        if invalid_names:
            raise ValueError('Step names must not contain __: got '
                             '{0!r}'.format(invalid_names))


class Pipeline(_BasePipeline):
    """Pipeline of transforms with a final estimator.

    Sequentially apply a list of transforms and a final estimator.
    Intermediate steps of the pipeline must be 'transforms', that is, they
    must implement fit and transform methods.
    The final estimator only needs to implement fit.
    The transformers in the pipeline can be cached using ```memory`` argument.

    The purpose of the pipeline is to assemble several steps that can be
    cross-validated together while setting different parameters.
    For this, it enables setting parameters of the various steps using their
    names and the parameter name separated by a '__', as in the example below.
    A step's estimator may be replaced entirely by setting the parameter
    with its name to another estimator, or a transformer removed by setting
    to None.

    Read more in the :ref:`User Guide <pipeline>`.

    Parameters
    ----------
    steps : list
        List of (name, transform) tuples (implementing fit/transform) that are
        chained, in the order in which they are chained, with the last object
        an estimator.

    memory : Instance of joblib.Memory or string, optional (default=None)
        Used to cache the fitted transformers of the pipeline. By default,
        no caching is performed. If a string is given, it is the path to
        the caching directory. Enabling caching triggers a clone of
        the transformers before fitting. Therefore, the transformer
        instance given to the pipeline cannot be inspected
        directly. Use the attribute ``named_steps`` or ``steps`` to
        inspect estimators within the pipeline. Caching the
        transformers is advantageous when fitting is time consuming.

    Attributes
    ----------
    named_steps : bunch object, a dictionary with attribute access
        Read-only attribute to access any step parameter by user given name.
        Keys are step names and values are steps parameters.

    Examples
{'fraction_non_alphanumeric': 0.07560035576638008, 'fraction_numerical': 0.0016800079059195573, 'mean_word_length': 2.632447954055994, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4583510', 'n_tokens_mistral': 2666, 'n_tokens_neox': 2570, 'n_words': 1054}