<commit_before>from __future__ import division

from operator import getitem

import dask.array as da
import dask.dataframe as dd
import numpy as np
import pandas as pd
import scipy.sparse
from sklearn.preprocessing import label as sklabel
from sklearn.utils.validation import check_is_fitted


class LabelEncoder(sklabel.LabelEncoder):
    """Encode labels with value between 0 and n_classes-1.

    .. note::

       This differs from the scikit-learn version for Categorical data.
       When passed a categorical `y`, this implementation will use the
       categorical information for the label encoding and transformation.
       You will receive different answers when

       1. Your categories are not monotonically increasing
       2. You have unobserved categories

       Specify ``use_categorical=False`` to recover the scikit-learn behavior.

    Parameters
    ----------
    use_categorical : bool, default True
        Whether to use the categorical dtype information when `y` is a
        dask or pandas Series with a categorical dtype.

    Attributes
    ----------
    classes_ : array of shape (n_class,)
        Holds the label for each class.
    dtype_ : Optional CategoricalDtype
        For Categorical `y`, the dtype is stored here.

    Examples
    --------
    `LabelEncoder` can be used to normalize labels.

    >>> from dask_ml import preprocessing
    >>> le = preprocessing.LabelEncoder()
    >>> le.fit([1, 2, 2, 6])
    LabelEncoder()
    >>> le.classes_
    array([1, 2, 6])
    >>> le.transform([1, 1, 2, 6]) #doctest: +ELLIPSIS
    array([0, 0, 1, 2]...)
    >>> le.inverse_transform([0, 0, 1, 2])
    array([1, 1, 2, 6])

    It can also be used to transform non-numerical labels (as long as they are
    hashable and comparable) to numerical labels.

    >>> le = preprocessing.LabelEncoder()
    >>> le.fit(["paris", "paris", "tokyo", "amsterdam"])
    LabelEncoder()
    >>> list(le.classes_)
    ['amsterdam', 'paris', 'tokyo']
    >>> le.transform(["tokyo", "tokyo", "paris"]) #doctest: +ELLIPSIS
    array([2, 2, 1]...)
    >>> list(le.inverse_transform([2, 2, 1]))
    ['tokyo', 'tokyo', 'paris']

    When using Dask, we strongly recommend using a Categorical dask Series if
    possible. This avoids a (potentially expensive) scan of the values and
    enables a faster `transform` algorithm.

    >>> import dask.dataframe as dd
    >>> import pandas as pd
    >>> data = dd.from_pandas(pd.Series(['a', 'a', 'b'], dtype='category'),
    ...                       npartitions=2)
    >>> le.fit_transform(data)
    dask.array<values, shape=(nan,), dtype=int8, chunksize=(nan,)>
    >>> le.fit_transform(data).compute()
    array([0, 0, 1], dtype=int8)
    """

    def __init__(self, use_categorical=True):
        self.use_categorical = use_categorical
        super(LabelEncoder, self).__init__()

    def _check_array(self, y):
        if isinstance(y, (dd.Series, pd.DataFrame)):
            y = y.squeeze()

            if y.ndim > 1:
                raise ValueError("Expected a 1-D array or Series.")

        if not self.use_categorical:
            if isinstance(y, dd.Series):
                y = da.asarray(y)
            elif isinstance(y, pd.Series):
                y = np.asarray(y)

        if isinstance(y, dd.Series):
            if pd.api.types.is_categorical_dtype(y):
                # TODO(dask-3784): just call y.cat.as_known()
                # https://github.com/dask/dask/issues/3784
                if not y.cat.known:
                    y = y.cat.as_known()
            else:
                y = da.asarray(y)
        return y

    def fit(self, y):
        y = self._check_array(y)

        if isinstance(y, da.Array):
            classes_ = _encode_dask_array(y)
            self.classes_ = classes_.compute()
            self.dtype_ = None
        elif _is_categorical(y):
            self.classes_ = _encode_categorical(y)
            self.dtype_ = y.dtype
        else:
            self.dtype_ = None
            return super(LabelEncoder, self).fit(y)

        return self

    def fit_transform(self, y):<commit_msg>Fixed CI failures in label encoder (#336)<commit_after>from __future__ import division

from operator import getitem

import dask.array as da
import dask.dataframe as dd
import numpy as np
import pandas as pd
import scipy.sparse
from sklearn.preprocessing import label as sklabel
from sklearn.utils.validation import check_is_fitted


class LabelEncoder(sklabel.LabelEncoder):
    """Encode labels with value between 0 and n_classes-1.

    .. note::

       This differs from the scikit-learn version for Categorical data.
       When passed a categorical `y`, this implementation will use the
       categorical information for the label encoding and transformation.
       You will receive different answers when

       1. Your categories are not monotonically increasing
       2. You have unobserved categories

       Specify ``use_categorical=False`` to recover the scikit-learn behavior.

    Parameters
    ----------
    use_categorical : bool, default True
        Whether to use the categorical dtype information when `y` is a
        dask or pandas Series with a categorical dtype.

    Attributes
    ----------
    classes_ : array of shape (n_class,)
        Holds the label for each class.
    dtype_ : Optional CategoricalDtype
        For Categorical `y`, the dtype is stored here.

    Examples
    --------
    `LabelEncoder` can be used to normalize labels.

    >>> from dask_ml import preprocessing
    >>> le = preprocessing.LabelEncoder()
    >>> le.fit([1, 2, 2, 6])
    LabelEncoder()
    >>> le.classes_
    array([1, 2, 6])
    >>> le.transform([1, 1, 2, 6]) #doctest: +ELLIPSIS
    array([0, 0, 1, 2]...)
    >>> le.inverse_transform([0, 0, 1, 2])
    array([1, 1, 2, 6])

    It can also be used to transform non-numerical labels (as long as they are
    hashable and comparable) to numerical labels.

    >>> le = preprocessing.LabelEncoder()
    >>> le.fit(["paris", "paris", "tokyo", "amsterdam"])
    LabelEncoder()
    >>> list(le.classes_)
    ['amsterdam', 'paris', 'tokyo']
    >>> le.transform(["tokyo", "tokyo", "paris"]) #doctest: +ELLIPSIS
    array([2, 2, 1]...)
    >>> list(le.inverse_transform([2, 2, 1]))
    ['tokyo', 'tokyo', 'paris']

    When using Dask, we strongly recommend using a Categorical dask Series if
    possible. This avoids a (potentially expensive) scan of the values and
    enables a faster `transform` algorithm.

    >>> import dask.dataframe as dd
    >>> import pandas as pd
    >>> data = dd.from_pandas(pd.Series(['a', 'a', 'b'], dtype='category'),
    ...                       npartitions=2)
    >>> le.fit_transform(data)
    dask.array<values, shape=(nan,), dtype=int8, chunksize=(nan,)>
    >>> le.fit_transform(data).compute()
    array([0, 0, 1], dtype=int8)
    """

    def __init__(self, use_categorical=True):
        self.use_categorical = use_categorical
        super(LabelEncoder, self).__init__()

    def _check_array(self, y):
        if isinstance(y, (dd.Series, pd.DataFrame)):
            y = y.squeeze()

            if y.ndim > 1:
                raise ValueError("Expected a 1-D array or Series.")

        if not self.use_categorical:
            if isinstance(y, dd.Series):
                y = y.to_dask_array(lengths=True)
            elif isinstance(y, pd.Series):
                y = np.asarray(y)

        if isinstance(y, dd.Series):
            if pd.api.types.is_categorical_dtype(y):
                # TODO(dask-3784): just call y.cat.as_known()
                # https://github.com/dask/dask/issues/3784
                if not y.cat.known:
                    y = y.cat.as_known()
            else:
                y = y.to_dask_array(lengths=True)
        return y

    def fit(self, y):
        y = self._check_array(y)

        if isinstance(y, da.Array):
            classes_ = _encode_dask_array(y)
            self.classes_ = classes_.compute()
            self.dtype_ = None
        elif _is_categorical(y):
            self.classes_ = _encode_categorical(y)
            self.dtype_ = y.dtype
        else:
            self.dtype_ = None
            return super(LabelEncoder, self).fit(y)

        return self

    def fit_transform(self, y):
{'fraction_non_alphanumeric': 0.11843697072344544, 'fraction_numerical': 0.01221872731671909, 'mean_word_length': 2.3267605633802817, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 103, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5191618', 'n_tokens_mistral': 2606, 'n_tokens_neox': 2459, 'n_words': 753}