<commit_before>"""Utilities for input validation"""
# Authors: <NAME> and <NAME> and others (please update me)
# License: BSD 3 clause

import warnings
import numbers

import numpy as np
from scipy import sparse

from ..externals import six
from .fixes import safe_copy


class DataConversionWarning(UserWarning):
    "A warning on implicit data conversions happening in the code"
    pass

warnings.simplefilter("always", DataConversionWarning)


class NonBLASDotWarning(UserWarning):
    "A warning on implicit dispatch to numpy.dot"
    pass


# Silenced by default to reduce verbosity. Turn on at runtime for
# performance profiling.
warnings.simplefilter('ignore', NonBLASDotWarning)


def _assert_all_finite(X):
    """Like assert_all_finite, but only for ndarray."""
    if (X.dtype.char in np.typecodes['AllFloat'] and not np.isfinite(X.sum())
            and not np.isfinite(X).all()):
        raise ValueError("Array contains NaN or infinity.")


def assert_all_finite(X):
    """Throw a ValueError if X contains NaN or infinity.

    Input MUST be an np.ndarray instance or a scipy.sparse matrix."""

    # First try an O(n) time, O(1) space solution for the common case that
    # there everything is finite; fall back to O(n) space np.isfinite to
    # prevent false positives from overflow in sum method.
    _assert_all_finite(X.data if sparse.issparse(X) else X)


def safe_asarray(X, dtype=None, order=None, copy=False):
    """Convert X to an array or sparse matrix.

    Prevents copying X when possible; sparse matrices are passed through."""
    if sparse.issparse(X):
        if copy:
            X = X.copy()
        assert_all_finite(X.data)
        # enforces dtype on data array (order should be kept the same).
        X.data = np.asarray(X.data, dtype=dtype)
    else:
        X = np.array(X, dtype=dtype, order=order, copy=copy)
        assert_all_finite(X)
    return X


def as_float_array(X, copy=True):
    """Converts an array-like to an array of floats

    The new dtype will be np.float32 or np.float64, depending on the original
    type. The function can create a copy or modify the argument depending
    on the argument copy.

    Parameters
    ----------
    X : {array-like, sparse matrix}

    copy : bool, optional
        If True, a copy of X will be created. If False, a copy may still be
        returned if X's dtype is not a floating point type.

    Returns
    -------
    XT : {array, sparse matrix}
        An array of type np.float
    """
    if isinstance(X, np.matrix) or (not isinstance(X, np.ndarray)
                                    and not sparse.issparse(X)):
        return safe_asarray(X, dtype=np.float64, copy=copy)
    elif sparse.issparse(X) and X.dtype in [np.float32, np.float64]:
        return X.copy() if copy else X
    elif X.dtype in [np.float32, np.float64]:  # is numpy array
        return X.copy('F' if X.flags['F_CONTIGUOUS'] else 'C') if copy else X
    else:
        return X.astype(np.float32 if X.dtype == np.int32 else np.float64)


def array2d(X, dtype=None, order=None, copy=False, force_all_finite=True):
    """Returns at least 2-d array with data from X"""
    if sparse.issparse(X):
        raise TypeError('A sparse matrix was passed, but dense data '
                        'is required. Use X.toarray() to convert to dense.')
    X_2d = np.asarray(np.atleast_2d(X), dtype=dtype, order=order)
    if force_all_finite:
        _assert_all_finite(X_2d)
    if X is X_2d and copy:
        X_2d = safe_copy(X_2d)
    return X_2d


def _atleast2d_or_sparse(X, dtype, order, copy, sparse_class, convmethod,
                         force_all_finite):
    if sparse.issparse(X):
        if dtype is None or X.dtype == dtype:
            X = getattr(X, convmethod)()
        else:
            X = sparse_class(X, dtype=dtype)
        if force_all_finite:
            _assert_all_finite(X.data)
        X.data = np.array(X.data, copy=False, order=order)
    else:
        X = array2d(X, dtype=dtype, order=order, copy=copy,
                    force_all_finite=force_all_finite)
        if force_all_finite:
            _assert_all_finite(X)
    return X


def atleast2d_or_csc(X, dtype=None, order=None, copy=False,
                     force_all_finite=True):
    """Like numpy.atleast_2d, but converts sparse matrices to CSC format.

    Also, converts np.matrix to np.ndarray.
    """
    return _atleast2d_or_sparse(X, dtype, order, copy, sparse.csc_matrix,
                                "tocsc", force_all_finite)


def atleast2d_or_csr(X, dtype=None, order=None, copy=False,
                     force_all_finite=True):
    """Like numpy.atleast_2d, but converts sparse matrices to CSR format<commit_msg>BUG duplicate finity check in input validation<commit_after>"""Utilities for input validation"""
# Authors: <NAME> and <NAME> and others (please update me)
# License: BSD 3 clause

import warnings
import numbers

import numpy as np
from scipy import sparse

from ..externals import six
from .fixes import safe_copy


class DataConversionWarning(UserWarning):
    "A warning on implicit data conversions happening in the code"
    pass

warnings.simplefilter("always", DataConversionWarning)


class NonBLASDotWarning(UserWarning):
    "A warning on implicit dispatch to numpy.dot"
    pass


# Silenced by default to reduce verbosity. Turn on at runtime for
# performance profiling.
warnings.simplefilter('ignore', NonBLASDotWarning)


def _assert_all_finite(X):
    """Like assert_all_finite, but only for ndarray."""
    if (X.dtype.char in np.typecodes['AllFloat'] and not np.isfinite(X.sum())
            and not np.isfinite(X).all()):
        raise ValueError("Array contains NaN or infinity.")


def assert_all_finite(X):
    """Throw a ValueError if X contains NaN or infinity.

    Input MUST be an np.ndarray instance or a scipy.sparse matrix."""

    # First try an O(n) time, O(1) space solution for the common case that
    # there everything is finite; fall back to O(n) space np.isfinite to
    # prevent false positives from overflow in sum method.
    _assert_all_finite(X.data if sparse.issparse(X) else X)


def safe_asarray(X, dtype=None, order=None, copy=False):
    """Convert X to an array or sparse matrix.

    Prevents copying X when possible; sparse matrices are passed through."""
    if sparse.issparse(X):
        if copy:
            X = X.copy()
        assert_all_finite(X.data)
        # enforces dtype on data array (order should be kept the same).
        X.data = np.asarray(X.data, dtype=dtype)
    else:
        X = np.array(X, dtype=dtype, order=order, copy=copy)
        assert_all_finite(X)
    return X


def as_float_array(X, copy=True):
    """Converts an array-like to an array of floats

    The new dtype will be np.float32 or np.float64, depending on the original
    type. The function can create a copy or modify the argument depending
    on the argument copy.

    Parameters
    ----------
    X : {array-like, sparse matrix}

    copy : bool, optional
        If True, a copy of X will be created. If False, a copy may still be
        returned if X's dtype is not a floating point type.

    Returns
    -------
    XT : {array, sparse matrix}
        An array of type np.float
    """
    if isinstance(X, np.matrix) or (not isinstance(X, np.ndarray)
                                    and not sparse.issparse(X)):
        return safe_asarray(X, dtype=np.float64, copy=copy)
    elif sparse.issparse(X) and X.dtype in [np.float32, np.float64]:
        return X.copy() if copy else X
    elif X.dtype in [np.float32, np.float64]:  # is numpy array
        return X.copy('F' if X.flags['F_CONTIGUOUS'] else 'C') if copy else X
    else:
        return X.astype(np.float32 if X.dtype == np.int32 else np.float64)


def array2d(X, dtype=None, order=None, copy=False, force_all_finite=True):
    """Returns at least 2-d array with data from X"""
    if sparse.issparse(X):
        raise TypeError('A sparse matrix was passed, but dense data '
                        'is required. Use X.toarray() to convert to dense.')
    X_2d = np.asarray(np.atleast_2d(X), dtype=dtype, order=order)
    if force_all_finite:
        _assert_all_finite(X_2d)
    if X is X_2d and copy:
        X_2d = safe_copy(X_2d)
    return X_2d


def _atleast2d_or_sparse(X, dtype, order, copy, sparse_class, convmethod,
                         force_all_finite):
    if sparse.issparse(X):
        if dtype is None or X.dtype == dtype:
            X = getattr(X, convmethod)()
        else:
            X = sparse_class(X, dtype=dtype)
        if force_all_finite:
            _assert_all_finite(X.data)
        X.data = np.array(X.data, copy=False, order=order)
    else:
        X = array2d(X, dtype=dtype, order=order, copy=copy,
                    force_all_finite=force_all_finite)
    return X


def atleast2d_or_csc(X, dtype=None, order=None, copy=False,
                     force_all_finite=True):
    """Like numpy.atleast_2d, but converts sparse matrices to CSC format.

    Also, converts np.matrix to np.ndarray.
    """
    return _atleast2d_or_sparse(X, dtype, order, copy, sparse.csc_matrix,
                                "tocsc", force_all_finite)


def atleast2d_or_csr(X, dtype=None, order=None, copy=False,
                     force_all_finite=True):
    """Like numpy.atleast_2d, but converts sparse matrices to CSR format
{'fraction_non_alphanumeric': 0.09564939219449776, 'fraction_numerical': 0.008104073363190446, 'mean_word_length': 2.7159270998415215, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6616938', 'n_tokens_mistral': 3024, 'n_tokens_neox': 2885, 'n_words': 972}