<commit_before>﻿# Copyright 2014-2016 The ODL development group
#
# This file is part of ODL.
#
# ODL is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ODL is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with ODL.  If not, see <http://www.gnu.org/licenses/>.

"""Utilities for normalization of user input."""

# Imports for common Python 2/3 codebase
from __future__ import print_function, division, absolute_import
from future import standard_library
standard_library.install_aliases()

import numpy as np


__all__ = ('normalized_index_expression', 'normalized_scalar_param_list')


def normalized_scalar_param_list(param, length, param_conv=None,
                                 keep_none=True):
    """Return a list of given length from a scalar parameter.

    The typical use case is when a single value or a sequence of
    values is accepted as input. This function makes a list from
    a given sequence or a list of identical elements from a single
    value, with cast to a given parameter type if desired.

    To distinguish a parameter sequence from a single parameter, the
    following rules are applied:

    * If ``param`` is not a sequence, it is treated as a single
    parameter (e.g. ``1``).
    * If ``len(param) == length == 1``, then ``param`` is interpreted
      as a single parameter (e.g. ``[1]`` or ``'1'``).
    * If ``len(param) == length != 1``, then ``param`` is interpreted as
      sequence of parameters.
    * Otherwise, ``param`` is interpreted as a single parameter.

    Note that this function is not applicable to parameters which are
    themselves iterable (e.g. ``'abc'`` with ``length=3`` will be
    interpreted as equivalent to ``['a', 'b', 'c']``).

    Parameters
    ----------
    param :
        Input parameter to turn into a list.
    length : positive int
        Desired length of the output list.
    param_conv : callable, optional
        Conversion applied to each list element. None means no conversion.
    keep_none : bool, optional
        If True, None is not converted.

    Returns
    -------
    plist : list
        Input parameter turned into a list of length ``length``.

    Examples
    --------
    Turn input into a list of given length, possibly by broadcasting.
    By default, no conversion is performed.

    >>> normalized_scalar_param_list((1, 2, 3), 3)
    [1, 2, 3]
    >>> normalized_scalar_param_list((1, None, 3.0), 3)
    [1, None, 3.0]

    Single parameters are broadcast to the given length.

    >>> normalized_scalar_param_list(1, 3)
    [1, 1, 1]
    >>> normalized_scalar_param_list('10', 3)
    ['10', '10', '10']
    >>> normalized_scalar_param_list(None, 3)
    [None, None, None]

    List entries can be explicitly converted using ``param_conv``. If
    ``None`` should be kept, set ``keep_none`` to True:

    >>> normalized_scalar_param_list(1, 3, param_conv=float)
    [1.0, 1.0, 1.0]
    >>> normalized_scalar_param_list('10', 3, param_conv=int)
    [10, 10, 10]
    >>> normalized_scalar_param_list((1, None, 3.0), 3, param_conv=int,
    ...                              keep_none=True)  # default
    [1, None, 3]

    The conversion parameter can be any callable:

    >>> def myconv(x):
    ...     return False if x is None else bool(x)
    >>> normalized_scalar_param_list((0, None, 3.0), 3, param_conv=myconv,
    ...                              keep_none=False)
    [False, False, True]
    """
    length, length_in = int(length), length
    if length <= 0:
        raise ValueError('`length` must be positive, got {}'.format(length_in))

    try:
        # TODO: always use this when numpy >= 1.10 can be assumed
        param = np.array(param, dtype=object, copy=True, ndmin=1)
        out_list = list(np.broadcast_to(param, (length,)))
    except AttributeError:
        # numpy.broadcast_to not available
        if np.isscalar(param):
            # Try this first, will work better with iterable input like '10'
            out_list = [param] * length
        else:
            try:
                param_len = len(param)
            except TypeError:
                # Not a sequence -> single parameter
                out_list = [param] * length
            else:
                if param_len == 1:
                    out_list = list(param) * length
                elif param_len == length:
                    out_list = list(param)
                else:
                    raise ValueError('sequence `param` has length {}, '
                                     'expected {}'.format(param_len, length))

    if len(out_list) != length:
        raise ValueError('sequence `param` has length {}, expected {}'
                         ''.format(len(out_list), length))

    if param_conv is not None:
        for i, p in enumerate(out_list):
            if p is None and keep_none:
                pass
            else:
                out_list[i] = param_conv(p)

    return out_list


def normalized_index_expression(indices, shape, int_to_slice=False):
    """Enable indexing with almost Numpy-like capabilities.

    Implements the following features:

    - Usage of general slices and sequences of slices
    - Conversion of `Ellipsis` into an adequate number of ``slice(None)``
      objects
    - Fewer indices than axes by filling up with an `Ellipsis`
    - Error checking with respect to a given shape
    - Conversion of integer indices into corresponding slices

    Parameters
    ----------
    indices : int, slice, Ellipsis or sequence of these<commit_msg>ENH: add option to return unconverted list to param helper<commit_after>﻿# Copyright 2014-2016 The ODL development group
#
# This file is part of ODL.
#
# ODL is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ODL is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with ODL.  If not, see <http://www.gnu.org/licenses/>.

"""Utilities for normalization of user input."""

# Imports for common Python 2/3 codebase
from __future__ import print_function, division, absolute_import
from future import standard_library
standard_library.install_aliases()

import numpy as np


__all__ = ('normalized_index_expression', 'normalized_scalar_param_list')


def normalized_scalar_param_list(param, length, param_conv=None,
                                 keep_none=True, return_nonconv=False):
    """Return a list of given length from a scalar parameter.

    The typical use case is when a single value or a sequence of
    values is accepted as input. This function makes a list from
    a given sequence or a list of identical elements from a single
    value, with cast to a given parameter type if desired.

    To distinguish a parameter sequence from a single parameter, the
    following rules are applied:

    * If ``param`` is not a sequence, it is treated as a single
    parameter (e.g. ``1``).
    * If ``len(param) == length == 1``, then ``param`` is interpreted
      as a single parameter (e.g. ``[1]`` or ``'1'``).
    * If ``len(param) == length != 1``, then ``param`` is interpreted as
      sequence of parameters.
    * Otherwise, ``param`` is interpreted as a single parameter.

    Note that this function is not applicable to parameters which are
    themselves iterable (e.g. ``'abc'`` with ``length=3`` will be
    interpreted as equivalent to ``['a', 'b', 'c']``).

    Parameters
    ----------
    param :
        Input parameter to turn into a list.
    length : positive int
        Desired length of the output list.
    param_conv : callable, optional
        Conversion applied to each list element. None means no conversion.
    keep_none : bool, optional
        If True, None is not converted.
    return_nonconv : bool, optional
        If True, return also the list where no conversion has been
        applied.

    Returns
    -------
    plist : list
        Input parameter turned into a list of length ``length``.
    nonconv : list
        The same as ``plist``, but without conversion. This is only
        returned if ``return_nonconv == True``.

    Examples
    --------
    Turn input into a list of given length, possibly by broadcasting.
    By default, no conversion is performed.

    >>> normalized_scalar_param_list((1, 2, 3), 3)
    [1, 2, 3]
    >>> normalized_scalar_param_list((1, None, 3.0), 3)
    [1, None, 3.0]

    Single parameters are broadcast to the given length.

    >>> normalized_scalar_param_list(1, 3)
    [1, 1, 1]
    >>> normalized_scalar_param_list('10', 3)
    ['10', '10', '10']
    >>> normalized_scalar_param_list(None, 3)
    [None, None, None]

    List entries can be explicitly converted using ``param_conv``. If
    ``None`` should be kept, set ``keep_none`` to True:

    >>> normalized_scalar_param_list(1, 3, param_conv=float)
    [1.0, 1.0, 1.0]
    >>> normalized_scalar_param_list('10', 3, param_conv=int)
    [10, 10, 10]
    >>> normalized_scalar_param_list((1, None, 3.0), 3, param_conv=int,
    ...                              keep_none=True)  # default
    [1, None, 3]

    The conversion parameter can be any callable:

    >>> def myconv(x):
    ...     return False if x is None else bool(x)
    >>> normalized_scalar_param_list((0, None, 3.0), 3, param_conv=myconv,
    ...                              keep_none=False)
    [False, False, True]
    """
    length, length_in = int(length), length
    if length <= 0:
        raise ValueError('`length` must be positive, got {}'.format(length_in))

    try:
        # TODO: always use this when numpy >= 1.10 can be assumed
        param = np.array(param, dtype=object, copy=True, ndmin=1)
        nonconv_list = list(np.broadcast_to(param, (length,)))
    except AttributeError:
        # numpy.broadcast_to not available
        if np.isscalar(param):
            # Try this first, will work better with iterable input like '10'
            nonconv_list = [param] * length
        else:
            try:
                param_len = len(param)
            except TypeError:
                # Not a sequence -> single parameter
                nonconv_list = [param] * length
            else:
                if param_len == 1:
                    nonconv_list = list(param) * length
                elif param_len == length:
                    nonconv_list = list(param)
                else:
                    raise ValueError('sequence `param` has length {}, '
                                     'expected {}'.format(param_len, length))

    if len(nonconv_list) != length:
        raise ValueError('sequence `param` has length {}, expected {}'
                         ''.format(len(nonconv_list), length))

    if param_conv is None:
        out_list = nonconv_list.copy()
    else:
        out_list = []
        for p in nonconv_list:
            if p is None and keep_none:
                out_list.append(p)
            else:
                out_list.append(param_conv(p))

    if return_nonconv:
        return out_list, nonconv_list
    else:
        return out_list


def normalized_index_expression(indices, shape, int_to_slice=False):
    """Enable indexing with almost Numpy-like capabilities.

    Implements the following features:

    - Usage of general slices and sequences of slices
    - Conversion of `Ellipsis` into an adequate number of ``slice(None)``
      objects
    - Fewer indices than axes by filling up with an `Ellipsis`
    - Error checking with respect to a given shape
    - Conversion of integer indices into corresponding slices

    Parameters
    ----------
    indices : int, slice, Ellipsis or sequence of these
{'fraction_non_alphanumeric': 0.09750585196545322, 'fraction_numerical': 0.013076115909274356, 'mean_word_length': 2.4628283957518167, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2843314', 'n_tokens_mistral': 3540, 'n_tokens_neox': 3353, 'n_words': 1349}