<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import numpy as np
from astropy.stats import sigma_clip

# from scikit-image
DTYPE_RANGE = {np.bool_: (False, True),
               np.bool8: (False, True),
               np.uint8: (0, 255),
               np.uint16: (0, 65535),
               np.int8: (-128, 127),
               np.int16: (-32768, 32767),
               np.float32: (-1, 1),
               np.float64: (-1, 1)}
DTYPE_RANGE.update((d.__name__, limits) for d, limits in list(DTYPE_RANGE.items()))
DTYPE_RANGE.update({'uint10': (0, 2**10 - 1), 'uint12': (0, 2**12 - 1),
                    'uint14': (0, 2**14 - 1), 'bool': DTYPE_RANGE[np.bool_],
                    'float': DTYPE_RANGE[np.float64]})


def find_imgcuts(image, min_cut=None, max_cut=None, min_percent=None,
                 max_percent=None, percent=None):
    """
    Find minimum and maximum image cut levels from percentiles of the
    image values.

    Parameters
    ----------
    image : array_like
        The 2D array of the image.

    min_cut : float, optional
        The minimum cut level.  Data values less than ``min_cut`` will
        set to ``min_cut`` before scaling the image.

    max_cut : float, optional
        The maximum cut level.  Data values greater than ``max_cut``
        will set to ``max_cut`` before scaling the image.

    min_percent : float, optional
        The minimum cut level as a percentile of the values in the
        image.  If ``min_cut`` is input, then ``min_percent`` will be
        ignored.

    max_percent : float, optional
        The maximum cut level as a percentile of the values in the
        image.  If ``max_cut`` is input, then ``max_percent`` will be
        ignored.

    percent : float, optional
        The percentage of the image values to scale.  The lower cut
        level will set at the ``(100 - percent) / 2`` percentile, while
        the upper cut level will be set at the ``(100 + percent) / 2``
        percentile.  This value overrides the values of ``min_percent``
        and ``max_percent``, but is ignored if ``min_cut`` and
        ``max_cut`` are both input.
    """

    if min_cut is not None and max_cut is not None:
        return min_cut, max_cut
    if percent:
        assert (percent >= 0) and (percent <= 100.0), 'percent must be >= 0 and <= 100.0'
        if not min_percent and not max_percent:
            min_percent = (100.0 - float(percent)) / 2.0
            max_percent = 100.0 - min_percent
    if min_cut is None:
        if min_percent is None:
            min_percent = 0.0
        assert min_percent >= 0, 'min_percent must be >= 0'
        min_cut = np.percentile(image, min_percent)
    if max_cut is None:
        if max_percent is None:
            max_percent = 100.0
        assert max_percent <= 100.0, 'max_percent must be <= 100.0'
        max_cut = np.percentile(image, max_percent)
    assert min_cut <= max_cut, 'min_cut must be <= max_cut'
    return min_cut, max_cut


def img_stats(image, image_mask=None, mask_val=None, sig=3.0, iters=None):
    """
    Perform sigma-clipped statistics on an image.

    Parameters
    ----------
    image : array_like
        The 2D array of the image.

    image_mask : boolean, array_like, optional
        The 2D array of image mask values.  Bad pixels have a value of
        `True` and bad pixels have a value of `False.`

    mask_val : float, optional
        An image data value (e.g. 0.0) that is ignored when computing
        the image statistics (crude!).  ``mask_val`` is ignored if
        ``image_mask`` is input.

    sig : float, optional
        The number of standard deviations to use as the clipping limit.

    iters : float, optional
       The number of iterations to perform clipping, or `None` to clip
       until convergence is achieved (i.e. continue until the last
       iteration clips nothing).
    """

    if image_mask:
        image = image[~image_mask]
    if mask_val and not image_mask:
        idx = (image != mask_val).nonzero()
        image = image[idx]
    image_clip = sigma_clip(image, sig=sig, iters=iters)
    goodvals = image_clip.data[~image_clip.mask]
    return np.mean(goodvals), np.median(goodvals), np.std(goodvals)<commit_msg>updated scale_img docs<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import numpy as np
from astropy.stats import sigma_clip

# from scikit-image
DTYPE_RANGE = {np.bool_: (False, True),
               np.bool8: (False, True),
               np.uint8: (0, 255),
               np.uint16: (0, 65535),
               np.int8: (-128, 127),
               np.int16: (-32768, 32767),
               np.float32: (-1, 1),
               np.float64: (-1, 1)}
DTYPE_RANGE.update((d.__name__, limits) for d, limits in list(DTYPE_RANGE.items()))
DTYPE_RANGE.update({'uint10': (0, 2**10 - 1), 'uint12': (0, 2**12 - 1),
                    'uint14': (0, 2**14 - 1), 'bool': DTYPE_RANGE[np.bool_],
                    'float': DTYPE_RANGE[np.float64]})


def find_imgcuts(image, min_cut=None, max_cut=None, min_percent=None,
                 max_percent=None, percent=None):
    """
    Find minimum and maximum image cut levels from percentiles of the
    image values.

    Parameters
    ----------
    image : array_like
        The 2D array of the image.

    min_cut : float, optional
        The minimum cut level.  Data values less than ``min_cut`` will
        set to ``min_cut`` before scaling the image.

    max_cut : float, optional
        The maximum cut level.  Data values greater than ``max_cut``
        will set to ``max_cut`` before scaling the image.

    min_percent : float, optional
        The minimum cut level as a percentile of the values in the
        image.  If ``min_cut`` is input, then ``min_percent`` will be
        ignored.

    max_percent : float, optional
        The maximum cut level as a percentile of the values in the
        image.  If ``max_cut`` is input, then ``max_percent`` will be
        ignored.

    percent : float, optional
        The percentage of the image values to scale.  The lower cut
        level will set at the ``(100 - percent) / 2`` percentile, while
        the upper cut level will be set at the ``(100 + percent) / 2``
        percentile.  This value overrides the values of ``min_percent``
        and ``max_percent``, but is ignored if ``min_cut`` and
        ``max_cut`` are both input.
    """

    if min_cut is not None and max_cut is not None:
        return min_cut, max_cut
    if percent:
        assert (percent >= 0) and (percent <= 100.0), 'percent must be >= 0 and <= 100.0'
        if not min_percent and not max_percent:
            min_percent = (100.0 - float(percent)) / 2.0
            max_percent = 100.0 - min_percent
    if min_cut is None:
        if min_percent is None:
            min_percent = 0.0
        assert min_percent >= 0, 'min_percent must be >= 0'
        min_cut = np.percentile(image, min_percent)
    if max_cut is None:
        if max_percent is None:
            max_percent = 100.0
        assert max_percent <= 100.0, 'max_percent must be <= 100.0'
        max_cut = np.percentile(image, max_percent)
    assert min_cut <= max_cut, 'min_cut must be <= max_cut'
    return min_cut, max_cut


def img_stats(image, image_mask=None, mask_val=None, sig=3.0, iters=None):
    """
    Perform sigma-clipped statistics on an image.

    Parameters
    ----------
    image : array_like
        The 2D array of the image.

    image_mask : array_like, boolean, optional
        A boolean mask with the same shape as `image`, where a `True`
        value indicates the corresponding element of `image` is invalid.
        Masked pixels are ignored when computing the image statistics.

    mask_val : float, optional
        An image data value (e.g., ``0.0``) that is ignored when
        computing the image statistics.  ``mask_val`` will be ignored if
        ``image_mask`` is input.

    sig : float, optional
        The number of standard deviations to use as the clipping limit.

    iters : float, optional
       The number of iterations to perform clipping, or `None` to clip
       until convergence is achieved (i.e. continue until the last
       iteration clips nothing).
    """

    if image_mask:
        image = image[~image_mask]
    if mask_val and not image_mask:
        idx = (image != mask_val).nonzero()
        image = image[idx]
    image_clip = sigma_clip(image, sig=sig, iters=iters)
    goodvals = image_clip.data[~image_clip.mask]
    return np.mean(goodvals), np.median(goodvals), np.std(goodvals)
{'fraction_non_alphanumeric': 0.10045300113250283, 'fraction_numerical': 0.026160815402038504, 'mean_word_length': 2.3099700149925035, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5293830', 'n_tokens_mistral': 2803, 'n_tokens_neox': 2603, 'n_words': 961}