<commit_before>"""
Augmenters that don't apply augmentations themselves, but are needed
for meta usage.

Do not import directly from this file, as the categorization is not final.
Use instead
    `from imgaug import augmenters as iaa`
and then e.g.
    `seq = iaa.Sequential([...])`

List of augmenters:
    * Augmenter (base class for all augmenters)
    * Sequential
    * SomeOf
    * OneOf
    * Sometimes
    * WithChannels
    * Noop
    * Lambda
    * AssertLambda
    * AssertShape

Note that WithColorspace is in `color.py`.
"""
from __future__ import print_function, division, absolute_import
from .. import imgaug as ia
# TODO replace these imports with iap.XYZ
from ..parameters import StochasticParameter, Binomial, DiscreteUniform
from abc import ABCMeta, abstractmethod
import numpy as np
import copy as copy_module
import re
from scipy import misc
import itertools
import six
import six.moves as sm
import warnings

def copy_dtypes_for_restore(images, force_list=False):
    if ia.is_np_array(images):
        if force_list:
            return [images.dtype for _ in sm.xrange(len(images))]
        else:
            return images.dtype
    else:
        return [image.dtype for image in images]

def restore_augmented_image_dtype_(image, orig_dtype):
    return image.astype(orig_dtype, copy=False)

def restore_augmented_image_dtype(image, orig_dtype):
    return image.astype(orig_dtype, copy=True)

def restore_augmented_images_dtypes_(images, orig_dtypes):
    if ia.is_np_array(images):
        return images.astype(orig_dtypes, copy=False)
    else:
        return [image.astype(orig_dtype, copy=False) for image, orig_dtype in zip(images, orig_dtypes)]

def restore_augmented_images_dtypes(images, orig_dtypes):
    if ia.is_np_array(images):
        images = np.copy(images)
    else:
        images = [np.copy(image) for image in images]
    return restore_augmented_images_dtypes_(images, orig_dtypes)

def clip_augmented_image_(image, minval, maxval):
    return clip_augmented_images_(image, minval, maxval)

def clip_augmented_image(image, minval, maxval):
    return clip_augmented_images(image, minval, maxval)

def clip_augmented_images_(images, minval, maxval):
    if ia.is_np_array(images):
        return np.clip(images, minval, maxval, out=images)
    else:
        return [np.clip(image, minval, maxval, out=image) for image in images]

def clip_augmented_images(images, minval, maxval):
    if ia.is_np_array(images):
        images = np.copy(images)
    else:
        images = [np.copy(image) for image in images]
    return clip_augmented_images_(images, minval, maxval)

@six.add_metaclass(ABCMeta)
class Augmenter(object): # pylint: disable=locally-disabled, unused-variable, line-too-long
    """
    Base class for Augmenter objects.
    All augmenters derive from this class.
    """

    def __init__(self, name=None, deterministic=False, random_state=None):
        """
        Create a new Augmenter instance.

        Parameters
        ----------
        name : None or string, optional(default=None)
            Name given to an Augmenter object. This name is used in print()
            statements as well as find and remove functions.
            If None, `UnnamedX` will be used as the name, where X is the
            Augmenter's class name.

        deterministic : bool, optional(default=False)
            Whether the augmenter instance's random state will be saved before
            augmenting images and then reset to that saved state after an
            augmentation (of multiple images/keypoints) is finished.
            I.e. if set to True, each batch of images will be augmented in the
            same way (e.g. first image might always be flipped horizontally,
            second image will never be flipped etc.).
            This is useful when you want to transform multiple batches of images
            in the same way, or when you want to augment images and keypoints
            on these images.
            Usually, there is no need to set this variable by hand. Instead,
            instantiate the augmenter with the defaults and then use
            `augmenter.to_deterministic()`.

        random_state : None or int or np.random.RandomState, optional(default=None)
            The random state to use for this
            augmenter.
                * If int, a new np.random.RandomState will be created using this
                  value as the seed.
                * If np.random.RandomState instance, the instance will be used directly.
                * If None, imgaug's default RandomState will be used, which's state can
                  be controlled using imgaug.seed(int).
            Usually there is no need to set this variable by hand. Instead,
            instantiate the augmenter with the defaults and then use
            `augmenter.to_deterministic()`.

        """
        super(Augmenter, self).__init__()

        if name is None:
            self.name = "Unnamed%s" % (self.__class__.__name__,)
        else:
            self.name = name

        self.deterministic = deterministic

        if random_state is None:
            if self.deterministic:
                self.random_state = ia.new_random_state()
            else:
                self.random_state = ia.current_random_state()
        elif isinstance(random_state, np.random.RandomState):
            self.random_state = random_state
        else:
            self.random_state = np.random.RandomState(random_state)

        self.activated = True

    def augment_batches(self, batches, hooks=None, background=False):
        """
        Augment multiple batches of images.

        Parameters
        ----------
        batches : list
            List of image batches to augment.
            The expected input is a list, with each entry having one of the
            following datatypes:
                * ia.Batch
                * []
                * list of ia.KeypointsOnImage
                * list of (H,W,C) ndarray
                * list of (H,W) ndarray
                * (N,H,W,C) ndarray
                * (N,H,W) ndarray
            where N = number of images, H = height, W = width,
            C = number of channels.
            Each image is recommended to have dtype uint8 (range 0-255).

        hooks : None or ia.HooksImages, optional(default=None)
            HooksImages object to dynamically interfere with the augmentation
            process.

        background : bool, optional(default=False)
            Whether to augment the batches in background processes.
            If true, hooks can currently not be used as that would require
            pickling functions.

        Yields
        -------
        augmented_batch : ia.Batch or list of ia.KeypointsOnImage or list of (H,W,C) ndarray or list of (H,W) ndarray or (N,H,W,C) ndarray or (N,H,W) ndarray
            Augmented images/keypoints.
            Datatype usually matches the input datatypes per list element.

        """
        ia.do_assert(isinstance(batches, list))
        ia.do_assert(len(batches) > 0)
        if background:
            ia.do_assert(hooks is None, "Hooks can not be used when background augmentation is activated.")

        batches_normalized = []
        batches_original_dts = []
        for i, batch in enumerate(batches):
            if isinstance(batch, ia.Batch):
                batch.data = (i, batch.data)
                batches_normalized.append(batch)
                batches_original_dts.append("imgaug.Batch")
            elif ia.is_np_array(batch):
                ia.do_assert(batch.ndim in (3, 4), "Expected numpy array to have shape (N, H, W) or (N, H, W, C), got %s." % (batch.shape,))
                batches_normalized.append(ia.Batch(images=batch, data=i))
                batches_original_dts.append("numpy_array")
            elif isinstance(batch, list):
                if len(batch) == 0:
                    batches_normalized.append(ia.Batch(data=i))
                    batches_original_dts.append("empty_list")
                elif ia.is_np_array(batch[0]):
                    batches_normalized.append(ia.Batch(images=batch, data=i))
                    batches_original_dts.append("list_of_numpy_arrays")
                elif isinstance(batch[0], ia.KeypointsOnImage):
                    batches_normalized.append(ia.Batch(keypoints=batch, data=i))
                    batches_original_dts.append("list_of_imgaug.KeypointsOnImage")
                else:
                    raise Exception("Unknown datatype in batch[0]. Expected numpy array or imgaug.KeypointsOnImage, got %s." % (type(batch[0]),))
            else:
                raise Exception("Unknown datatype of batch. Expected imgaug.Batch or numpy array or list of numpy arrays/imgaug.KeypointsOnImage. Got %s." % (type(batch),))

        def unnormalize_batch(batch_aug):
            #if batch_aug.data is None:
            #    return batch_aug
            #else:
            i = batch_aug.data
            # if input was ia.Batch, then .data has content (i, .data)
            if isinstance(i, tuple):
                i = i[0]
            dt_orig = batches_original_dts[i]
            if dt_orig == "imgaug.Batch":
                batch_unnormalized = batch_aug
                # change (i, .data) back to just .data
                batch_unnormalized.data = batch_unnormalized.data[1]
            elif dt_orig == "numpy_array":
                batch_unnormalized = batch_aug.images_aug
            elif dt_orig == "empty_list":
                batch_unnormalized = []
            elif dt_orig == "list_of_numpy_arrays":
                batch_unnormalized = batch_aug.images_aug
            elif dt_orig == "list_of_imgaug.KeypointsOnImage":
                batch_unnormalized = batch_aug.keypoints_aug
            else:
                raise Exception("Internal error. Unexpected value in dt_orig '%s'. This should never happen." % (dt_orig,))
            return batch_unnormalized

        if not background:
            for batch_normalized in batches_normalized:
                batch_augment_images = batch_normalized.images is not None
                batch_augment_keypoints = batch_normalized.keypoints is not None

                if batch_augment_images and batch_augment_keypoints:
                    augseq_det = self.to_deterministic() if not self.deterministic else self
                    batch_normalized.images_aug = augseq_det.augment_images(batch_normalized.images, hooks=hooks)
                    batch_normalized.keypoints_aug = augseq_det.augment_keypoints(batch_normalized.keypoints, hooks=hooks)
                elif batch_augment_images:<commit_msg>Make if/else statement fully testable<commit_after>"""
Augmenters that don't apply augmentations themselves, but are needed
for meta usage.

Do not import directly from this file, as the categorization is not final.
Use instead
    `from imgaug import augmenters as iaa`
and then e.g.
    `seq = iaa.Sequential([...])`

List of augmenters:
    * Augmenter (base class for all augmenters)
    * Sequential
    * SomeOf
    * OneOf
    * Sometimes
    * WithChannels
    * Noop
    * Lambda
    * AssertLambda
    * AssertShape

Note that WithColorspace is in `color.py`.
"""
from __future__ import print_function, division, absolute_import
from .. import imgaug as ia
# TODO replace these imports with iap.XYZ
from ..parameters import StochasticParameter, Binomial, DiscreteUniform
from abc import ABCMeta, abstractmethod
import numpy as np
import copy as copy_module
import re
from scipy import misc
import itertools
import six
import six.moves as sm
import warnings

def copy_dtypes_for_restore(images, force_list=False):
    if ia.is_np_array(images):
        if force_list:
            return [images.dtype for _ in sm.xrange(len(images))]
        else:
            return images.dtype
    else:
        return [image.dtype for image in images]

def restore_augmented_image_dtype_(image, orig_dtype):
    return image.astype(orig_dtype, copy=False)

def restore_augmented_image_dtype(image, orig_dtype):
    return image.astype(orig_dtype, copy=True)

def restore_augmented_images_dtypes_(images, orig_dtypes):
    if ia.is_np_array(images):
        return images.astype(orig_dtypes, copy=False)
    else:
        return [image.astype(orig_dtype, copy=False) for image, orig_dtype in zip(images, orig_dtypes)]

def restore_augmented_images_dtypes(images, orig_dtypes):
    if ia.is_np_array(images):
        images = np.copy(images)
    else:
        images = [np.copy(image) for image in images]
    return restore_augmented_images_dtypes_(images, orig_dtypes)

def clip_augmented_image_(image, minval, maxval):
    return clip_augmented_images_(image, minval, maxval)

def clip_augmented_image(image, minval, maxval):
    return clip_augmented_images(image, minval, maxval)

def clip_augmented_images_(images, minval, maxval):
    if ia.is_np_array(images):
        return np.clip(images, minval, maxval, out=images)
    else:
        return [np.clip(image, minval, maxval, out=image) for image in images]

def clip_augmented_images(images, minval, maxval):
    if ia.is_np_array(images):
        images = np.copy(images)
    else:
        images = [np.copy(image) for image in images]
    return clip_augmented_images_(images, minval, maxval)

@six.add_metaclass(ABCMeta)
class Augmenter(object): # pylint: disable=locally-disabled, unused-variable, line-too-long
    """
    Base class for Augmenter objects.
    All augmenters derive from this class.
    """

    def __init__(self, name=None, deterministic=False, random_state=None):
        """
        Create a new Augmenter instance.

        Parameters
        ----------
        name : None or string, optional(default=None)
            Name given to an Augmenter object. This name is used in print()
            statements as well as find and remove functions.
            If None, `UnnamedX` will be used as the name, where X is the
            Augmenter's class name.

        deterministic : bool, optional(default=False)
            Whether the augmenter instance's random state will be saved before
            augmenting images and then reset to that saved state after an
            augmentation (of multiple images/keypoints) is finished.
            I.e. if set to True, each batch of images will be augmented in the
            same way (e.g. first image might always be flipped horizontally,
            second image will never be flipped etc.).
            This is useful when you want to transform multiple batches of images
            in the same way, or when you want to augment images and keypoints
            on these images.
            Usually, there is no need to set this variable by hand. Instead,
            instantiate the augmenter with the defaults and then use
            `augmenter.to_deterministic()`.

        random_state : None or int or np.random.RandomState, optional(default=None)
            The random state to use for this
            augmenter.
                * If int, a new np.random.RandomState will be created using this
                  value as the seed.
                * If np.random.RandomState instance, the instance will be used directly.
                * If None, imgaug's default RandomState will be used, which's state can
                  be controlled using imgaug.seed(int).
            Usually there is no need to set this variable by hand. Instead,
            instantiate the augmenter with the defaults and then use
            `augmenter.to_deterministic()`.

        """
        super(Augmenter, self).__init__()

        if name is None:
            self.name = "Unnamed%s" % (self.__class__.__name__,)
        else:
            self.name = name

        self.deterministic = deterministic

        if random_state is None:
            if self.deterministic:
                self.random_state = ia.new_random_state()
            else:
                self.random_state = ia.current_random_state()
        elif isinstance(random_state, np.random.RandomState):
            self.random_state = random_state
        else:
            self.random_state = np.random.RandomState(random_state)

        self.activated = True

    def augment_batches(self, batches, hooks=None, background=False):
        """
        Augment multiple batches of images.

        Parameters
        ----------
        batches : list
            List of image batches to augment.
            The expected input is a list, with each entry having one of the
            following datatypes:
                * ia.Batch
                * []
                * list of ia.KeypointsOnImage
                * list of (H,W,C) ndarray
                * list of (H,W) ndarray
                * (N,H,W,C) ndarray
                * (N,H,W) ndarray
            where N = number of images, H = height, W = width,
            C = number of channels.
            Each image is recommended to have dtype uint8 (range 0-255).

        hooks : None or ia.HooksImages, optional(default=None)
            HooksImages object to dynamically interfere with the augmentation
            process.

        background : bool, optional(default=False)
            Whether to augment the batches in background processes.
            If true, hooks can currently not be used as that would require
            pickling functions.

        Yields
        -------
        augmented_batch : ia.Batch or list of ia.KeypointsOnImage or list of (H,W,C) ndarray or list of (H,W) ndarray or (N,H,W,C) ndarray or (N,H,W) ndarray
            Augmented images/keypoints.
            Datatype usually matches the input datatypes per list element.

        """
        ia.do_assert(isinstance(batches, list))
        ia.do_assert(len(batches) > 0)
        if background:
            ia.do_assert(hooks is None, "Hooks can not be used when background augmentation is activated.")

        batches_normalized = []
        batches_original_dts = []
        for i, batch in enumerate(batches):
            if isinstance(batch, ia.Batch):
                batch.data = (i, batch.data)
                batches_normalized.append(batch)
                batches_original_dts.append("imgaug.Batch")
            elif ia.is_np_array(batch):
                ia.do_assert(batch.ndim in (3, 4), "Expected numpy array to have shape (N, H, W) or (N, H, W, C), got %s." % (batch.shape,))
                batches_normalized.append(ia.Batch(images=batch, data=i))
                batches_original_dts.append("numpy_array")
            elif isinstance(batch, list):
                if len(batch) == 0:
                    batches_normalized.append(ia.Batch(data=i))
                    batches_original_dts.append("empty_list")
                elif ia.is_np_array(batch[0]):
                    batches_normalized.append(ia.Batch(images=batch, data=i))
                    batches_original_dts.append("list_of_numpy_arrays")
                elif isinstance(batch[0], ia.KeypointsOnImage):
                    batches_normalized.append(ia.Batch(keypoints=batch, data=i))
                    batches_original_dts.append("list_of_imgaug.KeypointsOnImage")
                else:
                    raise Exception("Unknown datatype in batch[0]. Expected numpy array or imgaug.KeypointsOnImage, got %s." % (type(batch[0]),))
            else:
                raise Exception("Unknown datatype of batch. Expected imgaug.Batch or numpy array or list of numpy arrays/imgaug.KeypointsOnImage. Got %s." % (type(batch),))

        def unnormalize_batch(batch_aug):
            #if batch_aug.data is None:
            #    return batch_aug
            #else:
            i = batch_aug.data
            # if input was ia.Batch, then .data has content (i, .data)
            if isinstance(i, tuple):
                i = i[0]
            dt_orig = batches_original_dts[i]
            if dt_orig == "imgaug.Batch":
                batch_unnormalized = batch_aug
                # change (i, .data) back to just .data
                batch_unnormalized.data = batch_unnormalized.data[1]
            elif dt_orig == "numpy_array":
                batch_unnormalized = batch_aug.images_aug
            elif dt_orig == "empty_list":
                batch_unnormalized = []
            elif dt_orig == "list_of_numpy_arrays":
                batch_unnormalized = batch_aug.images_aug
            else:
                ia.do_assert(dt_orig == "list_of_imgaug.KeypointsOnImage")  # only option left
                batch_unnormalized = batch_aug.keypoints_aug
            return batch_unnormalized

        if not background:
            for batch_normalized in batches_normalized:
                batch_augment_images = batch_normalized.images is not None
                batch_augment_keypoints = batch_normalized.keypoints is not None

                if batch_augment_images and batch_augment_keypoints:
                    augseq_det = self.to_deterministic() if not self.deterministic else self
                    batch_normalized.images_aug = augseq_det.augment_images(batch_normalized.images, hooks=hooks)
                    batch_normalized.keypoints_aug = augseq_det.augment_keypoints(batch_normalized.keypoints, hooks=hooks)
                elif batch_augment_images:
{'fraction_non_alphanumeric': 0.07750188111361926, 'fraction_numerical': 0.0014108352144469526, 'mean_word_length': 2.2416158536585367, 'pattern_counts': {'":': 9, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3275803', 'n_tokens_mistral': 6149, 'n_tokens_neox': 5871, 'n_words': 1796}