<commit_before>from __future__ import (division, print_function, absolute_import,
                        unicode_literals)

from collections import OrderedDict
import os
import warnings

from astropy.modeling import models
import ccdproc
from astropy.extern import six

import numpy as np

from . import gui
from .ipython_version_helper import ipython_version_as_string

if ipython_version_as_string().startswith('3'):
    from IPython.html import widgets
else:
    import ipywidgets as widgets


__all__ = [
    'Reduction',
    'Combiner',
    'CosmicRaySettings',
    'Slice',
    'CalibrationStep',
    'BiasSubtract',
    'DarkSubtract',
    'FlatCorrect',
    'Overscan',
    'Trim'
]

DEFAULT_IMAGE_UNIT = "adu"


class ReducerBase(gui.ToggleGo):
    """
    Base class for reduction and combination widgets that provides a couple
    of properties common to both.

    Parameters
    ----------

    apply_to : dict
        Key-value pair(s) that select images that will be acted on by the
        widget.

    destination : str
        Directory in which reduced images will be stored.
    """
    def __init__(self, *arg, **kwd):
        self._apply_to = kwd.pop('apply_to', None)
        self._destination = kwd.pop('destination', None)
        super(ReducerBase, self).__init__(*arg, **kwd)

    @property
    def destination(self):
        return self._destination

    @property
    def apply_to(self):
        return self._apply_to


class Reduction(ReducerBase):
    """
    Primary widget for performing a logical reduction step (e.g. dark
    subtraction or flat correction).
    """
    def __init__(self, *arg, **kwd):
        allow_flat = kwd.pop('allow_flat', True)
        allow_dark = kwd.pop('allow_dark', True)
        allow_bias = kwd.pop('allow_bias', True)
        self.image_collection = kwd.pop('input_image_collection', None)
        self._master_source = kwd.pop('master_source', None)
        super(Reduction, self).__init__(*arg, **kwd)
        self._overscan = Overscan(description='Subtract overscan?')
        self._trim = Trim(description='Trim (specify region to keep)?')
        self._cosmic_ray = CosmicRaySettings()
        self._bias_calib = BiasSubtract(master_source=self._master_source)
        self._dark_calib = DarkSubtract(master_source=self._master_source)
        self._flat_calib = FlatCorrect(master_source=self._master_source)
        self.add_child(self._overscan)
        self.add_child(self._trim)
        self.add_child(self._cosmic_ray)

        if allow_bias:
            self.add_child(self._bias_calib)
        if allow_dark:
            self.add_child(self._dark_calib)
        if allow_flat:
            self.add_child(self._flat_calib)

    @property
    def reduced_images(self):
        """
        List of reduced images; each image is a `ccdproc.CCDData`` object.
        """
        return self._reduced_images

    def action(self):
        if not self.image_collection:
            raise ValueError("No images to reduce")
        self.progress_bar.visible = True

        # Refresh in case files have been added since the widget was created.
        self.image_collection.refresh()

        reduced_images = []
        # Suppress warnings that come up here...mostly about HIERARCH keywords
        warnings.filterwarnings('ignore')
        try:
            n_files = \
                len(self.image_collection.files_filtered(**self.apply_to))
            current_file = 0
            for hdu, fname in self.image_collection.hdus(return_fname=True,
                                                         save_location=self.destination,
                                                         **self.apply_to):
                current_file += 1
                try:
                    unit = hdu.header['BUNIT']
                except KeyError:
                    unit = DEFAULT_IMAGE_UNIT
                ccd = ccdproc.CCDData(hdu.data, meta=hdu.header, unit=unit)

                for child in self.container.children:
                    if not child.toggle.value:
                        # Nothing to do for this child, so keep going.
                        continue
                    ccd = child.action(ccd)
                hdu_tmp = ccd.to_hdu()[0]
                hdu.header = hdu_tmp.header
                hdu.data = hdu_tmp.data
                reduced_images.append(ccd)
                self.progress_bar.description = \
                    ("Processed file {} of {}".format(current_file, n_files))
                self.progress_bar.value = current_file/n_files
        except IOError:
            print("One or more of the reduced images already exists. Delete "
                  "those files and try again. This notebook will NOT "
                  "overwrite existing files.")
        finally:
            self.progress_bar.visible = False

        self._reduced_images = reduced_images


class Clipping(gui.ToggleContainer):
    """docstring for Clipping"""
    def __init__(self, *args, **kwd):<commit_msg>Workaround for improper handling of uint16 images<commit_after>from __future__ import (division, print_function, absolute_import,
                        unicode_literals)

from collections import OrderedDict
import os
import warnings

from astropy.modeling import models
import ccdproc
from astropy.extern import six

import numpy as np

from . import gui
from .ipython_version_helper import ipython_version_as_string

if ipython_version_as_string().startswith('3'):
    from IPython.html import widgets
else:
    import ipywidgets as widgets


__all__ = [
    'Reduction',
    'Combiner',
    'CosmicRaySettings',
    'Slice',
    'CalibrationStep',
    'BiasSubtract',
    'DarkSubtract',
    'FlatCorrect',
    'Overscan',
    'Trim'
]

DEFAULT_IMAGE_UNIT = "adu"


class ReducerBase(gui.ToggleGo):
    """
    Base class for reduction and combination widgets that provides a couple
    of properties common to both.

    Parameters
    ----------

    apply_to : dict
        Key-value pair(s) that select images that will be acted on by the
        widget.

    destination : str
        Directory in which reduced images will be stored.
    """
    def __init__(self, *arg, **kwd):
        self._apply_to = kwd.pop('apply_to', None)
        self._destination = kwd.pop('destination', None)
        super(ReducerBase, self).__init__(*arg, **kwd)

    @property
    def destination(self):
        return self._destination

    @property
    def apply_to(self):
        return self._apply_to


class Reduction(ReducerBase):
    """
    Primary widget for performing a logical reduction step (e.g. dark
    subtraction or flat correction).
    """
    def __init__(self, *arg, **kwd):
        allow_flat = kwd.pop('allow_flat', True)
        allow_dark = kwd.pop('allow_dark', True)
        allow_bias = kwd.pop('allow_bias', True)
        self.image_collection = kwd.pop('input_image_collection', None)
        self._master_source = kwd.pop('master_source', None)
        super(Reduction, self).__init__(*arg, **kwd)
        self._overscan = Overscan(description='Subtract overscan?')
        self._trim = Trim(description='Trim (specify region to keep)?')
        self._cosmic_ray = CosmicRaySettings()
        self._bias_calib = BiasSubtract(master_source=self._master_source)
        self._dark_calib = DarkSubtract(master_source=self._master_source)
        self._flat_calib = FlatCorrect(master_source=self._master_source)
        self.add_child(self._overscan)
        self.add_child(self._trim)
        self.add_child(self._cosmic_ray)

        if allow_bias:
            self.add_child(self._bias_calib)
        if allow_dark:
            self.add_child(self._dark_calib)
        if allow_flat:
            self.add_child(self._flat_calib)

    @property
    def reduced_images(self):
        """
        List of reduced images; each image is a `ccdproc.CCDData`` object.
        """
        return self._reduced_images

    def action(self):
        if not self.image_collection:
            raise ValueError("No images to reduce")
        self.progress_bar.visible = True

        # Refresh in case files have been added since the widget was created.
        self.image_collection.refresh()

        reduced_images = []
        # Suppress warnings that come up here...mostly about HIERARCH keywords
        warnings.filterwarnings('ignore')
        try:
            n_files = \
                len(self.image_collection.files_filtered(**self.apply_to))
            current_file = 0
            for hdu, fname in self.image_collection.hdus(return_fname=True,
                                                         save_location=self.destination,
                                                         **self.apply_to):
                current_file += 1
                try:
                    unit = hdu.header['BUNIT']
                except KeyError:
                    unit = DEFAULT_IMAGE_UNIT
                ccd = ccdproc.CCDData(hdu.data, meta=hdu.header, unit=unit)
                for child in self.container.children:
                    if not child.toggle.value:
                        # Nothing to do for this child, so keep going.
                        continue
                    ccd = child.action(ccd)
                hdu_tmp = ccd.to_hdu()[0]
                hdu.header = hdu_tmp.header
                hdu.data = hdu_tmp.data

                # Workaround to ensure uint16 images are handled properly.
                if 'bzero' in hdu.header:
                    # Check for the unsigned int16 case, and if our data type
                    # is no longer uint16, delete BZERO and BSCALE
                    header_unsigned_int = ((hdu.header['bscale'] == 1) and
                                           (hdu.header['bzero'] == 32768))
                    if (header_unsigned_int and
                        (hdu.data.dtype != np.dtype('uint16'))):

                        del hdu.header['bzero'], hdu.header['bscale']

                reduced_images.append(ccd)
                self.progress_bar.description = \
                    ("Processed file {} of {}".format(current_file, n_files))
                self.progress_bar.value = current_file/n_files
        except IOError:
            print("One or more of the reduced images already exists. Delete "
                  "those files and try again. This notebook will NOT "
                  "overwrite existing files.")
        finally:
            self.progress_bar.visible = False

        self._reduced_images = reduced_images


class Clipping(gui.ToggleContainer):
    """docstring for Clipping"""
    def __init__(self, *args, **kwd):
{'fraction_non_alphanumeric': 0.0886801315171442, 'fraction_numerical': 0.002254579614842649, 'mean_word_length': 2.03045829775121, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4585305', 'n_tokens_mistral': 3045, 'n_tokens_neox': 2908, 'n_words': 734}