<commit_before>from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import ccdproc
import datetime
import glob
import logging
import numpy as np
import matplotlib.pyplot as plt
import random
import re
import os

from astropy import units as u
from ccdproc import CCDData
from ..core import (astroscrappy_lacosmic,
                    call_cosmic_rejection,
                    combine_data,
                    get_best_flat,
                    get_slit_trim_section,
                    image_overscan,
                    image_trim,
                    normalize_master_flat,
                    read_fits,
                    write_fits)

from ..core import SaturationValues, SpectroscopicMode


class ImageProcessor(object):
    """Image processing class

    This class contains methods for performing CCD image reduction for
    spectroscopy and imaging.

    """

    def __init__(self, args, data_container):
        """Initialization method for ImageProcessor class

        Args:
            args (Namespace): argparse instance
            data_container (DataFrame): Contains relevant information of the
            night and the data itself.
        """
        # TODO (simon): Check how inheritance could be used here.
        self.log = logging.getLogger(__name__)
        self.args = args
        self.instrument = data_container.instrument
        self.technique = data_container.technique
        self.gain = data_container.gain
        self.rdnoise = data_container.rdnoise
        self.roi = data_container.roi
        self.bias = data_container.bias
        self.day_flats = data_container.day_flats
        self.dome_flats = data_container.dome_flats
        self.sky_flats = data_container.sky_flats
        self.data_groups = data_container.data_groups
        self.spec_groups = data_container.spec_groups
        self.sun_set = data_container.sun_set_time
        self.sun_rise = data_container.sun_rise_time
        self.morning_twilight = data_container.morning_twilight
        self.evening_twilight = data_container.evening_twilight
        self.pixel_scale = 0.15 * u.arcsec
        self.queue = None
        self.trim_section = self.define_trim_section(
            technique=self.technique)
        self.overscan_region = self.get_overscan_region()
        self.saturation_values = SaturationValues()
        self.spec_mode = SpectroscopicMode()
        self.master_bias = None
        self.master_bias_name = None
        self.out_prefix = None

    def __call__(self):
        """Call method for ImageProcessor class

        This method manages the image processing by calling the appropriate
        methods.

        """

        for group in [self.bias,
                      self.day_flats,
                      self.dome_flats,
                      self.sky_flats,
                      self.data_groups,
                      self.spec_groups]:
            if group is not None:
                for sub_group in group:
                    group_obstype = sub_group.obstype.unique()

                    if len(group_obstype) == 1 and \
                        group_obstype[0] == 'BIAS' and \
                            not self.args.ignore_bias:

                        self.log.debug('Creating Master Bias')
                        self.create_master_bias(sub_group)
                    elif len(group_obstype) == 1 and group_obstype[0] == 'FLAT':
                        self.log.debug('Create Master FLATS')
                        self.create_master_flats(sub_group)
                    else:
                        self.log.debug('Process Data Group')
                        if self.technique == 'Spectroscopy':
                            self.process_spectroscopy_science(sub_group)
                        else:
                            self.log.info('Processing Imaging Science Data')
                            self.process_imaging_science(sub_group)

    def _is_file_saturated(self, ccd):
        """Detects a saturated image

        It counts the number of pixels above the saturation level, then finds
        which percentage they represents and if it is above the threshold it
        will return True. The percentage threshold can be set using the command
        line argument ``--saturation``.

        Args:
            ccd (CCDData): Image to be tested for saturation

        Returns:
            True for saturated and False for non-saturated

        """

        pixels_above_saturation = np.count_nonzero(
            ccd.data[np.where(
                ccd.data > self.saturation_values.get_saturation_value(
                    ccd=ccd))])

        total_pixels = np.count_nonzero(ccd.data)

        saturated_percent = (pixels_above_saturation * 100.) / total_pixels

        if saturated_percent >= float(self.args.saturation_threshold):
            self.log.warning("The current image has more than {:s} percent of "
                             "pixels above saturation level".format(
                self.args.saturation_threshold))
            return True
        else:
            return False

    def define_trim_section(self, technique=None):
        """Get the initial trim section

        The initial trim section is usually defined in the header with the
        keyword ``TRIMSEC`` but in the case of Goodman HTS this does not work well.<commit_msg>fixing bug in formatting<commit_after>from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import ccdproc
import datetime
import glob
import logging
import numpy as np
import matplotlib.pyplot as plt
import random
import re
import os

from astropy import units as u
from ccdproc import CCDData
from ..core import (astroscrappy_lacosmic,
                    call_cosmic_rejection,
                    combine_data,
                    get_best_flat,
                    get_slit_trim_section,
                    image_overscan,
                    image_trim,
                    normalize_master_flat,
                    read_fits,
                    write_fits)

from ..core import SaturationValues, SpectroscopicMode


class ImageProcessor(object):
    """Image processing class

    This class contains methods for performing CCD image reduction for
    spectroscopy and imaging.

    """

    def __init__(self, args, data_container):
        """Initialization method for ImageProcessor class

        Args:
            args (Namespace): argparse instance
            data_container (DataFrame): Contains relevant information of the
            night and the data itself.
        """
        # TODO (simon): Check how inheritance could be used here.
        self.log = logging.getLogger(__name__)
        self.args = args
        self.instrument = data_container.instrument
        self.technique = data_container.technique
        self.gain = data_container.gain
        self.rdnoise = data_container.rdnoise
        self.roi = data_container.roi
        self.bias = data_container.bias
        self.day_flats = data_container.day_flats
        self.dome_flats = data_container.dome_flats
        self.sky_flats = data_container.sky_flats
        self.data_groups = data_container.data_groups
        self.spec_groups = data_container.spec_groups
        self.sun_set = data_container.sun_set_time
        self.sun_rise = data_container.sun_rise_time
        self.morning_twilight = data_container.morning_twilight
        self.evening_twilight = data_container.evening_twilight
        self.pixel_scale = 0.15 * u.arcsec
        self.queue = None
        self.trim_section = self.define_trim_section(
            technique=self.technique)
        self.overscan_region = self.get_overscan_region()
        self.saturation_values = SaturationValues()
        self.spec_mode = SpectroscopicMode()
        self.master_bias = None
        self.master_bias_name = None
        self.out_prefix = None

    def __call__(self):
        """Call method for ImageProcessor class

        This method manages the image processing by calling the appropriate
        methods.

        """

        for group in [self.bias,
                      self.day_flats,
                      self.dome_flats,
                      self.sky_flats,
                      self.data_groups,
                      self.spec_groups]:
            if group is not None:
                for sub_group in group:
                    group_obstype = sub_group.obstype.unique()

                    if len(group_obstype) == 1 and \
                        group_obstype[0] == 'BIAS' and \
                            not self.args.ignore_bias:

                        self.log.debug('Creating Master Bias')
                        self.create_master_bias(sub_group)
                    elif len(group_obstype) == 1 and group_obstype[0] == 'FLAT':
                        self.log.debug('Create Master FLATS')
                        self.create_master_flats(sub_group)
                    else:
                        self.log.debug('Process Data Group')
                        if self.technique == 'Spectroscopy':
                            self.process_spectroscopy_science(sub_group)
                        else:
                            self.log.info('Processing Imaging Science Data')
                            self.process_imaging_science(sub_group)

    def _is_file_saturated(self, ccd):
        """Detects a saturated image

        It counts the number of pixels above the saturation level, then finds
        which percentage they represents and if it is above the threshold it
        will return True. The percentage threshold can be set using the command
        line argument ``--saturation``.

        Args:
            ccd (CCDData): Image to be tested for saturation

        Returns:
            True for saturated and False for non-saturated

        """

        pixels_above_saturation = np.count_nonzero(
            ccd.data[np.where(
                ccd.data > self.saturation_values.get_saturation_value(
                    ccd=ccd))])

        total_pixels = np.count_nonzero(ccd.data)

        saturated_percent = (pixels_above_saturation * 100.) / total_pixels

        if saturated_percent >= float(self.args.saturation_threshold):
            self.log.warning("The current image has more than {:.2f} percent "
                             "of pixels above saturation level".format(
                self.args.saturation_threshold))
            return True
        else:
            return False

    def define_trim_section(self, technique=None):
        """Get the initial trim section

        The initial trim section is usually defined in the header with the
        keyword ``TRIMSEC`` but in the case of Goodman HTS this does not work well.
{'fraction_non_alphanumeric': 0.06202051945651169, 'fraction_numerical': 0.001941029670024956, 'mean_word_length': 1.8228541612314115, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '357902', 'n_tokens_mistral': 2910, 'n_tokens_neox': 2700, 'n_words': 742}