<commit_before>from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import ccdproc
import datetime
import glob
import logging
import numpy as np
import random
import re
import os

from astropy import units as u
from ccdproc import CCDData
from ..core import (astroscrappy_lacosmic,
                    call_cosmic_rejection,
                    combine_data,
                    get_best_flat,
                    get_slit_trim_section,
                    image_overscan,
                    image_trim,
                    normalize_master_flat,
                    read_fits,
                    write_fits)

from ..core import SpectroscopicMode


class ImageProcessor(object):
    """Image processing class

    This class contains methods for performing CCD image reduction for
    spectroscopy and imaging.

    """

    def __init__(self, args, data_container):
        """Initialization method for ImageProcessor class

        Args:
            args (object): argparse instance
            data_container (object): Contains relevant information of the night
                and the data itself.
        """
        # TODO (simon): Check how inheritance could be used here.
        self.log = logging.getLogger(__name__)
        self.args = args
        self.instrument = data_container.instrument
        self.technique = data_container.technique
        self.gain = data_container.gain
        self.rdnoise = data_container.rdnoise
        self.roi = data_container.roi
        self.bias = data_container.bias
        self.day_flats = data_container.day_flats
        self.dome_flats = data_container.dome_flats
        self.sky_flats = data_container.sky_flats
        self.data_groups = data_container.data_groups
        self.sun_set = data_container.sun_set_time
        self.sun_rise = data_container.sun_rise_time
        self.morning_twilight = data_container.morning_twilight
        self.evening_twilight = data_container.evening_twilight
        self.pixel_scale = 0.15 * u.arcsec
        self.queue = None
        self.trim_section = self.define_trim_section(
            technique=self.technique)
        self.overscan_region = self.get_overscan_region()
        self.spec_mode = SpectroscopicMode()
        self.master_bias = None
        self.master_bias_name = None
        self.out_prefix = None

    def __call__(self):
        """Call method for ImageProcessor class

        This method manages the image processing by calling the appropriate
        methods.

        """

        for group in [self.bias,
                      self.day_flats,
                      self.dome_flats,
                      self.sky_flats,
                      self.data_groups]:
            if group is not None:
                for sub_group in group:
                    group_obstype = sub_group.obstype.unique()

                    if len(group_obstype) == 1 and \
                        group_obstype[0] == 'BIAS' and \
                            not self.args.ignore_bias:

                        self.log.debug('Creating Master Bias')
                        self.create_master_bias(sub_group)
                    elif len(group_obstype) == 1 and group_obstype[0] == 'FLAT':
                        self.log.debug('Create Master FLATS')
                        self.create_master_flats(sub_group)
                    else:
                        self.log.debug('Process Data Group')
                        if self.technique == 'Spectroscopy':
                            self.process_spectroscopy_science(sub_group)
                        else:
                            self.log.info('Processing Imaging Science Data')
                            self.process_imaging_science(sub_group)

    def define_trim_section(self, technique=None):
        """Get the initial trim section

        The initial trim section is usually defined in the header with the
        keyword ``TRIMSEC`` but in the case of Goodman HTS this does not work well.
        In particular for spectroscopy where is more likely to have combined
        binning and so on.
<commit_msg>Added ``self.spec_group`` attribute from ``DataContainer.spec_group``<commit_after>from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import ccdproc
import datetime
import glob
import logging
import numpy as np
import random
import re
import os

from astropy import units as u
from ccdproc import CCDData
from ..core import (astroscrappy_lacosmic,
                    call_cosmic_rejection,
                    combine_data,
                    get_best_flat,
                    get_slit_trim_section,
                    image_overscan,
                    image_trim,
                    normalize_master_flat,
                    read_fits,
                    write_fits)

from ..core import SpectroscopicMode


class ImageProcessor(object):
    """Image processing class

    This class contains methods for performing CCD image reduction for
    spectroscopy and imaging.

    """

    def __init__(self, args, data_container):
        """Initialization method for ImageProcessor class

        Args:
            args (object): argparse instance
            data_container (object): Contains relevant information of the night
                and the data itself.
        """
        # TODO (simon): Check how inheritance could be used here.
        self.log = logging.getLogger(__name__)
        self.args = args
        self.instrument = data_container.instrument
        self.technique = data_container.technique
        self.gain = data_container.gain
        self.rdnoise = data_container.rdnoise
        self.roi = data_container.roi
        self.bias = data_container.bias
        self.day_flats = data_container.day_flats
        self.dome_flats = data_container.dome_flats
        self.sky_flats = data_container.sky_flats
        self.data_groups = data_container.data_groups
        self.spec_groups = data_container.spec_groups
        self.sun_set = data_container.sun_set_time
        self.sun_rise = data_container.sun_rise_time
        self.morning_twilight = data_container.morning_twilight
        self.evening_twilight = data_container.evening_twilight
        self.pixel_scale = 0.15 * u.arcsec
        self.queue = None
        self.trim_section = self.define_trim_section(
            technique=self.technique)
        self.overscan_region = self.get_overscan_region()
        self.spec_mode = SpectroscopicMode()
        self.master_bias = None
        self.master_bias_name = None
        self.out_prefix = None

    def __call__(self):
        """Call method for ImageProcessor class

        This method manages the image processing by calling the appropriate
        methods.

        """

        for group in [self.bias,
                      self.day_flats,
                      self.dome_flats,
                      self.sky_flats,
                      self.data_groups,
                      self.spec_groups]:
            if group is not None:
                for sub_group in group:
                    group_obstype = sub_group.obstype.unique()

                    if len(group_obstype) == 1 and \
                        group_obstype[0] == 'BIAS' and \
                            not self.args.ignore_bias:

                        self.log.debug('Creating Master Bias')
                        self.create_master_bias(sub_group)
                    elif len(group_obstype) == 1 and group_obstype[0] == 'FLAT':
                        self.log.debug('Create Master FLATS')
                        self.create_master_flats(sub_group)
                    else:
                        self.log.debug('Process Data Group')
                        if self.technique == 'Spectroscopy':
                            self.process_spectroscopy_science(sub_group)
                        else:
                            self.log.info('Processing Imaging Science Data')
                            self.process_imaging_science(sub_group)

    def define_trim_section(self, technique=None):
        """Get the initial trim section

        The initial trim section is usually defined in the header with the
        keyword ``TRIMSEC`` but in the case of Goodman HTS this does not work well.
        In particular for spectroscopy where is more likely to have combined
        binning and so on.

{'fraction_non_alphanumeric': 0.06058809580270334, 'fraction_numerical': 0.0016599478302110505, 'mean_word_length': 1.774671052631579, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1192587', 'n_tokens_mistral': 2256, 'n_tokens_neox': 2098, 'n_words': 563}