<commit_before>from __future__ import absolute_import

from unittest import TestCase, skip
from ccdproc import CCDData
from astropy.convolution import convolve, Gaussian1DKernel, Box1DKernel
from astropy.io import fits
from astropy.modeling import Model
from astropy.modeling import (models,
                              fitting)
import astropy.units as u
import collections
import mock
import numpy as np
import os
import pandas
import random
import re
import shutil
import logging

logging.disable(logging.CRITICAL)

# import all classes in core.py
from ..core import (GenerateDcrParFile,
                    NightDataContainer,
                    NoMatchFound,
                    NotEnoughLinesDetected,
                    NoTargetException,
                    ReferenceData,
                    SaturationValues,
                    SpectroscopicMode)


# import of functions in core.py
from ..core import (astroscrappy_lacosmic,
                    add_linear_wavelength_solution,
                    add_wcs_keys,
                    bin_reference_data,
                    call_cosmic_rejection,
                    classify_spectroscopic_data,
                    combine_data,
                    convert_time,
                    create_master_bias,
                    create_master_flats,
                    cross_correlation,
                    dcr_cosmicray_rejection,
                    define_trim_section,
                    extraction,
                    extract_fractional_pixel,
                    extract_optimal,
                    evaluate_wavelength_solution,
                    fix_keywords,
                    fractional_sum,
                    get_best_flat,
                    get_central_wavelength,
                    get_lines_in_lamp,
                    get_overscan_region,
                    get_spectral_characteristics,
                    get_slit_trim_section,
                    get_twilight_time,
                    identify_targets,
                    image_overscan,
                    image_trim,
                    interpolate,
                    is_file_saturated,
                    linearize_spectrum,
                    name_master_flats,
                    normalize_master_flat,
                    ra_dec_to_deg,
                    read_fits,
                    record_trace_information,
                    save_extracted,
                    search_comp_group,
                    setup_logging,
                    trace,
                    trace_targets,
                    validate_ccd_region,
                    write_fits)


def fake_subprocess_popen(*args, stdout, stderr):
    raise OSError


class AddLinearWavelengthSolutionTest(TestCase):

    def setUp(self):
        self.ccd = CCDData(data=np.random.random_sample(200),
                           meta=fits.Header(),
                           unit='adu')
        self.ccd = add_wcs_keys(ccd=self.ccd)
        self.ccd.header.set('SLIT',
                            value='1.0_LONG_SLIT',
                            comment="slit [arcsec]")

    def test_add_wavelength_solution(self):

        calibration_lamp = 'non-existent.fits'

        crval1 = 3977.948
        npix = 4060
        cdelt = 0.9910068

        x_axis = np.linspace(crval1,
                             crval1 + cdelt * npix,
                             npix)

        self.ccd = add_linear_wavelength_solution(
            ccd=self.ccd,
            x_axis=x_axis,
            reference_lamp=calibration_lamp)

        self.assertEqual(self.ccd.header['CTYPE1'], 'LINEAR')
        self.assertEqual(self.ccd.header['CRVAL1'], crval1)
        self.assertEqual(self.ccd.header['CRPIX1'], 1)
        self.assertAlmostEqual(self.ccd.header['CDELT1'], cdelt, places=3)
        self.assertEqual(self.ccd.header['DCLOG1'],
                         'REFSPEC1 = {:s}'.format(calibration_lamp))


class AddWCSKeywordsTest(TestCase):

    def setUp(self):
        self.test_ccd = CCDData(data=np.ones((100, 100)),
                                meta=fits.Header(),
                                unit='adu')

    def test_add_wcs_keys(self):
        wcs_keys = ['BANDID1',
                    'APNUM1',
                    'WCSDIM',
                    'CTYPE1',
                    'CRVAL1',
                    'CRPIX1',
                    'CDELT1',
                    'CD1_1',
                    'LTM1_1',
                    'WAT0_001',
                    'WAT1_001',
                    'DC-FLAG',
                    'DCLOG1']



        self.test_ccd = add_wcs_keys(ccd=self.test_ccd)
        for key in wcs_keys:
            self.assertIn(key, self.test_ccd.header)

    @skip
    def test_add_wcs_keys_error(self):
        wcs_keys = ['BANDID1',
                    'APNUM1',
                    'WCSDIM',
                    'CTYPE1',
                    'CRVAL1',
                    'CRPIX1',
                    'CDELT1',
                    'CD1_1',
                    'LTM1_1',
                    'WAT0_001',
                    'WAT1_001',
                    'DC-FLAG',
                    'DCLOG1']


class BinningTest(TestCase):

    def test__bin_reference_data(self):
        wavelength = np.linspace(3000, 7000, 4000)
        intensity = np.random.random_sample(4000)

        for i in range(1, 4):
            new_wavelength, new_intensity = bin_reference_data(
                wavelength=wavelength,
                intensity=intensity,
                serial_binning=i)

            self.assertEqual(len(wavelength), len(intensity))
            self.assertEqual(len(new_wavelength), len(new_intensity))
            self.assertEqual(len(new_wavelength), np.floor(len(wavelength) / i))




class CentralWavelength(TestCase):

    def setUp(self):
        # 400m2
        self.grating = '400'
        self.grating_angle = 7.5
        self.camera_angle = 16.1
        self.reference_central_wavelength = 7001.54 * u.angstrom

    def test_get_central_wavelength(self):
        central_wavelength = get_central_wavelength(grating=self.grating,
                                                    grt_ang=self.grating_angle,
                                                    cam_ang=self.camera_angle)
        self.assertAlmostEqual(central_wavelength.value,
                               self.reference_central_wavelength.value,
                               places=2)


class ClassifySpectroscopicData(TestCase):

    def setUp(self):
        self.path = os.path.join(
            os.getcwd(),
            'goodman_pipeline/data/test_data/test_classify_spectroscopic');
        if not os.path.isdir(self.path):
            os.mkdir(self.path)

    def tearDown(self):
        if os.path.isdir(self.path):
            shutil.rmtree(self.path)

    def create_fake_spectroscopic_data(self):
        if os.path.isdir(self.path):
            card_values = [
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:24.285', 'obsdec': '-39:12:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:26:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:27:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:28:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:34.285', 'obsdec': ' 39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:34.285', 'obsdec': ' 39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:34.285', 'obsdec': ' 39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:34.285', 'obsdec': ' 39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:34.285', 'obsdec': ' 39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:22:34.285', 'obsdec': '-39:23:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:26:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:27:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:28:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:23:34.285', 'obsdec': '39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:23:34.285', 'obsdec': '39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:23:34.285', 'obsdec': '39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:23:34.285', 'obsdec': '39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:23:34.285', 'obsdec': '39:13:53.954'}]
            for i in range(len(card_values)):
                ccd = CCDData(data=np.ones((3, 3)),
                              meta=fits.Header(),
                              unit='adu')

                ccd.header.set('DATE', value='2019-03-22', comment='nc')
                ccd.header.set('SLIT', value='1.0" long slit', comment='nc')
                ccd.header.set('DATE-OBS', value='2019-03-22T09:59:33.654', comment='nc')
                ccd.header.set('OBSTYPE', value=card_values[i]['obstype'], comment='nc')
                ccd.header.set('OBJECT', value=card_values[i]['object'], comment='nc')
                ccd.header.set('EXPTIME', value='10', comment='nc')
                ccd.header.set('OBSRA', value=card_values[i]['obsra'], comment='nc')
                ccd.header.set('OBSDEC', value=card_values[i]['obsdec'], comment='nc')
                ccd.header.set('GRATING', value='SYZY_400', comment='nc')
                ccd.header.set('CAM_TARG', value='16.1', comment='nc')
                ccd.header.set('GRT_TARG', value='7.5', comment='nc')
                ccd.header.set('FILTER', value='<NO FILTER>', comment='nc')
                ccd.header.set('FILTER2', value='GG455', comment='nc')
                ccd.header.set('GAIN', value='1.48', comment='nc')
                ccd.header.set('RDNOISE', value='3.89', comment='nc')

                ccd.write(os.path.join(self.path, 'test_file_{:03d}.fits'.format(i)))


    def test_classify_spectroscopic_data__no_data(self):
        self.assertRaises(SystemExit, classify_spectroscopic_data, self.path, '*fits')

    def test_classify_spectroscopic_data(self):
        self.create_fake_spectroscopic_data()
        result = classify_spectroscopic_data(path=self.path, search_pattern='test_file')
        self.assertIsInstance(result, NightDataContainer)
        self.assertFalse(result.is_empty)


class CombineDataTest(TestCase):

    def setUp(self):
        self.ccd1 = CCDData(data=np.ones((100, 100)),
                           meta=fits.Header(),
                           unit='adu')
        self.ccd1.header.set('OBJECT', value='TestObject')
        self.ccd1.header.set('GRATING', value='Grating')
        self.ccd1.header.set('SLIT', value='1.05SlitSize')

        self.ccd2 = self.ccd1.copy()
        self.ccd2.data *= 2
        self.ccd3 = self.ccd1.copy()
        self.ccd3.data *= 5
        self.ccd1.header.set('GSP_FNAM', value='image_1.fits')
        self.ccd2.header.set('GSP_FNAM', value='image_2.fits')
        self.ccd3.header.set('GSP_FNAM', value='image_3.fits')
        self.image_list = [self.ccd1, self.ccd2, self.ccd3]
        self.dest_path = os.getcwd()
        self.prefix = 'testing_'
        self.output_name = 'combine_data.fits'

    def tearDown(self):
        using_output_name_file_name = os.path.join(
            self.dest_path,
            self.output_name)
        if os.path.isfile(using_output_name_file_name):
            os.unlink(using_output_name_file_name)

        not_using_output_name_file_name_list = os.listdir(self.dest_path)
        if not_using_output_name_file_name_list:
            for _file in not_using_output_name_file_name_list:
                if '{:s}combined'.format(self.prefix) in _file:
                    os.unlink(_file)

    def test_combine_data_median_prefix_ignored(self):
        combined = combine_data(
            image_list=self.image_list,
            dest_path=self.dest_path,
            prefix=self.prefix,
            output_name=self.output_name,
            method='median',
            save=True)

        np.testing.assert_array_equal(combined.data, np.ones((100, 100)) * 1.5)
        self.assertEqual(len(combined.header['GSP_IC*']), 3)
        self.assertTrue(self.prefix not in combined.header['GSP_FNAM'])

    def test_combine_data_median_prefix_used(self):
        combined = combine_data(
            image_list=self.image_list,
            dest_path=self.dest_path,
            prefix=self.prefix,
            method='median',
            save=True)

        np.testing.assert_array_equal(combined.data, np.ones((100, 100)) * 1.5)
        self.assertEqual(len(combined.header['GSP_IC*']), 3)
        self.assertTrue(self.prefix in combined.header['GSP_FNAM'])



class CosmicRayRejectionTest(TestCase):

    def setUp(self):
        self.ccd = CCDData(data=np.ones((100, 100)),
                           meta=fits.Header(),
                           unit='adu')
        self.file_name = 'cr_test.fits'

        self.ccd.header.set('CCDSUM', value='1 1')
        self.ccd.header.set('OBSTYPE', value='OBJECT')
        self.ccd.header.set('INSTCONF', value='Red')
        self.ccd.header.set('GSP_FNAM', value=self.file_name)
        self.ccd.header.set('GSP_COSM', value='none')

        self.red_path = os.getcwd()
        self.out_prefix = 'prefix'

    @skip
    def test_dcr_cosmicray_rejection(self):
        pass

    def test_call_cosmic_rejection_default_1x1(self):
        prefix = 'new_'
        initial_value = self.ccd.data[50, 50]
        self.ccd.data[50, 50] = 50000

        ccd, out_prefix = call_cosmic_rejection(ccd=self.ccd,
                                                image_name=self.file_name,
                                                out_prefix=self.out_prefix,
                                                red_path=self.red_path,
                                                dcr_par=os.getcwd(),
                                                keep_files=True,
                                                prefix=prefix,
                                                method='default',
                                                save=True)
        self.assertAlmostEqual(initial_value, ccd.data[50, 50])
        self.assertEqual(out_prefix, prefix + self.out_prefix)
        self.assertEqual(ccd.header['GSP_FNAM'],
                         prefix + self.out_prefix + self.file_name)
        self.assertEqual(ccd.header['GSP_COSM'], 'DCR')

        self.assertTrue(os.path.isfile('dcr.par'))
        self.assertTrue(os.path.isfile('new_prefixcr_test.fits'))

    @mock.patch('subprocess.Popen', side_effect=fake_subprocess_popen)
    def test_call_cosmic_rejection_default_1x1_no_dcr_par(self,
                                                          subprocess_Popen_function):

        self.assertRaises(SystemExit,
                          call_cosmic_rejection,
                          self.ccd,
                          self.file_name,
                          self.out_prefix,
                          self.red_path,
                          os.getcwd())


    def test_call_cosmic_rejection_default_2x2(self):
        self.ccd.header.set('CCDSUM', value='2 2')
        prefix = 'new_'
        initial_value = self.ccd.data[50, 50]
        self.ccd.data[50, 50] = 50000

        ccd, out_prefix = call_cosmic_rejection(ccd=self.ccd,
                                                image_name=self.file_name,
                                                out_prefix=self.out_prefix,
                                                red_path=self.red_path,
                                                dcr_par=os.getcwd(),
                                                keep_files=True,
                                                prefix=prefix,
                                                method='default',
                                                save=True)
        self.assertAlmostEqual(initial_value, ccd.data[50, 50])
        self.assertEqual(out_prefix, prefix + self.out_prefix)
        self.assertEqual(ccd.header['GSP_FNAM'],
                         prefix + self.out_prefix + self.file_name)
        self.assertEqual(ccd.header['GSP_COSM'], 'LACosmic')
        self.assertTrue(os.path.isfile('new_prefixcr_test.fits'))
<commit_msg>added test for case when dcr is not found by subprocess<commit_after>from __future__ import absolute_import

from unittest import TestCase, skip
from ccdproc import CCDData
from astropy.convolution import convolve, Gaussian1DKernel, Box1DKernel
from astropy.io import fits
from astropy.modeling import Model
from astropy.modeling import (models,
                              fitting)
import astropy.units as u
import collections
import mock
import numpy as np
import os
import pandas
import random
import re
import shutil
import logging

logging.disable(logging.CRITICAL)

# import all classes in core.py
from ..core import (GenerateDcrParFile,
                    NightDataContainer,
                    NoMatchFound,
                    NotEnoughLinesDetected,
                    NoTargetException,
                    ReferenceData,
                    SaturationValues,
                    SpectroscopicMode)


# import of functions in core.py
from ..core import (astroscrappy_lacosmic,
                    add_linear_wavelength_solution,
                    add_wcs_keys,
                    bin_reference_data,
                    call_cosmic_rejection,
                    classify_spectroscopic_data,
                    combine_data,
                    convert_time,
                    create_master_bias,
                    create_master_flats,
                    cross_correlation,
                    dcr_cosmicray_rejection,
                    define_trim_section,
                    extraction,
                    extract_fractional_pixel,
                    extract_optimal,
                    evaluate_wavelength_solution,
                    fix_keywords,
                    fractional_sum,
                    get_best_flat,
                    get_central_wavelength,
                    get_lines_in_lamp,
                    get_overscan_region,
                    get_spectral_characteristics,
                    get_slit_trim_section,
                    get_twilight_time,
                    identify_targets,
                    image_overscan,
                    image_trim,
                    interpolate,
                    is_file_saturated,
                    linearize_spectrum,
                    name_master_flats,
                    normalize_master_flat,
                    ra_dec_to_deg,
                    read_fits,
                    record_trace_information,
                    save_extracted,
                    search_comp_group,
                    setup_logging,
                    trace,
                    trace_targets,
                    validate_ccd_region,
                    write_fits)


def fake_subprocess_popen(*args, stdout, stderr):
    raise OSError


def fake_dcr_communicate_stderr_no_dcr():
    return b'some message', b'dcr: not found'


class AddLinearWavelengthSolutionTest(TestCase):

    def setUp(self):
        self.ccd = CCDData(data=np.random.random_sample(200),
                           meta=fits.Header(),
                           unit='adu')
        self.ccd = add_wcs_keys(ccd=self.ccd)
        self.ccd.header.set('SLIT',
                            value='1.0_LONG_SLIT',
                            comment="slit [arcsec]")

    def test_add_wavelength_solution(self):

        calibration_lamp = 'non-existent.fits'

        crval1 = 3977.948
        npix = 4060
        cdelt = 0.9910068

        x_axis = np.linspace(crval1,
                             crval1 + cdelt * npix,
                             npix)

        self.ccd = add_linear_wavelength_solution(
            ccd=self.ccd,
            x_axis=x_axis,
            reference_lamp=calibration_lamp)

        self.assertEqual(self.ccd.header['CTYPE1'], 'LINEAR')
        self.assertEqual(self.ccd.header['CRVAL1'], crval1)
        self.assertEqual(self.ccd.header['CRPIX1'], 1)
        self.assertAlmostEqual(self.ccd.header['CDELT1'], cdelt, places=3)
        self.assertEqual(self.ccd.header['DCLOG1'],
                         'REFSPEC1 = {:s}'.format(calibration_lamp))


class AddWCSKeywordsTest(TestCase):

    def setUp(self):
        self.test_ccd = CCDData(data=np.ones((100, 100)),
                                meta=fits.Header(),
                                unit='adu')

    def test_add_wcs_keys(self):
        wcs_keys = ['BANDID1',
                    'APNUM1',
                    'WCSDIM',
                    'CTYPE1',
                    'CRVAL1',
                    'CRPIX1',
                    'CDELT1',
                    'CD1_1',
                    'LTM1_1',
                    'WAT0_001',
                    'WAT1_001',
                    'DC-FLAG',
                    'DCLOG1']



        self.test_ccd = add_wcs_keys(ccd=self.test_ccd)
        for key in wcs_keys:
            self.assertIn(key, self.test_ccd.header)

    @skip
    def test_add_wcs_keys_error(self):
        wcs_keys = ['BANDID1',
                    'APNUM1',
                    'WCSDIM',
                    'CTYPE1',
                    'CRVAL1',
                    'CRPIX1',
                    'CDELT1',
                    'CD1_1',
                    'LTM1_1',
                    'WAT0_001',
                    'WAT1_001',
                    'DC-FLAG',
                    'DCLOG1']


class BinningTest(TestCase):

    def test__bin_reference_data(self):
        wavelength = np.linspace(3000, 7000, 4000)
        intensity = np.random.random_sample(4000)

        for i in range(1, 4):
            new_wavelength, new_intensity = bin_reference_data(
                wavelength=wavelength,
                intensity=intensity,
                serial_binning=i)

            self.assertEqual(len(wavelength), len(intensity))
            self.assertEqual(len(new_wavelength), len(new_intensity))
            self.assertEqual(len(new_wavelength), np.floor(len(wavelength) / i))




class CentralWavelength(TestCase):

    def setUp(self):
        # 400m2
        self.grating = '400'
        self.grating_angle = 7.5
        self.camera_angle = 16.1
        self.reference_central_wavelength = 7001.54 * u.angstrom

    def test_get_central_wavelength(self):
        central_wavelength = get_central_wavelength(grating=self.grating,
                                                    grt_ang=self.grating_angle,
                                                    cam_ang=self.camera_angle)
        self.assertAlmostEqual(central_wavelength.value,
                               self.reference_central_wavelength.value,
                               places=2)


class ClassifySpectroscopicData(TestCase):

    def setUp(self):
        self.path = os.path.join(
            os.getcwd(),
            'goodman_pipeline/data/test_data/test_classify_spectroscopic');
        if not os.path.isdir(self.path):
            os.mkdir(self.path)

    def tearDown(self):
        if os.path.isdir(self.path):
            shutil.rmtree(self.path)

    def create_fake_spectroscopic_data(self):
        if os.path.isdir(self.path):
            card_values = [
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:24.285', 'obsdec': '-39:12:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:26:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:27:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:28:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:34.285', 'obsdec': ' 39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:34.285', 'obsdec': ' 39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:34.285', 'obsdec': ' 39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:34.285', 'obsdec': ' 39:13:53.954'},
                {'obstype': 'OBJECT', 'object': 'NGC2070', 'obsra': '16:23:34.285', 'obsdec': ' 39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:22:34.285', 'obsdec': '-39:23:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:24:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:25:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:26:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:27:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:28:34.285', 'obsdec': '-39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:23:34.285', 'obsdec': '39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:23:34.285', 'obsdec': '39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:23:34.285', 'obsdec': '39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:23:34.285', 'obsdec': '39:13:53.954'},
                {'obstype': 'COMP',   'object': 'HgArNe',  'obsra': '16:23:34.285', 'obsdec': '39:13:53.954'}]
            for i in range(len(card_values)):
                ccd = CCDData(data=np.ones((3, 3)),
                              meta=fits.Header(),
                              unit='adu')

                ccd.header.set('DATE', value='2019-03-22', comment='nc')
                ccd.header.set('SLIT', value='1.0" long slit', comment='nc')
                ccd.header.set('DATE-OBS', value='2019-03-22T09:59:33.654', comment='nc')
                ccd.header.set('OBSTYPE', value=card_values[i]['obstype'], comment='nc')
                ccd.header.set('OBJECT', value=card_values[i]['object'], comment='nc')
                ccd.header.set('EXPTIME', value='10', comment='nc')
                ccd.header.set('OBSRA', value=card_values[i]['obsra'], comment='nc')
                ccd.header.set('OBSDEC', value=card_values[i]['obsdec'], comment='nc')
                ccd.header.set('GRATING', value='SYZY_400', comment='nc')
                ccd.header.set('CAM_TARG', value='16.1', comment='nc')
                ccd.header.set('GRT_TARG', value='7.5', comment='nc')
                ccd.header.set('FILTER', value='<NO FILTER>', comment='nc')
                ccd.header.set('FILTER2', value='GG455', comment='nc')
                ccd.header.set('GAIN', value='1.48', comment='nc')
                ccd.header.set('RDNOISE', value='3.89', comment='nc')

                ccd.write(os.path.join(self.path, 'test_file_{:03d}.fits'.format(i)))


    def test_classify_spectroscopic_data__no_data(self):
        self.assertRaises(SystemExit, classify_spectroscopic_data, self.path, '*fits')

    def test_classify_spectroscopic_data(self):
        self.create_fake_spectroscopic_data()
        result = classify_spectroscopic_data(path=self.path, search_pattern='test_file')
        self.assertIsInstance(result, NightDataContainer)
        self.assertFalse(result.is_empty)


class CombineDataTest(TestCase):

    def setUp(self):
        self.ccd1 = CCDData(data=np.ones((100, 100)),
                           meta=fits.Header(),
                           unit='adu')
        self.ccd1.header.set('OBJECT', value='TestObject')
        self.ccd1.header.set('GRATING', value='Grating')
        self.ccd1.header.set('SLIT', value='1.05SlitSize')

        self.ccd2 = self.ccd1.copy()
        self.ccd2.data *= 2
        self.ccd3 = self.ccd1.copy()
        self.ccd3.data *= 5
        self.ccd1.header.set('GSP_FNAM', value='image_1.fits')
        self.ccd2.header.set('GSP_FNAM', value='image_2.fits')
        self.ccd3.header.set('GSP_FNAM', value='image_3.fits')
        self.image_list = [self.ccd1, self.ccd2, self.ccd3]
        self.dest_path = os.getcwd()
        self.prefix = 'testing_'
        self.output_name = 'combine_data.fits'

    def tearDown(self):
        using_output_name_file_name = os.path.join(
            self.dest_path,
            self.output_name)
        if os.path.isfile(using_output_name_file_name):
            os.unlink(using_output_name_file_name)

        not_using_output_name_file_name_list = os.listdir(self.dest_path)
        if not_using_output_name_file_name_list:
            for _file in not_using_output_name_file_name_list:
                if '{:s}combined'.format(self.prefix) in _file:
                    os.unlink(_file)

    def test_combine_data_median_prefix_ignored(self):
        combined = combine_data(
            image_list=self.image_list,
            dest_path=self.dest_path,
            prefix=self.prefix,
            output_name=self.output_name,
            method='median',
            save=True)

        np.testing.assert_array_equal(combined.data, np.ones((100, 100)) * 1.5)
        self.assertEqual(len(combined.header['GSP_IC*']), 3)
        self.assertTrue(self.prefix not in combined.header['GSP_FNAM'])

    def test_combine_data_median_prefix_used(self):
        combined = combine_data(
            image_list=self.image_list,
            dest_path=self.dest_path,
            prefix=self.prefix,
            method='median',
            save=True)

        np.testing.assert_array_equal(combined.data, np.ones((100, 100)) * 1.5)
        self.assertEqual(len(combined.header['GSP_IC*']), 3)
        self.assertTrue(self.prefix in combined.header['GSP_FNAM'])



class CosmicRayRejectionTest(TestCase):

    def setUp(self):
        self.ccd = CCDData(data=np.ones((100, 100)),
                           meta=fits.Header(),
                           unit='adu')
        self.file_name = 'cr_test.fits'

        self.ccd.header.set('CCDSUM', value='1 1')
        self.ccd.header.set('OBSTYPE', value='OBJECT')
        self.ccd.header.set('INSTCONF', value='Red')
        self.ccd.header.set('GSP_FNAM', value=self.file_name)
        self.ccd.header.set('GSP_COSM', value='none')

        self.red_path = os.getcwd()
        self.out_prefix = 'prefix'

    @skip
    def test_dcr_cosmicray_rejection(self):
        pass

    def test_call_cosmic_rejection_default_1x1(self):
        prefix = 'new_'
        initial_value = self.ccd.data[50, 50]
        self.ccd.data[50, 50] = 50000

        ccd, out_prefix = call_cosmic_rejection(ccd=self.ccd,
                                                image_name=self.file_name,
                                                out_prefix=self.out_prefix,
                                                red_path=self.red_path,
                                                dcr_par=os.getcwd(),
                                                keep_files=True,
                                                prefix=prefix,
                                                method='default',
                                                save=True)
        self.assertAlmostEqual(initial_value, ccd.data[50, 50])
        self.assertEqual(out_prefix, prefix + self.out_prefix)
        self.assertEqual(ccd.header['GSP_FNAM'],
                         prefix + self.out_prefix + self.file_name)
        self.assertEqual(ccd.header['GSP_COSM'], 'DCR')

        self.assertTrue(os.path.isfile('dcr.par'))
        self.assertTrue(os.path.isfile('new_prefixcr_test.fits'))

    @mock.patch('subprocess.Popen', side_effect=fake_subprocess_popen)
    def test_call_cosmic_rejection_default_1x1_no_dcr_par(self,
                                                          subprocess_Popen_function):

        self.assertRaises(SystemExit,
                          call_cosmic_rejection,
                          self.ccd,
                          self.file_name,
                          self.out_prefix,
                          self.red_path,
                          os.getcwd())

    @mock.patch('subprocess.Popen.communicate',
                side_effect=fake_dcr_communicate_stderr_no_dcr)
    def test_dcr_cosmicray_rejection_no_dcr_executable(
            self, subprocess_Popen_communicate):

        self.assertRaises(SystemExit,
                          dcr_cosmicray_rejection,
                          self.red_path,
                          self.file_name,
                          'c',
                          os.getcwd())


    def test_call_cosmic_rejection_default_2x2(self):
        self.ccd.header.set('CCDSUM', value='2 2')
        prefix = 'new_'
        initial_value = self.ccd.data[50, 50]
        self.ccd.data[50, 50] = 50000

        ccd, out_prefix = call_cosmic_rejection(ccd=self.ccd,
                                                image_name=self.file_name,
                                                out_prefix=self.out_prefix,
                                                red_path=self.red_path,
                                                dcr_par=os.getcwd(),
                                                keep_files=True,
                                                prefix=prefix,
                                                method='default',
                                                save=True)
        self.assertAlmostEqual(initial_value, ccd.data[50, 50])
        self.assertEqual(out_prefix, prefix + self.out_prefix)
        self.assertEqual(ccd.header['GSP_FNAM'],
                         prefix + self.out_prefix + self.file_name)
        self.assertEqual(ccd.header['GSP_COSM'], 'LACosmic')
        self.assertTrue(os.path.isfile('new_prefixcr_test.fits'))

{'fraction_non_alphanumeric': 0.13450372106970532, 'fraction_numerical': 0.0473243736880844, 'mean_word_length': 1.7310899344848123, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 76, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3734606', 'n_tokens_mistral': 12695, 'n_tokens_neox': 11161, 'n_words': 1686}