<commit_before>import triage_fits_files as tff
import ccd_characterization as ccd_char
from astropysics import ccd
from os import path
import pyfits
from datetime import datetime
import numpy as np

combiner = ccd.ImageCombiner()

def master_frame(data, img_type, T, Terr, sample=None,combiner=None):
    copy_from_sample = ['xbinning', 'ybinning',
                        'xpixsz', 'ypixsz', 'exptime','filter']
    img = ccd.FitsImage(data)
    hdr = img.fitsfile[0].header
    hdr.update('imagetyp',img_type)
    now = datetime.utcnow()
    now = now.replace(microsecond=0)
    hdr.update('date', now.isoformat(),
               'Creation date of file')
    hdr.update('ccd-temp', T, 'Average temperature of CCD')
    hdr.update('temp-dev', Terr,
               'Standard deviation of CCD temperature')
    if combiner is not None:
        hdr.update('cmbn-mth',combiner.method,
                   'Combination method for producing master')
        
    if sample is not None:
        if not isinstance(sample, pyfits.Header):
            raise TypeError
        cards = sample.ascard
        for key in copy_from_sample:
            hdr.update(key,cards[key].value,cards[key].comment)
    return img
    
def master_flat(directories):
    """
    Construct master flats by combining individual flats.

    :param directories: List of directories.

    Each directory must contain master darks whose exposure time
    matches the flats in the directory. A separate master flat will be
    constructed for each filter band for which there are flats in the directory.
    """
    for currentDir in directories:
        keywords = ['imagetyp', 'exptime', 'filter', 'ccd-temp']
        image_collection = tff.ImageFileCollection(location=currentDir,
                                                   keywords=keywords,
                                                   info_file=None)
        images = image_collection.summary_info
        master_dark_files = images.where(images['imagetyp'] == 'MASTER DARK')
        all_flats = images.where(images['imagetyp'] == 'FLAT')
        exposure_times = set(all_flats['exptime'])
        print exposure_times
        for time in exposure_times:
            flats_time = all_flats.where(all_flats['exptime'] == time)
            filters = np.unique(flats_time['filter'])
            for flat_filter in filters:
                these_flats = flats_time.where(flats_time['filter'] == flat_filter)
                same_filter = (these_flats['filter'] == flat_filter)
                if not same_filter.all():
                    raise RuntimeError('Holy crap, my flats have mixed filters!')
                master_dark = master_dark_files.where(master_dark_files['exptime']==time)
                if not master_dark:
                    print 'Sorry, no dark for the exposure %f, skipping....' %time
                    continue
                master_dark = ccd.FitsImage(path.join(currentDir,master_dark['file'][0]))
                flats = []
                for flat_file in these_flats['file']:
                    flat = ccd.FitsImage(path.join(currentDir,flat_file))
                    flats.append(flat.data - master_dark.data)
                master_flat = combiner.combineImages(flats)
                avg_temp = these_flats['ccd-temp'].mean()
                temp_dev = these_flats['ccd-temp'].std()
                sample = pyfits.open(path.join(currentDir,these_flats['file'][0]))
                flat_im = master_frame(master_flat, 'MASTER FLAT', avg_temp,
                                       temp_dev, sample=sample[0].header,
                                       combiner=combiner)
                flat_fn = 'Master_Flat_%s_band.fit' % flat_filter
                flat_im.save(path.join(currentDir,flat_fn))

if __name__ == "__main__":
    from sys import argv
    master_flat(argv[1:])<commit_msg>Start merging master flat routines with master bias/dark.<commit_after>import triage_fits_files as tff
import ccd_characterization as ccd_char
from astropysics import ccd
from os import path
import pyfits
from datetime import datetime
import numpy as np
from master_bias_dark import master_frame, add_files_info

combiner = ccd.ImageCombiner()

def master_flat(directories):
    """
    Construct master flats by combining individual flats.

    :param directories: List of directories.

    Each directory must contain master darks whose exposure time
    matches the flats in the directory. A separate master flat will be
    constructed for each filter band for which there are flats in the directory.
    """
    for currentDir in directories:
        keywords = ['imagetyp', 'exptime', 'filter', 'ccd-temp']
        image_collection = tff.ImageFileCollection(location=currentDir,
                                                   keywords=keywords,
                                                   info_file=None)
        images = image_collection.summary_info
        master_dark_files = images.where(images['imagetyp'] == 'MASTER DARK')
        all_flats = images.where(images['imagetyp'] == 'FLAT')
        exposure_times = set(all_flats['exptime'])
        print exposure_times
        for time in exposure_times:
            flats_time = all_flats.where(all_flats['exptime'] == time)
            filters = np.unique(flats_time['filter'])
            for flat_filter in filters:
                these_flats = flats_time.where(flats_time['filter'] == flat_filter)
                same_filter = (these_flats['filter'] == flat_filter)
                if not same_filter.all():
                    raise RuntimeError('Holy crap, my flats have mixed filters!')
                master_dark = master_dark_files.where(master_dark_files['exptime']==time)
                if not master_dark:
                    print 'Sorry, no dark for the exposure %f, skipping....' %time
                    continue
                master_dark = ccd.FitsImage(path.join(currentDir,master_dark['file'][0]))
                flats = []
                for flat_file in these_flats['file']:
                    flat = ccd.FitsImage(path.join(currentDir,flat_file))
                    flats.append(flat.data - master_dark.data)
                combiner.method = "median"
                master_flat = combiner.combineImages(flats)
                avg_temp = these_flats['ccd-temp'].mean()
                temp_dev = these_flats['ccd-temp'].std()
                sample = pyfits.open(path.join(currentDir,these_flats['file'][0]))
                flat_im = master_frame(master_flat, 'MASTER FLAT', avg_temp,
                                       temp_dev, sample=sample[0].header,
                                       combiner=combiner)
                flat_fn = 'Master_Flat_%s_band.fit' % flat_filter
                add_files_info(flat_im, these_flats['file'])
                flat_im.save(path.join(currentDir,flat_fn))

if __name__ == "__main__":
    from sys import argv
    master_flat(argv[1:])
{'fraction_non_alphanumeric': 0.09381294964028777, 'fraction_numerical': 0.0014388489208633094, 'mean_word_length': 2.1841502519468623, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4960236', 'n_tokens_mistral': 1978, 'n_tokens_neox': 1882, 'n_words': 460}