<commit_before>
# coding: utf-8

"""
reduce_arctic.py
<NAME>
2016

Automatic reduction pipeline for the Astrophysical Research Consortium Imaging Camera (ARCTIC) at Apache Point Observatory (APO).


to use:
python reduce_arctic.py [your directory of data]

OR place .py in your folder with data and run with no argument:
python reduce_arctic.py

creates /reduced/cals/ and /reduced/data/

"""

import astropy.io.fits as pyfits
import numpy as np
import matplotlib.pyplot as plt
import pyfits
import pylab
import glob
import os
import re
import sys
import pandas as pd


if len(sys.argv) > 0:
    direc = sys.argv[1]
else:
    direc = '.'
    
files = glob.glob(direc+"/*.fits")


if not os.path.exists(direc+'/reduced/cals'):
    os.makedirs(direc+'/reduced/cals')
if not os.path.exists(direc+'/reduced/data'):
    os.makedirs(direc+'/reduced/data')

    
df = pd.DataFrame(files,columns=['fname'])
df['objtype'] = pd.Series("", index=df.index)
df['filt'] = pd.Series("", index=df.index)
df['exp'] = pd.Series("", index=df.index)
df['objname'] = pd.Series("", index=df.index)


for ff,fname in enumerate(files):
    try:
        df['objtype'][ff] = pyfits.open(fname)[0].header['IMAGETYP']
        df['filt'][ff] = pyfits.open(fname)[0].header['FILTER']
        df['exp'][ff] = pyfits.open(fname)[0].header['EXPTIME']
        df['objname'][ff] = pyfits.open(fname)[0].header['OBJNAME']
    except IOError:    
        print('\n File corrupt or missing: ' + fname)


def trim_image(f):
    datfile = pyfits.getdata(f, header=True)
    dat_raw = datfile[0]
    dat_head = datfile[1]
    
    amp = pyfits.open(f)[0].header['READAMPS']
    
    if amp == "Quad":
        # ll, ul, lr, ur
        quads = ['DSEC11', 'DSEC21', 'DSEC12', 'DSEC22']

        dat = [[],[],[],[]]
        for i,quad in enumerate(quads):
            idx_string = pyfits.open(f)[0].header[quad]
            idx = re.split('[: ,]',idx_string.rstrip(']').lstrip('['))
            dat[i] = dat_raw[int(idx[2]):int(idx[3]),int(idx[0]):int(idx[1])]
    
        sci_lo = np.concatenate((dat[2], dat[3]), axis = 1)
        sci_up = np.concatenate((dat[0], dat[1]), axis = 1)
        sci = np.concatenate((sci_up, sci_lo), axis = 0)
    
    if amp == 'LL':
        idx_string = pyfits.open(f)[0].header['DSEC11']
        idx = re.split('[: ,]',idx_string.rstrip(']').lstrip('['))
        sci = dat_raw[int(idx[2]):int(idx[3]),int(idx[0]):int(idx[1])]
    
    return [sci,dat_head]



### CREATE MASTER BIAS #######################################

print('\n >>> Starting bias combine...')

bias_idx = df[df['objtype'] == 'Bias'].index.tolist()
biases = np.array([trim_image(df['fname'][n])[0] for n in bias_idx])

bias = np.median(biases,axis=0)
pyfits.writeto(direc+'/reduced/cals/master_bias.fits',bias,clobber=True)

print('     > Created master bias')



### CREATE MASTER DARKS ######################################
### these are bias subtracted

times = pd.unique(df.exp.ravel())

print('\n >>> Starting darks (if you have them)...')

for ii in range(0,len(times)):
    dark_idx = df[(df['exp'] == times[ii]) & 
                (df['objtype'] == 'Dark')].index.tolist()
    if len(dark_idx) == 0:<commit_msg>starting to do median/mean flag<commit_after>
# coding: utf-8

"""
reduce_arctic.py
<NAME>
2016

Automatic reduction pipeline for the Astrophysical Research Consortium Imaging Camera (ARCTIC) at Apache Point Observatory (APO).


to use:
python reduce_arctic.py [your directory of data]

OR place .py in your folder with data and run with no argument:
python reduce_arctic.py

creates /reduced/cals/ and /reduced/data/

"""

import astropy.io.fits as pyfits
import numpy as np
import matplotlib.pyplot as plt
import pyfits
import pylab
import glob
import os
import re
import sys
import pandas as pd


if len(sys.argv) > 0:
    direc = sys.argv[1]
else:
    direc = '.'
    
files = glob.glob(direc+"/*.fits")


if not os.path.exists(direc+'/reduced/cals'):
    os.makedirs(direc+'/reduced/cals')
if not os.path.exists(direc+'/reduced/data'):
    os.makedirs(direc+'/reduced/data')

    
df = pd.DataFrame(files,columns=['fname'])
df['objtype'] = pd.Series("", index=df.index)
df['filt'] = pd.Series("", index=df.index)
df['exp'] = pd.Series("", index=df.index)
df['objname'] = pd.Series("", index=df.index)


for ff,fname in enumerate(files):
    try:
        df['objtype'][ff] = pyfits.open(fname)[0].header['IMAGETYP']
        df['filt'][ff] = pyfits.open(fname)[0].header['FILTER']
        df['exp'][ff] = pyfits.open(fname)[0].header['EXPTIME']
        df['objname'][ff] = pyfits.open(fname)[0].header['OBJNAME']
    except IOError:    
        print('\n File corrupt or missing: ' + fname)


def trim_image(f):
    datfile = pyfits.getdata(f, header=True)
    dat_raw = datfile[0]
    dat_head = datfile[1]
    
    amp = pyfits.open(f)[0].header['READAMPS']
    
    if amp == "Quad":
        # ll, ul, lr, ur
        quads = ['DSEC11', 'DSEC21', 'DSEC12', 'DSEC22']

        dat = [[],[],[],[]]
        for i,quad in enumerate(quads):
            idx_string = pyfits.open(f)[0].header[quad]
            idx = re.split('[: ,]',idx_string.rstrip(']').lstrip('['))
            dat[i] = dat_raw[int(idx[2]):int(idx[3]),int(idx[0]):int(idx[1])]
    
        sci_lo = np.concatenate((dat[2], dat[3]), axis = 1)
        sci_up = np.concatenate((dat[0], dat[1]), axis = 1)
        sci = np.concatenate((sci_up, sci_lo), axis = 0)
    
    if amp == 'LL':
        idx_string = pyfits.open(f)[0].header['DSEC11']
        idx = re.split('[: ,]',idx_string.rstrip(']').lstrip('['))
        sci = dat_raw[int(idx[2]):int(idx[3]),int(idx[0]):int(idx[1])]
    
    return [sci,dat_head]


### CREATE MASTER BIAS #######################################

print('\n >>> Starting bias combine...')

bias_idx = df[df['objtype'] == 'Bias'].index.tolist()
biases = np.array([trim_image(df['fname'][n])[0] for n in bias_idx])

bias = np.median(biases,axis=0)
pyfits.writeto(direc+'/reduced/cals/master_bias.fits',bias,clobber=True)

print('     > Created master bias')



### CREATE MASTER DARKS ######################################
### these are bias subtracted

times = pd.unique(df.exp.ravel())

print('\n >>> Starting darks (if you have them)...')

for ii in range(0,len(times)):
    dark_idx = df[(df['exp'] == times[ii]) & 
                (df['objtype'] == 'Dark')].index.tolist()
    if len(dark_idx) == 0:
{'fraction_non_alphanumeric': 0.21109543958627175, 'fraction_numerical': 0.014104372355430184, 'mean_word_length': 3.555317630264097, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5853431', 'n_tokens_mistral': 2475, 'n_tokens_neox': 2346, 'n_words': 487}