<commit_before># -*- coding: utf-8 -*-
"""
Created on Thu Jul 27 16:39:59 2017

@author: asadm2
"""
from __future__ import division, print_function, absolute_import
import astropy
from glob import glob
from astropy.io import fits
from astropy.io.fits import getdata
import warnings
from astropy.utils.exceptions import AstropyUserWarning
import pandas as pd
import numpy as np
import os
import subprocess
import imp

foo = imp.load_source('ECO_funcs', '../mods_swarp/ECO_funcs.py')

#Formatted ECO catalog with filters and new_filename column and list of 
#good ECO objects
ECOnew  = '../../../data/interim/ECO_formatted.txt'
goodObj = '../../../data/interim/goodObj.txt'

#Using the make_dict function to return a dictionary whose keys are ECOID and
#filter name and whose values are filenames, return the list of sorted keys,
#and the formatted ECO catalog
ECO_dict,ECO_keys,ECOcat = foo.make_dict(ECOnew,goodObj)

for index,key in enumerate(ECO_keys):
    print('Key {0}/{1}'.format(index+1,len(ECO_keys)))
    print(key)
    obj = key[0]
    filter_i = key[1]
    dir_path = os.getcwd()
    #Since this file is in src/ and we need to access the images we have to 
    #move to the data/raw folder the first time this code runs
    if os.path.basename(dir_path) == 'main':
        os.chdir('../../../data/raw/'+obj)
    #After the first time all that has to be done is to move out of the  
    #interim ECO object dir and into the raw ECO object dir
    elif os.path.basename(dir_path) != obj:
        print(dir_path)
        os.chdir('../raw/'+obj)
    #Using the ECOID,filter key to acess the values in the new_filename column
    #i.e. get a list of the images associated with this key pair
    imgs = ECO_dict.get_group(key)['new_filename'].values
    #Using the update_header function to output the name of the text file
    #that contains the names of the new versions of the fits images being
    #input into the function
    img_list_txt = foo.update_header(imgs,obj,filter_i)

    #To check for case where all images were in Except clause of update_header
    #and run swarp even for single images
    if len(imgs) >= 1 and img_list_txt != 'error': 
        ra = np.unique(ECOcat.RA.loc[ECOcat.ECOID==obj])[0]
        dec = np.unique(ECOcat.DEC.loc[ECOcat.ECOID==obj])[0]
        
        #Running swarp with parameters. The only ones that change are the list
        #of images, the RA and DEC of the object, the object name and the 
        #filter name
        print('Starting to swarp')    
        subprocess.call(['swarp','@'+img_list_txt,'-CENTER_TYPE','MANUAL', \
        '-CENTER',str(ra)+","+str(dec),'-NTHREADS','32', \
        '-COMBINE_BUFSIZE','10000','-MEM_MAX','1000','-VMEM_MAX','5000',\
        '-IMAGE_SIZE','3960','-GAIN_KEYWORD','CCDGAIN','-GAIN_DEFAULT','1.0',\
        '-SUBTRACT_BACK','Y','-IMAGEOUT_NAME',obj+'_'+filter_i+'_'+'coadd.fits',\
        '-WEIGHTOUT_NAME',obj+'_'+filter_i+'_'+'coadd.weight.fits','-XML_NAME',\
        obj+'_'+filter_i+'_'+'swarp.xml'])
        print('Finished swarping')
        
        coadd = fits.open(obj+'_'+filter_i+'_'+'coadd.fits')
        #Open the coadd and use the percent_blank function 
        foo.percent_blank(coadd,obj,filter_i)
        #So that you can distinguish between interim object folder and raw 
        #object filder
        
    else:
        os.chdir('..') # Moving one level up when no imgs are found
        <commit_msg>Changes to add BUNIT check<commit_after># -*- coding: utf-8 -*-
"""
Created on Thu Jul 27 16:39:59 2017

@author: asadm2
"""
from __future__ import division, print_function, absolute_import
import astropy
from glob import glob
from astropy.io import fits
from astropy.io.fits import getdata
import warnings
from astropy.utils.exceptions import AstropyUserWarning
import pandas as pd
import numpy as np
import os
import subprocess
import imp

foo = imp.load_source('ECO_funcs', '../mods_swarp/ECO_funcs.py')

#Formatted ECO catalog with filters and new_filename column and list of 
#good ECO objects
ECOnew  = '../../../data/interim/ECO_formatted.txt'
goodObj = '../../../data/interim/goodObj.txt'

#Using the make_dict function to return a dictionary whose keys are ECOID and
#filter name and whose values are filenames, return the list of sorted keys,
#and the formatted ECO catalog
ECO_dict,ECO_keys,ECOcat = foo.make_dict(ECOnew,goodObj)

for index,key in enumerate(ECO_keys[0:1]):
    print('Key {0}/{1}'.format(index+1,len(ECO_keys)))
    print(key)
    obj = key[0]
    filter_i = key[1]
    dir_path = os.getcwd()
    #Since this file is in src/ and we need to access the images we have to 
    #move to the data/raw folder the first time this code runs
    if os.path.basename(dir_path) == 'main':
        os.chdir('../../../data/raw/'+obj)
    #After the first time all that has to be done is to move out of the  
    #interim ECO object dir and into the raw ECO object dir
    elif os.path.basename(dir_path) != obj:
        print(dir_path)
        os.chdir('../raw/'+obj)
    #Using the ECOID,filter key to acess the values in the new_filename column
    #i.e. get a list of the images associated with this key pair
    imgs = ECO_dict.get_group(key)['new_filename'].values
    #Using the update_header function to output the name of the text file
    #that contains the names of the new versions of the fits images being
    #input into the function
    img_list_txt = foo.update_header(imgs,obj,filter_i)

    #To check for case where all images were in Except clause of update_header
    #and run swarp even for single images
    if len(imgs) >= 1 and img_list_txt != 'error': 
        ra = np.unique(ECOcat.RA.loc[ECOcat.ECOID==obj])[0]
        dec = np.unique(ECOcat.DEC.loc[ECOcat.ECOID==obj])[0]
        
        #Running swarp with parameters. The only ones that change are the list
        #of images, the RA and DEC of the object, the object name and the 
        #filter name
        print('Starting to swarp')    
        subprocess.call(['swarp','@'+img_list_txt,'-CENTER_TYPE','MANUAL', \
        '-CENTER',str(ra)+","+str(dec),'-NTHREADS','32', '-COPY_KEYWORDS', \
        'PHOTFLAM,PHOTZPT','-COMBINE_BUFSIZE','10000','-MEM_MAX','1000',\
        '-VMEM_MAX','5000','-IMAGE_SIZE','3960','-GAIN_KEYWORD','CCDGAIN',\
        '-GAIN_DEFAULT','1.0','-SUBTRACT_BACK','Y','-IMAGEOUT_NAME',\
        obj+'_'+filter_i+'_'+'coadd.fits','-WEIGHTOUT_NAME',\
        obj+'_'+filter_i+'_'+'coadd.weight.fits','-XML_NAME',\
        obj+'_'+filter_i+'_'+'swarp.xml'])
        print('Finished swarping')
        
        coadd = fits.open(obj+'_'+filter_i+'_'+'coadd.fits')
        #Open the coadd and use the percent_blank function 
        foo.percent_blank(coadd,obj,filter_i)
        #So that you can distinguish between interim object folder and raw 
        #object filder
        
    else:
        os.chdir('..') # Moving one level up when no imgs are found
        
{'fraction_non_alphanumeric': 0.12371432710415761, 'fraction_numerical': 0.012748080544690715, 'mean_word_length': 3.2433927473878303, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1035812', 'n_tokens_mistral': 2339, 'n_tokens_neox': 2245, 'n_words': 694}