<jupyter_start><jupyter_text>This notebook prepare the catalogues that will be analysed by CIGALE for SED fitting and physical parameter estimation.<jupyter_code>import numpy as np
import os
os.environ['LOG_LEVEL'] = 'INFO'
import glob
from astropy.table import Table

from herschelhelp.filters import correct_galactic_extinction
from herschelhelp.external import convert_table_for_cigale
SUFFIX = '20180221'
file_list = glob.glob('../../dmu32/dmu32_XMM-LSS/data/tiles/xmm-lss_{}_cigale_*.fits'.format(SUFFIX))
#len(master_catalogue)<jupyter_output><empty_output><jupyter_text>Best sourcesDefine a good far-IR measurement as:- an existing flux in the band;- the flag from XID+ must not be set;- the signal to noise ratio must be over 2.<jupyter_code># good = {}
# for band in ['pacs_green', 'pacs_red', 'spire_250', 'spire_350', 'spire_500']:
#     good[band] = (~np.isnan(master_catalogue['f_{}'.format(band)]) & 
#                                    ~master_catalogue['flag_{}'.format(band)])
#     good[band][good[band]] &= (master_catalogue[good[band]]['f_{}'.format(band)] /
#                               master_catalogue[good[band]]['ferr_{}'.format(band)] >= 2)<jupyter_output><empty_output><jupyter_text>We will keep only sources with at leat 2 good far-IR measurements (we may actually use less sources are not all may have a redshift).<jupyter_code># combined_good = np.sum(list(good.values()), axis=0) >= 2
# print("Number of good sources: {}".format(np.sum(combined_good)))<jupyter_output><empty_output><jupyter_text>Main catalogue for CIGALE<jupyter_code># best_catalogue = master_catalogue[combined_good].copy()
# Correction for galactic extinction
#best_catalogue = correct_galactic_extinction(best_catalogue, inplace=True)
# Convertion to CIGALE format
#best_catalogue = convert_table_for_cigale(best_catalogue, inplace=True, remove_zerofluxes=True)<jupyter_output><empty_output><jupyter_text>Band selectionWe want to use only one filter for similar bands. We define an order of preference and set to NaN the flux in the lower prefered bands when a prefered band is available. Some band may have a 0 flux, we set there values to NaN.<jupyter_code>u_bands = [                         "megacam_u"]
g_bands = [            "suprime_g", "megacam_g", "decam_g", "gpc1_g" ]
r_bands = [            "suprime_r", "megacam_r", "decam_r", "gpc1_r" ]
i_bands = [            "suprime_i", "megacam_i", "decam_i", "gpc1_i" ]
z_bands = ["vista_z",  "suprime_z", "megacam_z", "decam_z", "gpc1_z" ]
y_bands = ["vista_y",  "suprime_y", "megacam_y", "decam_y", "gpc1_y" ]
j_bands = ["vista_j",  "wircam_j",  "ukidss_j"]
h_bands = ["vista_h",  "wircam_h",  "ukidss_h"]
k_bands = ["vista_ks", "wircam_ks"] #, "ukidss_k"]

def remove_unneeded_fluxes(list_of_bands):
    for band_idx, band in enumerate(list_of_bands[:-1]):
        mask = ~np.isnan(best_catalogue[band])
        for lower_band in list_of_bands[band_idx+1:]:
            best_catalogue[lower_band][mask] = np.nan
            best_catalogue["{}_err".format(lower_band)][mask] = np.nan
# remove_unneeded_fluxes(g_bands)
# remove_unneeded_fluxes(u_bands)
# remove_unneeded_fluxes(r_bands)
# remove_unneeded_fluxes(i_bands)
# remove_unneeded_fluxes(z_bands)
# remove_unneeded_fluxes(y_bands)
#best_catalogue.write("data_tmp/CDFS-SWIRE_cigale_best_extcor_20180129.fits")<jupyter_output><empty_output><jupyter_text>Catalogue using spectroscopic redshift<jupyter_code>for tile in file_list:
        
    file_num = tile.split('_')[4].split('.')[0]
    best_catalogue = Table.read(tile)
    optnir = ((best_catalogue['flag_optnir_det'] == 3) 
          | (best_catalogue['flag_optnir_det'] == 7))
    
    best_catalogue = best_catalogue[optnir]


    best_catalogue = best_catalogue[~np.isnan(best_catalogue["redshift"])]

    print("Number of sources with z-spec in file {}: {}".format(file_num, len(best_catalogue)))

    # Correction for galactic extinction
    best_catalogue = correct_galactic_extinction(best_catalogue, inplace=True)

    # Convertion to CIGALE format

    best_catalogue = convert_table_for_cigale(best_catalogue, inplace=True, remove_zerofluxes=True)

    remove_unneeded_fluxes(g_bands)
    remove_unneeded_fluxes(u_bands)
    remove_unneeded_fluxes(r_bands)
    remove_unneeded_fluxes(i_bands)
    remove_unneeded_fluxes(z_bands)
    remove_unneeded_fluxes(y_bands)
    remove_unneeded_fluxes(j_bands)
    remove_unneeded_fluxes(h_bands)
    remove_unneeded_fluxes(k_bands)

    best_catalogue.write("data_tmp/tiles/XMM-LSS_cigale_optnir_extcor_zspec_{}_{}.fits".format(SUFFIX, file_num), overwrite=True)<jupyter_output>Number of sources with z-spec in file 40: 1800
{'fraction_non_alphanumeric': 0.12099031216361679, 'fraction_numerical': 0.011194833153928956, 'mean_word_length': 5.286874154262517, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '626217', 'license': 'MIT', 'n_tokens_mistral': 1788, 'n_tokens_neox': 1672, 'n_words': 315, 'path': 'dmu28/dmu28_XMM-LSS/CIGALE_catalogue_preparation.ipynb', 'repo_name': 'djbsmith/dmu_products', 'size': '4645'}