<jupyter_start><jupyter_text>Final ProcessingThe final processing stage requires:1. Quick validation of catalogues and Bayesian Pvalue maps2. Skewness level3. Adding flag to catalogue<jupyter_code>import seaborn as sns
from astropy.table import Table
%matplotlib inline
import numpy as np
import pylab as plt<jupyter_output><empty_output><jupyter_text>Read tables<jupyter_code>cat=Table.read('../data/CDFS-SWIRE/PACS/dmu26_XID+PACS_CDFS-SWIRE_20171019.fits')

cat[0:10]
from astropy import units as u
cat['RA'].unit=u.degree
cat['Dec'].unit=u.degree<jupyter_output><empty_output><jupyter_text>Look at Symmetry of PDFs to determine depth level of catalogue<jupyter_code>skew=(cat['FErr_PACS_100_u']-cat['F_PACS_100'])/(cat['F_PACS_100']-cat['FErr_PACS_100_l'])
skew.name='(84th-50th)/(50th-16th) percentile'
g=sns.jointplot(x=np.log10(cat['F_PACS_100']),y=skew, kind='scatter')<jupyter_output><empty_output><jupyter_text>For 100 $\mathrm{\mu m}$ depth is ~ 30mJy<jupyter_code>skew=(cat['FErr_PACS_160_u']-cat['F_PACS_160'])/(cat['F_PACS_160']-cat['FErr_PACS_160_l'])
skew.name='(84th-50th)/(50th-16th) percentile'
g=sns.jointplot(x=np.log10(cat['F_PACS_160']),y=skew, kind='scatter')<jupyter_output><empty_output><jupyter_text>For 160 $\mathrm{\mu m}$ depth is ~ 30mJy  Add flag to catalogue<jupyter_code>from astropy.table import Column
cat.add_column(Column(np.zeros(len(cat), dtype=bool),name='flag_PACS_100'))
cat.add_column(Column(np.zeros(len(cat), dtype=bool),name='flag_PACS_160'))


ind_100=(cat['Pval_res_100']>0.5) | (cat['F_PACS_100'] < 30)
ind_160=(cat['Pval_res_160']>0.5) | (cat['F_PACS_160'] < 30) 

print(ind_100.sum(),ind_160.sum(),len(cat))
cat['flag_PACS_100'][ind_100]=True
cat['flag_PACS_160'][ind_160]=True

cat.write('../data/CDFS-SWIRE/PACS/dmu26_XID+PACS_CDFS-SWIRE_cat_20171019.fits', format='fits',overwrite=True)<jupyter_output><empty_output><jupyter_text>Check maps<jupyter_code>from astropy.io import fits
pacs100_pval=fits.open('../data/CDFS-SWIRE/PACS/dmu26_XID+PACS_PACS_100_CDFS-SWIRE_Bayes_Pval_20171005.fits')
pacs160_pval=fits.open('../data/CDFS-SWIRE/PACS/dmu26_XID+PACS_PACS_160_CDFS-SWIRE_Bayes_Pval_20171005.fits')
plt.hist(pacs100_pval[1].data[np.isfinite(pacs100_pval[1].data)],bins=np.arange(-6,6,0.05));
plt.hist(pacs160_pval[1].data[np.isfinite(pacs160_pval[1].data)],bins=np.arange(-6,6,0.05));<jupyter_output><empty_output><jupyter_text>Check cat<jupyter_code>old_cat=Table.read('../data/CDFS-SWIRE/PACS/WP4-CDFS-SWIRE-PACSxID24-v1.fits.gz')
old_cat['RA'].unit=u.degree
old_cat['Dec'].unit=u.degree
old_cat
from astropy.coordinates import SkyCoord
from astropy import units as u
c = SkyCoord(ra=cat['RA'], dec=cat['Dec'])  
catalog = SkyCoord(ra=old_cat['RA'], dec=old_cat['Dec'])  
idx, d2d, d3d = c.match_to_catalog_sky(catalog)  
plt.plot(cat['F_PACS_100'],old_cat['F_PACS_100'][idx],'o' )
plt.plot(np.arange(0,1000),np.arange(0,1000))
plt.plot(cat['F_PACS_160'],old_cat['F_PACS_160'][idx],'o' )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.176747311827957, 'fraction_numerical': 0.07459677419354839, 'mean_word_length': 9.747292418772563, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '312109', 'license': 'MIT', 'n_tokens_mistral': 1496, 'n_tokens_neox': 1292, 'n_words': 135, 'path': 'dmu26/dmu26_XID+PACS_CDFS-SWIRE/XID+PACS_CDFS_SWIRE_final_processing.ipynb', 'repo_name': 'djbsmith/dmu_products', 'size': '2976'}