<jupyter_start><jupyter_text>`/tigress/MERIAN/ancillary/decals/sweep/9.0`<jupyter_code>len(cat_fdfc)
cat_dir = '/Users/song/astro6/hsc/s20a/decals'
dr9_list = glob.glob(cat_dir + '/*.fits')
cat_test = Table.read(os.path.join(cat_dir, 'sweep-000m005-010p000_s20a.fits'))
cols = ['OBJID', 'TYPE', 'RA', 'DEC', 
        'FLUX_G', 'FLUX_R', 'FLUX_Z', 
        'FLUX_IVAR_G', 'FLUX_IVAR_R', 'FLUX_IVAR_Z', 
        'MW_TRANSMISSION_G', 'MW_TRANSMISSION_R', 'MW_TRANSMISSION_Z', 
        'NOBS_G', 'NOBS_R', 'NOBS_Z', 
        'RCHISQ_G', 'RCHISQ_R', 'RCHISQ_Z', 
        'FRACFLUX_G', 'FRACFLUX_R', 'FRACFLUX_Z', 
        'FRACMASKED_G', 'FRACMASKED_R', 'FRACMASKED_Z', 
        'FRACIN_G', 'FRACIN_R', 'FRACIN_Z', 
        'ANYMASK_G', 'ANYMASK_R', 'ANYMASK_Z', 
        'ALLMASK_G', 'ALLMASK_R', 'ALLMASK_Z', 
        'PSFSIZE_G', 'PSFSIZE_R', 'PSFSIZE_Z', 
        'GALDEPTH_G', 'GALDEPTH_R', 'GALDEPTH_Z', 
        'SHAPE_R', 'SHAPE_R_IVAR', 'SHAPE_E1', 'SHAPE_E1_IVAR', 'SHAPE_E2', 'SHAPE_E2_IVAR', 
        'FIBERFLUX_G', 'FIBERFLUX_R', 'FIBERFLUX_Z', 
        'FIBERTOTFLUX_G', 'FIBERTOTFLUX_R', 'FIBERTOTFLUX_Z', 
        'SERSIC', 'SERSIC_IVAR']
def reduce_cat(cat):
    gal_mask = cat['TYPE'] != 'PSF'
    
    flux_mask = (
        (cat['FLUX_G'] >= 0) & (cat['FLUX_R'] >= 0) & (cat['FLUX_Z'] >= 0) &
        (np.isfinite(cat['FLUX_G'])) & (np.isfinite(cat['FLUX_R'])) & 
        (np.isfinite(cat['FLUX_Z']))
    )

    r_mag = 22.5 - 2.5 * np.log10(cat['FLUX_R'])
    r_mask = r_mag <= 24.0

    print((gal_mask & flux_mask & r_mask).sum(), len(cat))
    
    return cat[(gal_mask & flux_mask & r_mask)][cols]
    
dr9_reduced = [reduce_cat(Table.read(c)) for c in dr9_list]
dr9_hsc = vstack(dr9_reduced)
len(dr9_hsc)
dr9_hsc.write(os.path.join(cat_dir, 'decals_dr9_hsc_s20a_gal_r24.fits'), overwrite=True)<jupyter_output>WARNING: UnitsWarning: 'nanomaggy' did not parse as fits unit: At col 0, Unit 'nanomaggy' not supported by the FITS standard.  If this is meant to be a custom unit, define it with 'u.def_unit'. To have it recognized inside a file reader or other code, enable it with 'u.add_enabled_units'. For details, see https://docs.astropy.org/en/latest/units/combining_and_defining.html [astropy.units.core]
WARNING: UnitsWarning: '1/nanomaggy^2' did not parse as fits unit: Numeric factor not supported by FITS If this is meant to be a custom unit, define it with 'u.def_unit'. To have it recognized inside a file reader or other code, enable it with 'u.add_enabled_units'. For details, see https://docs.astropy.org/en/latest/units/combining_and_defining.html [astropy.units.core]
WARNING: UnitsWarning: '1/arcsec^2' did not parse as fits unit: Numeric factor not supported by FITS If this is meant to be a custom unit, define it with 'u.def_unit'. To have it recognized inside a file reader or other code, ena[...]
{'fraction_non_alphanumeric': 0.152787579393084, 'fraction_numerical': 0.0187014820042343, 'mean_word_length': 4.026595744680851, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '125871', 'license': 'CC0-1.0', 'n_tokens_mistral': 1223, 'n_tokens_neox': 1158, 'n_words': 246, 'path': 'notebook/sandbox/decals_dr9_match_hsc.ipynb', 'repo_name': 'dr-guangtou/caterpillar', 'size': '2834'}