<commit_before>"""
This file collects a set of common cuts we apply to objects.
See https://github.com/yymao/easyquery for further documentation.

In Jupyter, use ?? to see the definitions of the cuts

Examples
--------
>>> is_sat.filter(base_table)
"""

from easyquery import Query, QueryMaker

COLUMNS_USED = [
    "ZQUALITY",
    "REMOVE",
    "PHOTPTYPE",
    "FIBERMAG_R",
    "SPEC_Z",
    "RHOST_KPC",
    "HOST_VHOST",
    "SATS",
    "OBJ_NSAID",
    "HOST_NSAID",
    "SPEC_REPEAT",
    "r_mag",
    "ug",
    "ug_err",
    "gr",
    "gr_err",
    "ri",
    "ri_err",
]

COLUMNS_USED2 = [
    "ZQUALITY",
    "REMOVE",
    "is_galaxy",
    "SPEC_Z",
    "RHOST_KPC",
    "HOST_VHOST",
    "SATS",
    "SPEC_REPEAT",
    "r_mag",
    "i_mag",
    "ug",
    "ug_err",
    "gr",
    "gr_err",
    "ri",
    "ri_err",
    "rz",
    "rz_err",
    "sb_r",
]

has_spec = Query("ZQUALITY >= 3")
is_clean = Query("REMOVE == -1")
is_clean2 = Query("REMOVE == 0")
is_galaxy = Query("PHOTPTYPE == 3")
is_galaxy2 = Query("is_galaxy")
fibermag_r_cut = Query("FIBERMAG_R <= 23.0")

faint_end_limit = Query("r_mag < 20.75")
sdss_limit = Query("r_mag < 17.77")
lowz_mag_cut = Query("r_mag > 18", "r_mag < 20")
r_abs_limit = Query("Mr < -12.295")

faint_end_limit_abs = Query("r_mag - log10(HOST_DIST)*5 - 25 < -12")

sat_vcut = Query("abs(SPEC_Z * 2.99792458e5 - HOST_VHOST) < 250.0")
sat_rcut = Query("RHOST_KPC < 300.0")

valid_u_mag = Query("u_mag > 0", "u_mag < 30")
valid_g_mag = Query("g_mag > 0", "g_mag < 30")
valid_i_mag = Query("i_mag > 0", "i_mag < 30")
valid_z_mag = Query("z_mag > 0", "z_mag < 30")

gr_cut = Query("gr-abs(gr_err)*2.0 < 0.85")
ri_cut = Query("ri-abs(ri_err)*2.0 < 0.55")
rz_cut = Query("rz-abs(rz_err)*2.0 < 1.0")

ug_cut = Query("(ug+abs(ug_err)*2.0) > (gr-abs(gr_err)*2.0)*1.5")
gri_cut = gr_cut & ri_cut
ugri_cut = gri_cut & ug_cut
grz_cut = gr_cut & rz_cut

gri_or_grz_cut = Query(
    gr_cut | (~valid_g_mag), ri_cut | (~valid_i_mag), rz_cut | (~valid_z_mag),
)

high_priority_ug = Query(
    "ug - abs(ug_err) < 1.8 + 0.05*(r_mag-14)",
    "ug + abs(ug_err) > 0.1 + 0.05*(r_mag-14)",<commit_msg>update to faint_end_limit_strict<commit_after>"""
This file collects a set of common cuts we apply to objects.
See https://github.com/yymao/easyquery for further documentation.

In Jupyter, use ?? to see the definitions of the cuts

Examples
--------
>>> is_sat.filter(base_table)
"""

from easyquery import Query, QueryMaker

COLUMNS_USED = [
    "ZQUALITY",
    "REMOVE",
    "PHOTPTYPE",
    "FIBERMAG_R",
    "SPEC_Z",
    "RHOST_KPC",
    "HOST_VHOST",
    "SATS",
    "OBJ_NSAID",
    "HOST_NSAID",
    "SPEC_REPEAT",
    "r_mag",
    "ug",
    "ug_err",
    "gr",
    "gr_err",
    "ri",
    "ri_err",
]

COLUMNS_USED2 = [
    "ZQUALITY",
    "REMOVE",
    "is_galaxy",
    "SPEC_Z",
    "RHOST_KPC",
    "HOST_VHOST",
    "SATS",
    "SPEC_REPEAT",
    "r_mag",
    "i_mag",
    "ug",
    "ug_err",
    "gr",
    "gr_err",
    "ri",
    "ri_err",
    "rz",
    "rz_err",
    "sb_r",
]

has_spec = Query("ZQUALITY >= 3")
is_clean = Query("REMOVE == -1")
is_clean2 = Query("REMOVE == 0")
is_galaxy = Query("PHOTPTYPE == 3")
is_galaxy2 = Query("is_galaxy")
fibermag_r_cut = Query("FIBERMAG_R <= 23.0")

faint_end_limit = Query("r_mag < 20.75")
sdss_limit = Query("r_mag < 17.77")
lowz_mag_cut = Query("r_mag > 18", "r_mag < 20")
r_abs_limit = Query("Mr < -12.295")

faint_end_limit_strict = Query("r_mag - log10(HOST_DIST)*5 - 25 < -12", faint_end_limit)

sat_vcut = Query("abs(SPEC_Z * 2.99792458e5 - HOST_VHOST) < 250.0")
sat_rcut = Query("RHOST_KPC < 300.0")

valid_u_mag = Query("u_mag > 0", "u_mag < 30")
valid_g_mag = Query("g_mag > 0", "g_mag < 30")
valid_i_mag = Query("i_mag > 0", "i_mag < 30")
valid_z_mag = Query("z_mag > 0", "z_mag < 30")

gr_cut = Query("gr-abs(gr_err)*2.0 < 0.85")
ri_cut = Query("ri-abs(ri_err)*2.0 < 0.55")
rz_cut = Query("rz-abs(rz_err)*2.0 < 1.0")

ug_cut = Query("(ug+abs(ug_err)*2.0) > (gr-abs(gr_err)*2.0)*1.5")
gri_cut = gr_cut & ri_cut
ugri_cut = gri_cut & ug_cut
grz_cut = gr_cut & rz_cut

gri_or_grz_cut = Query(
    gr_cut | (~valid_g_mag), ri_cut | (~valid_i_mag), rz_cut | (~valid_z_mag),
)

high_priority_ug = Query(
    "ug - abs(ug_err) < 1.8 + 0.05*(r_mag-14)",
    "ug + abs(ug_err) > 0.1 + 0.05*(r_mag-14)",
{'fraction_non_alphanumeric': 0.19104268719384185, 'fraction_numerical': 0.0457196174480989, 'mean_word_length': 3.4759916492693113, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2806864', 'n_tokens_mistral': 2107, 'n_tokens_neox': 1964, 'n_words': 356}