<jupyter_start><jupyter_text>read single exposures from Blanc reduction<NAME> is currently running redrock outputs for single exposures<jupyter_code>exps = sv1.blanc_nexp1_exposures()
# lets only keep the nightly exposures where we have a corresponding 
# deep exposure that we can use as the truth table
deep_exp = sv1.blanc_deep_exposures()

keep = np.zeros(len(exps)).astype(bool)
for i in range(len(exps)): 
    if exps[i]['TILEID'] in deep_exp['TILEID']: 
        keep[i] = True
print('%i exposures with corresponding deep exposures' % np.sum(keep))
exps[keep]<jupyter_output><empty_output><jupyter_text>read redrock output for nightly combined exposures<jupyter_code>zbest_deep = sv1.get_zbest(80614, 'deep', targetclass='brightfaint')
zbest_exp = sv1.get_zbest(80614, 20201218, expid=68685, targetclass='brightfaint')
zbest_deep[:5]
zbest_exp[:5]<jupyter_output><empty_output><jupyter_text>caclulate $z$ success rate using the deep exposures as the truth table<jupyter_code>def zsuccess_redrock_criteria(zbest): 
    ''' redshift success criteria based only on redrock output
    '''
    crit_zwarn = (zbest['ZWARN'] == 0)
    crit_dchi2 = (zbest['DELTACHI2']  > 40.) 
    crit_stype = (zbest['SPECTYPE'] != "STAR") # only galaxy spectra
    crit_z_lim = (zbest['Z'] > 0.0) & (zbest['Z'] < 0.6) # rough BGS redshift limit
    crit_z_err = (zbest['ZERR'] < (0.0005 * (1. + zbest['Z'])))

    # combine all criteria
    zsuccess = crit_zwarn & crit_dchi2 & crit_stype & crit_z_lim & crit_z_err
    return zsuccess

def zsuccess_truth_criteria(zbest, z_true, silent=True): 
    ''' redshift success criteria based on redrock output *and* comparison
    to "true" redshift
    '''
    crit_zwarn = (zbest['ZWARN'] == 0)
    crit_dchi2 = (zbest['DELTACHI2']  > 40.) 
    crit_stype = (zbest['SPECTYPE'] != "STAR") # only galaxy spectra
    crit_z_lim = (zbest['Z'] > 0.0) & (zbest['Z'] < 0.6) # rough BGS redshift limit
    crit_z_err = (zbest['ZERR'] < (0.0005 * (1. + zbest['Z'])))
    
    dz_1pz = np.abs(z_true - zbest['Z']) / (1. + z_true)
    crit_ztrue = (dz_1pz < 0.003)

    # combine all criteria
    zsuccess = crit_zwarn & crit_dchi2 & crit_stype & crit_z_lim & crit_z_err & crit_ztrue
    if not silent: 
        print('%i of %i pass the redshift success criteria' % (np.sum(zsuccess), len(zsuccess)))
        print('%.2f redshift success rate' % (np.sum(zsuccess)/len(zsuccess)))
    return zsuccess
def zsuccess_rate(prop, zsuccess_cond, range=None, nbins=20, bin_min=2):
    ''' measure the redshift success rate along with property `prop`

    :params prop: 
        array of properties (i.e. Legacy r-band magnitude) 

    :params zsuccess_cond:
        boolean array indicating redshift success 

    :params range: (default: None) 
        range of the `prop` 

    :params nbins: (default: 20) 
        number of bins to divide `prop` by 
    
    :params bin_min: (default: 2)  
        minimum number of objects in bin to exlcude it 

    :return wmean: 
        weighted mean of `prop` in the bins 

    :return e1: 
        redshift success rate in the bins

    :return ee1: 
        simple poisson error on the success rate
    '''
    h0, bins = np.histogram(prop, bins=nbins, range=range)
    hv, _ = np.histogram(prop, bins=bins, weights=prop)
    h1, _ = np.histogram(prop[zsuccess_cond], bins=bins)
    
    good = h0 > bin_min
    hv = hv[good]
    h0 = h0[good]
    h1 = h1[good]

    wmean = hv / h0 # weighted mean 
    rate = h1.astype("float") / (h0.astype('float') + (h0==0))
    e_rate = np.sqrt(rate * (1 - rate)) / np.sqrt(h0.astype('float') + (h0 == 0))
    return wmean, rate, e_rate
# only use targets where we have accurate redshifts in the deep exposure
_zbest_deep = sv1.get_zbest(80614, 'deep', targetclass='brightfaint')
zs_deep = zsuccess_redrock_criteria(_zbest_deep)

zbest_deep = _zbest_deep[zs_deep]['TARGETID', 'Z']
zbest_deep.rename_column('Z', 'Z_TRUE') # deep exposure redshift is the "true redshift"
_zbest_exp = sv1.get_zbest(80614, 20201218, expid=68685, targetclass='brightfaint')
zbest_exp = atable.join(zbest_deep, _zbest_exp, keys='TARGETID', join_type='left')
# calculate redshift successs rate
zs_exp = zsuccess_truth_criteria(zbest_exp, zbest_exp['Z_TRUE'])
r_mag = 22.5 - 2.5 * np.log10(zbest_exp['FLUX_R'])
r_mid, zs, zs_err = zsuccess_rate(r_mag, zs_exp)

fig = plt.figure(figsize=(8,6))
sub = fig.add_subplot(111)
sub.errorbar(r_mid, zs, yerr=zs_err, fmt='.C0')
sub.plot(r_mid, zs)
sub.set_xlabel('$r$ mag', fontsize=25)
sub.set_ylabel('$z$ success rate', fontsize=25)<jupyter_output><empty_output><jupyter_text>redshift success for the single exposures for all tiles with deep exposure<jupyter_code>#for tileid in [80612, 80613, 80614, 80616, 80617, 80618, 80619]: 
zss_tiles, iexps_tiles = [], [] 
for tileid in deep_exp['TILEID']: 
    # only use targets where we have accurate redshifts in the deep exposure
    try: 
        _zbest_deep = sv1.get_zbest(tileid, 'deep', targetclass='brightfaint');
    except:
        continue 
    zs_deep = zsuccess_redrock_criteria(_zbest_deep)
    
    zbest_deep = _zbest_deep[zs_deep]['TARGETID', 'Z']
    zbest_deep.rename_column('Z', 'Z_TRUE') # deep exposure redshift is the "true redshift"
    
    if len(zbest_deep) < 200: continue
    
    zss, iexps = [], []
    nights = np.unique(exps['NIGHT'][exps['TILEID'] == tileid])
    for i, night in enumerate(nights): 
        expids = np.unique(exps['EXPID'][(exps['TILEID'] == tileid) & (exps['NIGHT'] == night)])
        for ii, expid in enumerate(expids): 
            # only look at TRANSP > 0.9 with GFA SKY MAG < 20.075
            iexp = np.where(exps['EXPID'] == expid)[0]
            if exps['GFA_TRANSPARENCY_MED'][iexp] <= 0.9:# or exps['GFA_SKY_MAG_AB_MED'][isexp] > 20.075:
                continue 
            try: 
                _zbest_exp = sv1.get_zbest(tileid, night, expid=expid, targetclass='brightfaint');
            except: 
                continue
            zbest_exp = atable.join(zbest_deep, _zbest_exp, keys='TARGETID', join_type='inner')

            # calculate redshift successs rate
            zs_exp = zsuccess_truth_criteria(zbest_exp, zbest_exp['Z_TRUE'])

            r_mag = 22.5 - 2.5 * np.log10(zbest_exp['FLUX_R'])
            zss.append(zsuccess_rate(r_mag, zs_exp, range=[17.5, 20.5]))
            iexps.append(iexp)
    if len(zss) > 0:
        zss_tiles.append(zss)
        iexps_tiles.append(iexps)
fig = plt.figure(figsize=(30,15))
for i, zss, iexps in zip(range(len(iexps_tiles)), zss_tiles, iexps_tiles): 
    sub = fig.add_subplot(2,4,i+1)
    nexp = 0
    sub.plot([17.5, 20.5], [1., 1.], c='k', ls='--')
    sub.vlines(19.5, 0.6, 1.0, color='k', linestyle=':')
    for zs, iexp in zip(zss, iexps): 
        r_mid, zs, zs_err = zs
        sub.errorbar(r_mid, zs, yerr=zs_err, fmt='.C%i' % nexp)
        sub.plot(r_mid, zs, c='C%i' % nexp, 
                 label=r'TEXP=%.fs, SKY MAG=%.2f' % #\nMOON ILL=%.1f,SEP=%.f,ALT=%.f,AIRMASS=%.1f' % 
                 (exps['EXPTIME'][iexp], 
                  exps['GFA_SKY_MAG_AB_MED'][iexp]))#, exps['GFA_MOON_ILLUMINATION_MED'][iexp], exps['GFA_MOON_SEP_DEG_MED'][iexp], 90.-exps['GFA_MOON_ZD_DEG_MED'][iexp], exps['GFA_AIRMASS_MED'][iexp]))
        nexp += 1
        sub.legend(loc='lower left', fontsize=15)
    sub.text(0.95, 0.95, str(exps['TILEID'][iexp][0]), transform=sub.transAxes, ha='right', va='top', fontsize=25) 
    sub.set_xlim(17.5, 20.5)
    sub.set_ylim(0.6, 1.1)
bkgd = fig.add_subplot(111, frameon=False)
bkgd.set_xlabel('$r$ mag', fontsize=30)
bkgd.set_ylabel('$z$ success rate', fontsize=30)
bkgd.set_title('single exposure $z$ success rates (deep exp. truth) $\Delta \chi^2 > 40$', fontsize=40)
bkgd.tick_params(labelcolor='none', top=False, bottom=False, left=False, right=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14488854727132974, 'fraction_numerical': 0.03420445810914681, 'mean_word_length': 3.663679808841099, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '18861', 'license': 'MIT', 'n_tokens_mistral': 3202, 'n_tokens_neox': 2973, 'n_words': 649, 'path': 'doc/nb/blanc_nexp1_zsuccess.ipynb', 'repo_name': 'qmxp55/bgs-cmxsv', 'size': '7811'}