<commit_before>from __future__ import print_function
import matplotlib
matplotlib.use('Agg')
#matplotlib.rc('text', usetex=True)
matplotlib.rc('font', family='serif')
import pylab as plt
import numpy as np
from astrometry.util.fits import *
from astrometry.libkd.spherematch import match_radec
from astrometry.util.plotutils import *
import fitsio
from glob import glob
from collections import Counter
from tractor.brightness import NanoMaggies

def main():

    ps = PlotSequence('dr3-summary')

    #fns = glob('/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-*.fits')
    fns = glob('/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-240p005-250p010.fits')
    TT = []
    for fn in fns:
        T = fits_table(fn)
        print(len(T), 'from', fn)
        TT.append(T)
    T = merge_tables(TT)
    del TT

    print('Types:', Counter(T.type))

    T.flux_g = T.decam_flux[:,1]
    T.flux_r = T.decam_flux[:,2]
    T.flux_z = T.decam_flux[:,4]
    T.flux_ivar_g = T.decam_flux_ivar[:,1]
    T.flux_ivar_r = T.decam_flux_ivar[:,2]
    T.flux_ivar_z = T.decam_flux_ivar[:,4]
    
    T.mag_g, T.magerr_g = NanoMaggies.fluxErrorsToMagErrors(T.flux_g, T.flux_ivar_g)
    T.mag_r, T.magerr_r = NanoMaggies.fluxErrorsToMagErrors(T.flux_r, T.flux_ivar_r)
    T.mag_z, T.magerr_z = NanoMaggies.fluxErrorsToMagErrors(T.flux_z, T.flux_ivar_z)

    T.typex = np.array([t[0] for t in T.type])

    I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0))
    print(len(I), 'with g,r,z fluxes')
    I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) *
                       (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0))
    print(len(I), 'with positive g,r,z fluxes')
    I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) *
                       (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
                       (T.magerr_g < 0.2) * (T.magerr_r < 0.2) * (T.magerr_z < 0.2))
    print(len(I), 'with < 0.2-mag errs')
    I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) *
                       (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
                       (T.magerr_g < 0.1) * (T.magerr_r < 0.1) * (T.magerr_z < 0.1))
    print(len(I), 'with < 0.1-mag errs')

    ha = dict(nbins=200, range=((-0.5, 2.5), (-0.5, 2.5)))
    plt.clf()
    #plothist(T.mag_g[I] - T.mag_r[I], T.mag_r[I] - T.mag_z[I], 200)
    loghist(T.mag_g[I] - T.mag_r[I], T.mag_r[I] - T.mag_z[I], **ha)
    plt.xlabel('g - r (mag)')
    plt.ylabel('r - z (mag)')
    ps.savefig()

    hists = []
    for tt,name in [('P', 'PSF'), ('E', 'EXP'), ('D', 'DeV')]:
        I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) *
                           (T.flux_ivar_z > 0) *
                           (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
                           (T.magerr_g < 0.1) * (T.magerr_r < 0.1) *(T.magerr_z < 0.1) *
                           (T.typex == tt))
        print(len(I), 'with < 0.1-mag errs and type', name)

        plt.clf()
        H,xe,ye = loghist(T.mag_g[I] - T.mag_r[I], T.mag_r[I] - T.mag_z[I], **ha)
        hists.append(H)
        plt.xlabel('g - r (mag)')
        plt.ylabel('r - z (mag)')
        plt.title('Type = %s' % name)
        ps.savefig()


    for H in hists:
        H /= H.max()
    rgbmap = np.dstack((hists[2], hists[0], hists[1]))
    plt.clf()
    ((xlo,xhi),(ylo,yhi)) = ha['range']
    plt.imshow(rgbmap, interpolation='nearest', origin='lower', extent=[xlo,xhi,ylo,yhi])
    plt.axis([0,2,0,2])
    plt.xlabel('g - r (mag)')
    plt.ylabel('r - z (mag)')
    plt.title('Red: DeV, Blue: Exp, Green: PSF')
    ps.savefig()

    plt.clf()
    plt.imshow(np.sqrt(rgbmap), interpolation='nearest', origin='lower', extent=[xlo,xhi,ylo,yhi])
    plt.axis([0,2,0,2])
    plt.xlabel('g - r (mag)')
    plt.ylabel('r - z (mag)')
    plt.title('Red: DeV, Blue: Exp, Green: PSF')
    ps.savefig()

    np.random.seed(42)

    T.gr = T.mag_g - T.mag_r
    T.rz = T.mag_r - T.mag_z

    for tt,name in [('P', 'PSF'), ('E', 'EXP'), ('D', 'DeV')]:
        I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) *
                           (T.flux_ivar_z > 0) *
                           (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
                           (T.magerr_g < 0.1) * (T.magerr_r < 0.1) *(T.magerr_z < 0.1) *
                           # Bright cut
                           (T.mag_r > 16.) *
                           (T.typex == tt))
        print(len(I), 'with < 0.1-mag errs and type', name)

        J = np.random.permutation(I)
        if tt == 'D':
            J = J[T.shapedev_r[J] < 3.]
        J = J[:100]
        rzbin = np.argsort(T.rz[J])
        rzblock = (rzbin / 10).astype(int)
        K = np.lexsort((T.gr[J], rzblock))
        print('sorted rzblock', rzblock[K])
        print('sorted rzbin', rzbin[K])
        J = J[K]

        imgs = []
        imgrow = []
        stackrows = []
        for i in J:
            fn = 'cutouts/cutout_%.4f_%.4f.jpg' % (T.ra[i], T.dec[i])
            if not os.path.exists(fn):
                url = 'http://legacysurvey.org/viewer/jpeg-cutout/?layer=decals-dr3&pixscale=0.262&size=25&ra=%f&dec=%f' % (T.ra[i], T.dec[i])
                print('URL', url)
                cmd = 'wget -O %s "%s"' % (fn, url)
                os.system(cmd)
            img = plt.imread(fn)
            #print('Image', img.shape)
            if tt == 'P':
                print(name, 'g/r/z %.2f, %.2f, %.2f' % (T.mag_g[i], T.mag_r[i], T.mag_z[i]))
            elif tt == 'D':
                print(name, 'g/r/z %.2f, %.2f, %.2f, shapedev_r %.2f' % (T.mag_g[i], T.mag_r[i], T.mag_z[i], T.shapedev_r[i]))
            elif tt == 'E':
                print(name, 'g/r/z %.2f, %.2f, %.2f, shapeexp_r %.2f' % (T.mag_g[i], T.mag_r[i], T.mag_z[i], T.shapeexp_r[i]))



            imgrow.append(img)
            if len(imgrow) == 10:
                stack = np.hstack(imgrow)
                print('stacked row:', stack.shape)
                stackrows.append(stack)

                imgs.append(imgrow)
                imgrow = []


        stack = np.vstack(stackrows)
        print('Stack', stack.shape)
        plt.clf()
        plt.imshow(stack, interpolation='nearest', origin='lower')
        plt.xticks([])
        plt.yticks([])
        plt.title('Type %s' % name)
        ps.savefig()
        

    # Wrap-around at RA=300
    # rax = T.ra + (-360 * T.ra > 300.)
    # 
    # rabins = np.arange(-60., 300., 0.2)<commit_msg>dr-summary: plot dereddened colors; clean up samples<commit_after>from __future__ import print_function
import matplotlib
matplotlib.use('Agg')
#matplotlib.rc('text', usetex=True)
matplotlib.rc('font', family='serif')
import pylab as plt
import numpy as np
from astrometry.util.fits import *
from astrometry.libkd.spherematch import match_radec
from astrometry.util.plotutils import *
import fitsio
from glob import glob
from collections import Counter
from tractor.brightness import NanoMaggies

def main():

    ps = PlotSequence('dr3-summary')

    #fns = glob('/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-*.fits')
    fns = glob('/project/projectdirs/cosmo/data/legacysurvey/dr3.1/sweep/3.1/sweep-240p005-250p010.fits')
    TT = []
    for fn in fns:
        T = fits_table(fn)
        print(len(T), 'from', fn)
        TT.append(T)
    T = merge_tables(TT)
    del TT

    print('Types:', Counter(T.type))

    T.flux_g = T.decam_flux[:,1]
    T.flux_r = T.decam_flux[:,2]
    T.flux_z = T.decam_flux[:,4]
    T.flux_ivar_g = T.decam_flux_ivar[:,1]
    T.flux_ivar_r = T.decam_flux_ivar[:,2]
    T.flux_ivar_z = T.decam_flux_ivar[:,4]
    
    T.mag_g, T.magerr_g = NanoMaggies.fluxErrorsToMagErrors(T.flux_g, T.flux_ivar_g)
    T.mag_r, T.magerr_r = NanoMaggies.fluxErrorsToMagErrors(T.flux_r, T.flux_ivar_r)
    T.mag_z, T.magerr_z = NanoMaggies.fluxErrorsToMagErrors(T.flux_z, T.flux_ivar_z)

    T.deflux_g = T.flux_g / T.decam_mw_transmission[:,1]
    T.deflux_r = T.flux_r / T.decam_mw_transmission[:,2]
    T.deflux_z = T.flux_z / T.decam_mw_transmission[:,4]
    T.deflux_ivar_g = T.flux_ivar_g * T.decam_mw_transmission[:,1]**2
    T.deflux_ivar_r = T.flux_ivar_r * T.decam_mw_transmission[:,2]**2
    T.deflux_ivar_z = T.flux_ivar_z * T.decam_mw_transmission[:,4]**2

    T.demag_g, T.demagerr_g = NanoMaggies.fluxErrorsToMagErrors(T.deflux_g, T.deflux_ivar_g)
    T.demag_r, T.demagerr_r = NanoMaggies.fluxErrorsToMagErrors(T.deflux_r, T.deflux_ivar_r)
    T.demag_z, T.demagerr_z = NanoMaggies.fluxErrorsToMagErrors(T.deflux_z, T.deflux_ivar_z)

    T.typex = np.array([t[0] for t in T.type])

    I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0))
    print(len(I), 'with g,r,z fluxes')
    I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) *
                       (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0))
    print(len(I), 'with positive g,r,z fluxes')
    I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) *
                       (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
                       (T.magerr_g < 0.2) * (T.magerr_r < 0.2) * (T.magerr_z < 0.2))
    print(len(I), 'with < 0.2-mag errs')
    I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) * (T.flux_ivar_z > 0) *
                       (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
                       (T.magerr_g < 0.1) * (T.magerr_r < 0.1) * (T.magerr_z < 0.1))
    print(len(I), 'with < 0.1-mag errs')

    ha = dict(nbins=200, range=((-0.5, 2.5), (-0.5, 2.5)))
    plt.clf()
    #plothist(T.mag_g[I] - T.mag_r[I], T.mag_r[I] - T.mag_z[I], 200)
    #loghist(T.mag_g[I] - T.mag_r[I], T.mag_r[I] - T.mag_z[I], **ha)
    loghist(T.demag_g[I] - T.demag_r[I], T.demag_r[I] - T.demag_z[I], **ha)
    plt.xlabel('g - r (mag)')
    plt.ylabel('r - z (mag)')
    ps.savefig()

    hists = []
    for tt,name in [('P', 'PSF'), ('E', 'EXP'), ('D', 'DeV')]:
        I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) *
                           (T.flux_ivar_z > 0) *
                           (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
                           (T.magerr_g < 0.1) * (T.magerr_r < 0.1) *(T.magerr_z < 0.1) *
                           (T.typex == tt))
        print(len(I), 'with < 0.1-mag errs and type', name)

        plt.clf()
        H,xe,ye = loghist(T.mag_g[I] - T.mag_r[I], T.mag_r[I] - T.mag_z[I], **ha)
        plt.xlabel('g - r (mag)')
        plt.ylabel('r - z (mag)')
        plt.title('Type = %s' % name)
        ps.savefig()

        plt.clf()
        H,xe,ye = loghist(T.demag_g[I] - T.demag_r[I], T.demag_r[I] - T.demag_z[I], **ha)

        # Keeping the dereddened color-color histograms
        hists.append(H)

        plt.xlabel('g - r (mag)')
        plt.ylabel('r - z (mag)')
        plt.title('Type = %s, dereddened' % name)
        ps.savefig()


    for H in hists:
        H /= H.max()
    rgbmap = np.dstack((hists[2], hists[0], hists[1]))
    plt.clf()
    ((xlo,xhi),(ylo,yhi)) = ha['range']
    plt.imshow(rgbmap, interpolation='nearest', origin='lower', extent=[xlo,xhi,ylo,yhi])
    plt.axis([0,2,0,2])
    plt.xlabel('g - r (mag)')
    plt.ylabel('r - z (mag)')
    plt.title('Red: DeV, Blue: Exp, Green: PSF')
    ps.savefig()

    plt.clf()
    plt.imshow(np.sqrt(rgbmap), interpolation='nearest', origin='lower', extent=[xlo,xhi,ylo,yhi])
    plt.axis([0,2,0,2])
    plt.xlabel('g - r (mag)')
    plt.ylabel('r - z (mag)')
    plt.title('Red: DeV, Blue: Exp, Green: PSF')
    ps.savefig()

    # For a white-background plot, mix colors like paint
    hr,hg,hb = (hists[2], hists[0], hists[1])
    H,W = hr.shape
    for mapping in [lambda x: x]: #, lambda x: np.sqrt(x), lambda x: x**2]:
        red = np.ones((H,W,3))
        red[:,:,1] = 1 - mapping(hr)
        red[:,:,2] = 1 - mapping(hr)
        # in matplotlib, 'green'->(0, 0.5, 0)
        green = np.ones((H,W,3))
        green[:,:,0] = 1 - mapping(hg)
        green[:,:,1] = 1 - 0.5*mapping(hg)
        green[:,:,2] = 1 - mapping(hg)
        blue = np.ones((H,W,3))
        blue[:,:,0] = 1 - mapping(hb)
        blue[:,:,1] = 1 - mapping(hb)
        plt.clf()
        plt.imshow(red * green * blue, origin='lower', interpolation='nearest',
                   extent=[xlo,xhi,ylo,yhi])
        plt.xlabel('g - r (mag)')
        plt.ylabel('r - z (mag)')
        plt.xticks(np.arange(0, 2.1, 0.5))
        plt.yticks(np.arange(0, 2.1, 0.5))
        plt.axis([0,2,0,2])
        plt.title('Red: DeV, Blue: Exp, Green: PSF')
        ps.savefig()

    np.random.seed(42)

    T.gr = T.mag_g - T.mag_r
    T.rz = T.mag_r - T.mag_z

    for tt,name in [('P', 'PSF'), ('E', 'EXP'), ('D', 'DeV')]:
        I = np.flatnonzero((T.flux_ivar_g > 0) * (T.flux_ivar_r > 0) *
                           (T.flux_ivar_z > 0) *
                           (T.flux_g > 0) * (T.flux_r > 0) * (T.flux_z > 0) *
                           (T.magerr_g < 0.1) * (T.magerr_r < 0.1) *(T.magerr_z < 0.1) *
                           # Bright cut
                           (T.mag_r > 16.) *

                           # Contamination cut
                           (T.decam_fracflux[:,2] < 0.5) *

                           # Mag range cut
                           (T.mag_r > 19.) *
                           (T.mag_r < 20.) *
                           
                           (T.typex == tt))
        print(len(I), 'with < 0.1-mag errs and type', name)

        # plt.clf()
        # ha = dict(histtype='step', range=(16, 24), bins=100)
        # plt.hist(T.mag_g[I], color='g', **ha)
        # plt.hist(T.mag_r[I], color='r', **ha)
        # plt.hist(T.mag_z[I], color='m', **ha)
        # plt.title('Type %s' % name)
        # ps.savefig()

        J = np.random.permutation(I)

        if tt == 'D':
            J = J[T.shapedev_r[J] < 3.]

        J = J[:100]
        rzbin = np.argsort(T.rz[J])
        rzblock = (rzbin / 10).astype(int)
        K = np.lexsort((T.gr[J], rzblock))
        print('sorted rzblock', rzblock[K])
        print('sorted rzbin', rzbin[K])
        J = J[K]

        # plt.clf()
        # plt.hist(T.decam_fracflux[J,2], label='fracflux_r', histtype='step', bins=20)
        # plt.hist(T.decam_fracmasked[J,2], label='fracmasked_r', histtype='step', bins=20)
        # plt.legend()
        # plt.title('Type %s' % name)
        # ps.savefig()

        #print('fracflux r', T.decam_fracflux[J,2])
        #print('fracmasked r', T.decam_fracmasked[J,2])
        #print('anymasked r', T.decam_anymask[J,2])

        imgs = []
        imgrow = []
        stackrows = []
        for i in J:
            fn = 'cutouts/cutout_%.4f_%.4f.jpg' % (T.ra[i], T.dec[i])
            if not os.path.exists(fn):
                url = 'http://legacysurvey.org/viewer/jpeg-cutout/?layer=decals-dr3&pixscale=0.262&size=25&ra=%f&dec=%f' % (T.ra[i], T.dec[i])
                print('URL', url)
                cmd = 'wget -O %s "%s"' % (fn, url)
                os.system(cmd)
            img = plt.imread(fn)
            #print('Image', img.shape)
            extra = ''
            if tt == 'D':
                extra = ', shapedev_r %.2f' % T.shapedev_r[i]
            elif tt == 'E':
                extra = ', shapeexp_r %.2f' % T.shapeexp_r[i]

            print(name, 'g/r/z %.2f, %.2f, %.2f, fracflux_r %.3f, fracmasked_r %.3f, anymasked_r %4i%s' % (
                T.mag_g[i], T.mag_r[i], T.mag_z[i],
                T.decam_fracflux[i,2], T.decam_fracmasked[i,2], T.decam_anymask[i,2], extra))

            imgrow.append(img)
            if len(imgrow) == 10:
                stack = np.hstack(imgrow)
                print('stacked row:', stack.shape)
                stackrows.append(stack)

                imgs.append(imgrow)
                imgrow = []


        stack = np.vstack(stackrows)
        print('Stack', stack.shape)
        plt.clf()
        plt.imshow(stack, interpolation='nearest', origin='lower')
        plt.xticks([])
        plt.yticks([])
        plt.title('Type %s' % name)
        ps.savefig()
        

    # Wrap-around at RA=300
    # rax = T.ra + (-360 * T.ra > 300.)
    # 
    # rabins = np.arange(-60., 300., 0.2)
{'fraction_non_alphanumeric': 0.1888957055214724, 'fraction_numerical': 0.024846625766871164, 'mean_word_length': 1.974092318919905, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2863259', 'n_tokens_mistral': 7329, 'n_tokens_neox': 6832, 'n_words': 1167}