astrobase/checkplot/pkl_xmatch.py
def xmatch_external_catalogs(checkplotdict,
                             xmatchinfo,
                             xmatchradiusarcsec=2.0,
                             returndirect=False,
                             updatexmatch=True,
                             savepickle=None):
    '''This matches the current object in the checkplotdict to all of the
    external match catalogs specified.

    Parameters
    ----------

    checkplotdict : dict
        This is a checkplotdict, generated by either the `checkplot_dict`
        function, or read in from a `_read_checkplot_picklefile` function. This
        must have a structure somewhat like the following, where the indicated
        keys below are required::

            {'objectid': the ID assigned to this object
             'objectinfo': {'objectid': ID assigned to this object,
                            'ra': right ascension of the object in decimal deg,
                            'decl': declination of the object in decimal deg}}

    xmatchinfo : str or dict
        This is either the xmatch dict produced by the function
        :py:func:`astrobase.checkplot.pkl_xmatch.load_xmatch_external_catalogs`
        above, or the path to the xmatch info pickle file produced by that
        function.

    xmatchradiusarcsec : float
        This is the cross-matching radius to use in arcseconds.

    returndirect : bool
        If this is True, will only return the xmatch results as a dict. If this
        False, will return the checkplotdict with the xmatch results added in as
        a key-val pair.

    updatexmatch : bool
        This function will look for an existing 'xmatch' key in the input
        checkplotdict indicating that an xmatch has been performed before. If
        `updatexmatch` is set to True, the xmatch results will be added onto
        (e.g. when xmatching to additional catalogs after the first run). If
        this is set to False, the xmatch key-val pair will be completely
        overwritten.

    savepickle : str or None
        If this is None, it must be a path to where the updated checkplotdict
        will be written to as a new checkplot pickle. If this is False, only the
        updated checkplotdict is returned.

    Returns
    -------

    dict or str
        If `savepickle` is False, this returns a checkplotdict, with the xmatch
        results added in. An 'xmatch' key will be added to this dict, with
        something like the following dict as the value::

            {'xmatchradiusarcsec':xmatchradiusarcsec,
             'catalog1':{'name':'Catalog of interesting things',
                        'found':True,
                        'distarcsec':0.7,
                        'info':{'objectid':...,'ra':...,'decl':...,'desc':...}},
             'catalog2':{'name':'Catalog of more interesting things',
                         'found':False,
                         'distarcsec':nan,
                         'info':None},
            .
            .
            .
            ....}

        This will contain the matches of the object in the input checkplotdict
        to all of the catalogs provided in `xmatchinfo`.

        If `savepickle` is True, will return the path to the saved checkplot
        pickle file.

    '''

    # load the xmatch info
    if isinstance(xmatchinfo, str) and os.path.exists(xmatchinfo):
        with open(xmatchinfo,'rb') as infd:
            xmatchdict = pickle.load(infd)
    elif isinstance(xmatchinfo, dict):
        xmatchdict = xmatchinfo
    else:
        LOGERROR("can't figure out xmatch info, can't xmatch, skipping...")
        return checkplotdict

    #
    # generate the xmatch spec
    #

    # get our ra, decl
    objra = checkplotdict['objectinfo']['ra']
    objdecl = checkplotdict['objectinfo']['decl']

    cosdecl = np.cos(np.radians(objdecl))
    sindecl = np.sin(np.radians(objdecl))
    cosra = np.cos(np.radians(objra))
    sinra = np.sin(np.radians(objra))

    objxyz = np.column_stack((cosra*cosdecl,
                              sinra*cosdecl,
                              sindecl))

    # this is the search distance in xyz unit vectors
    xyzdist = 2.0 * np.sin(np.radians(xmatchradiusarcsec/3600.0)/2.0)

    #
    # now search in each external catalog
    #

    xmatchresults = {}

    extcats = sorted(list(xmatchdict.keys()))

    for ecat in extcats:

        # get the kdtree
        kdt = xmatchdict[ecat]['kdtree']

        # look up the coordinates
        kdt_dist, kdt_ind = kdt.query(objxyz,
                                      k=1,
                                      distance_upper_bound=xyzdist)

        # sort by matchdist
        mdsorted = np.argsort(kdt_dist)
        matchdists = kdt_dist[mdsorted]
        matchinds = kdt_ind[mdsorted]

        if matchdists[np.isfinite(matchdists)].size == 0:

            xmatchresults[ecat] = {'name':xmatchdict[ecat]['name'],
                                   'desc':xmatchdict[ecat]['desc'],
                                   'found':False,
                                   'distarcsec':None,
                                   'info':None}

        else:

            for md, mi in zip(matchdists, matchinds):

                if np.isfinite(md) and md < xyzdist:

                    infodict = {}

                    distarcsec = _xyzdist_to_distarcsec(md)

                    for col in xmatchdict[ecat]['columns']:

                        coldata = xmatchdict[ecat]['data'][col][mi]

                        if isinstance(coldata, str):
                            coldata = coldata.strip()

                        infodict[col] = coldata

                    xmatchresults[ecat] = {
                        'name':xmatchdict[ecat]['name'],
                        'desc':xmatchdict[ecat]['desc'],
                        'found':True,
                        'distarcsec':distarcsec,
                        'info':infodict,
                        'colkeys':xmatchdict[ecat]['columns'],
                        'colnames':xmatchdict[ecat]['colnames'],
                        'colunit':xmatchdict[ecat]['colunits'],
                    }
                    break

    #
    # should now have match results for all external catalogs
    #

    if returndirect:

        return xmatchresults

    else:

        if updatexmatch and 'xmatch' in checkplotdict:
            checkplotdict['xmatch'].update(xmatchresults)
        else:
            checkplotdict['xmatch'] = xmatchresults


        if savepickle:

            cpf = _write_checkplot_picklefile(checkplotdict,
                                              outfile=savepickle,
                                              protocol=4)
            return cpf

        else:
            return checkplotdict
def xmatch_external_catalogs(checkplotdict,
                             xmatchinfo,
                             xmatchradiusarcsec=2.0,
                             returndirect=False,
                             updatexmatch=True,
                             savepickle=None):
    '''This matches the current object in the checkplotdict to all of the
    external match catalogs specified.

    Parameters
    ----------

    checkplotdict : dict
        This is a checkplotdict, generated by either the `checkplot_dict`
        function, or read in from a `_read_checkplot_picklefile` function. This
        must have a structure somewhat like the following, where the indicated
        keys below are required::

            {'objectid': the ID assigned to this object
             'objectinfo': {'objectid': ID assigned to this object,
                            'ra': right ascension of the object in decimal deg,
                            'decl': declination of the object in decimal deg}}

    xmatchinfo : str or dict
        This is either the xmatch dict produced by the function
        :py:func:`astrobase.checkplot.pkl_xmatch.load_xmatch_external_catalogs`
        above, or the path to the xmatch info pickle file produced by that
        function.

    xmatchradiusarcsec : float
        This is the cross-matching radius to use in arcseconds.

    returndirect : bool
        If this is True, will only return the xmatch results as a dict. If this
        False, will return the checkplotdict with the xmatch results added in as
        a key-val pair.

    updatexmatch : bool
        This function will look for an existing 'xmatch' key in the input
        checkplotdict indicating that an xmatch has been performed before. If
        `updatexmatch` is set to True, the xmatch results will be added onto
        (e.g. when xmatching to additional catalogs after the first run). If
        this is set to False, the xmatch key-val pair will be completely
        overwritten.

    savepickle : str or None
        If this is None, it must be a path to where the updated checkplotdict
        will be written to as a new checkplot pickle. If this is False, only the
        updated checkplotdict is returned.

    Returns
    -------

    dict or str
        If `savepickle` is False, this returns a checkplotdict, with the xmatch
        results added in. An 'xmatch' key will be added to this dict, with
        something like the following dict as the value::

            {'xmatchradiusarcsec':xmatchradiusarcsec,
             'catalog1':{'name':'Catalog of interesting things',
                        'found':True,
                        'distarcsec':0.7,
                        'info':{'objectid':...,'ra':...,'decl':...,'desc':...}},
             'catalog2':{'name':'Catalog of more interesting things',
                         'found':False,
                         'distarcsec':nan,
                         'info':None},
            .
            .
            .
            ....}

        This will contain the matches of the object in the input checkplotdict
        to all of the catalogs provided in `xmatchinfo`.

        If `savepickle` is True, will return the path to the saved checkplot
        pickle file.

    '''

    # load the xmatch info
    if isinstance(xmatchinfo, str) and os.path.exists(xmatchinfo):
        with open(xmatchinfo,'rb') as infd:
            xmatchdict = pickle.load(infd)
    elif isinstance(xmatchinfo, dict):
        xmatchdict = xmatchinfo
    else:
        LOGERROR("can't figure out xmatch info, can't xmatch, skipping...")
        return checkplotdict

    #
    # generate the xmatch spec
    #

    # get our ra, decl
    objra = checkplotdict['objectinfo']['ra']
    objdecl = checkplotdict['objectinfo']['decl']

    cosdecl = np.cos(np.radians(objdecl))
    sindecl = np.sin(np.radians(objdecl))
    cosra = np.cos(np.radians(objra))
    sinra = np.sin(np.radians(objra))

    objxyz = np.column_stack((cosra*cosdecl,
                              sinra*cosdecl,
                              sindecl))

    # this is the search distance in xyz unit vectors
    xyzdist = 2.0 * np.sin(np.radians(xmatchradiusarcsec/3600.0)/2.0)

    #
    # now search in each external catalog
    #

    xmatchresults = {}

    extcats = sorted(list(xmatchdict.keys()))

    for ecat in extcats:

        # get the kdtree
        kdt = xmatchdict[ecat]['kdtree']

        # look up the coordinates
        kdt_dist, kdt_ind = kdt.query(objxyz,
                                      k=1,
                                      distance_upper_bound=xyzdist)

        # sort by matchdist
        mdsorted = np.argsort(kdt_dist)
        matchdists = kdt_dist[mdsorted]
        matchinds = kdt_ind[mdsorted]

        if matchdists[np.isfinite(matchdists)].size == 0:

            xmatchresults[ecat] = {'name':xmatchdict[ecat]['name'],
                                   'desc':xmatchdict[ecat]['desc'],
                                   'found':False,
                                   'distarcsec':None,
                                   'info':None}

        else:

            for md, mi in zip(matchdists, matchinds):

                if np.isfinite(md) and md < xyzdist:

                    infodict = {}

                    distarcsec = _xyzdist_to_distarcsec(md)

                    for col in xmatchdict[ecat]['columns']:

                        coldata = xmatchdict[ecat]['data'][col][mi]

                        if isinstance(coldata, str):
                            coldata = coldata.strip()

                        infodict[col] = coldata

                    xmatchresults[ecat] = {
                        'name':xmatchdict[ecat]['name'],
                        'desc':xmatchdict[ecat]['desc'],
                        'found':True,
                        'distarcsec':distarcsec,
                        'info':infodict,
                        'colkeys':xmatchdict[ecat]['columns'],
                        'colnames':xmatchdict[ecat]['colnames'],
                        'colunit':xmatchdict[ecat]['colunits'],
                    }
                    break

    #
    # should now have match results for all external catalogs
    #

    if returndirect:

        return xmatchresults

    else:

        if updatexmatch and 'xmatch' in checkplotdict:
            checkplotdict['xmatch'].update(xmatchresults)
        else:
            checkplotdict['xmatch'] = xmatchresults


        if savepickle:

            cpf = _write_checkplot_picklefile(checkplotdict,
                                              outfile=savepickle,
                                              protocol=4)
            return cpf

        else:
            return checkplotdict
This matches the current object in the checkplotdict to all of the
    external match catalogs specified.

    Parameters
    ----------

    checkplotdict : dict
        This is a checkplotdict, generated by either the `checkplot_dict`
        function, or read in from a `_read_checkplot_picklefile` function. This
        must have a structure somewhat like the following, where the indicated
        keys below are required::

            {'objectid': the ID assigned to this object
             'objectinfo': {'objectid': ID assigned to this object,
                            'ra': right ascension of the object in decimal deg,
                            'decl': declination of the object in decimal deg}}

    xmatchinfo : str or dict
        This is either the xmatch dict produced by the function
        :py:func:`astrobase.checkplot.pkl_xmatch.load_xmatch_external_catalogs`
        above, or the path to the xmatch info pickle file produced by that
        function.

    xmatchradiusarcsec : float
        This is the cross-matching radius to use in arcseconds.

    returndirect : bool
        If this is True, will only return the xmatch results as a dict. If this
        False, will return the checkplotdict with the xmatch results added in as
        a key-val pair.

    updatexmatch : bool
        This function will look for an existing 'xmatch' key in the input
        checkplotdict indicating that an xmatch has been performed before. If
        `updatexmatch` is set to True, the xmatch results will be added onto
        (e.g. when xmatching to additional catalogs after the first run). If
        this is set to False, the xmatch key-val pair will be completely
        overwritten.

    savepickle : str or None
        If this is None, it must be a path to where the updated checkplotdict
        will be written to as a new checkplot pickle. If this is False, only the
        updated checkplotdict is returned.

    Returns
    -------

    dict or str
        If `savepickle` is False, this returns a checkplotdict, with the xmatch
        results added in. An 'xmatch' key will be added to this dict, with
        something like the following dict as the value::

            {'xmatchradiusarcsec':xmatchradiusarcsec,
             'catalog1':{'name':'Catalog of interesting things',
                        'found':True,
                        'distarcsec':0.7,
                        'info':{'objectid':...,'ra':...,'decl':...,'desc':...}},
             'catalog2':{'name':'Catalog of more interesting things',
                         'found':False,
                         'distarcsec':nan,
                         'info':None},
            .
            .
            .
            ....}

        This will contain the matches of the object in the input checkplotdict
        to all of the catalogs provided in `xmatchinfo`.

        If `savepickle` is True, will return the path to the saved checkplot
        pickle file.