astrobase/services/dust.py
def extinction_query(lon, lat,
                     coordtype='equatorial',
                     sizedeg=5.0,
                     forcefetch=False,
                     cachedir='~/.astrobase/dust-cache',
                     verbose=True,
                     timeout=10.0,
                     jitter=5.0):
    '''This queries the 2MASS DUST service to find the extinction parameters
    for the given `lon`, `lat`.

    Parameters
    ----------

    lon,lat: float
        These are decimal right ascension and declination if `coordtype =
        'equatorial'`. These are are decimal Galactic longitude and latitude if
        `coordtype = 'galactic'`.

    coordtype : {'equatorial','galactic'}
        Sets the type of coordinates passed in as `lon`, `lat`.

    sizedeg : float
        This is the width of the image returned by the DUST service. This can
        usually be left as-is if you're interested in the extinction only.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our request.

    jitter : float
        This is used to control the scale of the random wait in seconds before
        starting the query. Useful in parallelized situations.

    Returns
    -------

    dict
        A dict of the following form is returned::

            {'Amag':{dict of extinction A_v values for several mag systems},
             'table': array containing the full extinction table,
             'tablefile': the path to the full extinction table file on disk,
             'provenance': 'cached' or 'new download',
             'request': string repr of the request made to 2MASS DUST}

    '''

    dustparams = DUST_PARAMS.copy()

    # convert the lon, lat to the required format
    # and generate the param dict
    if coordtype == 'equatorial':
        locstr = '%.3f %.3f Equ J2000' % (lon, lat)
    elif coordtype == 'galactic':
        locstr = '%.3f %.3f gal' % (lon, lat)
    else:
        LOGERROR('unknown coordinate type: %s' % coordtype)
        return None

    dustparams['locstr'] = locstr
    dustparams['regSize'] = '%.3f' % sizedeg

    # see if the cachedir exists
    if '~' in cachedir:
        cachedir = os.path.expanduser(cachedir)
    if not os.path.exists(cachedir):
        os.makedirs(cachedir)

    # generate the cachekey and cache filename
    cachekey = '%s - %.1f' % (locstr, sizedeg)
    cachekey = hashlib.sha256(cachekey.encode()).hexdigest()
    cachefname = os.path.join(cachedir, '%s.txt' % cachekey)
    provenance = 'cache'

    # if this does not exist in cache or if we're forcefetching, do the query
    if forcefetch or (not os.path.exists(cachefname)):

        time.sleep(random.randint(1,jitter))
        provenance = 'new download'

        try:

            if verbose:
                LOGINFO('submitting 2MASS DUST request for '
                        'lon = %.3f, lat = %.3f, type = %s, size = %.1f' %
                        (lon, lat, coordtype, sizedeg))

            req = requests.get(DUST_URL, dustparams, timeout=timeout)
            req.raise_for_status()

            resp = req.text

            # see if we got an extinction table URL in the response
            tableurl = DUST_REGEX.search(resp)

            # if we did, download it to the cache directory
            if tableurl:

                tableurl = tableurl.group(0)

                req2 = requests.get(tableurl, timeout=timeout)

                # write the table to the cache directory
                with open(cachefname,'wb') as outfd:
                    outfd.write(req2.content)

                tablefname = cachefname

            else:
                LOGERROR('could not get extinction parameters for '
                         '%s (%.3f, %.3f) with size = %.1f' % (coordtype,
                                                               lon,lat,sizedeg))
                LOGERROR('error from DUST service follows:\n%s' % resp)
                return None

        except requests.exceptions.Timeout as e:
            LOGERROR('DUST request timed out for '
                     '%s (%.3f, %.3f) with size = %.1f' % (coordtype,
                                                           lon,lat,sizedeg))
            return None

        except Exception as e:
            LOGEXCEPTION('DUST request failed for '
                         '%s (%.3f, %.3f) with size = %.1f' % (coordtype,
                                                               lon,lat,sizedeg))
            return None

    # if this result is available in the cache, get it from there
    else:

        if verbose:
            LOGINFO('getting cached 2MASS DUST result for '
                    'lon = %.3f, lat = %.3f, coordtype = %s, size = %.1f' %
                    (lon, lat, coordtype, sizedeg))

        tablefname = cachefname

    #
    # now we should have the extinction table in some form
    #
    # read and parse the extinction table using astropy.Table
    extinction_table = Table.read(tablefname, format='ascii.ipac')

    # get the columns we need
    filters = np.array(extinction_table['Filter_name'])
    a_sf11_byfilter = np.array(extinction_table['A_SandF'])
    a_sfd98_byfilter = np.array(extinction_table['A_SFD'])

    # generate the output dict
    extdict = {'Amag':{x:{'sf11':y, 'sfd98':z} for
                       x,y,z in zip(filters,a_sf11_byfilter,a_sfd98_byfilter)},
               'table':np.array(extinction_table),
               'tablefile':os.path.abspath(cachefname),
               'provenance':provenance,
               'request':'%s (%.3f, %.3f) with size = %.1f' % (coordtype,
                                                               lon,lat,
                                                               sizedeg)}

    return extdict
def extinction_query(lon, lat,
                     coordtype='equatorial',
                     sizedeg=5.0,
                     forcefetch=False,
                     cachedir='~/.astrobase/dust-cache',
                     verbose=True,
                     timeout=10.0,
                     jitter=5.0):
    '''This queries the 2MASS DUST service to find the extinction parameters
    for the given `lon`, `lat`.

    Parameters
    ----------

    lon,lat: float
        These are decimal right ascension and declination if `coordtype =
        'equatorial'`. These are are decimal Galactic longitude and latitude if
        `coordtype = 'galactic'`.

    coordtype : {'equatorial','galactic'}
        Sets the type of coordinates passed in as `lon`, `lat`.

    sizedeg : float
        This is the width of the image returned by the DUST service. This can
        usually be left as-is if you're interested in the extinction only.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our request.

    jitter : float
        This is used to control the scale of the random wait in seconds before
        starting the query. Useful in parallelized situations.

    Returns
    -------

    dict
        A dict of the following form is returned::

            {'Amag':{dict of extinction A_v values for several mag systems},
             'table': array containing the full extinction table,
             'tablefile': the path to the full extinction table file on disk,
             'provenance': 'cached' or 'new download',
             'request': string repr of the request made to 2MASS DUST}

    '''

    dustparams = DUST_PARAMS.copy()

    # convert the lon, lat to the required format
    # and generate the param dict
    if coordtype == 'equatorial':
        locstr = '%.3f %.3f Equ J2000' % (lon, lat)
    elif coordtype == 'galactic':
        locstr = '%.3f %.3f gal' % (lon, lat)
    else:
        LOGERROR('unknown coordinate type: %s' % coordtype)
        return None

    dustparams['locstr'] = locstr
    dustparams['regSize'] = '%.3f' % sizedeg

    # see if the cachedir exists
    if '~' in cachedir:
        cachedir = os.path.expanduser(cachedir)
    if not os.path.exists(cachedir):
        os.makedirs(cachedir)

    # generate the cachekey and cache filename
    cachekey = '%s - %.1f' % (locstr, sizedeg)
    cachekey = hashlib.sha256(cachekey.encode()).hexdigest()
    cachefname = os.path.join(cachedir, '%s.txt' % cachekey)
    provenance = 'cache'

    # if this does not exist in cache or if we're forcefetching, do the query
    if forcefetch or (not os.path.exists(cachefname)):

        time.sleep(random.randint(1,jitter))
        provenance = 'new download'

        try:

            if verbose:
                LOGINFO('submitting 2MASS DUST request for '
                        'lon = %.3f, lat = %.3f, type = %s, size = %.1f' %
                        (lon, lat, coordtype, sizedeg))

            req = requests.get(DUST_URL, dustparams, timeout=timeout)
            req.raise_for_status()

            resp = req.text

            # see if we got an extinction table URL in the response
            tableurl = DUST_REGEX.search(resp)

            # if we did, download it to the cache directory
            if tableurl:

                tableurl = tableurl.group(0)

                req2 = requests.get(tableurl, timeout=timeout)

                # write the table to the cache directory
                with open(cachefname,'wb') as outfd:
                    outfd.write(req2.content)

                tablefname = cachefname

            else:
                LOGERROR('could not get extinction parameters for '
                         '%s (%.3f, %.3f) with size = %.1f' % (coordtype,
                                                               lon,lat,sizedeg))
                LOGERROR('error from DUST service follows:\n%s' % resp)
                return None

        except requests.exceptions.Timeout as e:
            LOGERROR('DUST request timed out for '
                     '%s (%.3f, %.3f) with size = %.1f' % (coordtype,
                                                           lon,lat,sizedeg))
            return None

        except Exception as e:
            LOGEXCEPTION('DUST request failed for '
                         '%s (%.3f, %.3f) with size = %.1f' % (coordtype,
                                                               lon,lat,sizedeg))
            return None

    # if this result is available in the cache, get it from there
    else:

        if verbose:
            LOGINFO('getting cached 2MASS DUST result for '
                    'lon = %.3f, lat = %.3f, coordtype = %s, size = %.1f' %
                    (lon, lat, coordtype, sizedeg))

        tablefname = cachefname

    #
    # now we should have the extinction table in some form
    #
    # read and parse the extinction table using astropy.Table
    extinction_table = Table.read(tablefname, format='ascii.ipac')

    # get the columns we need
    filters = np.array(extinction_table['Filter_name'])
    a_sf11_byfilter = np.array(extinction_table['A_SandF'])
    a_sfd98_byfilter = np.array(extinction_table['A_SFD'])

    # generate the output dict
    extdict = {'Amag':{x:{'sf11':y, 'sfd98':z} for
                       x,y,z in zip(filters,a_sf11_byfilter,a_sfd98_byfilter)},
               'table':np.array(extinction_table),
               'tablefile':os.path.abspath(cachefname),
               'provenance':provenance,
               'request':'%s (%.3f, %.3f) with size = %.1f' % (coordtype,
                                                               lon,lat,
                                                               sizedeg)}

    return extdict
This queries the 2MASS DUST service to find the extinction parameters
    for the given `lon`, `lat`.

    Parameters
    ----------

    lon,lat: float
        These are decimal right ascension and declination if `coordtype =
        'equatorial'`. These are are decimal Galactic longitude and latitude if
        `coordtype = 'galactic'`.

    coordtype : {'equatorial','galactic'}
        Sets the type of coordinates passed in as `lon`, `lat`.

    sizedeg : float
        This is the width of the image returned by the DUST service. This can
        usually be left as-is if you're interested in the extinction only.

    forcefetch : bool
        If this is True, the query will be retried even if cached results for
        it exist.

    cachedir : str
        This points to the directory where results will be downloaded.

    verbose : bool
        If True, will indicate progress and warn of any issues.

    timeout : float
        This sets the amount of time in seconds to wait for the service to
        respond to our request.

    jitter : float
        This is used to control the scale of the random wait in seconds before
        starting the query. Useful in parallelized situations.

    Returns
    -------

    dict
        A dict of the following form is returned::

            {'Amag':{dict of extinction A_v values for several mag systems},
             'table': array containing the full extinction table,
             'tablefile': the path to the full extinction table file on disk,
             'provenance': 'cached' or 'new download',
             'request': string repr of the request made to 2MASS DUST}