astrobase/lcproc/catalogs.py
def add_cpinfo_to_lclist(
        checkplots,  # list or a directory path
        initial_lc_catalog,
        magcol,  # to indicate checkplot magcol
        outfile,
        checkplotglob='checkplot*.pkl*',
        infokeys=CPINFO_DEFAULTKEYS,
        nworkers=NCPUS
):
    '''This adds checkplot info to the initial light curve catalogs generated by
    `make_lclist`.

    This is used to incorporate all the extra info checkplots can have for
    objects back into columns in the light curve catalog produced by
    `make_lclist`. Objects are matched between the checkplots and the light
    curve catalog using their `objectid`. This then allows one to search this
    'augmented' light curve catalog by these extra columns. The 'augmented'
    light curve catalog also forms the basis for search interface provided by
    the LCC-Server.

    The default list of keys that will be extracted from a checkplot and added
    as columns in the initial light curve catalog is listed above in the
    `CPINFO_DEFAULTKEYS` list.

    Parameters
    ----------

    checkplots : str or list
        If this is a str, is interpreted as a directory which will be searched
        for checkplot pickle files using `checkplotglob`. If this is a list, it
        will be interpreted as a list of checkplot pickle files to process.

    initial_lc_catalog : str
        This is the path to the light curve catalog pickle made by
        `make_lclist`.

    magcol : str
        This is used to indicate the light curve magnitude column to extract
        magnitude column specific information. For example, Stetson variability
        indices can be generated using magnitude measurements in separate
        photometric apertures, which appear in separate `magcols` in the
        checkplot. To associate each such feature of the object with its
        specific `magcol`, pass that `magcol` in here. This `magcol` will then
        be added as a prefix to the resulting column in the 'augmented' LC
        catalog, e.g. Stetson J will appear as `magcol1_stetsonj` and
        `magcol2_stetsonj` for two separate magcols.

    outfile : str
        This is the file name of the output 'augmented' light curve catalog
        pickle file that will be written.

    infokeys : list of tuples

        This is a list of keys to extract from the checkplot and some info on
        how this extraction is to be done. Each key entry is a six-element
        tuple of the following form:

        - key name in the checkplot
        - numpy dtype of the value of this key
        - False if key is associated with a magcol or True otherwise
        - False if subsequent updates to the same column name will append to
          existing key values in the output augmented light curve catalog or
          True if these will overwrite the existing key value
        - character to use to substitute a None value of the key in the
          checkplot in the output light curve catalog column
        - character to use to substitute a nan value of the key in the
          checkplot in the output light curve catalog column

        See the `CPFINFO_DEFAULTKEYS` list above for examples.

    nworkers : int
        The number of parallel workers to launch to extract checkplot
        information.

    Returns
    -------

    str
        Returns the path to the generated 'augmented' light curve catalog pickle
        file.

    '''

    # get the checkplots from the directory if one is provided
    if not isinstance(checkplots, list) and os.path.exists(checkplots):
        checkplots = sorted(glob.glob(os.path.join(checkplots, checkplotglob)))

    tasklist = [(cpf, infokeys) for cpf in checkplots]

    with ProcessPoolExecutor(max_workers=nworkers) as executor:
        resultfutures = executor.map(_cpinfo_key_worker, tasklist)

    results = [x for x in resultfutures]
    executor.shutdown()

    # now that we have all the checkplot info, we need to match to the
    # objectlist in the lclist

    # open the lclist
    with open(initial_lc_catalog,'rb') as infd:
        lc_catalog = pickle.load(infd)

    catalog_objectids = np.array(lc_catalog['objects']['objectid'])
    checkplot_objectids = np.array([x[0] for x in results])

    # add the extra key arrays in the lclist dict
    extrainfokeys = []
    actualkeys = []

    # set up the extrainfokeys list
    for keyspec in infokeys:

        key, dtype, firstlevel, overwrite_append, nonesub, nansub = keyspec

        if firstlevel:
            eik = key
        else:
            eik = '%s.%s' % (magcol, key)

        extrainfokeys.append(eik)

        # now handle the output dicts and column list
        eactual = eik.split('.')

        # this handles dereferenced list indices
        if not eactual[-1].isdigit():

            if not firstlevel:
                eactual = '.'.join([eactual[0], eactual[-1]])
            else:
                eactual = eactual[-1]

        else:
            elastkey = eactual[-2]

            # for list columns, this converts stuff like errs -> err,
            # and parallaxes -> parallax
            if elastkey.endswith('es'):
                elastkey = elastkey[:-2]
            elif elastkey.endswith('s'):
                elastkey = elastkey[:-1]

            if not firstlevel:
                eactual = '.'.join([eactual[0], elastkey])
            else:
                eactual = elastkey

        actualkeys.append(eactual)

        # add a new column only if required
        if eactual not in lc_catalog['columns']:
            lc_catalog['columns'].append(eactual)

        # we'll overwrite earlier existing columns in any case
        lc_catalog['objects'][eactual] = []


    # now go through each objectid in the catalog and add the extra keys to
    # their respective arrays
    for catobj in tqdm(catalog_objectids):

        cp_objind = np.where(checkplot_objectids == catobj)

        if len(cp_objind[0]) > 0:

            # get the info line for this checkplot
            thiscpinfo = results[cp_objind[0][0]]

            # the first element is the objectid which we remove
            thiscpinfo = thiscpinfo[1:]

            # update the object catalog entries for this object
            for ekind, ek in enumerate(actualkeys):

                # add the actual thing to the output list
                lc_catalog['objects'][ek].append(
                    thiscpinfo[ekind]
                )

        else:

            # update the object catalog entries for this object
            for ekind, ek in enumerate(actualkeys):

                thiskeyspec = infokeys[ekind]
                nonesub = thiskeyspec[-2]

                lc_catalog['objects'][ek].append(
                    nonesub
                )

    # now we should have all the new keys in the object catalog
    # turn them into arrays
    for ek in actualkeys:

        lc_catalog['objects'][ek] = np.array(
            lc_catalog['objects'][ek]
        )

    # add the magcol to the lc_catalog
    if 'magcols' in lc_catalog:
        if magcol not in lc_catalog['magcols']:
            lc_catalog['magcols'].append(magcol)
    else:
        lc_catalog['magcols'] = [magcol]

    # write back the new object catalog
    with open(outfile, 'wb') as outfd:
        pickle.dump(lc_catalog, outfd, protocol=pickle.HIGHEST_PROTOCOL)

    return outfile
def add_cpinfo_to_lclist(
        checkplots,  # list or a directory path
        initial_lc_catalog,
        magcol,  # to indicate checkplot magcol
        outfile,
        checkplotglob='checkplot*.pkl*',
        infokeys=CPINFO_DEFAULTKEYS,
        nworkers=NCPUS
):
    '''This adds checkplot info to the initial light curve catalogs generated by
    `make_lclist`.

    This is used to incorporate all the extra info checkplots can have for
    objects back into columns in the light curve catalog produced by
    `make_lclist`. Objects are matched between the checkplots and the light
    curve catalog using their `objectid`. This then allows one to search this
    'augmented' light curve catalog by these extra columns. The 'augmented'
    light curve catalog also forms the basis for search interface provided by
    the LCC-Server.

    The default list of keys that will be extracted from a checkplot and added
    as columns in the initial light curve catalog is listed above in the
    `CPINFO_DEFAULTKEYS` list.

    Parameters
    ----------

    checkplots : str or list
        If this is a str, is interpreted as a directory which will be searched
        for checkplot pickle files using `checkplotglob`. If this is a list, it
        will be interpreted as a list of checkplot pickle files to process.

    initial_lc_catalog : str
        This is the path to the light curve catalog pickle made by
        `make_lclist`.

    magcol : str
        This is used to indicate the light curve magnitude column to extract
        magnitude column specific information. For example, Stetson variability
        indices can be generated using magnitude measurements in separate
        photometric apertures, which appear in separate `magcols` in the
        checkplot. To associate each such feature of the object with its
        specific `magcol`, pass that `magcol` in here. This `magcol` will then
        be added as a prefix to the resulting column in the 'augmented' LC
        catalog, e.g. Stetson J will appear as `magcol1_stetsonj` and
        `magcol2_stetsonj` for two separate magcols.

    outfile : str
        This is the file name of the output 'augmented' light curve catalog
        pickle file that will be written.

    infokeys : list of tuples

        This is a list of keys to extract from the checkplot and some info on
        how this extraction is to be done. Each key entry is a six-element
        tuple of the following form:

        - key name in the checkplot
        - numpy dtype of the value of this key
        - False if key is associated with a magcol or True otherwise
        - False if subsequent updates to the same column name will append to
          existing key values in the output augmented light curve catalog or
          True if these will overwrite the existing key value
        - character to use to substitute a None value of the key in the
          checkplot in the output light curve catalog column
        - character to use to substitute a nan value of the key in the
          checkplot in the output light curve catalog column

        See the `CPFINFO_DEFAULTKEYS` list above for examples.

    nworkers : int
        The number of parallel workers to launch to extract checkplot
        information.

    Returns
    -------

    str
        Returns the path to the generated 'augmented' light curve catalog pickle
        file.

    '''

    # get the checkplots from the directory if one is provided
    if not isinstance(checkplots, list) and os.path.exists(checkplots):
        checkplots = sorted(glob.glob(os.path.join(checkplots, checkplotglob)))

    tasklist = [(cpf, infokeys) for cpf in checkplots]

    with ProcessPoolExecutor(max_workers=nworkers) as executor:
        resultfutures = executor.map(_cpinfo_key_worker, tasklist)

    results = [x for x in resultfutures]
    executor.shutdown()

    # now that we have all the checkplot info, we need to match to the
    # objectlist in the lclist

    # open the lclist
    with open(initial_lc_catalog,'rb') as infd:
        lc_catalog = pickle.load(infd)

    catalog_objectids = np.array(lc_catalog['objects']['objectid'])
    checkplot_objectids = np.array([x[0] for x in results])

    # add the extra key arrays in the lclist dict
    extrainfokeys = []
    actualkeys = []

    # set up the extrainfokeys list
    for keyspec in infokeys:

        key, dtype, firstlevel, overwrite_append, nonesub, nansub = keyspec

        if firstlevel:
            eik = key
        else:
            eik = '%s.%s' % (magcol, key)

        extrainfokeys.append(eik)

        # now handle the output dicts and column list
        eactual = eik.split('.')

        # this handles dereferenced list indices
        if not eactual[-1].isdigit():

            if not firstlevel:
                eactual = '.'.join([eactual[0], eactual[-1]])
            else:
                eactual = eactual[-1]

        else:
            elastkey = eactual[-2]

            # for list columns, this converts stuff like errs -> err,
            # and parallaxes -> parallax
            if elastkey.endswith('es'):
                elastkey = elastkey[:-2]
            elif elastkey.endswith('s'):
                elastkey = elastkey[:-1]

            if not firstlevel:
                eactual = '.'.join([eactual[0], elastkey])
            else:
                eactual = elastkey

        actualkeys.append(eactual)

        # add a new column only if required
        if eactual not in lc_catalog['columns']:
            lc_catalog['columns'].append(eactual)

        # we'll overwrite earlier existing columns in any case
        lc_catalog['objects'][eactual] = []


    # now go through each objectid in the catalog and add the extra keys to
    # their respective arrays
    for catobj in tqdm(catalog_objectids):

        cp_objind = np.where(checkplot_objectids == catobj)

        if len(cp_objind[0]) > 0:

            # get the info line for this checkplot
            thiscpinfo = results[cp_objind[0][0]]

            # the first element is the objectid which we remove
            thiscpinfo = thiscpinfo[1:]

            # update the object catalog entries for this object
            for ekind, ek in enumerate(actualkeys):

                # add the actual thing to the output list
                lc_catalog['objects'][ek].append(
                    thiscpinfo[ekind]
                )

        else:

            # update the object catalog entries for this object
            for ekind, ek in enumerate(actualkeys):

                thiskeyspec = infokeys[ekind]
                nonesub = thiskeyspec[-2]

                lc_catalog['objects'][ek].append(
                    nonesub
                )

    # now we should have all the new keys in the object catalog
    # turn them into arrays
    for ek in actualkeys:

        lc_catalog['objects'][ek] = np.array(
            lc_catalog['objects'][ek]
        )

    # add the magcol to the lc_catalog
    if 'magcols' in lc_catalog:
        if magcol not in lc_catalog['magcols']:
            lc_catalog['magcols'].append(magcol)
    else:
        lc_catalog['magcols'] = [magcol]

    # write back the new object catalog
    with open(outfile, 'wb') as outfd:
        pickle.dump(lc_catalog, outfd, protocol=pickle.HIGHEST_PROTOCOL)

    return outfile
This adds checkplot info to the initial light curve catalogs generated by
    `make_lclist`.

    This is used to incorporate all the extra info checkplots can have for
    objects back into columns in the light curve catalog produced by
    `make_lclist`. Objects are matched between the checkplots and the light
    curve catalog using their `objectid`. This then allows one to search this
    'augmented' light curve catalog by these extra columns. The 'augmented'
    light curve catalog also forms the basis for search interface provided by
    the LCC-Server.

    The default list of keys that will be extracted from a checkplot and added
    as columns in the initial light curve catalog is listed above in the
    `CPINFO_DEFAULTKEYS` list.

    Parameters
    ----------

    checkplots : str or list
        If this is a str, is interpreted as a directory which will be searched
        for checkplot pickle files using `checkplotglob`. If this is a list, it
        will be interpreted as a list of checkplot pickle files to process.

    initial_lc_catalog : str
        This is the path to the light curve catalog pickle made by
        `make_lclist`.

    magcol : str
        This is used to indicate the light curve magnitude column to extract
        magnitude column specific information. For example, Stetson variability
        indices can be generated using magnitude measurements in separate
        photometric apertures, which appear in separate `magcols` in the
        checkplot. To associate each such feature of the object with its
        specific `magcol`, pass that `magcol` in here. This `magcol` will then
        be added as a prefix to the resulting column in the 'augmented' LC
        catalog, e.g. Stetson J will appear as `magcol1_stetsonj` and
        `magcol2_stetsonj` for two separate magcols.

    outfile : str
        This is the file name of the output 'augmented' light curve catalog
        pickle file that will be written.

    infokeys : list of tuples

        This is a list of keys to extract from the checkplot and some info on
        how this extraction is to be done. Each key entry is a six-element
        tuple of the following form:

        - key name in the checkplot
        - numpy dtype of the value of this key
        - False if key is associated with a magcol or True otherwise
        - False if subsequent updates to the same column name will append to
          existing key values in the output augmented light curve catalog or
          True if these will overwrite the existing key value
        - character to use to substitute a None value of the key in the
          checkplot in the output light curve catalog column
        - character to use to substitute a nan value of the key in the
          checkplot in the output light curve catalog column

        See the `CPFINFO_DEFAULTKEYS` list above for examples.

    nworkers : int
        The number of parallel workers to launch to extract checkplot
        information.

    Returns
    -------

    str
        Returns the path to the generated 'augmented' light curve catalog pickle
        file.