<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''hatlc.py - <NAME> (<EMAIL>) - Jan 2016
License: MIT - see LICENSE for the full text.

This contains functions to read HAT sqlite ("sqlitecurves") and CSV light curves
generated by the new HAT data server.

The most probably useful functions in this module are:

read_csvlc(lcfile):
    This reads the HAT data server producd CSV light curve into an lcdict.

    lcfile is the HAT gzipped CSV LC (with a .hatlc.csv.gz extension)


read_and_filter_sqlitecurve(lcfile, columns=None, sqlfilters=None,
                            raiseonfail=False, forcerecompress=False):
    This reads the sqlitecurve and optionally filters it, returns an lcdict.

    Returns columns requested in columns. If None, then returns all columns
    present in the latest columnlist in the lightcurve. See COLUMNDEFS for the
    full list of HAT LC columns.

    If sqlfilters is not None, it must be a list of text sql filters that apply
    to the columns in the lightcurve.

    This returns an lcdict with an added 'lcfiltersql' key that indicates what
    the parsed SQL filter string was.

    If forcerecompress = True, will recompress the un-gzipped sqlitecurve even
    if the gzipped form exists on disk already.


Two other functions that might be useful:

normalize_lcdict(lcdict, timecol='rjd', magcols='all', mingap=4.0,
                 normto='sdssr', debugmode=False):

    This normalizes magnitude columns (specified in the magcols keyword
    argument) in an lcdict obtained from reading a HAT light curve. This
    normalization is done by finding 'timegroups' in each magnitude column,
    assuming that these belong to different 'eras' separated by a specified gap
    in the mingap keyword argument, and thus may be offset vertically from one
    another. Measurements within a timegroup are normalized to zero using the
    meidan magnitude of the timegroup. Once all timegroups have been processed
    this way, the whole time series is then re-normalized to the specified value
    in the normto keyword argument.


normalize_lcdict_byinst(lcdict, magcols='all', normto='sdssr', debugmode=False)

    This normalized magnitude columns (specified in the magcols keyword
    argument) in an lcdict obtained from reading a HAT light curve. This
    normalization is done by generating a normalization key using columns in the
    lcdict that specify various instrument properties. The normalization key is
    a combination of:

    - HAT station IDs
    - camera filters
    - observed HAT field names
    - HAT project IDs
    - camera exposure times

    with the assumption that measurements with identical normalization keys
    belong to a single 'era'. Measurements within an era are normalized to zero
    using the median magnitude of the era. Once all eras have been processed
    this way, the whole time series is then re-normalized to the specified value
    in the normto keyword argument.


There's an IPython notebook describing the use of this module and accompanying
modules from the astrobase package at:

https://github.com/waqasbhatti/astrobase/blob/master/notebooks/lightcurve-work.ipynb

'''

# put this in here because hatlc can be used as a standalone module
__version__ = '0.3.8'


#############
## LOGGING ##
#############

import logging
from datetime import datetime
from traceback import format_exc

# setup a logger
LOGGER = None
LOGMOD = __name__
DEBUG = False

def set_logger_parent(parent_name):
    globals()['LOGGER'] = logging.getLogger('%s.%s' % (parent_name, LOGMOD))

def LOGDEBUG(message):
    if LOGGER:
        LOGGER.debug(message)<commit_msg>Fix link to example notebook<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''hatlc.py - <NAME> (<EMAIL>) - Jan 2016
License: MIT - see LICENSE for the full text.

This contains functions to read HAT sqlite ("sqlitecurves") and CSV light curves
generated by the new HAT data server.

The most probably useful functions in this module are:

read_csvlc(lcfile):
    This reads the HAT data server producd CSV light curve into an lcdict.

    lcfile is the HAT gzipped CSV LC (with a .hatlc.csv.gz extension)


read_and_filter_sqlitecurve(lcfile, columns=None, sqlfilters=None,
                            raiseonfail=False, forcerecompress=False):
    This reads the sqlitecurve and optionally filters it, returns an lcdict.

    Returns columns requested in columns. If None, then returns all columns
    present in the latest columnlist in the lightcurve. See COLUMNDEFS for the
    full list of HAT LC columns.

    If sqlfilters is not None, it must be a list of text sql filters that apply
    to the columns in the lightcurve.

    This returns an lcdict with an added 'lcfiltersql' key that indicates what
    the parsed SQL filter string was.

    If forcerecompress = True, will recompress the un-gzipped sqlitecurve even
    if the gzipped form exists on disk already.


Two other functions that might be useful:

normalize_lcdict(lcdict, timecol='rjd', magcols='all', mingap=4.0,
                 normto='sdssr', debugmode=False):

    This normalizes magnitude columns (specified in the magcols keyword
    argument) in an lcdict obtained from reading a HAT light curve. This
    normalization is done by finding 'timegroups' in each magnitude column,
    assuming that these belong to different 'eras' separated by a specified gap
    in the mingap keyword argument, and thus may be offset vertically from one
    another. Measurements within a timegroup are normalized to zero using the
    meidan magnitude of the timegroup. Once all timegroups have been processed
    this way, the whole time series is then re-normalized to the specified value
    in the normto keyword argument.


normalize_lcdict_byinst(lcdict, magcols='all', normto='sdssr', debugmode=False)

    This normalized magnitude columns (specified in the magcols keyword
    argument) in an lcdict obtained from reading a HAT light curve. This
    normalization is done by generating a normalization key using columns in the
    lcdict that specify various instrument properties. The normalization key is
    a combination of:

    - HAT station IDs
    - camera filters
    - observed HAT field names
    - HAT project IDs
    - camera exposure times

    with the assumption that measurements with identical normalization keys
    belong to a single 'era'. Measurements within an era are normalized to zero
    using the median magnitude of the era. Once all eras have been processed
    this way, the whole time series is then re-normalized to the specified value
    in the normto keyword argument.


There's an IPython notebook describing the use of this module and accompanying
modules from the astrobase package at:

https://github.com/waqasbhatti/astrobase-notebooks/blob/master/lightcurve-work.ipynb

'''

# put this in here because hatlc can be used as a standalone module
__version__ = '0.3.8'


#############
## LOGGING ##
#############

import logging
from datetime import datetime
from traceback import format_exc

# setup a logger
LOGGER = None
LOGMOD = __name__
DEBUG = False

def set_logger_parent(parent_name):
    globals()['LOGGER'] = logging.getLogger('%s.%s' % (parent_name, LOGMOD))

def LOGDEBUG(message):
    if LOGGER:
        LOGGER.debug(message)
{'fraction_non_alphanumeric': 0.060937286514551166, 'fraction_numerical': 0.0027326137450471378, 'mean_word_length': 3.6594525779758116, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5498766', 'n_tokens_mistral': 2041, 'n_tokens_neox': 1939, 'n_words': 931}