<commit_before>
class ConfigFileError(Exception):
    pass

class TGKSync:
    """Sync observations with LCO.

    """

    _defaults = {
        'username': 'your@email',
        'password': '<PASSWORD>',
        'proposals': 'LCO2016B-109',
        'download path': '/full/path/to/your/download/directory'
    }
    

    def __init__(self, config_file):
        import json
        
        self.config_file = config_file
        self.last_download = None

        try:
            self._read_config()
        except json.JSONDecodeError as e:
            raise ConfigFileError('Error reading config file: {}\n{}'.format(config_file, e))
        
        self._setup_logger()

    @classmethod
    def show_config(cls, config_file):
        import os
        import sys
        import json

        if config_file is None:
            print(json.dumps(cls._defaults, indent=2))
        elif os.path.exists(config_file):
            with open(config_file) as inf:
                config = json.load(inf)
            print(json.dumps(config, indent=2))
        else:
            raise FileNotFoundError('Config file does not exist: {}'.format(config_file))
        
    def _read_config(self):
        import os
        import json

        if os.path.exists(self.config_file):
            with open(self.config_file) as inf:
                self.config = json.load(inf)
        else:
            raise FileNotFoundError("""Configuration file not found: {}
Use --show-config for an example.""".format(self.config_file))

        # verify download path
        if not os.path.isdir(self.config['download path']):
            raise OSError('Download path does not exist: {}'.format(self.config['download path']))

    def _setup_logger(self):
        import os
        import logging
        from datetime import datetime
        
        fn = os.sep.join([self.config['download path'], 'tgk-sync.log'])
        
        self.logger = logging.Logger('TGK Sync')
        self.logger.setLevel(logging.DEBUG)
        if len(self.logger.handlers) == 0:
            formatter = logging.Formatter('%(levelname)s: %(message)s')
            console = logging.StreamHandler(sys.stdout)
            console.setLevel(logging.DEBUG)
            console.setFormatter(formatter)
            self.logger.addHandler(console)

            logfile = logging.FileHandler(fn)
            logfile.setLevel(logging.INFO)
            logfile.setFormatter(formatter)
            self.logger.addHandler(logfile)

        self.logger.info('#' * 73)
        self.logger.info(datetime.now().isoformat())
        self.logger.info('Logging to console and {}'.format(fn))
        self.logger.info('Loaded configuration file: {}'.format(
            self.config_file))
        
    def run(self):
        from datetime import datetime
        from astropy.time import Time
        import astropy.units as u

        now = Time(datetime.now())
        
    def sync(self, **kwargs):
        import requests

if __name__ == '__main__':
    import os
    import sys
    import argparse

    default_config = os.sep.join([os.path.expanduser('~'), '.config',
                                  '41p-lco', 'sync.cfg'])
    
    parser = argparse.ArgumentParser(description='Master control program for monitoring LCO images of 41P.', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('--config', default=default_config, help='Use this configuration file.')
    parser.add_argument('--show-config', action='store_true', help='Read and print the configuration file.')
    parser.add_argument('--show-defaults', action='store_true', help='Print the default configuration file.')

    args = parser.parse_args()

    if args.show_config:
        TGKSync.show_config(args.config)
        sys.exit()

    if args.show_defaults:
        TGKSync.show_config(None)
        sys.exit()

    try:
        sync = TGKSync(args.config)
        sync.run()
    except Exception as e:
        print(e)
        exit()
<commit_msg>Reorg logging; Requests working; Started framework for archive monitoring.<commit_after>"""tgk-sync - Sync observations with LCO.

A portion of this code is based on Nestor Espinoza's lcogtDD.

"""

import logging

class ConfigFileError(Exception):
    pass

class AuthorizationError(Exception):
    pass

########################################################################
class Logger(logging.Logger):
    def __init__(self):
        import os
        from datetime import datetime
        
        logging.Logger.__init__(self, 'TGK Sync')

        self.setLevel(logging.DEBUG)

        formatter = logging.Formatter('%(levelname)s: %(message)s')
        console = logging.StreamHandler(sys.stdout)
        console.setLevel(logging.DEBUG)
        console.setFormatter(formatter)
        self.addHandler(console)
        self.info('Logging to console.')

        self.info('#' * 73)
        self.info(datetime.now().isoformat())

    def open_file(self, log_file):
        formatter = logging.Formatter('%(levelname)s: %(message)s')
        logfile = logging.FileHandler(log_file)
        logfile.setLevel(logging.INFO)
        logfile.setFormatter(formatter)
        self.addHandler(logfile)
        self.info('Logging to {}'.format(log_file))

    def shutdown(self):
        from astropy.time import Time
        self.info('Logging termnated at {}'.format(Time.now().utc))
        logging.shutdown()
        
########################################################################
class TGKSync:
    """Sync observations with LCO.

    Parameters
    ----------
    config_file : string
      The name of a configuration file to read.
    logger : logging.Logger, optional
      Log to this `Logger` instance, otherwise log to the python
      default.

    """

    _defaults = {
        'username': 'your@email',
        'password': '<PASSWORD>',
        'proposal': 'LCO2016B-109',
        'download path': '/full/path/to/your/download/directory'
    }
    

    def __init__(self, config_file, logger=None):
        import astropy.units as u
        from astropy.time import Time
        from json import JSONDecodeError

        self.config_file = config_file
        self.request_delay = 1 * u.s
        self.last_request = Time.now() - self.request_delay
        self.last_download = None

        # read configuration file
        try:
            self._read_config()
        except JSONDecodeError as e:
            raise ConfigFileError('Error reading config file: {}\n{}'.format(config_file, e))

        # begin logging
        if logger is None:
            self.logger = logging
        else:
            self.logger = logger
            fn = os.sep.join([self.config['download path'], 'tgk-sync.log'])
            self.logger.open_file(fn)
            self.logger.info('Loaded configuration file: {}'.format(
                self.config_file))

        # get http authoration token from LCO
        self._get_auth_token()

    @classmethod
    def show_config(cls, config_file):
        import os
        import sys
        import json

        if config_file is None:
            print(json.dumps(cls._defaults, indent=2))
        elif os.path.exists(config_file):
            with open(config_file) as inf:
                config = json.load(inf)
            print(json.dumps(config, indent=2))
        else:
            raise FileNotFoundError('Config file does not exist: {}'.format(config_file))
        
    def _read_config(self):
        import os
        import json

        if os.path.exists(self.config_file):
            with open(self.config_file) as inf:
                self.config = json.load(inf)
        else:
            raise FileNotFoundError("""Configuration file not found: {}
Use --show-config for an example.""".format(self.config_file))

        # verify download path
        if not os.path.isdir(self.config['download path']):
            raise OSError('Download path does not exist: {}'.format(self.config['download path']))

    def run(self):
        from astropy.time import Time
        import astropy.units as u

        now = Time.now()
        dt = -1 * u.day  # search window

        self.last_download = self.sync((now + dt))
        
    def sync(self, start, rlevels=[91], **kwargs):
        """Request frames list from LCO and download, if needed.

        Parameters
        ----------
        start : Time
          Check for frames since `start`.  [UTC]
        rlevels : list of int
          Which reduction levels to check.

        """
        from astropy.time import Time

        for rlevel in rlevels:
            query = {
                'PROPID': self.config['proposal'],
                'limit': 50,
                'RLEVEL': rlevel,
                'start': start.iso[:10],
            }
            r = self.request('https://archive-api.lco.global/frames/',
                             query=query)

    def _get_auth_token(self):
        import requests

        data = {
            'username': self.config['username'],
            'password': self.config['password']
        }
        r = requests.post('https://archive-api.lco.global/api-token-auth/',
                          data=data).json()
        token = r.get('token')
        if token is None:
            raise AuthorizationError('Authorization token not returned.')

        self.auth = {'Authorization': 'Token ' + token}
        self.logger.info('Obtained authoriation token.')

    def request(self, url, query={}):
        """Send HTTP request and return the output.

        Limits the overall number of requests to slow down any runaway
        code and prevent from exceeding the request limit.

        Parameters
        ----------
        url : string
          The URL.
        param : dict, optional
          The HTTP get parameters.

        """
        import time
        from astropy.time import Time
        import requests
        
        while (Time.now() - self.last_request) < self.request_delay:
            time.sleep(1)

        self.logger.info('Request: {}, {}'.format(url, query))
        response = requests.get(url, params=query, headers=self.auth)
        self.logger.debug(response.url)

        data = response.json()
        self.logger.info('Found {} frames.'.format(data['count']))
        
        return data

########################################################################
if __name__ == '__main__':
    import os
    import sys
    import argparse
    from astropy.time import Time

    default_config = os.sep.join([os.path.expanduser('~'), '.config',
                                  '41p-lco', 'sync.cfg'])
    
    parser = argparse.ArgumentParser(description='Master control program for monitoring LCO images of 41P.', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('--config', default=default_config, help='Use this configuration file.')
    parser.add_argument('--show-config', action='store_true', help='Read and print the configuration file.')
    parser.add_argument('--show-defaults', action='store_true', help='Print the default configuration file.')
    parser.add_argument('-v', action='store_true', help='Increase verbosity.')

    args = parser.parse_args()

    if args.show_config:
        TGKSync.show_config(args.config)
        sys.exit()

    if args.show_defaults:
        TGKSync.show_config(None)
        sys.exit()

    try:
        logger = Logger()
        sync = TGKSync(args.config, logger=logger)
        sync.run()
        logging.shutdown()
    except Exception as e:
        err = '{}: {}'.format(type(e).__name__, e)
        #logger.info(err)
        logging.shutdown()

        if args.v:
            raise(e)
        else:
            print(err)
            exit()

{'fraction_non_alphanumeric': 0.13043107854830466, 'fraction_numerical': 0.0034077355597205657, 'mean_word_length': 2.159892328398385, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3461990', 'n_tokens_mistral': 3285, 'n_tokens_neox': 3161, 'n_words': 779}