<commit_before>'''utilities.py: worker functions for CREST calls'''

import os
import logging
from logging.handlers import TimedRotatingFileHandler, SMTPHandler
import configparser
from configparser import ExtendedInterpolation

#CONFIG_FILES = ['prosperAPI_local.cfg','prosperAPI.cfg'] #TODO: change to .cfg?

LOGGER_DEFAULTS = {
    'log_level': 'DEBUG',
    'log_freq': 'd',
    'log_total': '60',
    'log_email_level': 'CRITICAL'
}

def get_config(
        config_filepath
):
    '''returns config object for parsing global values'''
    config = configparser.ConfigParser(
        interpolation=ExtendedInterpolation(),
        allow_no_value=True,
        delimiters=('=')
    )

    local_config_filepath = config_filepath.replace('.cfg', '_local.cfg')

    real_config_filepath = ''
    if os.path.isfile(local_config_filepath):
        #if _local.cfg version exists, use it instead
        real_config_filepath = local_config_filepath
    else:
        #else use tracked default
        real_config_filepath = config_filepath

    with open(real_config_filepath, 'r') as filehandle:
        config.read_file(filehandle)
    return config
    #TODO: add cfg error handling just in case

def create_logger(
        logName,
        logPath,
        configObject = None,
        logLevel_override = ''
):
    '''creates logging handle for programs'''
    if not configObject:
        #build up mini-config off in-script defaults
        tmpconfig = configparser.ConfigParser()
        tmpconfig['LOGGING'] = LOGGER_DEFAULTS
        configObject = tmpconfig

    if not os.path.exists(logPath):
        os.makedirs(logPath)
    #logFolder = os.path.join('../', tmpConfig.get('LOGGING', 'logFolder'))
    #if not os.path.exists(logFolder):
    #    os.makedirs(logFolder)

    Logger = logging.getLogger(logName)

    logLevel  = configObject.get('LOGGING', 'log_level')
    logFreq   = configObject.get('LOGGING', 'log_freq')
    logTotal  = configObject.get('LOGGING', 'log_total')

    logName   = logName + '.log'
    logFormat = '[%(asctime)s;%(levelname)s;%(filename)s;%(lineno)s] %(message)s'
    #print(logName + ':' + logLevel)
    if logLevel_override:
        logLevel = logLevel_override

    logFullpath = os.path.join(logPath, logName)

    Logger.setLevel(logLevel)
    generalHandler = TimedRotatingFileHandler(
        logFullpath,
        when        = logFreq,
        interval    = 1,
        backupCount = logTotal
    )
    formatter = logging.Formatter(logFormat)
    generalHandler.setFormatter(formatter)
    Logger.addHandler(generalHandler)

#    bool_doEmail = False
#    try:
#        emailSource     = configObject.get('LOGGING', 'emailSource')
#        emailRecipients = configObject.get('LOGGING', 'emailRecipients')
#        emailUsername   = configObject.get('LOGGING', 'emailUsername')
#        emailFromaddr   = configObject.get('LOGGING', 'emailFromaddr')
#        emailSecret     = configObject.get('LOGGING', 'emailSecret')
#        emailServer     = configObject.get('LOGGING', 'emailServer')
#        emailPort       = configObject.get('LOGGING', 'emailPort')
#        emailTitle      = logName + ' CRITICAL ERROR'
#
#        bool_doEmail = (
#            emailSource     and
#            emailRecipients and
#            emailUsername   and
#            emailFromaddr   and
#            emailSecret     and
#            emailServer     and
#            emailPort
#        )
#    except (KeyError, configparser.NoOptionError) as error:
#        #email keys not included, don't set up SMTPHandler
#        bool_doEmail = False
#
#    if bool_doEmail:
#        emailHandler = SMTPHandler(
#            mailhost    = emailServer + ':' + emailPort,
#            fromaddr    = emailFromaddr,
#            toaddrs     = str(emailRecipients).split(','),
#            subject     = emailTitle,
#            credentials = (emailUsername, emailSecret)
#        )
#        emailHandler.setFormatter(formatter)
#        Logger.addHandler(emailHandler)

    return Logger

def email_body_builder(errorMsg, helpMsg):
    '''Builds email message for easier reading with SMTPHandler'''
    #todo: format emails better
    return errorMsg + '\n' + helpMsg

def quandlfy_json(jsonObj):
    '''turn object from JSON into QUANDL-style JSON'''
    pass

def quandlfy_xml(xmlObj):
    '''turn object from XML into QUANDL-style XML'''
    pass

def log_and_debug(debug_str, debug=False, logger=None, log_level="DEBUG"):
    '''handles debug logger/print statements'''
    if debug:
        print(debug_str)
    if logger:
        logger.log(log_level.upper(), debug_str)

class LoggerDebugger(object):
    '''container for executing print/debug/log calls'''
    def __init__(self, debug, logger):
        self.debug=debug,
        self.logger=logger
        self.do_debug=bool(debug)
        self.do_logger=bool(logger)

    def message(self, message_str, log_level):
        '''actually do the thing'''
        if self.do_debug:
            print(message_str)
        if self.do_logger:
            self.logger.log(message_str, log_level)<commit_msg>expanded methods on LoggerDebugger to get logger obj<commit_after>'''utilities.py: worker functions for CREST calls'''

import os
import logging
from logging.handlers import TimedRotatingFileHandler, SMTPHandler
import configparser
from configparser import ExtendedInterpolation

#CONFIG_FILES = ['prosperAPI_local.cfg','prosperAPI.cfg'] #TODO: change to .cfg?

LOGGER_DEFAULTS = {
    'log_level': 'DEBUG',
    'log_freq': 'd',
    'log_total': '60',
    'log_email_level': 'CRITICAL'
}

def get_config(
        config_filepath
):
    '''returns config object for parsing global values'''
    config = configparser.ConfigParser(
        interpolation=ExtendedInterpolation(),
        allow_no_value=True,
        delimiters=('=')
    )

    local_config_filepath = config_filepath.replace('.cfg', '_local.cfg')

    real_config_filepath = ''
    if os.path.isfile(local_config_filepath):
        #if _local.cfg version exists, use it instead
        real_config_filepath = local_config_filepath
    else:
        #else use tracked default
        real_config_filepath = config_filepath

    with open(real_config_filepath, 'r') as filehandle:
        config.read_file(filehandle)
    return config
    #TODO: add cfg error handling just in case

def create_logger(
        logName,
        logPath,
        configObject = None,
        logLevel_override = ''
):
    '''creates logging handle for programs'''
    if not configObject:
        #build up mini-config off in-script defaults
        tmpconfig = configparser.ConfigParser()
        tmpconfig['LOGGING'] = LOGGER_DEFAULTS
        configObject = tmpconfig

    if not os.path.exists(logPath):
        os.makedirs(logPath)
    #logFolder = os.path.join('../', tmpConfig.get('LOGGING', 'logFolder'))
    #if not os.path.exists(logFolder):
    #    os.makedirs(logFolder)

    Logger = logging.getLogger(logName)

    logLevel  = configObject.get('LOGGING', 'log_level')
    logFreq   = configObject.get('LOGGING', 'log_freq')
    logTotal  = configObject.get('LOGGING', 'log_total')

    logName   = logName + '.log'
    logFormat = '[%(asctime)s;%(levelname)s;%(filename)s;%(lineno)s] %(message)s'
    #print(logName + ':' + logLevel)
    if logLevel_override:
        logLevel = logLevel_override

    logFullpath = os.path.join(logPath, logName)

    Logger.setLevel(logLevel)
    generalHandler = TimedRotatingFileHandler(
        logFullpath,
        when        = logFreq,
        interval    = 1,
        backupCount = logTotal
    )
    formatter = logging.Formatter(logFormat)
    generalHandler.setFormatter(formatter)
    Logger.addHandler(generalHandler)

#    bool_doEmail = False
#    try:
#        emailSource     = configObject.get('LOGGING', 'emailSource')
#        emailRecipients = configObject.get('LOGGING', 'emailRecipients')
#        emailUsername   = configObject.get('LOGGING', 'emailUsername')
#        emailFromaddr   = configObject.get('LOGGING', 'emailFromaddr')
#        emailSecret     = configObject.get('LOGGING', 'emailSecret')
#        emailServer     = configObject.get('LOGGING', 'emailServer')
#        emailPort       = configObject.get('LOGGING', 'emailPort')
#        emailTitle      = logName + ' CRITICAL ERROR'
#
#        bool_doEmail = (
#            emailSource     and
#            emailRecipients and
#            emailUsername   and
#            emailFromaddr   and
#            emailSecret     and
#            emailServer     and
#            emailPort
#        )
#    except (KeyError, configparser.NoOptionError) as error:
#        #email keys not included, don't set up SMTPHandler
#        bool_doEmail = False
#
#    if bool_doEmail:
#        emailHandler = SMTPHandler(
#            mailhost    = emailServer + ':' + emailPort,
#            fromaddr    = emailFromaddr,
#            toaddrs     = str(emailRecipients).split(','),
#            subject     = emailTitle,
#            credentials = (emailUsername, emailSecret)
#        )
#        emailHandler.setFormatter(formatter)
#        Logger.addHandler(emailHandler)

    return Logger

def email_body_builder(errorMsg, helpMsg):
    '''Builds email message for easier reading with SMTPHandler'''
    #todo: format emails better
    return errorMsg + '\n' + helpMsg

def quandlfy_json(jsonObj):
    '''turn object from JSON into QUANDL-style JSON'''
    pass

def quandlfy_xml(xmlObj):
    '''turn object from XML into QUANDL-style XML'''
    pass

def log_and_debug(debug_str, debug=False, logger=None, log_level="DEBUG"):
    '''handles debug logger/print statements'''
    if debug:
        print(debug_str)
    if logger:
        logger.log(log_level.upper(), debug_str)

class LoggerDebugger(object):
    '''container for executing print/debug/log calls'''
    def __init__(self, debug, logger):
        self.debug=debug,
        self.logger=logger
        self.do_debug=bool(debug)
        self.do_logger=bool(logger)

    def get_logger(self):
        return self.logger

    def message(self, message_str, log_level):
        '''actually do the thing'''
        if self.do_debug:
            print(message_str)
        if self.do_logger:
            self.logger.log(message_str, log_level)
{'fraction_non_alphanumeric': 0.10741138560687433, 'fraction_numerical': 0.0005858802851284055, 'mean_word_length': 2.827354260089686, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2855920', 'n_tokens_mistral': 3114, 'n_tokens_neox': 3062, 'n_words': 670}