<commit_before># -*- coding: utf-8 -*-

"""

Move here the logic for any logs

"""

from __future__ import absolute_import
import os
import json
import logging
from logging.config import fileConfig
from json.decoder import JSONDecodeError
from . import AVOID_COLORS_ENV_LABEL, LOG_CONFIG

MAX_CHAR_LEN = 200
OBSCURE_VALUE = '****'
OBSCURED_FIELDS = ['password', '<PASSWORD>', 'token', 'file', 'filename']


################
# LOGGING

class LogMe(object):
    """ A common logger to be used all around development packages """

    def __init__(self, debug=None):

        #####################
        self._log_level = logging.INFO
        self._colors_enabled = True
        super(LogMe, self).__init__()

        # became useless:

        # #####################
        # # Set debug
        # if debug is None:
        #     debug = int(os.environ.get('API_DEBUG', False))
        # self.set_debug(debug)

        #####################
        if AVOID_COLORS_ENV_LABEL in os.environ:
            self._colors_enabled = False

        #####################
        # Set default logging handler to avoid "No handler found" warnings.
        try:  # Python 2.7+
            from logging import NullHandler
        except ImportError:
            class NullHandler(logging.Handler):
                def emit(self, record):
                    pass

        #####################
        # Make sure there is at least one logger
        logging.getLogger(__name__).addHandler(NullHandler())
        # Format
        fileConfig(LOG_CONFIG)

        #####################
        # modify logging labels colors
        if self._colors_enabled:
            logging.addLevelName(
                logging.CRITICAL, "\033[1;41m%s\033[1;0m"
                % logging.getLevelName(logging.CRITICAL))
            logging.addLevelName(
                logging.ERROR, "\033[1;31m%s\033[1;0m"
                % logging.getLevelName(logging.ERROR))
            logging.addLevelName(
                logging.WARNING, "\033[1;33m%s\033[1;0m"
                % logging.getLevelName(logging.WARNING))
            logging.addLevelName(
                logging.INFO, "\033[1;32m%s\033[1;0m"
                % logging.getLevelName(logging.INFO))
            logging.addLevelName(
                logging.DEBUG, "\033[1;35m%s\033[1;0m"
                % logging.getLevelName(logging.DEBUG))

    def set_debug(self, debug=True):
        # print("DEBUG IS", debug)
        # if debug is None:
        #     return

        self.debug = debug
        if self.debug:
            self._log_level = logging.DEBUG
        else:
            self._log_level = logging.INFO

        return self._log_level

    def get_new_logger(self, name):
        """ Recover the right logger + set a proper specific level """
        if self._colors_enabled:
            name = "\033[1;90m%s\033[1;0m" % name
        logger = logging.getLogger(name)<commit_msg>Adding verbose levels for logging<commit_after># -*- coding: utf-8 -*-

"""

Move here the logic for any logs

"""

from __future__ import absolute_import
import os
import json
import logging
from logging.config import fileConfig
from json.decoder import JSONDecodeError
from . import AVOID_COLORS_ENV_LABEL, LOG_CONFIG

VERBOSE_LEVEL = 5
VERY_VERBOSE_LEVEL = 1


def verbose(self, message, *args, **kws):
    # Yes, logger takes its '*args' as 'args'.
    if self.isEnabledFor(VERBOSE_LEVEL):
        self._log(VERBOSE_LEVEL, message, args, **kws)


def very_verbose(self, message, *args, **kws):
    # Yes, logger takes its '*args' as 'args'.
    if self.isEnabledFor(VERBOSE_LEVEL):
        self._log(VERBOSE_LEVEL, message, args, **kws)


logging.addLevelName(VERY_VERBOSE_LEVEL, "VERY_VERBOSE")
logging.Logger.very_verbose = very_verbose

logging.addLevelName(VERBOSE_LEVEL, "VERBOSE")
logging.Logger.verbose = verbose


MAX_CHAR_LEN = 200
OBSCURE_VALUE = '****'
OBSCURED_FIELDS = ['password', '<PASSWORD>', 'token', 'file', 'filename']


################
# LOGGING

class LogMe(object):
    """ A common logger to be used all around development packages """

    def __init__(self, debug=None):

        #####################
        self._log_level = logging.INFO
        self._colors_enabled = True
        super(LogMe, self).__init__()

        # became useless:

        # #####################
        # # Set debug
        # if debug is None:
        #     debug = int(os.environ.get('API_DEBUG', False))
        # self.set_debug(debug)

        #####################
        if AVOID_COLORS_ENV_LABEL in os.environ:
            self._colors_enabled = False

        #####################
        # Set default logging handler to avoid "No handler found" warnings.
        try:  # Python 2.7+
            from logging import NullHandler
        except ImportError:
            class NullHandler(logging.Handler):
                def emit(self, record):
                    pass

        #####################
        # Make sure there is at least one logger
        logging.getLogger(__name__).addHandler(NullHandler())
        # Format
        fileConfig(LOG_CONFIG)

        #####################
        # modify logging labels colors
        if self._colors_enabled:
            logging.addLevelName(
                logging.CRITICAL, "\033[1;41m%s\033[1;0m"
                % logging.getLevelName(logging.CRITICAL))
            logging.addLevelName(
                logging.ERROR, "\033[1;31m%s\033[1;0m"
                % logging.getLevelName(logging.ERROR))
            logging.addLevelName(
                logging.WARNING, "\033[1;33m%s\033[1;0m"
                % logging.getLevelName(logging.WARNING))
            logging.addLevelName(
                logging.INFO, "\033[1;32m%s\033[1;0m"
                % logging.getLevelName(logging.INFO))
            logging.addLevelName(
                logging.DEBUG, "\033[1;35m%s\033[1;0m"
                % logging.getLevelName(logging.DEBUG))
            logging.addLevelName(
                VERBOSE_LEVEL, "\033[1;35m%s\033[1;0m"
                % logging.getLevelName(VERBOSE_LEVEL))
            logging.addLevelName(
                VERY_VERBOSE_LEVEL, "\033[1;35m%s\033[1;0m"
                % logging.getLevelName(VERY_VERBOSE_LEVEL))

    def set_debug(self, debug=True, level=None):
        # print("DEBUG IS", debug)
        # if debug is None:
        #     return

        self.debug = debug
        if self.debug:
            if level is not None:
                self._log_level = level
            else:
                self._log_level = logging.DEBUG
        else:
            self._log_level = logging.INFO

        return self._log_level

    def get_new_logger(self, name):
        """ Recover the right logger + set a proper specific level """
        if self._colors_enabled:
            name = "\033[1;90m%s\033[1;0m" % name
        logger = logging.getLogger(name)
{'fraction_non_alphanumeric': 0.14046822742474915, 'fraction_numerical': 0.024429256943434636, 'mean_word_length': 2.182785747339195, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '864538', 'n_tokens_mistral': 2116, 'n_tokens_neox': 1966, 'n_words': 436}