<commit_before># coding=utf-8
import re
import logging
from datetime import datetime
from logging import captureWarnings, Formatter, Logger, Handler
from logging.handlers import RotatingFileHandler
from typing import Dict, Match

from .exceptions.config import InvalidLiteralError


class ConvertedObject:
    pass

def convertDictToObject(x: Dict) -> object:
    y: ConvertedObject = ConvertedObject()

    for k, v in x.items():
        if isinstance(v, dict):
            v: Dict = convertDictToObject(v)

        if not isSafeLiteral(k):
            raise InvalidLiteralError(f"`{k}` is invalid literal.")
        setattr(y, k, v)

    return y

def listAttr(x: object) -> list:
    return [y for y in dir(x) if not y.startswith("__") and not y.endswith("__")]

def isSafeLiteral(x: str) -> bool:
    m: Match = re.match("^\w+$", x)
    m2: Match = re.match("^[^\d]", x)
    return m is not None and m2 is not None

def dumpVar(x: object) -> None:
    width: int = max([len(y) for y in dir(x)])
    [print(y.ljust(width + 10), "=", getattr(x, y)) for y in dir(x)]

def getLogger(directory: str, logLevel: int) -> Logger:
    logger: Logger = logging.getLogger()
    captureWarnings(capture=True)

    handler: Handler = RotatingFileHandler(
            f"{directory}/{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.log",
            maxBytes=2 ** 20, backupCount=10000, encoding="utf-8"
    )
    formatter: Formatter = Formatter(
            "[%(asctime)s][%(threadName)s %(name)s/%(levelname)s]: %(message)s",
            "%H:%M:%S"
    )
    handler.setFormatter(formatter)

    logger.setLevel(logLevel)
    logger.addHandler(handler)
    return logger<commit_msg>Add: StreamHandler to show logs in console<commit_after># coding=utf-8
import re
import logging
from datetime import datetime
from logging import captureWarnings, Formatter, Logger, Handler, StreamHandler
from logging.handlers import RotatingFileHandler
from typing import Dict, Match

from .exceptions.config import InvalidLiteralError


class ConvertedObject:
    pass

def convertDictToObject(x: Dict) -> object:
    y: ConvertedObject = ConvertedObject()

    for k, v in x.items():
        if isinstance(v, dict):
            v: Dict = convertDictToObject(v)

        if not isSafeLiteral(k):
            raise InvalidLiteralError(f"`{k}` is invalid literal.")
        setattr(y, k, v)

    return y

def listAttr(x: object) -> list:
    return [y for y in dir(x) if not y.startswith("__") and not y.endswith("__")]

def isSafeLiteral(x: str) -> bool:
    m: Match = re.match("^\w+$", x)
    m2: Match = re.match("^[^\d]", x)
    return m is not None and m2 is not None

def dumpVar(x: object) -> None:
    width: int = max([len(y) for y in dir(x)])
    [print(y.ljust(width + 10), "=", getattr(x, y)) for y in dir(x)]

def getLogger(directory: str, logLevel: int) -> Logger:
    logger: Logger = logging.getLogger()
    captureWarnings(capture=True)

    handler: Handler = RotatingFileHandler(
            f"{directory}/{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.log",
            maxBytes=2 ** 20, backupCount=10000, encoding="utf-8"
    )
    handler2: Handler = StreamHandler()
    formatter: Formatter = Formatter(
            "[%(asctime)s][%(threadName)s %(name)s/%(levelname)s]: %(message)s",
            "%H:%M:%S"
    )
    handler.setFormatter(formatter)
    handler2.setFormatter(formatter)

    logger.setLevel(logLevel)
    logger.addHandler(handler)
    logger.addHandler(handler2)
    return logger
{'fraction_non_alphanumeric': 0.1402002861230329, 'fraction_numerical': 0.008869814020028613, 'mean_word_length': 3.337468982630273, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6458296', 'n_tokens_mistral': 1174, 'n_tokens_neox': 1133, 'n_words': 313}