<commit_before>import luigi
import logging
import util

def setup_logging():
    util.ensuredir('log')
    log_path = 'log/sciluigi_run_%s.log' % util.timepath()

    # Formatter
    stream_formatter = logging.Formatter('[%(asctime)s] %(message)s','%H:%M:%S')
    luigi_log_formatter = logging.Formatter('%(asctime)s %(levelname)8s    LUIGI %(message)s','%Y-%m-%d %H:%M:%S')
    sciluigi_log_formatter = logging.Formatter('%(asctime)s %(levelname)8s SCILUIGI %(message)s','%Y-%m-%d %H:%M:%S')

    # Stream handler (for STDERR)
    stream_handler = logging.StreamHandler()
    stream_handler.setFormatter(stream_formatter)
    stream_handler.setLevel(logging.INFO)

    # File handler
    luigi_file_handler = logging.FileHandler(log_path)
    luigi_file_handler.setFormatter(luigi_log_formatter)<commit_msg>Include full date and levelnames in stdout log<commit_after>import luigi
import logging
import util

def setup_logging():
    util.ensuredir('log')
    log_path = 'log/sciluigi_run_%s.log' % util.timepath()

    # Formatter
    stream_formatter = logging.Formatter('%(asctime)s | %(levelname)8s | %(message)s','%Y-%m-%d %H:%M:%S')
    luigi_log_formatter = logging.Formatter('%(asctime)s %(levelname)8s    LUIGI %(message)s','%Y-%m-%d %H:%M:%S')
    sciluigi_log_formatter = logging.Formatter('%(asctime)s %(levelname)8s SCILUIGI %(message)s','%Y-%m-%d %H:%M:%S')

    # Stream handler (for STDERR)
    stream_handler = logging.StreamHandler()
    stream_handler.setFormatter(stream_formatter)
    stream_handler.setLevel(logging.INFO)

    # File handler
    luigi_file_handler = logging.FileHandler(log_path)
    luigi_file_handler.setFormatter(luigi_log_formatter)
{'fraction_non_alphanumeric': 0.15044776119402986, 'fraction_numerical': 0.0029850746268656717, 'mean_word_length': 5.3007518796992485, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4401367', 'n_tokens_mistral': 623, 'n_tokens_neox': 597, 'n_words': 90}