<commit_before># logger module

from datetime import datetime
import os

logfile = None
logbuf = []

def init(analysis_path):
    global logfile
    if not os.path.isdir(analysis_path):
        log("logger: analysis_path missing:", analysis_path)
    use_log_dir = False
    if use_log_dir:
        logdir = os.path.join(analysis_path, "log")
        if not os.path.isdir(logdir):
            log("logger: creating log directory:", logdir)
            os.makedirs(logdir)
        logfile = os.path.join(logdir, "messages")
    else:
        logfile = os.path.join(analysis_path, "messages")

# log a message to messages files (and to stdout by default)
def log(*args, quiet=False):
    global logbuf
    # assemble message line
    msg = []
    now = datetime.now()
    msg.append(str(now) + ":")
    for a in args:
        msg.append(str(a))
    logbuf.append(" ".join(msg))
    if logfile:
        # flush log buffer
        f = open(logfile, "a")
        for line in logbuf:
            f.write(line)
            f.write("\n")
        f.close()
        logbuf = []
    if not quiet:
        print(*msg)
<commit_msg>Only timestamp the log file lines (for stdout, just display the message.)<commit_after># logger module

from datetime import datetime
import os

logfile = None
logbuf = []

def init(analysis_path):
    global logfile
    if not os.path.isdir(analysis_path):
        log("logger: analysis_path missing:", analysis_path)
    use_log_dir = False
    if use_log_dir:
        logdir = os.path.join(analysis_path, "log")
        if not os.path.isdir(logdir):
            log("logger: creating log directory:", logdir)
            os.makedirs(logdir)
        logfile = os.path.join(logdir, "messages")
    else:
        logfile = os.path.join(analysis_path, "messages")

# log a message to messages files (and to stdout by default)
def log(*args, quiet=False):
    global logbuf
    # timestamp
    now = datetime.now()
    timestamp = str(now) + ": "
    # assemble message line
    msg = []
    for a in args:
        msg.append(str(a))
    logbuf.append(timestamp + " ".join(msg))
    if logfile:
        # flush log buffer
        f = open(logfile, "a")
        for line in logbuf:
            f.write(line)
            f.write("\n")
        f.close()
        logbuf = []
    if not quiet:
        print(*msg)

{'fraction_non_alphanumeric': 0.11612903225806452, 'fraction_numerical': 0.0, 'mean_word_length': 2.3276108726752502, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2447313', 'n_tokens_mistral': 731, 'n_tokens_neox': 728, 'n_words': 205}