<commit_before>import boto
import datetime
import logging
import os
from Queue import Empty
import sys

LOG_LEVELS = {
    "critical": logging.CRITICAL,
    "error": logging.ERROR,
    "warning": logging.WARNING,
    "info": logging.INFO,
    "debug": logging.DEBUG
}

def get_config(config_name):
    if config_name.endswith(".py"):
        config_name = config_name[:-3]
    elif config_name.endswith(".pyc"):
        config_name = config_name[:-4]

    directory, config_module = os.path.split(config_name)

    # If the directory isn't in the PYTHONPATH, add it so our import will work
    if directory not in sys.path:
        sys.path.insert(0, directory)

    config = __import__(config_module)

    # settings that are not overriden need to be set to defaults
    from . import default_config
    for k, v in default_config.__dict__.iteritems():
        if k.startswith("_"):
            continue
        if not hasattr(config, k):
            setattr(config, k, v)

    now = datetime.datetime.now()
    config.OUTPUT_FILENAME = config.OUTPUT_FILENAME % {"config_name": config_module, "time": now}
    config.LOG_FILENAME = config.LOG_FILENAME % {"config_name": config_module}

    return config

def configure_logging(config):
    level_str = config.LOG_LEVEL.lower()
    level = LOG_LEVELS.get(level_str, logging.INFO)
    logging.basicConfig(level=level, format=config.LOG_FORMAT, filename=config.LOG_FILENAME)

    if level_str not in LOG_LEVELS:
        logging.warn("invalid value for LOG_LEVEL: %s", config.LOG_LEVEL)

    paramiko_level_str = config.PARAMIKO_LOG_LEVEL.lower()<commit_msg>ensure log file and results file is writable (and create dirs if necessary)<commit_after>import boto
import datetime
import logging
import os
from Queue import Empty
import sys

LOG_LEVELS = {
    "critical": logging.CRITICAL,
    "error": logging.ERROR,
    "warning": logging.WARNING,
    "info": logging.INFO,
    "debug": logging.DEBUG
}

def ensure_dir_exists(path):
    dir_name = os.path.dirname(path)
    if dir_name != '' and not os.path.exists(dir_name):
        os.makedirs(dir_name)

def get_config(config_name):
    if config_name.endswith(".py"):
        config_name = config_name[:-3]
    elif config_name.endswith(".pyc"):
        config_name = config_name[:-4]

    directory, config_module = os.path.split(config_name)

    # If the directory isn't in the PYTHONPATH, add it so our import will work
    if directory not in sys.path:
        sys.path.insert(0, directory)

    config = __import__(config_module)

    # settings that are not overriden need to be set to defaults
    from . import default_config
    for k, v in default_config.__dict__.iteritems():
        if k.startswith("_"):
            continue
        if not hasattr(config, k):
            setattr(config, k, v)

    now = datetime.datetime.now()
    config.OUTPUT_FILENAME = config.OUTPUT_FILENAME % {"config_name": config_module, "time": now}
    ensure_dir_exists(config.OUTPUT_FILENAME)
    config.LOG_FILENAME = config.LOG_FILENAME % {"config_name": config_module}

    return config

def configure_logging(config):
    level_str = config.LOG_LEVEL.lower()
    level = LOG_LEVELS.get(level_str, logging.INFO)
    ensure_dir_exists(config.LOG_FILENAME)
    logging.basicConfig(level=level, format=config.LOG_FORMAT, filename=config.LOG_FILENAME)

    if level_str not in LOG_LEVELS:
        logging.warn("invalid value for LOG_LEVEL: %s", config.LOG_LEVEL)

    paramiko_level_str = config.PARAMIKO_LOG_LEVEL.lower()
{'fraction_non_alphanumeric': 0.10130904951622083, 'fraction_numerical': 0.001707455890722823, 'mean_word_length': 3.5531088082901556, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1721141', 'n_tokens_mistral': 1128, 'n_tokens_neox': 1090, 'n_words': 277}