<commit_before>from mr.awsome.common import Hooks
from ConfigParser import RawConfigParser
from UserDict import DictMixin
from weakref import proxy
import inspect
import logging
import os
import sys
import warnings


log = logging.getLogger('mr.awsome')


def value_asbool(value):
    if isinstance(value, bool):
        return value
    if value.lower() in ('true', 'yes', 'on'):
        return True
    elif value.lower() in ('false', 'no', 'off'):
        return False


class BaseMassager(object):
    def __init__(self, sectiongroupname, key):
        self.sectiongroupname = sectiongroupname
        self.key = key

    def path(self, config, sectionname):
        return config._dict[self.key].path

    def __call__(self, config, sectionname):
        value = config._dict[self.key]
        if isinstance(value, ConfigValue):
            return value.value
        return value


class BooleanMassager(BaseMassager):
    def __call__(self, config, sectionname):
        value = BaseMassager.__call__(self, config, sectionname)
        value = value_asbool(value)
        if value is None:
            raise ValueError("Unknown value %s for %s in %s:%s." % (value, self.key, self.sectiongroupname, sectionname))
        return value


class IntegerMassager(BaseMassager):
    def __call__(self, config, sectionname):
        value = BaseMassager.__call__(self, config, sectionname)
        return int(value)


class PathMassager(BaseMassager):
    def __call__(self, config, sectionname):
        value = BaseMassager.__call__(self, config, sectionname)
        value = os.path.expanduser(value)
        if not os.path.isabs(value):
            value = os.path.join(self.path(config, sectionname), value)
        return value


def resolve_dotted_name(value):
    if '.' in value:
        prefix, name = value.rsplit('.', 1)
        _temp = __import__(prefix, globals(), locals(), [name], -1)
        return getattr(_temp, name)
    else:
        return __import__(value, globals(), locals(), [], -1)


class HooksMassager(BaseMassager):
    def __call__(self, config, sectionname):
        value = BaseMassager.__call__(self, config, sectionname)
        hooks = Hooks()
        for hook_spec in value.split():
            hooks.add(resolve_dotted_name(hook_spec)())
        return hooks


class StartupScriptMassager(BaseMassager):
    def __call__(self, config, sectionname):
        value = BaseMassager.__call__(self, config, sectionname)
        result = dict()
        if value.startswith('gzip:'):
            value = value[5:]
            result['gzip'] = True
        if not os.path.isabs(value):
            value = os.path.join(self.path(config, sectionname), value)
        result['path'] = value<commit_msg>Add expand_path utility function.<commit_after>from mr.awsome.common import Hooks
from ConfigParser import RawConfigParser
from UserDict import DictMixin
from weakref import proxy
import inspect
import logging
import os
import sys
import warnings


log = logging.getLogger('mr.awsome')


def value_asbool(value):
    if isinstance(value, bool):
        return value
    if value.lower() in ('true', 'yes', 'on'):
        return True
    elif value.lower() in ('false', 'no', 'off'):
        return False


class BaseMassager(object):
    def __init__(self, sectiongroupname, key):
        self.sectiongroupname = sectiongroupname
        self.key = key

    def path(self, config, sectionname):
        return config._dict[self.key].path

    def __call__(self, config, sectionname):
        value = config._dict[self.key]
        if isinstance(value, ConfigValue):
            return value.value
        return value


class BooleanMassager(BaseMassager):
    def __call__(self, config, sectionname):
        value = BaseMassager.__call__(self, config, sectionname)
        value = value_asbool(value)
        if value is None:
            raise ValueError("Unknown value %s for %s in %s:%s." % (value, self.key, self.sectiongroupname, sectionname))
        return value


class IntegerMassager(BaseMassager):
    def __call__(self, config, sectionname):
        value = BaseMassager.__call__(self, config, sectionname)
        return int(value)


def expand_path(value, base):
    value = os.path.expanduser(value)
    if not os.path.isabs(value):
        value = os.path.join(base, value)
    return value


class PathMassager(BaseMassager):
    def __call__(self, config, sectionname):
        value = BaseMassager.__call__(self, config, sectionname)
        return expand_path(value, self.path(config, sectionname))


def resolve_dotted_name(value):
    if '.' in value:
        prefix, name = value.rsplit('.', 1)
        _temp = __import__(prefix, globals(), locals(), [name], -1)
        return getattr(_temp, name)
    else:
        return __import__(value, globals(), locals(), [], -1)


class HooksMassager(BaseMassager):
    def __call__(self, config, sectionname):
        value = BaseMassager.__call__(self, config, sectionname)
        hooks = Hooks()
        for hook_spec in value.split():
            hooks.add(resolve_dotted_name(hook_spec)())
        return hooks


class StartupScriptMassager(BaseMassager):
    def __call__(self, config, sectionname):
        value = BaseMassager.__call__(self, config, sectionname)
        result = dict()
        if value.startswith('gzip:'):
            value = value[5:]
            result['gzip'] = True
        if not os.path.isabs(value):
            value = os.path.join(self.path(config, sectionname), value)
        result['path'] = value
{'fraction_non_alphanumeric': 0.10918293564714389, 'fraction_numerical': 0.0014461315979754157, 'mean_word_length': 3.000723065798988, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7107360', 'n_tokens_mistral': 1601, 'n_tokens_neox': 1575, 'n_words': 434}