<commit_before>from __future__ import absolute_import, print_function
import types
import sys
import six
import inspect
import collections


def get_caller(offset=0):
    _, filename, linenum, funcname, _, _ = inspect.stack()[offset]
    return {'filename': filename,
            'linenum': linenum,
            'funcname': funcname}


def string_type():
    try:
        return basestring # noqa
    except:
        return str


def stringify(x):
    if six.PY2:
        return x
    if isinstance(x, bytes):
        return x.decode('utf-8')


class ModuleRedirector(object):
    def __init__(self, name, fn, redirect_everything=False):
        self._orig_module_ = sys.modules[name]
        sys.modules[name] = self
        self._everything_ = redirect_everything
        fn.__module__ = __name__
        self.__fn__ = fn

    def __getattr__(self, name):
        try:
            assert not self._everything_
            return getattr(object.__getattribute__(self, '_orig_module_'), name)
        except (AssertionError, AttributeError):
            return object.__getattribute__(self, '__fn__')(name)


def decorate(val, _name_, decorator):
    assert isinstance(val, dict)
    assert isinstance(_name_, string_type())
    assert callable(decorator)
    for k, v in list(val.items()):
        if callable(v) and v.__module__ == _name_:
            fn = decorator(v)
            val[k] = fn


def pformat_prep(val):
    import s
    if isinstance(val, tuple) and hasattr(val, '_fields'):
        return ['namedtuple'] + [{k: v} for k, v in zip(val._fields, val)]
    elif isinstance(val, collections.Counter):
        return ['counter'] + [{k: v} for k, v in sorted(val.items(), key=lambda x: x[1], reverse=True)]
    elif isinstance(val, dict):
        return {k: pformat_prep(v) for k, v in val.items()}
    elif isinstance(val, s.data._listy_types):
        return [pformat_prep(x) for x in val]
    elif isinstance(val, set):
        return {pformat_prep(x) for x in val}
    return val


def optionally_parameterized_decorator(fn):
    """
    wont work if you decorator can be invoked with a single function argument,
    which is the same signature as decorator usage
    """
    def decorated(*a, **kw):
        if len(a) == 1 and isinstance(a[0], types.FunctionType) and not kw:
            return fn()(*a, **kw)
        else:
            return fn(*a, **kw)
    return decorated<commit_msg>optionally_parameterized_decorator works with methods as well as functions<commit_after>from __future__ import absolute_import, print_function
import inspect
import types
import sys
import six
import collections


def get_caller(offset=0):
    _, filename, linenum, funcname, _, _ = inspect.stack()[offset]
    return {'filename': filename,
            'linenum': linenum,
            'funcname': funcname}


def string_type():
    try:
        return basestring # noqa
    except:
        return str


def stringify(x):
    if six.PY2:
        return x
    if isinstance(x, bytes):
        return x.decode('utf-8')


class ModuleRedirector(object):
    def __init__(self, name, decoratee, redirect_everything=False):
        self._orig_module_ = sys.modules[name]
        sys.modules[name] = self
        self._everything_ = redirect_everything
        decoratee.__module__ = __name__
        self.__fn__ = decoratee

    def __getattr__(self, name):
        try:
            assert not self._everything_
            return getattr(object.__getattribute__(self, '_orig_module_'), name)
        except (AssertionError, AttributeError):
            return object.__getattribute__(self, '__fn__')(name)


def decorate(val, _name_, decorator):
    assert isinstance(val, dict)
    assert isinstance(_name_, string_type())
    assert callable(decorator)
    for k, v in list(val.items()):
        if callable(v) and v.__module__ == _name_:
            decoratee = decorator(v)
            val[k] = decoratee


def pformat_prep(val):
    import s
    if isinstance(val, tuple) and hasattr(val, '_fields'):
        return ['namedtuple'] + [{k: v} for k, v in zip(val._fields, val)]
    elif isinstance(val, collections.Counter):
        return ['counter'] + [{k: v} for k, v in sorted(val.items(), key=lambda x: x[1], reverse=True)]
    elif isinstance(val, dict):
        return {k: pformat_prep(v) for k, v in val.items()}
    elif isinstance(val, s.data._listy_types):
        return [pformat_prep(x) for x in val]
    elif isinstance(val, set):
        return {pformat_prep(x) for x in val}
    return val


def optionally_parameterized_decorator(decoratee):
    """
    wont work if you decorator can be invoked with a single function argument,
    which is the same signature as decorator usage.
    """
    def decorated(*a, **kw):
        method = (len(a) == 2
                  and inspect.ismethod(getattr(a[0], decoratee.__name__, None))
                  and isinstance(a[1], types.FunctionType)
                  and not kw)
        function = (len(a) == 1
                    and isinstance(a[0], types.FunctionType)
                    and not kw)
        if method:
            return decoratee(a[0])(*a[1:], **kw)
        elif function:
            return decoratee()(*a, **kw)
        else:
            return decoratee(*a, **kw)
    return decorated
{'fraction_non_alphanumeric': 0.10955002847921018, 'fraction_numerical': 0.0032276438200113918, 'mean_word_length': 2.6813417190775684, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5640926', 'n_tokens_mistral': 1598, 'n_tokens_neox': 1552, 'n_words': 464}