<commit_before>import inspect
import types
from .delta import Autogenerator

__all__ = ["aliases", "expose", "make_callable", "HiiSpiderPlugin", "delta"]
EXPOSED_FUNCTIONS = {}
CALLABLE_FUNCTIONS = {}
MEMOIZED_FUNCTIONS = {}
FUNCTION_ALIASES = {}
DELTA_FUNCTIONS = {}


def aliases(*args):
    def decorator(f):
        FUNCTION_ALIASES[id(f)] = args
        return f
    return decorator


def autodelta(paths=None, includes=None, ignores=None):
    def decorator(f):
        DELTA_FUNCTIONS[id(f)] = Autogenerator(paths=paths, includes=includes, ignores=ignores)
        return f
    return decorator


def delta(handler):
    def decorator(f):
        DELTA_FUNCTIONS[id(f)] = handler
        return f
    return decorator


def expose(func=None, interval=0, category=None, name=None, memoize=False):
    if func is not None:
        EXPOSED_FUNCTIONS[id(func)] = {"interval": interval, "name": name, 'category': category}
        return func

    def decorator(f):
        EXPOSED_FUNCTIONS[id(f)] = {"interval": interval, "name": name, 'category': category}
        return f
    return decorator


def make_callable(func=None, interval=0, category=None, name=None, memoize=False):
    if func is not None:
        CALLABLE_FUNCTIONS[id(func)] = {"interval": interval, "name": name, 'category': category}
        return func

    def decorator(f):
        CALLABLE_FUNCTIONS[id(f)] = {"interval": interval, "name": name, 'category': category}
        return f
    return decorator


class HiiSpiderPlugin(object):

    def __init__(self, spider):
        self.spider = spider
        check_method = lambda x: isinstance(x[1], types.MethodType)
        instance_methods = filter(check_method, inspect.getmembers(self))
        for instance_method in instance_methods:
            instance_id = id(instance_method[1].__func__)
            if instance_id in DELTA_FUNCTIONS:
                self.spider.delta(
                    instance_method[1],
                    DELTA_FUNCTIONS[instance_id])
            if instance_id in EXPOSED_FUNCTIONS:
                self.spider.expose(
                    instance_method[1],
                    interval=EXPOSED_FUNCTIONS[instance_id]["interval"],
                    name=EXPOSED_FUNCTIONS[instance_id]["name"],
                    category=EXPOSED_FUNCTIONS[instance_id]["category"])
                if instance_id in FUNCTION_ALIASES:
                    for name in FUNCTION_ALIASES[instance_id]:
                        self.spider.expose(
                            instance_method[1],
                            interval=EXPOSED_FUNCTIONS[instance_id]["interval"],
                            name=name)
            if instance_id in CALLABLE_FUNCTIONS:
                self.spider.expose(
                    instance_method[1],
                    interval=CALLABLE_FUNCTIONS[instance_id]["interval"],
                    name=CALLABLE_FUNCTIONS[instance_id]["name"],
                    category=CALLABLE_FUNCTIONS[instance_id]["category"])
                if instance_id in FUNCTION_ALIASES:
                    for name in CALLABLE_FUNCTIONS[instance_id]:
                        self.spider.expose(
                            instance_method[1],
                            interval=CALLABLE_FUNCTIONS[instance_id]["interval"],
                            name=name)

    def setFastCache(self, uuid, data):
        return self.spider.setFastCache(uuid, data)

    def getPage(self, *args, **kwargs):
        return self.spider.getPage(*args, **kwargs)<commit_msg>Added abstract methods for identity server. Now branching.<commit_after>import inspect
import types
from .delta import Autogenerator

__all__ = ["aliases", "expose", "make_callable", "HiiSpiderPlugin", "delta"]
EXPOSED_FUNCTIONS = {}
CALLABLE_FUNCTIONS = {}
MEMOIZED_FUNCTIONS = {}
FUNCTION_ALIASES = {}
DELTA_FUNCTIONS = {}


def aliases(*args):
    def decorator(f):
        FUNCTION_ALIASES[id(f)] = args
        return f
    return decorator


def autodelta(paths=None, includes=None, ignores=None):
    def decorator(f):
        DELTA_FUNCTIONS[id(f)] = Autogenerator(paths=paths, includes=includes, ignores=ignores)
        return f
    return decorator


def delta(handler):
    def decorator(f):
        DELTA_FUNCTIONS[id(f)] = handler
        return f
    return decorator


def expose(func=None, interval=0, category=None, name=None, memoize=False):
    if func is not None:
        EXPOSED_FUNCTIONS[id(func)] = {"interval": interval, "name": name, 'category': category}
        return func

    def decorator(f):
        EXPOSED_FUNCTIONS[id(f)] = {"interval": interval, "name": name, 'category': category}
        return f
    return decorator


def make_callable(func=None, interval=0, category=None, name=None, memoize=False):
    if func is not None:
        CALLABLE_FUNCTIONS[id(func)] = {"interval": interval, "name": name, 'category': category}
        return func

    def decorator(f):
        CALLABLE_FUNCTIONS[id(f)] = {"interval": interval, "name": name, 'category': category}
        return f
    return decorator


class HiiSpiderPlugin(object):

    def __init__(self, spider):
        self.spider = spider
        check_method = lambda x: isinstance(x[1], types.MethodType)
        instance_methods = filter(check_method, inspect.getmembers(self))
        for instance_method in instance_methods:
            instance_id = id(instance_method[1].__func__)
            if instance_id in DELTA_FUNCTIONS:
                self.spider.delta(
                    instance_method[1],
                    DELTA_FUNCTIONS[instance_id])
            if instance_id in EXPOSED_FUNCTIONS:
                self.spider.expose(
                    instance_method[1],
                    interval=EXPOSED_FUNCTIONS[instance_id]["interval"],
                    name=EXPOSED_FUNCTIONS[instance_id]["name"],
                    category=EXPOSED_FUNCTIONS[instance_id]["category"])
                if instance_id in FUNCTION_ALIASES:
                    for name in FUNCTION_ALIASES[instance_id]:
                        self.spider.expose(
                            instance_method[1],
                            interval=EXPOSED_FUNCTIONS[instance_id]["interval"],
                            name=name)
            if instance_id in CALLABLE_FUNCTIONS:
                self.spider.expose(
                    instance_method[1],
                    interval=CALLABLE_FUNCTIONS[instance_id]["interval"],
                    name=CALLABLE_FUNCTIONS[instance_id]["name"],
                    category=CALLABLE_FUNCTIONS[instance_id]["category"])
                if instance_id in FUNCTION_ALIASES:
                    for name in CALLABLE_FUNCTIONS[instance_id]:
                        self.spider.expose(
                            instance_method[1],
                            interval=CALLABLE_FUNCTIONS[instance_id]["interval"],
                            name=name)

    def setFastCache(self, uuid, data):
        return self.spider.setFastCache(uuid, data)

    def getPage(self, *args, **kwargs):
        return self.spider.getPage(*args, **kwargs)
    
    @make_callable
    def getID(self):
        raise NotImplementedError()
        
    @make_callable
    def getConnections(self):
        raise NotImplementedError()
    
{'fraction_non_alphanumeric': 0.09935501578152875, 'fraction_numerical': 0.002470152326060107, 'mean_word_length': 2.311222171740118, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3699008', 'n_tokens_mistral': 2119, 'n_tokens_neox': 2030, 'n_words': 422}