<commit_before># Universal Translator base classes
# (c) krvss 2011-2014

from inspect import getargspec
from operator import attrgetter

from utils import *

# TODO remove
logging = False
import time
def set_logging(value):
    global logging
    logging = value


class Abstract(object):
    """
    Base abstract class for all communicable objects
    """
    def __call__(self, *message, **context):
        raise NotImplementedError('Method not implemented')


class Access(Abstract):
    """
    Access provides abstract-like access to any object
    """
    CALL = 'call'
    ABSTRACT = 'abstract'
    FUNCTION = 'function'
    VALUE = 'value'
    OTHER = 'other'

    CACHE_ATTR = '__access__'
    CACHEABLE = (CALL, FUNCTION)

    def __init__(self, value):
        self._value = value
        self._mode, self._spec = self.OTHER, self.OTHER
        self._call = self.call_direct

        self.setup()

    def __eq__(self, other):
        if isinstance(other, Access):
            return other._value == self._value

        return other == self._value

    def __repr__(self):
        return self.__str__()

    def __str__(self):
        return '%s, %s: %s' % (self._mode, self._spec, self._value)

    def setup(self):
        """
        Init type information
        """
        if isinstance(self._value, Abstract):
            self._mode, self._spec = self.CALL, self.ABSTRACT

        elif callable(self._value):
            self._mode, self._spec = self.FUNCTION, getargspec(self._value)

            if self._spec.varargs and not self._spec.keywords:
                self._call = self.call_args

            elif self._spec.keywords and not self._spec.varargs:
                self._call = self.call_kwargs

            elif self._spec.varargs and self._spec.keywords:
                return  # default call_direct is good here

            elif not self._spec.varargs and not self._spec.keywords and not self._spec.args:
                self._call = self.call_noargs

            elif self._spec.args:
                self._defaults_len = len(self._spec.defaults) - 1 if self._spec.defaults else -1
                self._reversed_args = list(reversed(self._spec.args))

                self._call = self.call_general

        else:
            self._mode = self.VALUE
            self._call = self.call_value

    def __call__(self, *message, **context):
        return self._call(message, context)

    def call_direct(self, message, context):
        return self._value(*message, **context)

    def call_args(self, message, context):
        return self._value(*message)

    def call_kwargs(self, message, context):
        return self._value(**context)

    def call_noargs(self, message, context):
        return self._value()

    def call_general(self, message, context):
        i, args = self._defaults_len, {}
        for arg in self._reversed_args:
            if arg != 'self':
                args[arg] = context[arg] if arg in context else self._spec.defaults[i] if i >= 0 else None
                i -= 1

        return self._value(**args)

    def call_value(self, message, context):
        return self._value

    value = property(attrgetter('_value'))
    mode = property(attrgetter('_mode'))
    spec = property(attrgetter('_spec'))

    @staticmethod
    def get_access(obj, cache=False):
        if hasattr(obj, Access.CACHE_ATTR):
            access = getattr(obj, Access.CACHE_ATTR)
        else:
            access = Access(obj)

            if cache and access._mode in Access.CACHEABLE:
                setattr(obj, Access.CACHE_ATTR, access)

        return access


class Condition(Access):
    """
    Condition is an Access that checks the possibility of the access according to the message and context
    """
    NUMBER = 'number'
    LIST = 'list'
    DICT = 'dict'
    STRING = 'string'
    REGEX = 'regex'
    BOOLEAN = 'bool'

    NO_CHECK = -1, None

    def __init__(self, value, ignore_case=False, *tags):
        self._ignore_case, self.tags = ignore_case, frozenset(tags)
        self.check, self._conditions = self.check_compare, tuple([self])
        super(Condition, self).__init__(value)

    def setup(self):
        """
        Setting additional mode info
        """
        super(Condition, self).setup()

        if self._mode == self.FUNCTION:
            self.check = self.check_function

        elif is_number(self._value):
            self._spec = self.NUMBER

        elif is_list(self._value):
            self._spec, self.check = self.LIST, self.check_list
            self._conditions = tuple([Condition(c, self._ignore_case) for c in self._value])

        elif is_string(self._value):
            self._spec, self.check = self.STRING, self.check_string
            self._value_len = len(self._value)

            if self._ignore_case:
                self._value = self._value.upper()

        elif is_regex(self._value):
            self._spec, self.check = self.REGEX, self.check_regex

        elif isinstance(self._value, dict):
            self._spec = self.DICT

        elif self._value is True or self._value is False:
            self._spec, self.check = self.BOOLEAN, self.check_boolean

    def check_function(self, message, context):
        check = self._call(message, context)

        # Do we work with (rank, check) format?
        if get_len(check) == 2:
            return check
        elif check:
            return 0 if check is True else check, check
        else:
            return self.NO_CHECK

    def check_regex(self, message, context):
        check = self._value.match(message[0]) if message else None

        if check:
            rank = check.end() - check.start()  # Match length is the rank
            check = check.group(0)
        else:
            rank = - 1

        return rank, check

    def check_string(self, message, context):
        try:
            message0 = message[0][:self._value_len]

            if self._ignore_case:
                message0 = message0.upper()

            if message0 == self._value:
                return self._value_len, self._value

        except (TypeError, IndexError):
            pass

        return self.NO_CHECK

    def check_boolean(self, message, context):
        if message and message[0] is self._value:
            return 0, self._value
        else:
            return self.NO_CHECK

    def check_compare(self, message, context):
        if has_first(message, self._value):
            return max(get_len(self._value), 0), self._value
        else:
            return self.NO_CHECK

    def check_list(self, message, context):
        rank, check = self.NO_CHECK

        for condition in self._conditions:
            c_rank, c_check = condition.check(message, context)

            if c_rank > rank:
                rank, check = c_rank, c_check

        return rank, check

    @property
    def list(self):
        return self._conditions


# Condition that is always satisfied
class TrueCondition(Condition):
    def __init__(self):
        super(TrueCondition, self).__init__(id(self))
        self.check = lambda message, context: (0, True)


TRUE_CONDITION = TrueCondition()


class Event(Access):
    """
    Event is an Access that allows to attach Pre and Post custom functions before and after access
    """
    RESULT = 'result'

    def __init__(self, value):
        super(Event, self).__init__(value)
        self.pre_event, self.post_event = None, None

    def run(self, message, context):
        if self.pre_event:
            pre_result = self.pre_event.run(message, context)

            if pre_result[0] is not None:
                return pre_result

        result = self._call(message, context)

        if self.post_event:
            context[self.RESULT] = result
            post_result = self.post_event.run(message, context)

            if post_result[0] is not None:
                return post_result

        return result, self._value

    @property
    def pre(self):
        return self.pre_event.value if self.pre_event else None

    @pre.setter
    def pre(self, value):
        self.pre_event = Event(value) if value is not None else None

    @property
    def post(self):
        return self.post_event.value if self.post_event else None

    @post.setter
    def post(self, value):
        self.post_event = Event(value) if value is not None else None


class Handler(Abstract):
    """
    Handler is a class for the routing of messages to processing functions (events) basing on specified conditions
    """
    ANSWER = 'answer'
    SENDER = 'sender'
    CONDITION = 'condition'
    EVENT = 'event'
    RANK = 'rank'

    NO_HANDLE = (False, -1, None)

    def __init__(self):
        Access.get_access(self, True)

        self._tags = set()
        self._events, self.active_events = [], tuple()
        self.unknown_event = None

    def on_access(self, condition_access, event_access):
        """
        Adding the accesses
        """
        if (condition_access, event_access) not in self._events:
            self._events.append((condition_access, event_access))

            self.update_events()

    def on(self, condition, event, *tags):
        """
        Adding the condition - event pair
        """
        self.on_access(Condition(condition, False, *tags), Event(event))

    def on_any(self, event):
        """
        Adding the event, without condition it will trigger on any message
        """
        self.on_access(TRUE_CONDITION, Event(event))

    def off(self, condition, event):
        """
        Removing the condition - event pair
        """
        if (condition, event) in self._events:
            self._events.remove((condition, event))

            self.update_events()

    def off_any(self, event):
        """
        Removing the event
        """
        self.off(TRUE_CONDITION, event)

    def off_condition(self, condition):
        """
        Removing all the events for the condition
        """
        self._events = filter(lambda e: not (e[0] == condition), self._events)

        self.update_events()

    def off_event(self, event):
        """
        Remove all occurrences of the event
        """
        self._events = filter(lambda e: not (e[1] == event), self._events)

        self.update_events()

    def get_events(self, condition=None):
        """
        Getting the events for the specified condition
        """
        if condition:
            return [e[1] for e in self._events if has_first(e, condition)]
        else:
            return [e[1] for e in self._events if e[0] == TRUE_CONDITION]

    def clear_events(self):
        del self._events[:]
        self.active_events = tuple()

    def update_tags(self):
        """
        Called when update needed, returns new tags
        """
        return self._tags

    def update_events(self):
        """
        Called when event update needed (for example, after new event was added or tags was changed)
        """
        self.active_events = tuple(filter(lambda e: e[0].tags.issubset(self._tags), self._events))

    def update(self):
        """
        Update is called by the object itself when something is changed and it could change tags
        """
        new_tags = self.update_tags()

        if new_tags != self._tags:
            self.tags = new_tags

            self.update_events()

    def handle(self, message, context):
        """
        Calling events basing on condition
        """
        global logging
        if logging:
            print ">>> %s.handle of |%s|, events %s" % (type(self), message, len(self.active_events))

        check, rank, event_found = self.NO_HANDLE

        if not self.SENDER in context:
            context[self.SENDER] = self

        # Searching for the best event
        for condition_access, event_access in self.active_events:
            if logging:
                t1 = time.time()

            # Condition check, if no condition the result is true with zero rank
            c_rank, c_check = condition_access.check(message, context)

            if logging:
                print "trying %s, spent %s" % (condition_access.value, time.time() - t1)

            if c_rank > rank:
                rank, check, event_found = c_rank, c_check, event_access

        # Running the best event
        if rank >= 0:
            context.update({self.RANK: rank, self.CONDITION: check, self.EVENT: event_found.value})

            # Call event, add the condition result to the context
            result, event_found = event_found.run(message, context)
        else:
            result = False

            # There is a way to handle unknown message
            if self.unknown_event:
                return self.unknown_event.run(message, context)

        return result, rank, event_found

    def __call__(self, *message, **context):
        """
        Answer depends on the context
        """
        answer_mode = context.pop(self.ANSWER, None)  # Applicable only for a top level
        result = self.handle(message, context)

        if answer_mode == self.RANK:
            return result[0], result[1]

        return result[0]  # No need to know the details

    @property
    def tags(self):
        return self._tags

    @tags.setter
    def tags(self, value):
        self._tags = frozenset(value)

    @property
    def events(self):
        return self._events


class Element(Handler):
    """
    Element is a part of a complex system (e.g. graph)
    """
    NEXT = 'next'
    PREVIOUS = 'previous'

    OWNER = 'owner'
    SET_PREFIX = 'set'
    NAME = 'name'
    OLD_VALUE = 'old-value'
    NEW_VALUE = 'new-value'

    SEP = '_'

    FORWARD = [NEXT]
    BACKWARD = [PREVIOUS]

    def __init__(self, owner=None):
        super(Element, self).__init__()

        self._owner, self.owner = None, owner

    def is_forward(self, message):
        return message and message[0] in self.FORWARD

    def can_go_forward(self, *message, **context):
        return self.is_forward(message)

    def on_forward(self, event):
        self.on(self.can_go_forward, event)

    def off_forward(self):
        self.off_condition(self.can_go_forward)

    def is_backward(self, message):
        return message and message[0] in self.BACKWARD

    def can_go_backward(self, *message, **context):
        return self.is_backward(message)

    def on_backward(self, event):
        self.on(self.can_go_backward, event)

    def off_backward(self):
        self.off_condition(self.can_go_backward)

    @staticmethod
    def add_prefix(msg, prefix):
        """
        Add prefix to the message
        """
        event_name = msg

        if not msg.startswith(prefix):
            event_name = Element.SEP.join([prefix, event_name])

        return event_name

    def change_property(self, name, value):
        """
        Set the property to the new value with notification, if needed
        """
        old_value = getattr(self, name)
        if old_value == value:
            return

        set_message, context = self.add_prefix(name, self.SET_PREFIX), \
                               {self.NEW_VALUE: value, self.OLD_VALUE: old_value, self.SENDER: self}

        if old_value and Access.get_access(old_value, True).spec == Access.ABSTRACT:
            old_value(set_message, **context)

        setattr(self, '_%s' % name, value)

        if value and Access.get_access(value, True).spec == Access.ABSTRACT:
            value(set_message,  **context)

        return True

    @property
    def owner(self):
        return self._owner

    @owner.setter
    def owner(self, value):
        self.change_property('owner', value)


class Notion(Element):
    """
    Notion is an element with name
    """
    def __init__(self, name, owner=None):
        super(Notion, self).__init__(owner)
        self._name, self.name = None, name

    def __str__(self):
        return '"%s"' % self.name

    def __repr__(self):
        rep = '<%s(%s' % (get_object_name(self.__class__), self.__str__())

        if self.owner:
            rep += ', %s' % self.owner

        return rep + ')>'

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, value):
        self.change_property(self.NAME, value)


class ActionNotion(Notion):
    """
    Action notion is a notion with specified forward handler
    """
    def __init__(self, name, action, owner=None):
        super(ActionNotion, self).__init__(name, owner)
        self.on_forward(action)

    @property
    def action(self):
        value = self.get_events(self.can_go_forward)
        return value[0] if value else None

    @action.setter
    def action(self, value):
        self.off_forward()
        self.on_forward(value)


class Relation(Element):
    """
    Relation is a connection between one or more elements: subject -> object
    """
    SUBJECT = 'subject'
    OBJECT = 'object'

    def __init__(self, subj, obj, owner=None):
        super(Relation, self).__init__(owner)
        self._object = self._subject = None
        self.subject, self.object = subj, obj

    @property
    def subject(self):
        return self._subject

    @subject.setter
    def subject(self, value):
        self.change_property(self.SUBJECT, value)

    @property
    def object(self):
        return self._object

    @object.setter
    def object(self, value):
        self.change_property(self.OBJECT, value)

    def __str__(self):
        return '<%s - %s>' % (self.subject, self.object)

    def __repr__(self):
        return self.__str__()


class ComplexNotion(Notion):
    """
    Complex notion is a notion that relates with other notions (objects)
    """
    def __init__(self, name, owner=None):
        super(ComplexNotion, self).__init__(name, owner)

        self._relations = []

        self.on(self.add_prefix(Relation.SUBJECT, self.SET_PREFIX), self.do_relation)
        self.on_forward(self.do_forward)

    def do_relation(self, *message, **context):
        relation = context.get(self.SENDER)

        if context[self.OLD_VALUE] == self and relation in self._relations:
            self._relations.remove(relation)
            return True

        elif context[self.NEW_VALUE] == self and relation not in self._relations:
            self._relations.append(relation)
            return True

    def do_forward(self, *message, **context):
        if self._relations:
            return self._relations[0] if len(self._relations) == 1 else tuple(self.relations)

    @property
    def relations(self):
        return self._relations


class NextRelation(Relation):
    """
    Next relation checks for additional condition when relation passed forward
    """
    def __init__(self, subj, obj, condition=None, ignore_case=False, owner=None):
        super(NextRelation, self).__init__(subj, obj, owner)
        self.ignore_case = ignore_case

        if condition:
            self.set_condition(condition)
        else:
            self.condition_access = TRUE_CONDITION

        self.on(self.can_pass, self.next_handler)

    def check_condition(self, message, context):
        return self.condition_access.check(message, context)

    def can_pass(self, *message, **context):
        if self.is_forward(message):  # We use 0 rank to make condition prevail other forward command
            return self.check_condition(message, context)

    def next_handler(self, *message, **context):
        return self.object

    def set_condition(self, value):
        self.condition_access = Condition(value, self.ignore_case)

    @property
    def condition(self):
        return self.condition_access.value

    @condition.setter
    def condition(self, value):
        self.set_condition(value)


class ActionRelation(Relation):
    """
    Action relation performs an action and moves forward
    """
    def __init__(self, subj, obj, action, owner=None):
        super(ActionRelation, self).__init__(subj, obj, owner)
        self._action_access = Access(action)

        self.on_forward(self.do_act)

    def do_act(self, *message, **context):
        action_result = self._action_access(*message, **context)

        if action_result is not None and self.object is not None:
            return action_result, self.object
        else:
            return action_result if action_result is not None else self.object

    @property
    def action(self):
        return self._action_access.value

    @action.setter
    def action(self, value):
        self._action_access = Access(value)


class Process(Handler):
    """
    Process is a walker from an abstract to abstract, asking them for the next one with a query
    It has the current abstract and the message to process; when new abstract appears,
    the new queue item with current and message is created
    """
    NEW = 'new'
    OK = 'ok'
    STOP = 'stop'

    CURRENT = 'current'
    MESSAGE = 'message'
    QUERY = 'query'

    EMPTY_MESSAGE = 'empty_' + MESSAGE

    def __init__(self):
        super(Process, self).__init__()

        self._queue, self.context = NotifyList(self.update_fields), {}
        self.new_queue_item({})

        self.query = Element.NEXT

        self.setup_handlers()

    def new_queue_item(self, values):
        """
        Generate the new queue item and add it to the queue updated with values
        """
        item = NotifyDict(self.update_fields,
                          ((self.CURRENT, values.get(self.CURRENT)),
                           (self.MESSAGE, values.get(self.MESSAGE, NotifyList(self.update_message)))))

        self._queue.append(item)

        return item

    def to_queue(self, values):
        """
        Put the new item in the queue, updating the empty one, if presents
        """
        if not self.message:
            self._queue[-1].update(values)  # No need to keep the empty one in the queue
        else:
            if not self.CURRENT in values:
                values[self.CURRENT] = self.current  # It is better to keep the current current

            self.new_queue_item(values)

    def set_message(self, message, insert=False):
        """
        Set the current message or inserts in the front of current message if insert = True
        """
        message = NotifyList(self.update_message, [message] if not is_list(message) else message)

        if insert:
            message.extend(self.message)

        self._queue[-1][self.MESSAGE] = message

    def update_message(self):
        self.message = self._queue[-1].get(self.MESSAGE)

    def update_fields(self):
        top = self._queue[-1]
        self.message = top.get(self.MESSAGE)
        self.current = top.get(self.CURRENT)

    def skip(self):
        """
        Skip: remove current and the next item from the queue
        """
        while not self.message and self._queue:  # Looking for the item to skip
            self.do_queue_pop()

        # It is ok if the message is empty to ignore skip
        if self.message:
            self.message.pop(0)

    # Events #
    def can_push_queue(self):
        """
        Queue push: if the head of the message is an Abstract - we make the new queue item and get ready to query it
        """
        return Access.get_access(self.message[0], True).mode in Access.CACHEABLE

    def do_queue_push(self):
        self.to_queue({self.CURRENT: self.message.pop(0),
                       self.MESSAGE: [self.QUERY]})  # Adding query command to start from asking

    def can_pop_queue(self):
        """
        Queue pop: when current queue item is empty we can remove it
        """
        return len(self._queue) > 1

    def do_queue_pop(self):
        self._queue.pop()

    def do_query(self):
        self.message.pop(0)

        # If abstract returns False/None, we just continue to the next one
        return getattr(self.current, Access.CACHE_ATTR)(self.query, **self.context) or True

    def can_clear_message(self, *message):
        """
        Cleanup: remove empty message item
        """
        return not message[0]

    def do_clear_message(self):
        self.message.pop(0)

    def do_finish(self):
        return self.message.pop(0)

    def update_tags(self):
        """
        Check and return the new tags
        """
        tags = set()

        if self.message:
            tags.add(self.MESSAGE)

            if is_string(self.message[0]):
                tags.add(Condition.STRING)
            elif self.message[0] is True or self.message[0] is False:
                tags.add(Condition.BOOLEAN)
            elif isinstance(self.message[0], dict):
                tags.add(Condition.DICT)

        else:
            tags.add(self.EMPTY_MESSAGE)

        if self.current:
            tags.add(self.CURRENT)

        return tags

    def setup_handlers(self):
        """
        Init handlers
        """
        self.on((self.STOP, self.OK), self.do_finish, Condition.STRING)
        self.on((True, False), self.do_finish, Condition.BOOLEAN)

        self.on(self.QUERY, self.do_query, Condition.STRING, self.CURRENT)

        self.on(self.can_push_queue, self.do_queue_push, self.MESSAGE)
        self.on(self.can_pop_queue, self.do_queue_pop, self.EMPTY_MESSAGE)

        self.on(self.can_clear_message, self.do_clear_message, self.MESSAGE)

    def on_handle(self, message, context):
        """
        Starting the handle loop
        """
        if has_first(message, self.NEW):  # Very special case
            message.pop(0)

            self.to_queue({self.MESSAGE: message, self.CURRENT: None})
            del self._queue[:-1]  # And kill the rest

            self.context = context
        else:
            self.to_queue({self.MESSAGE: message})
            self.context.update(context)

    def handle(self, message, context):
        """
        Process' handle works in step-by-step manner, processing message and then popping the queue
        """
        self.on_handle(list(message), context)

        result = self.NO_HANDLE
<commit_msg>More optimizations<commit_after># Universal Translator base classes
# (c) krvss 2011-2014

from inspect import getargspec
from operator import attrgetter

from utils import *

# TODO remove
logging = False
import time
def set_logging(value):
    global logging
    logging = value


class Abstract(object):
    """
    Base abstract class for all communicable objects
    """
    def __call__(self, *message, **context):
        raise NotImplementedError('Method not implemented')


class Access(Abstract):
    """
    Access provides abstract-like access to any object
    """
    CALL = 'call'
    ABSTRACT = 'abstract'
    FUNCTION = 'function'
    VALUE = 'value'
    OTHER = 'other'

    CACHE_ATTR = '__access__'
    CACHEABLE = (CALL, FUNCTION)

    def __init__(self, value):
        self._value = value
        self._mode, self._spec = self.OTHER, self.OTHER
        self._call = self.call_direct

        self.setup()

    def __eq__(self, other):
        if isinstance(other, Access):
            return other._value == self._value

        return other == self._value

    def __repr__(self):
        return self.__str__()

    def __str__(self):
        return '%s, %s: %s' % (self._mode, self._spec, self._value)

    def setup(self):
        """
        Init type information
        """
        if isinstance(self._value, Abstract):
            self._mode, self._spec = self.CALL, self.ABSTRACT

        elif callable(self._value):
            self._mode, self._spec = self.FUNCTION, getargspec(self._value)

            if self._spec.varargs and not self._spec.keywords:
                self._call = self.call_args

            elif self._spec.keywords and not self._spec.varargs:
                self._call = self.call_kwargs

            elif self._spec.varargs and self._spec.keywords:
                return  # default call_direct is good here

            elif not self._spec.varargs and not self._spec.keywords and not self._spec.args:
                self._call = self.call_noargs

            elif self._spec.args:
                self._defaults_len = len(self._spec.defaults) - 1 if self._spec.defaults else -1
                self._reversed_args = list(reversed(self._spec.args))

                self._call = self.call_general

        else:
            self._mode = self.VALUE
            self._call = self.call_value

    def __call__(self, *message, **context):
        return self._call(message, context)

    def call_direct(self, message, context):
        return self._value(*message, **context)

    def call_args(self, message, context):
        return self._value(*message)

    def call_kwargs(self, message, context):
        return self._value(**context)

    def call_noargs(self, message, context):
        return self._value()

    def call_general(self, message, context):
        i, args = self._defaults_len, {}
        for arg in self._reversed_args:
            if arg != 'self':
                args[arg] = context[arg] if arg in context else self._spec.defaults[i] if i >= 0 else None
                i -= 1

        return self._value(**args)

    def call_value(self, message, context):
        return self._value

    value = property(attrgetter('_value'))
    mode = property(attrgetter('_mode'))
    spec = property(attrgetter('_spec'))

    @staticmethod
    def get_access(obj, cache=False):
        if hasattr(obj, Access.CACHE_ATTR):
            access = getattr(obj, Access.CACHE_ATTR)
        else:
            access = Access(obj)

            if cache and access._mode in Access.CACHEABLE:
                setattr(obj, Access.CACHE_ATTR, access)

        return access


class Condition(Access):
    """
    Condition is an Access that checks the possibility of the access according to the message and context
    """
    NUMBER = 'number'
    LIST = 'list'
    DICT = 'dict'
    STRING = 'string'
    REGEX = 'regex'
    BOOLEAN = 'bool'

    NO_CHECK = -1, None

    def __init__(self, value, ignore_case=False, *tags):
        self._ignore_case, self.tags = ignore_case, frozenset(tags)
        self.check, self._conditions = self.check_compare, tuple([self])
        super(Condition, self).__init__(value)

    def setup(self):
        """
        Setting additional mode info
        """
        super(Condition, self).setup()

        if self._mode == self.FUNCTION:
            self.check = self.check_function

        elif is_number(self._value):
            self._spec = self.NUMBER

        elif is_list(self._value):
            self._spec, self.check = self.LIST, self.check_list
            self._conditions = tuple([Condition(c, self._ignore_case) for c in self._value])

        elif is_string(self._value):
            self._spec, self.check = self.STRING, self.check_string
            self._value_len = len(self._value)

            if self._ignore_case:
                self._value = self._value.upper()

        elif is_regex(self._value):
            self._spec, self.check = self.REGEX, self.check_regex

        elif isinstance(self._value, dict):
            self._spec = self.DICT

        elif type(self._value) == bool:
            self._spec, self.check = self.BOOLEAN, self.check_boolean

    def check_function(self, message, context):
        check = self._call(message, context)

        # Do we work with (rank, check) format?
        if get_len(check) == 2:
            return check
        elif check:
            return 0 if check is True else check, check
        else:
            return self.NO_CHECK

    def check_regex(self, message, context):
        check = self._value.match(message[0]) if message else None

        if check:
            rank = check.end() - check.start()  # Match length is the rank
            check = check.group(0)
        else:
            rank = - 1

        return rank, check

    def check_string(self, message, context):
        try:
            message0 = message[0][:self._value_len]

            if self._ignore_case:
                message0 = message0.upper()

            if message0 == self._value:
                return self._value_len, self._value

        except (TypeError, IndexError):
            pass

        return self.NO_CHECK

    def check_boolean(self, message, context):
        if message and message[0] is self._value:
            return 0, self._value
        else:
            return self.NO_CHECK

    def check_compare(self, message, context):
        if has_first(message, self._value):
            return max(get_len(self._value), 0), self._value
        else:
            return self.NO_CHECK

    def check_list(self, message, context):
        rank, check = self.NO_CHECK

        for condition in self._conditions:
            c_rank, c_check = condition.check(message, context)

            if c_rank > rank:
                rank, check = c_rank, c_check

        return rank, check

    @property
    def list(self):
        return self._conditions


# Condition that is always satisfied
class TrueCondition(Condition):
    def __init__(self):
        super(TrueCondition, self).__init__(id(self))
        self.check = lambda message, context: (0, True)


TRUE_CONDITION = TrueCondition()


class Event(Access):
    """
    Event is an Access that allows to attach Pre and Post custom functions before and after access
    """
    RESULT = 'result'

    def __init__(self, value):
        super(Event, self).__init__(value)
        self.pre_event, self.post_event = None, None

    def run(self, message, context):
        if self.pre_event:
            pre_result = self.pre_event.run(message, context)

            if pre_result[0] is not None:
                return pre_result

        result = self._call(message, context)

        if self.post_event:
            context[self.RESULT] = result
            post_result = self.post_event.run(message, context)

            if post_result[0] is not None:
                return post_result

        return result, self._value

    @property
    def pre(self):
        return self.pre_event.value if self.pre_event else None

    @pre.setter
    def pre(self, value):
        self.pre_event = Event(value) if value is not None else None

    @property
    def post(self):
        return self.post_event.value if self.post_event else None

    @post.setter
    def post(self, value):
        self.post_event = Event(value) if value is not None else None


class Handler(Abstract):
    """
    Handler is a class for the routing of messages to processing functions (events) basing on specified conditions
    """
    ANSWER = 'answer'
    SENDER = 'sender'
    CONDITION = 'condition'
    EVENT = 'event'
    RANK = 'rank'

    NO_HANDLE = (False, -1, None)

    def __init__(self):
        Access.get_access(self, True)

        self._tags = set()
        self._events, self.active_events = [], tuple()
        self.unknown_event = None

    def on_access(self, condition_access, event_access):
        """
        Adding the accesses
        """
        if (condition_access, event_access) not in self._events:
            self._events.append((condition_access, event_access))

            self.update_events()

    def on(self, condition, event, *tags):
        """
        Adding the condition - event pair
        """
        self.on_access(Condition(condition, False, *tags), Event(event))

    def on_any(self, event):
        """
        Adding the event, without condition it will trigger on any message
        """
        self.on_access(TRUE_CONDITION, Event(event))

    def off(self, condition, event):
        """
        Removing the condition - event pair
        """
        if (condition, event) in self._events:
            self._events.remove((condition, event))

            self.update_events()

    def off_any(self, event):
        """
        Removing the event
        """
        self.off(TRUE_CONDITION, event)

    def off_condition(self, condition):
        """
        Removing all the events for the condition
        """
        self._events = filter(lambda e: not (e[0] == condition), self._events)

        self.update_events()

    def off_event(self, event):
        """
        Remove all occurrences of the event
        """
        self._events = filter(lambda e: not (e[1] == event), self._events)

        self.update_events()

    def get_events(self, condition=None):
        """
        Getting the events for the specified condition
        """
        if condition:
            return [e[1] for e in self._events if has_first(e, condition)]
        else:
            return [e[1] for e in self._events if e[0] == TRUE_CONDITION]

    def clear_events(self):
        del self._events[:]
        self.active_events = tuple()

    def update_tags(self):
        """
        Called when update needed, returns new tags
        """
        return self._tags

    def update_events(self):
        """
        Called when event update needed (for example, after new event was added or tags was changed)
        """
        self.active_events = tuple(filter(lambda e: e[0].tags.issubset(self._tags), self._events))

    def update(self):
        """
        Update is called by the object itself when something is changed and it could change tags
        """
        new_tags = self.update_tags()

        if new_tags != self._tags:
            self.tags = new_tags

            self.update_events()

    def handle(self, message, context):
        """
        Calling events basing on condition
        """
        global logging
        if logging:
            print ">>> %s.handle of |%s|, events %s" % (type(self), message, len(self.active_events))

        check, rank, event_found = self.NO_HANDLE

        if not self.SENDER in context:
            context[self.SENDER] = self

        # Searching for the best event
        for condition_access, event_access in self.active_events:
            if logging:
                t1 = time.time()

            # Condition check, if no condition the result is true with zero rank
            c_rank, c_check = condition_access.check(message, context)

            if logging:
                print "trying %s, spent %s" % (condition_access.value, time.time() - t1)

            if c_rank > rank:
                rank, check, event_found = c_rank, c_check, event_access

        # Running the best event
        if rank >= 0:
            context.update({self.RANK: rank, self.CONDITION: check, self.EVENT: event_found.value})

            # Call event, add the condition result to the context
            result, event_found = event_found.run(message, context)
        else:
            result = False

            # There is a way to handle unknown message
            if self.unknown_event:
                return self.unknown_event.run(message, context)

        return result, rank, event_found

    def __call__(self, *message, **context):
        """
        Answer depends on the context
        """
        answer_mode = context.pop(self.ANSWER, None)  # Applicable only for a top level
        result = self.handle(message, context)

        if answer_mode == self.RANK:
            return result[0], result[1]

        return result[0]  # No need to know the details

    @property
    def tags(self):
        return self._tags

    @tags.setter
    def tags(self, value):
        self._tags = frozenset(value)

    @property
    def events(self):
        return self._events


class Element(Handler):
    """
    Element is a part of a complex system (e.g. graph)
    """
    NEXT = 'next'
    PREVIOUS = 'previous'

    OWNER = 'owner'
    SET_PREFIX = 'set'
    NAME = 'name'
    OLD_VALUE = 'old-value'
    NEW_VALUE = 'new-value'

    SEP = '_'

    FORWARD = [NEXT]
    BACKWARD = [PREVIOUS]

    def __init__(self, owner=None):
        super(Element, self).__init__()

        self._owner, self.owner = None, owner

    def is_forward(self, message):
        return message and message[0] in self.FORWARD

    def can_go_forward(self, *message, **context):
        return self.is_forward(message)

    def on_forward(self, event):
        self.on(self.can_go_forward, event)

    def off_forward(self):
        self.off_condition(self.can_go_forward)

    def is_backward(self, message):
        return message and message[0] in self.BACKWARD

    def can_go_backward(self, *message, **context):
        return self.is_backward(message)

    def on_backward(self, event):
        self.on(self.can_go_backward, event)

    def off_backward(self):
        self.off_condition(self.can_go_backward)

    @staticmethod
    def add_prefix(msg, prefix):
        """
        Add prefix to the message
        """
        event_name = msg

        if not msg.startswith(prefix):
            event_name = Element.SEP.join([prefix, event_name])

        return event_name

    def change_property(self, name, value):
        """
        Set the property to the new value with notification, if needed
        """
        old_value = getattr(self, name)
        if old_value == value:
            return

        set_message, context = self.add_prefix(name, self.SET_PREFIX), \
                               {self.NEW_VALUE: value, self.OLD_VALUE: old_value, self.SENDER: self}

        if old_value and Access.get_access(old_value, True).spec == Access.ABSTRACT:
            old_value(set_message, **context)

        setattr(self, '_%s' % name, value)

        if value and Access.get_access(value, True).spec == Access.ABSTRACT:
            value(set_message,  **context)

        return True

    @property
    def owner(self):
        return self._owner

    @owner.setter
    def owner(self, value):
        self.change_property('owner', value)


class Notion(Element):
    """
    Notion is an element with name
    """
    def __init__(self, name, owner=None):
        super(Notion, self).__init__(owner)
        self._name, self.name = None, name

    def __str__(self):
        return '"%s"' % self.name

    def __repr__(self):
        rep = '<%s(%s' % (get_object_name(self.__class__), self.__str__())

        if self.owner:
            rep += ', %s' % self.owner

        return rep + ')>'

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, value):
        self.change_property(self.NAME, value)


class ActionNotion(Notion):
    """
    Action notion is a notion with specified forward handler
    """
    def __init__(self, name, action, owner=None):
        super(ActionNotion, self).__init__(name, owner)
        self.on_forward(action)

    @property
    def action(self):
        value = self.get_events(self.can_go_forward)
        return value[0] if value else None

    @action.setter
    def action(self, value):
        self.off_forward()
        self.on_forward(value)


class Relation(Element):
    """
    Relation is a connection between one or more elements: subject -> object
    """
    SUBJECT = 'subject'
    OBJECT = 'object'

    def __init__(self, subj, obj, owner=None):
        super(Relation, self).__init__(owner)
        self._object = self._subject = None
        self.subject, self.object = subj, obj

    @property
    def subject(self):
        return self._subject

    @subject.setter
    def subject(self, value):
        self.change_property(self.SUBJECT, value)

    @property
    def object(self):
        return self._object

    @object.setter
    def object(self, value):
        self.change_property(self.OBJECT, value)

    def __str__(self):
        return '<%s - %s>' % (self.subject, self.object)

    def __repr__(self):
        return self.__str__()


class ComplexNotion(Notion):
    """
    Complex notion is a notion that relates with other notions (objects)
    """
    def __init__(self, name, owner=None):
        super(ComplexNotion, self).__init__(name, owner)

        self._relations = []

        self.on(self.add_prefix(Relation.SUBJECT, self.SET_PREFIX), self.do_relation)
        self.on_forward(self.do_forward)

    def do_relation(self, *message, **context):
        relation = context.get(self.SENDER)

        if context[self.OLD_VALUE] == self and relation in self._relations:
            self._relations.remove(relation)
            return True

        elif context[self.NEW_VALUE] == self and relation not in self._relations:
            self._relations.append(relation)
            return True

    def do_forward(self, *message, **context):
        if self._relations:
            return self._relations[0] if len(self._relations) == 1 else tuple(self.relations)

    @property
    def relations(self):
        return self._relations


class NextRelation(Relation):
    """
    Next relation checks for additional condition when relation passed forward
    """
    def __init__(self, subj, obj, condition=None, ignore_case=False, owner=None):
        super(NextRelation, self).__init__(subj, obj, owner)
        self.ignore_case = ignore_case

        if condition:
            self.set_condition(condition)
        else:
            self.condition_access = TRUE_CONDITION

        self.on(self.can_pass, self.next_handler)

    def check_condition(self, message, context):
        return self.condition_access.check(message, context)

    def can_pass(self, *message, **context):
        if self.is_forward(message):  # We use 0 rank to make condition prevail other forward command
            return self.check_condition(message, context)

    def next_handler(self, *message, **context):
        return self.object

    def set_condition(self, value):
        self.condition_access = Condition(value, self.ignore_case)

    @property
    def condition(self):
        return self.condition_access.value

    @condition.setter
    def condition(self, value):
        self.set_condition(value)


class ActionRelation(Relation):
    """
    Action relation performs an action and moves forward
    """
    def __init__(self, subj, obj, action, owner=None):
        super(ActionRelation, self).__init__(subj, obj, owner)
        self._action_access = Access(action)

        self.on_forward(self.do_act)

    def do_act(self, *message, **context):
        action_result = self._action_access(*message, **context)

        if action_result is not None and self.object is not None:
            return action_result, self.object
        else:
            return action_result if action_result is not None else self.object

    @property
    def action(self):
        return self._action_access.value

    @action.setter
    def action(self, value):
        self._action_access = Access(value)


class Process(Handler):
    """
    Process is a walker from an abstract to abstract, asking them for the next one with a query
    It has the current abstract and the message to process; when new abstract appears,
    the new queue item with current and message is created
    """
    NEW = 'new'
    OK = 'ok'
    STOP = 'stop'

    CURRENT = 'current'
    MESSAGE = 'message'
    QUERY = 'query'

    EMPTY_MESSAGE = 'empty_' + MESSAGE

    def __init__(self):
        super(Process, self).__init__()

        self._queue, self.context = NotifyList(self.update_fields), {}
        self.new_queue_item({})

        self.query = Element.NEXT

        self.setup_handlers()

    def new_queue_item(self, values):
        """
        Generate the new queue item and add it to the queue updated with values
        """
        item = NotifyDict(self.update_fields,
                          ((self.CURRENT, values.get(self.CURRENT)),
                           (self.MESSAGE, values.get(self.MESSAGE, NotifyList(self.update_message)))))

        self._queue.append(item)

        return item

    def to_queue(self, values):
        """
        Put the new item in the queue, updating the empty one, if presents
        """
        if not self.message:
            self._queue[-1].update(values)  # No need to keep the empty one in the queue
        else:
            if not self.CURRENT in values:
                values[self.CURRENT] = self.current  # It is better to keep the current current

            self.new_queue_item(values)

    def set_message(self, message, insert=False):
        """
        Set the current message or inserts in the front of current message if insert = True
        """
        message = NotifyList(self.update_message, [message] if not is_list(message) else message)

        if insert:
            message.extend(self.message)

        self._queue[-1][self.MESSAGE] = message

    def update_message(self):
        self.message = self._queue[-1].get(self.MESSAGE)

    def update_fields(self):
        top = self._queue[-1]
        self.message = top.get(self.MESSAGE)
        self.current = top.get(self.CURRENT)

    def skip(self):
        """
        Skip: remove current and the next item from the queue
        """
        while not self.message and self._queue:  # Looking for the item to skip
            self.do_queue_pop()

        # It is ok if the message is empty to ignore skip
        if self.message:
            self.message.pop(0)

    # Events #
    def can_push_queue(self):
        """
        Queue push: if the head of the message is an Abstract - we make the new queue item and get ready to query it
        """
        return Access.get_access(self.message[0], True).mode in Access.CACHEABLE

    def do_queue_push(self):
        self.to_queue({self.CURRENT: self.message.pop(0),
                       self.MESSAGE: [self.QUERY]})  # Adding query command to start from asking

    def can_pop_queue(self):
        """
        Queue pop: when current queue item is empty we can remove it
        """
        return len(self._queue) > 1

    def do_queue_pop(self):
        self._queue.pop()

    def do_query(self):
        self.message.pop(0)

        # If abstract returns False/None, we just continue to the next one
        return getattr(self.current, Access.CACHE_ATTR)(self.query, **self.context) or True

    def can_clear_message(self, *message):
        """
        Cleanup: remove empty message item
        """
        return not message[0]

    def do_clear_message(self):
        self.message.pop(0)

    def do_finish(self):
        return self.message.pop(0)

    def update_tags(self):
        """
        Check and return the new tags
        """
        tags = set()

        if self.message:
            tags.add(self.MESSAGE)

            if is_string(self.message[0]):
                tags.add(Condition.STRING)
            elif type(self.message[0]) == bool:
                tags.add(Condition.BOOLEAN)
            elif isinstance(self.message[0], dict):
                tags.add(Condition.DICT)
            else:
                tags.add(Condition.OTHER)

        else:
            tags.add(self.EMPTY_MESSAGE)

        if self.current:
            tags.add(self.CURRENT)

        return tags

    def setup_handlers(self):
        """
        Init handlers
        """
        self.on((self.STOP, self.OK), self.do_finish, Condition.STRING)
        self.on((True, False), self.do_finish, Condition.BOOLEAN)

        self.on(self.QUERY, self.do_query, Condition.STRING, self.CURRENT)

        self.on(self.can_push_queue, self.do_queue_push, Condition.OTHER)
        self.on(self.can_pop_queue, self.do_queue_pop, self.EMPTY_MESSAGE)

        self.on(self.can_clear_message, self.do_clear_message, self.MESSAGE)

    def on_handle(self, message, context):
        """
        Starting the handle loop
        """
        if has_first(message, self.NEW):  # Very special case
            message.pop(0)

            self.to_queue({self.MESSAGE: message, self.CURRENT: None})
            del self._queue[:-1]  # And kill the rest

            self.context = context
        else:
            self.to_queue({self.MESSAGE: message})
            self.context.update(context)

    def handle(self, message, context):
        """
        Process' handle works in step-by-step manner, processing message and then popping the queue
        """
        self.on_handle(list(message), context)

        result = self.NO_HANDLE

{'fraction_non_alphanumeric': 0.0998539753295162, 'fraction_numerical': 0.00251700418860239, 'mean_word_length': 2.1798020527859236, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2150850', 'n_tokens_mistral': 15174, 'n_tokens_neox': 14512, 'n_words': 4127}