<commit_before>import logging
import inspect

from botstory.ast import stack_utils
from . import parser, callable, forking
from .. import di, matchers
from ..integrations import mocktracker

logger = logging.getLogger(__name__)


@di.desc(reg=False)
class StoryProcessor:
    def __init__(self, parser_instance, library, middlewares=[]):
        self.library = library
        self.middlewares = middlewares
        self.parser_instance = parser_instance
        self.tracker = mocktracker.MockTracker()

    @di.inject()
    def add_tracker(self, tracker):
        logger.debug('add_tracker')
        logger.debug(tracker)
        if not tracker:
            return
        self.tracker = tracker

    async def match_message(self, message):
        """
        because match_message is recursive we split function to
        public match_message and private _match_message

        :param message:
        :return:
        """
        logger.debug('')
        logger.debug('> match_message <')
        logger.debug('')
        logger.debug('  {} '.format(message))
        logger.debug('self.tracker')
        logger.debug(self.tracker)
        self.tracker.new_message(
            user=message and message['user'],
            data=message['data'],
        )

        stack = message['session']['stack']

        waiting_for = None

        # TODO: it seems that it could be merge with loop in _match_message
        # because they do similar things - bubble up the stack
        while True:
            # but it seems that we have hierarchy of callable
            # stories so we should drop current stack element
            # because it is over and return to the previous story
            waiting_for = await self._match_message(message)

            if waiting_for and \
                    not isinstance(waiting_for, callable.EndOfStory) or \
                            len(stack) <= 1:
                break

            stack.pop()

        return waiting_for

    async def _match_message(self, message):
        session = message['session']
        logger.debug('_match_message')

        compiled_story = None
        validation_result = None
        stack_tail = None

        if len(session['stack']) > 0:
            logger.debug('  check stack')
            logger.debug('    session.stack = {}'.format(session['stack']))

            # looking for first valid matcher
            while True:
                if len(session['stack']) == 0:
                    # we have reach the bottom of stack
                    logger.debug('  we have reach the bottom of stack '
                                 'so no once has receive this message')
                    return None
                stack_tail = session['stack'].pop()
                logger.debug('stack_tail {}'.format(stack_tail))
                if not stack_tail['data']:
                    break
                validator = matchers.deserialize(stack_tail['data'])
                logger.debug('validator {}'.format(validator))
                if hasattr(validator, 'immediately') and validator.immediately:
                    logger.debug('return from  _match_message')
                    return True

                if getattr(validator, 'new_scope', False):
                    # we are start new story line here
                    # so we don't need to check whether we still have tail of story
                    break

                logger.debug("stack_tail['topic'] {}".format(stack_tail['topic']))
                logger.debug("session['stack'] {}".format(session['stack']))
                logger.debug("self.library.get_story_by_topic(stack_tail['topic'], stack=session['stack'])")
                logger.debug(self.library.get_story_by_topic(stack_tail['topic'], stack=session['stack']))
                if stack_tail['step'] < len(
                        self.library.get_story_by_topic(stack_tail['topic'], stack=session['stack']).story_line
                ):
                    # if we haven't reach last step in list of story so we can parse result
                    break

            logger.debug('    after check session.stack = {}'.format(session['stack']))
            logger.debug('      stack_tail = {}'.format(stack_tail))
            if stack_tail and stack_tail['data']:
                logger.debug('  got it!')
                validator = matchers.deserialize(stack_tail['data'])
                validation_result = validator.validate(message)
                logger.debug('      validation_result {}'.format(validation_result))
                if not not validation_result:
                    # it seems we find stack item that matches our message
                    compiled_story = self.library.get_story_by_topic(stack_tail['topic'], stack=session['stack'])
            else:
                # TODO: we shouldn't get such case
                # because it shows that we have stack that can't catch incoming message
                # (?) maybe possible case when it is in a middle of hierarchy
                # and comes because the top one didn't match message

                # assert stack_tail is None

                stack_tail = None

        # if len(session['stack']) == 0:
        #     session['stack'] = [build_empty_stack_item()]

        if not compiled_story:
            compiled_story = self.library.get_right_story(message)

        if not compiled_story:
            return True

        # TODO: try to have single process_next_part_of_story
        # and simplify loop in general

        if not stack_tail:
            session['stack'].append(stack_utils.build_empty_stack_item())
        else:
            session['stack'].append(stack_tail)

        # TODO: very likely that we won't use `stack_tail`
        # for example in case when we don't have right switch case
        _, waiting_for = await self.process_next_part_of_story({
            'step': session['stack'][-1]['step'],
            'story': compiled_story,
            'stack': message['session']['stack'],
        }, validation_result, message)

        return waiting_for

        # TODO: still here to show how it could be simplified
        # return await self.process_story(
        #     idx=0,
        #     message=message,
        #     compiled_story=compiled_story,
        #     session=session,
        # )

    async def process_next_part_of_story(self, received_data, validation_result, message):
        logger.debug('')
        logger.debug('process_next_part_of_story')
        logger.debug('')
        logger.debug('  topic {}'.format(received_data['story'].topic))
        logger.debug('  step {} ({})'.format(received_data['step'], len(received_data['story'].story_line)))

        for m in self.middlewares:
            if hasattr(m, 'process'):
                received_data = m.process(received_data, validation_result)

        logger.debug('  session.stack = {}'.format(message['session']['stack']))
        logger.debug('! after topic {}'.format(received_data['story'].topic))
        logger.debug('! after step {}'.format(received_data['step']))

        waiting_for = await self.process_story(
            idx=received_data['step'],
            message=message,
            compiled_story=received_data['story'],
            session=message['session'],
        )

        return received_data['story'], waiting_for

    async def process_story(self, session, message, compiled_story,
                            idx=0, story_args=[], story_kwargs={},
                            ):
        logger.debug('')
        logger.debug('process_story')
        logger.debug('')

        logger.debug('! topic {}'.format(compiled_story.topic))
        logger.debug('! step {}'.format(idx))
        logger.debug('  story {}'.format(compiled_story))
        logger.debug('  message {}'.format(message))
        logger.debug('  session.stack {} ({})'.format(session['stack'], len(session['stack'])))
        logger.debug(<commit_msg>remove extra comments<commit_after>import logging
import inspect

from botstory.ast import stack_utils
from . import parser, callable, forking
from .. import di, matchers
from ..integrations import mocktracker

logger = logging.getLogger(__name__)


@di.desc(reg=False)
class StoryProcessor:
    def __init__(self, parser_instance, library, middlewares=[]):
        self.library = library
        self.middlewares = middlewares
        self.parser_instance = parser_instance
        self.tracker = mocktracker.MockTracker()

    @di.inject()
    def add_tracker(self, tracker):
        logger.debug('add_tracker')
        logger.debug(tracker)
        if not tracker:
            return
        self.tracker = tracker

    async def match_message(self, message):
        """
        because match_message is recursive we split function to
        public match_message and private _match_message

        :param message:
        :return:
        """
        logger.debug('')
        logger.debug('> match_message <')
        logger.debug('')
        logger.debug('  {} '.format(message))
        logger.debug('self.tracker')
        logger.debug(self.tracker)
        self.tracker.new_message(
            user=message and message['user'],
            data=message['data'],
        )

        stack = message['session']['stack']

        waiting_for = None

        # TODO: it seems that it could be merge with loop in _match_message
        # because they do similar things - bubble up the stack
        while True:
            # but it seems that we have hierarchy of callable
            # stories so we should drop current stack element
            # because it is over and return to the previous story
            waiting_for = await self._match_message(message)

            if waiting_for and \
                    not isinstance(waiting_for, callable.EndOfStory) or \
                            len(stack) <= 1:
                break

            stack.pop()

        return waiting_for

    async def _match_message(self, message):
        session = message['session']
        logger.debug('_match_message')

        compiled_story = None
        validation_result = None
        stack_tail = None

        if len(session['stack']) > 0:
            logger.debug('  check stack')
            logger.debug('    session.stack = {}'.format(session['stack']))

            # looking for first valid matcher
            while True:
                if len(session['stack']) == 0:
                    # we have reach the bottom of stack
                    logger.debug('  we have reach the bottom of stack '
                                 'so no once has receive this message')
                    return None
                stack_tail = session['stack'].pop()
                logger.debug('stack_tail {}'.format(stack_tail))
                if not stack_tail['data']:
                    break
                validator = matchers.deserialize(stack_tail['data'])
                logger.debug('validator {}'.format(validator))
                if hasattr(validator, 'immediately') and validator.immediately:
                    logger.debug('return from  _match_message')
                    return True

                if getattr(validator, 'new_scope', False):
                    # we are start new story line here
                    # so we don't need to check whether we still have tail of story
                    break

                logger.debug("stack_tail['topic'] {}".format(stack_tail['topic']))
                logger.debug("session['stack'] {}".format(session['stack']))
                logger.debug("self.library.get_story_by_topic(stack_tail['topic'], stack=session['stack'])")
                logger.debug(self.library.get_story_by_topic(stack_tail['topic'], stack=session['stack']))
                if stack_tail['step'] < len(
                        self.library.get_story_by_topic(stack_tail['topic'], stack=session['stack']).story_line
                ):
                    # if we haven't reach last step in list of story so we can parse result
                    break

            logger.debug('    after check session.stack = {}'.format(session['stack']))
            logger.debug('      stack_tail = {}'.format(stack_tail))
            if stack_tail and stack_tail['data']:
                logger.debug('  got it!')
                validator = matchers.deserialize(stack_tail['data'])
                validation_result = validator.validate(message)
                logger.debug('      validation_result {}'.format(validation_result))
                if not not validation_result:
                    # it seems we find stack item that matches our message
                    compiled_story = self.library.get_story_by_topic(stack_tail['topic'], stack=session['stack'])
            else:
                # TODO: we shouldn't get such case
                # because it shows that we have stack that can't catch incoming message
                # (?) maybe possible case when it is in a middle of hierarchy
                # and comes because the top one didn't match message

                # assert stack_tail is None

                stack_tail = None

        if not compiled_story:
            compiled_story = self.library.get_right_story(message)

        if not compiled_story:
            return True

        if not stack_tail:
            session['stack'].append(stack_utils.build_empty_stack_item())
        else:
            session['stack'].append(stack_tail)

        # TODO: very likely that we won't use `stack_tail`
        # for example in case when we don't have right switch case
        _, waiting_for = await self.process_next_part_of_story({
            'step': session['stack'][-1]['step'],
            'story': compiled_story,
            'stack': message['session']['stack'],
        }, validation_result, message)

        return waiting_for

    async def process_next_part_of_story(self, received_data, validation_result, message):
        logger.debug('')
        logger.debug('process_next_part_of_story')
        logger.debug('')

        for m in self.middlewares:
            if hasattr(m, 'process'):
                received_data = m.process(received_data, validation_result)

        waiting_for = await self.process_story(
            idx=received_data['step'],
            message=message,
            compiled_story=received_data['story'],
            session=message['session'],
        )

        return received_data['story'], waiting_for

    async def process_story(self, session, message, compiled_story,
                            idx=0, story_args=[], story_kwargs={},
                            ):
        logger.debug('')
        logger.debug('process_story')
        logger.debug('')

        logger.debug('! topic {}'.format(compiled_story.topic))
        logger.debug('! step {}'.format(idx))
        logger.debug('  story {}'.format(compiled_story))
        logger.debug('  message {}'.format(message))
        logger.debug('  session.stack {} ({})'.format(session['stack'], len(session['stack'])))
        logger.debug(
{'fraction_non_alphanumeric': 0.1042509450228795, 'fraction_numerical': 0.000795808740632668, 'mean_word_length': 1.9908766362554542, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7025668', 'n_tokens_mistral': 3891, 'n_tokens_neox': 3713, 'n_words': 1004}