<commit_before>from __future__ import unicode_literals
from abc import ABCMeta, abstractmethod
from six import with_metaclass

from pygments.token import Token

from prompt_toolkit.document import Document
from prompt_toolkit.filters import Filter

from .utils import token_list_len

__all__ = (
    'HighlightSearchProcessor',
    'HighlightSelectionProcessor',
    'PasswordProcessor',
    'BracketsMismatchProcessor',
    'BeforeInput',
    'AfterInput',
    'ConditionalProcessor',
)


class Processor(with_metaclass(ABCMeta, object)):
    """
    Manipulate the tokenstream for a `BufferControl`.
    """
    @abstractmethod
    def run(self, cli, buffer, tokens):
        return tokens, lambda i: i

    def invalidation_hash(self, cli, buffer):
        return None


class HighlightSearchProcessor(Processor):
    """
    Processor that highlights search matches in the document.
    """
    def run(self, cli, buffer, tokens):
        isearch_state = buffer.isearch_state

        if isearch_state:
            # For each search match, replace the Token.
            for index in buffer.document.find_all(isearch_state.isearch_text):
                if index == buffer.cursor_position:
                    token = Token.SearchMatch.Current
                else:
                    token = Token.SearchMatch

                for x in range(index, index + len(isearch_state.isearch_text)):
                    tokens[x] = (token, tokens[x][1])

        return tokens, lambda i: i

    def invalidation_hash(self, cli, buffer):
        # When the search state changes, highlighting will be different.
        return (
            buffer.isearch_state,
            (buffer.isearch_state and buffer.isearch_state.isearch_text),

            # When we search for text, and the cursor position changes. The
            # processor has to be applied every time again, because the current match is highlighted
            # in another color.
            (buffer.isearch_state and buffer.isearch_state.isearch_text and buffer.cursor_position)
        )

<commit_msg>Never apply search highlighting when the input is returning.<commit_after>from __future__ import unicode_literals
from abc import ABCMeta, abstractmethod
from six import with_metaclass

from pygments.token import Token

from prompt_toolkit.document import Document
from prompt_toolkit.filters import Filter

from .utils import token_list_len

__all__ = (
    'HighlightSearchProcessor',
    'HighlightSelectionProcessor',
    'PasswordProcessor',
    'BracketsMismatchProcessor',
    'BeforeInput',
    'AfterInput',
    'ConditionalProcessor',
)


class Processor(with_metaclass(ABCMeta, object)):
    """
    Manipulate the tokenstream for a `BufferControl`.
    """
    @abstractmethod
    def run(self, cli, buffer, tokens):
        return tokens, lambda i: i

    def invalidation_hash(self, cli, buffer):
        return None


class HighlightSearchProcessor(Processor):
    """
    Processor that highlights search matches in the document.
    """
    def run(self, cli, buffer, tokens):
        isearch_state = buffer.isearch_state

        if isearch_state and not cli.is_returning:
            # For each search match, replace the Token.
            for index in buffer.document.find_all(isearch_state.isearch_text):
                if index == buffer.cursor_position:
                    token = Token.SearchMatch.Current
                else:
                    token = Token.SearchMatch

                for x in range(index, index + len(isearch_state.isearch_text)):
                    tokens[x] = (token, tokens[x][1])

        return tokens, lambda i: i

    def invalidation_hash(self, cli, buffer):
        # When the search state changes, highlighting will be different.
        return (
            buffer.isearch_state,
            cli.is_returning,
            (buffer.isearch_state and buffer.isearch_state.isearch_text),

            # When we search for text, and the cursor position changes. The
            # processor has to be applied every time again, because the current match is highlighted
            # in another color.
            (buffer.isearch_state and buffer.isearch_state.isearch_text and buffer.cursor_position)
        )


{'fraction_non_alphanumeric': 0.07183364839319471, 'fraction_numerical': 0.0004725897920604915, 'mean_word_length': 2.63659793814433, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5198461', 'n_tokens_mistral': 1141, 'n_tokens_neox': 1097, 'n_words': 344}