<commit_before>import aiohttp

import json
import logging
import re

import os
from typing import Text, List, Dict, Any, Union, Optional

from rasa.core import constants
from rasa.core.constants import INTENT_MESSAGE_PREFIX
from rasa.utils.endpoints import EndpointConfig

logger = logging.getLogger(__name__)


class NaturalLanguageInterpreter(object):
    async def parse(
        self, text: Text, message_id: Optional[Text] = None
    ) -> Dict[Text, Any]:
        raise NotImplementedError(
            "Interpreter needs to be able to parse messages into structured output."
        )

    @staticmethod
    def create(
        obj: Union[Text, "NaturalLanguageInterpreter"],
        endpoint: Optional[EndpointConfig] = None,
    ) -> "NaturalLanguageInterpreter":

        if isinstance(obj, NaturalLanguageInterpreter):
            return obj

        if not isinstance(obj, str):
            if obj is not None:
                logger.warning(
                    "Tried to create NLU interpreter "
                    "from '{}', which is not possible."
                    "Using RegexInterpreter instead."
                    "".format(obj)
                )
            return RegexInterpreter()

        if endpoint is None:
            if not os.path.exists(obj):
                logger.warning(
                    "No local NLU model '{}' found. Using RegexInterpreter instead.".format(
                        obj
                    )
                )
                return RegexInterpreter()
            else:
                return RasaNLUInterpreter(model_directory=obj)

        return RasaNLUHttpInterpreter(endpoint)


class RegexInterpreter(NaturalLanguageInterpreter):
    @staticmethod
    def allowed_prefixes():
        return INTENT_MESSAGE_PREFIX

    @staticmethod
    def _create_entities(parsed_entities: Dict[Text, Union[Text, List[Text]]], sidx: int, eidx: int) -> List[Dict[Text, Any]]:
        entities = []
        for k, vs in parsed_entities.items():
            if not isinstance(vs, list):
                vs = [vs]
            for value in vs:
                entities.append(
                    {
                        "entity": k,
                        "start": sidx,
                        "end": eidx,  # can't be more specific
                        "value": value,
                    }
                )
        return entities

    @staticmethod
    def _parse_parameters(
        entity_str: Text, sidx: int, eidx: int, user_input: Text
    ) -> List[Dict[Text, Any]]:
        if entity_str is None or not entity_str.strip():
            # if there is nothing to parse we will directly exit
            return []
<commit_msg>Formatting.<commit_after>import aiohttp

import json
import logging
import re

import os
from typing import Text, List, Dict, Any, Union, Optional

from rasa.core import constants
from rasa.core.constants import INTENT_MESSAGE_PREFIX
from rasa.utils.endpoints import EndpointConfig

logger = logging.getLogger(__name__)


class NaturalLanguageInterpreter(object):
    async def parse(
        self, text: Text, message_id: Optional[Text] = None
    ) -> Dict[Text, Any]:
        raise NotImplementedError(
            "Interpreter needs to be able to parse messages into structured output."
        )

    @staticmethod
    def create(
        obj: Union[Text, "NaturalLanguageInterpreter"],
        endpoint: Optional[EndpointConfig] = None,
    ) -> "NaturalLanguageInterpreter":

        if isinstance(obj, NaturalLanguageInterpreter):
            return obj

        if not isinstance(obj, str):
            if obj is not None:
                logger.warning(
                    "Tried to create NLU interpreter "
                    "from '{}', which is not possible."
                    "Using RegexInterpreter instead."
                    "".format(obj)
                )
            return RegexInterpreter()

        if endpoint is None:
            if not os.path.exists(obj):
                logger.warning(
                    "No local NLU model '{}' found. Using RegexInterpreter instead.".format(
                        obj
                    )
                )
                return RegexInterpreter()
            else:
                return RasaNLUInterpreter(model_directory=obj)

        return RasaNLUHttpInterpreter(endpoint)


class RegexInterpreter(NaturalLanguageInterpreter):
    @staticmethod
    def allowed_prefixes():
        return INTENT_MESSAGE_PREFIX

    @staticmethod
    def _create_entities(
        parsed_entities: Dict[Text, Union[Text, List[Text]]], sidx: int, eidx: int
    ) -> List[Dict[Text, Any]]:
        entities = []
        for k, vs in parsed_entities.items():
            if not isinstance(vs, list):
                vs = [vs]
            for value in vs:
                entities.append(
                    {
                        "entity": k,
                        "start": sidx,
                        "end": eidx,  # can't be more specific
                        "value": value,
                    }
                )
        return entities

    @staticmethod
    def _parse_parameters(
        entity_str: Text, sidx: int, eidx: int, user_input: Text
    ) -> List[Dict[Text, Any]]:
        if entity_str is None or not entity_str.strip():
            # if there is nothing to parse we will directly exit
            return []

{'fraction_non_alphanumeric': 0.07996289424860853, 'fraction_numerical': 0.0, 'mean_word_length': 1.7199798183652877, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5968982', 'n_tokens_mistral': 1414, 'n_tokens_neox': 1375, 'n_words': 411}